Added mutli-threaded extraction

src/archive.zig

@@ -84,7 +84,10 @@ pub fn extract(self: *Archive, alloc: std.mem.Allocator, io: Io, ext_loc: []cons
         self.super.root_ref,
     );
 
-    return Extract.extractSingleThreaded(alloc, io, root_inode, &self.cache, self.super, ext_loc, options);
+    return if (options.single_threaded)
+        Extract.extractSingleThreaded(alloc, io, root_inode, &self.cache, self.super, ext_loc, options)
+    else
+        Extract.extractAsync(alloc, io, root_inode, &self.cache, self.super, ext_loc, options);
 }
 
 // Superblock
@@ -188,7 +191,7 @@ test "FullExtraction" {
     var arc: Archive = try .init(alloc, io, archive_file);
     defer arc.deinit(io);
 
-    try arc.extract(alloc, io, TestFullExtractLocation, .default);
+    try arc.extract(alloc, io, TestFullExtractLocation, .default_single_threaded);
 }
 
 const LinuxPATestCorrectSuperblock: Superblock = .{

src/bin/unsquashfs.zig

@@ -57,18 +57,28 @@ pub fn main(init: std.process.Init) !void {
     defer fil.close(io);
     var arc: squashfs.Archive = try .initAdvanced(alloc, io, fil, offset, 0); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
     defer arc.deinit(io);
+
     const options: squashfs.ExtractionOptions = .{
+        .single_threaded = threads == 1,
         .verbose = verbose,
         .verbose_writer = if (verbose) &out.interface else null,
         .ignore_xattr = ignore_xattrs,
         .ignore_permissions = ignore_permissions,
     };
+
     if (force)
         try Io.Dir.cwd().deleteTree(io, ext_loc);
-    if (threads != 1) {
+
-        try arc.extract(alloc, io, ext_loc, options); //TODO: Handle error gracefully.
+    if (threads != 0 and threads != 1) {
+        var limited_threaded: Io.Threaded = .init(alloc, .{
+            .argv0 = .init(init.minimal.args),
+            .async_limit = @enumFromInt(threads),
+            .concurrent_limit = @enumFromInt(threads),
+            .environ = init.minimal.environ,
+        });
+        try arc.extract(alloc, limited_threaded.io(), ext_loc, options); //TODO: Handle error gracefully.
     } else {
-        try arc.extract(alloc, Io.Threaded.global_single_threaded.io(), ext_loc, options);
+        try arc.extract(alloc, io, ext_loc, options); //TODO: Handle error gracefully.
     }
 }
 

src/extract-multi.zig

@@ -0,0 +1,404 @@
+const std = @import("std");
+const Io = std.Io;
+const Atomic = std.atomic.Value;
+
+const DataExtractor = @import("data/extractor.zig");
+const DecompCache = @import("decomp_cache.zig");
+const Directory = @import("directory.zig");
+const ExtractionOptions = @import("options.zig");
+const Inode = @import("inode.zig");
+const Lookup = @import("lookup.zig");
+const Superblock = @import("archive.zig").Superblock;
+const XattrTable = @import("xattr.zig");
+
+pub fn extractAsync(
+    alloc: std.mem.Allocator,
+    io: Io,
+    inode: Inode,
+    cache: *DecompCache,
+    super: Superblock,
+    ext_loc: []const u8,
+    options: ExtractionOptions,
+) !void {
+    const path = std.mem.trim(u8, ext_loc, "/");
+
+    var id_table: Lookup.Table(u16) = .init(alloc, cache, super.id_start, super.id_count);
+    defer id_table.deinit(io);
+
+    var xattr_table: XattrTable = try .init(alloc, cache, super.xattr_start);
+    defer xattr_table.deinit(io);
+
+    var sel_buffer: [100]ReturnUnion = undefined;
+    var sel: Io.Select(ReturnUnion) = .init(io, &sel_buffer);
+    defer {
+        while (sel.cancel()) |ret| switch (ret) {
+            .file_ret => |f| {
+                const res: FileReturn = f catch continue;
+                if (res.parent != null)
+                    alloc.free(res.path);
+            },
+            .dir_ret => |d| {
+                const res: DirReturn = d catch continue;
+                if (res.parent != null)
+                    alloc.free(res.path);
+                alloc.destroy(res.children);
+            },
+            .num_ret => {},
+        };
+    }
+
+    var concurrent = true;
+
+    var loop = io.concurrent(returnLoop, .{ alloc, io, &sel, &id_table, &xattr_table, options, inode.hdr.num }) catch blk: {
+        concurrent = false;
+        break :blk io.async(returnLoop, .{ alloc, io, &sel, &id_table, &xattr_table, options, inode.hdr.num });
+    };
+
+    var frag_table: Lookup.Table(Lookup.FragmentEntry) = .init(alloc, cache, super.frag_start, super.frag_count);
+    defer frag_table.deinit(io);
+
+    try extractReal(alloc, io, cache, super, &frag_table, &sel, concurrent, null, path, inode);
+
+    try loop.await(io);
+}
+
+fn extractReal(
+    alloc: std.mem.Allocator,
+    io: Io,
+    cache: *DecompCache,
+    super: Superblock,
+    frag_table: *Lookup.Table(Lookup.FragmentEntry),
+    sel: *Io.Select(ReturnUnion),
+    concurrent: bool,
+    parent: ?*Atomic(u32),
+    path: []const u8,
+    inode: Inode,
+) !void {
+    errdefer if (parent != null) {
+        alloc.free(path);
+        inode.deinit(alloc);
+    };
+
+    switch (inode.hdr.type) {
+        .dir, .ext_dir => if (concurrent)
+            try sel.concurrent(.dir_ret, extractDir, .{ alloc, io, cache, super, frag_table, sel, concurrent, parent, path, inode })
+        else
+            sel.async(.dir_ret, extractDir, .{ alloc, io, cache, super, frag_table, sel, concurrent, parent, path, inode }),
+        .file, .ext_file => if (concurrent)
+            try sel.concurrent(.file_ret, extractFile, .{ alloc, io, cache, frag_table, super.block_size, parent, path, inode })
+        else
+            sel.async(.file_ret, extractFile, .{ alloc, io, cache, frag_table, super.block_size, parent, path, inode }),
+        .symlink, .ext_symlink => if (concurrent)
+            try sel.concurrent(.num_ret, extractSymlink, .{ alloc, io, parent, path, inode })
+        else
+            sel.async(.num_ret, extractSymlink, .{ alloc, io, parent, path, inode }),
+        else => if (concurrent)
+            try sel.concurrent(.file_ret, extractNod, .{ alloc, parent, path, inode })
+        else
+            sel.async(.file_ret, extractNod, .{ alloc, parent, path, inode }),
+    }
+}
+
+fn extractDir(
+    alloc: std.mem.Allocator,
+    io: Io,
+    cache: *DecompCache,
+    super: Superblock,
+    frag_table: *Lookup.Table(Lookup.FragmentEntry),
+    sel: *Io.Select(ReturnUnion),
+    concurrent: bool,
+    parent: ?*Atomic(u32),
+    path: []const u8,
+    inode: Inode,
+) Error!DirReturn {
+    errdefer if (parent != null) alloc.free(path);
+
+    var dir = inode.directory(alloc, io, cache, super.dir_start) catch |err| switch (err) {
+        error.NotDirectory => unreachable,
+        else => |e| return e,
+    };
+    defer dir.deinit(alloc);
+
+    const children = try alloc.create(Atomic(usize));
+    errdefer alloc.destroy(children);
+    children.* = .init(dir.entries.len);
+
+    for (dir.entries) |entry| {
+        const new_inode: Inode = try .initDirEntry(alloc, io, cache, super.inode_start, super.block_size, entry);
+
+        const new_path = std.mem.concat(alloc, u8, &.{ path, "/", entry.name }) catch |err| {
+            new_inode.deinit(alloc);
+            return err;
+        };
+
+        try extractReal(alloc, io, cache, super, frag_table, sel, concurrent, parent, new_path, new_inode);
+    }
+
+    return .{
+        .hdr = inode.hdr,
+        .path = path,
+        .parent = parent,
+        .children = children,
+
+        .xattr_idx = switch (inode.data) {
+            .ext_dir => |d| if (d.xattr_idx != 0xFFFFFFFF)
+                d.xattr_idx
+            else
+                null,
+            else => null,
+        },
+    };
+}
+fn extractFile(
+    alloc: std.mem.Allocator,
+    io: Io,
+    cache: *DecompCache,
+    frag_table: *Lookup.Table(Lookup.FragmentEntry),
+    block_size: u32,
+    parent: ?*Atomic(u32),
+    path: []const u8,
+    inode: Inode,
+) Error!FileReturn {
+    defer if (parent != null) {
+        inode.deinit(alloc);
+        _ = parent.?.fetchSub(1, .acquire);
+    };
+    errdefer alloc.free(path);
+
+    var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{});
+    defer atomic.deinit(io);
+
+    var ret: FileReturn = .{
+        .hdr = inode.hdr,
+        .path = path,
+        .parent = parent,
+    };
+
+    var data: DataExtractor = switch (inode.data) {
+        .file => |f| blk: {
+            var data: DataExtractor = .init(cache, block_size, f.size, f.data_start, f.blocks);
+            if (f.frag_idx != 0xFFFFFFFF) {
+                const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx);
+                if (entry.size.uncompressed) {
+                    data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset);
+                } else {
+                    const block = try cache.get(io, entry.start, entry.size.size, block_size);
+                    data.addFragment(block, f.frag_offset);
+                }
+            }
+            break :blk data;
+        },
+        .ext_file => |f| blk: {
+            var data: DataExtractor = .init(cache, block_size, f.size, f.data_start, f.blocks);
+            if (f.frag_idx != 0xFFFFFFFF) {
+                const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx);
+                if (entry.size.uncompressed) {
+                    data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset);
+                } else {
+                    const block = try cache.get(io, entry.start, entry.size.size, block_size);
+                    data.addFragment(block, f.frag_offset);
+                }
+            }
+            if (f.xattr_idx != 0xFFFFFFFF)
+                ret.xattr_idx = f.xattr_idx;
+            break :blk data;
+        },
+        else => unreachable,
+    };
+
+    try data.asyncExtract(io, atomic.file);
+
+    try atomic.link(io);
+
+    return ret;
+}
+fn extractSymlink(alloc: std.mem.Allocator, io: Io, parent: ?*Atomic(u32), path: []const u8, inode: Inode) Error!u32 {
+    defer {
+        alloc.free(path);
+        if (parent != null) {
+            inode.deinit(alloc);
+            _ = parent.?.fetchSub(1, .acquire);
+        }
+    }
+
+    const target = switch (inode.data) {
+        .symlink => |s| s.target,
+        .ext_symlink => |s| s.target,
+        else => unreachable,
+    };
+
+    try Io.Dir.cwd().symLink(io, target, path, .{});
+
+    return inode.hdr.num;
+}
+fn extractNod(alloc: std.mem.Allocator, parent: ?*Atomic(u32), path: []const u8, inode: Inode) Error!FileReturn {
+    errdefer if (parent != null) alloc.free(path);
+
+    var ret: FileReturn = .{
+        .hdr = inode.hdr,
+        .path = path,
+        .parent = parent,
+    };
+
+    const DT = std.os.linux.DT;
+
+    var dev: u32 = 0;
+    var mode: u32 = undefined;
+
+    switch (inode.data) {
+        .char_dev => |d| {
+            dev = d.device;
+            mode = DT.CHR;
+        },
+        .ext_char_dev => |d| {
+            dev = d.device;
+            mode = DT.CHR;
+            if (d.xattr_idx != 0xFFFFFFFF)
+                ret.xattr_idx = d.xattr_idx;
+        },
+        .block_dev => |d| {
+            dev = d.device;
+            mode = DT.BLK;
+        },
+        .ext_block_dev => |d| {
+            dev = d.device;
+            mode = DT.BLK;
+            if (d.xattr_idx != 0xFFFFFFFF)
+                ret.xattr_idx = d.xattr_idx;
+        },
+        .fifo => mode = DT.FIFO,
+        .ext_fifo => |f| {
+            mode = DT.FIFO;
+            if (f.xattr_idx != 0xFFFFFFFF)
+                ret.xattr_idx = f.xattr_idx;
+        },
+        .socket => mode = DT.SOCK,
+        .ext_socket => |s| {
+            mode = DT.SOCK;
+            if (s.xattr_idx != 0xFFFFFFFF)
+                ret.xattr_idx = s.xattr_idx;
+        },
+        else => unreachable,
+    }
+
+    const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &.{path}, 0);
+    defer alloc.free(sentinel_path);
+
+    const res = std.os.linux.mknod(sentinel_path, mode, dev);
+    if (res != 0)
+        return Error.MknodError;
+
+    return ret;
+}
+
+fn returnLoop(
+    alloc: std.mem.Allocator,
+    io: Io,
+    sel: *Io.Select(ReturnUnion),
+    id_table: *Lookup.Table(u16),
+    xattr_table: *XattrTable,
+    options: ExtractionOptions,
+    origin_num: u32,
+) !void {
+    var origin_finished = false;
+    while (!origin_finished) {
+        const ret = try sel.await();
+
+        switch (ret) {
+            .file_ret => |f| {
+                const res: FileReturn = try f;
+                if (res.hdr.num == origin_num)
+                    origin_finished = true;
+                try res.finish(alloc, io, id_table, xattr_table, options);
+            },
+            .dir_ret => |d| {
+                const res: DirReturn = try d;
+                if (res.children.load(.unordered) > 0) {
+                    try sel.queue.putOne(io, .{ .dir_ret = res });
+                    continue;
+                }
+                if (res.hdr.num == origin_num)
+                    origin_finished = true;
+                try res.finish(alloc, io, id_table, xattr_table, options);
+            },
+            .num_ret => |n| {
+                const res: u32 = try n;
+                if (res == origin_num)
+                    origin_finished = true;
+            },
+        }
+    }
+}
+
+fn setMetadata(alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions, hdr: Inode.Header, xattr_idx: ?u32, path: []const u8) Error!void {
+    if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == null)) return;
+
+    const file = try Io.Dir.cwd().openFile(io, path, .{});
+    defer file.close(io);
+
+    if (!options.ignore_permissions) {
+        try file.setTimestamps(io, .{ .modify_timestamp = .init(Io.Timestamp.fromNanoseconds(@as(i96, @intCast(hdr.mod_time)) * std.time.ns_per_s)) });
+        try file.setPermissions(io, @enumFromInt(hdr.permission));
+        try file.setOwner(io, try id_table.get(io, hdr.uid_idx), try id_table.get(io, hdr.gid_idx));
+    }
+
+    if (!options.ignore_permissions and xattr_idx != null) {
+        var xattrs = try xattr_table.get(alloc, io, xattr_idx.?);
+        defer xattrs.deinit(alloc);
+
+        for (xattrs.xattrs) |x| {
+            const res = std.os.linux.fsetxattr(file.handle, x.key, x.value.ptr, x.value.len, 0);
+            if (res != 0)
+                return Error.SetXattrError;
+        }
+    }
+}
+
+// Types
+
+const Error = error{ Canceled, MknodError, SetXattrError, ConcurrencyUnavailable } || Directory.Error || Io.Dir.CreateFileAtomicError ||
+    Io.File.Atomic.LinkError || Io.Dir.SymLinkError || Io.Dir.CreateDirPathError || Io.Reader.StreamError || Io.File.SetPermissionsError ||
+    Io.File.Writer.Error || DataExtractor.Error;
+
+const ReturnUnion = union(enum) {
+    file_ret: Error!FileReturn,
+    dir_ret: Error!DirReturn,
+    num_ret: Error!u32,
+};
+
+const FileReturn = struct {
+    hdr: Inode.Header,
+    path: []const u8,
+    parent: ?*Atomic(u32),
+
+    xattr_idx: ?u32 = null,
+
+    fn finish(self: FileReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void {
+        defer if (self.parent != null) {
+            alloc.free(self.path);
+            _ = self.parent.?.fetchSub(1, .acquire);
+        };
+        return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path);
+    }
+};
+const DirReturn = struct {
+    hdr: Inode.Header,
+    path: []const u8,
+    parent: ?*Atomic(u32),
+    children: *Atomic(usize),
+
+    xattr_idx: ?u32 = null,
+
+    fn finish(self: DirReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void {
+        std.debug.assert(self.children.load(.unordered) == 0);
+
+        defer {
+            if (self.parent != null) {
+                alloc.free(self.path);
+                _ = self.parent.?.fetchSub(1, .acquire);
+            }
+            alloc.destroy(self.children);
+        }
+        return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path);
+    }
+};

src/extract-single.zig

@@ -0,0 +1,287 @@
+const std = @import("std");
+const Io = std.Io;
+const Atomic = std.atomic.Value;
+
+const DataReader = @import("data/reader.zig");
+const DecompCache = @import("decomp_cache.zig");
+const Directory = @import("directory.zig");
+const ExtractionOptions = @import("options.zig");
+const Inode = @import("inode.zig");
+const Lookup = @import("lookup.zig");
+const Superblock = @import("archive.zig").Superblock;
+const XattrTable = @import("xattr.zig");
+
+pub fn extractSingleThreaded(alloc: std.mem.Allocator, io: Io, inode: Inode, cache: *DecompCache, super: Superblock, ext_loc: []const u8, options: ExtractionOptions) !void {
+    const path = std.mem.trim(u8, ext_loc, "/");
+
+    var id_table: Lookup.Table(u16) = .init(alloc, cache, super.id_start, super.id_count);
+    defer id_table.deinit(io);
+
+    var xattr_table: XattrTable = try .init(alloc, cache, super.xattr_start);
+    defer xattr_table.deinit(io);
+
+    var frag_table: Lookup.Table(Lookup.FragmentEntry) = .init(alloc, cache, super.frag_start, super.frag_count);
+    defer frag_table.deinit(io);
+
+    try extractReal(alloc, io, cache, super, &frag_table, &id_table, &xattr_table, options, path, inode, true);
+}
+
+fn extractReal(
+    alloc: std.mem.Allocator,
+    io: Io,
+    cache: *DecompCache,
+    super: Superblock,
+    frag_table: *Lookup.Table(Lookup.FragmentEntry),
+    id_table: *Lookup.Table(u16),
+    xattr_table: *XattrTable,
+    options: ExtractionOptions,
+    path: []const u8,
+    inode: Inode,
+    origin: bool,
+) Error!void {
+    defer if (!origin) {
+        inode.deinit(alloc);
+        alloc.free(path);
+    };
+    switch (inode.data) {
+        .dir, .ext_dir => {
+            const res = try extractDir(alloc, io, cache, super, frag_table, id_table, xattr_table, options, path, inode);
+            try res.finish(alloc, io, id_table, xattr_table, options);
+        },
+        .file, .ext_file => {
+            const res = try extractFile(io, cache, super.block_size, frag_table, path, inode);
+            try res.finish(alloc, io, id_table, xattr_table, options);
+        },
+        .symlink, .ext_symlink => try extractSymlink(io, path, inode),
+        else => {
+            const res = try extractNod(alloc, path, inode);
+            try res.finish(alloc, io, id_table, xattr_table, options);
+        },
+    }
+}
+
+fn extractDir(
+    alloc: std.mem.Allocator,
+    io: Io,
+    cache: *DecompCache,
+    super: Superblock,
+    frag_table: *Lookup.Table(Lookup.FragmentEntry),
+    id_table: *Lookup.Table(u16),
+    xattr_table: *XattrTable,
+    options: ExtractionOptions,
+    path: []const u8,
+    inode: Inode,
+) Error!DirReturn {
+    try Io.Dir.cwd().createDirPath(io, path);
+
+    const dir = inode.directory(alloc, io, cache, super.dir_start) catch |err| switch (err) {
+        error.NotDirectory => unreachable,
+        else => |e| return e,
+    };
+    defer dir.deinit(alloc);
+
+    const ret: DirReturn = .{
+        .hdr = inode.hdr,
+        .path = path,
+
+        .xattr_idx = switch (inode.data) {
+            .ext_dir => |d| if (d.xattr_idx != 0xFFFFFFFF) d.xattr_idx else null,
+            else => null,
+        },
+    };
+
+    for (dir.entries) |entry| {
+        const new_inode: Inode = try .initDirEntry(alloc, io, cache, super.inode_start, super.block_size, entry);
+
+        const new_path = std.mem.concat(alloc, u8, &.{ path, "/", entry.name }) catch |err| {
+            new_inode.deinit(alloc);
+            return err;
+        };
+
+        try extractReal(
+            alloc,
+            io,
+            cache,
+            super,
+            frag_table,
+            id_table,
+            xattr_table,
+            options,
+            new_path,
+            new_inode,
+            false,
+        );
+    }
+    return ret;
+}
+fn extractFile(
+    io: Io,
+    cache: *DecompCache,
+    block_size: u32,
+    frag_table: *Lookup.Table(Lookup.FragmentEntry),
+    path: []const u8,
+    inode: Inode,
+) Error!FileReturn {
+    var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{});
+    defer atomic.deinit(io);
+
+    var ret: FileReturn = .{
+        .hdr = inode.hdr,
+        .path = path,
+    };
+
+    var data: DataReader = switch (inode.data) {
+        .file => |f| blk: {
+            var data: DataReader = .init(io, cache, block_size, f.size, f.data_start, f.blocks);
+            if (f.frag_idx != 0xFFFFFFFF) {
+                const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx);
+                if (entry.size.uncompressed) {
+                    data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset);
+                } else {
+                    const block = try cache.get(io, entry.start, entry.size.size, block_size);
+                    data.addFragment(block, f.frag_offset);
+                }
+            }
+            break :blk data;
+        },
+        .ext_file => |f| blk: {
+            var data: DataReader = .init(io, cache, block_size, f.size, f.data_start, f.blocks);
+            if (f.frag_idx != 0xFFFFFFFF) {
+                const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx);
+                if (entry.size.uncompressed) {
+                    data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset);
+                } else {
+                    const block = try cache.get(io, entry.start, entry.size.size, block_size);
+                    data.addFragment(block, f.frag_offset);
+                }
+            }
+            if (f.xattr_idx != 0xFFFFFFFF)
+                ret.xattr_idx = f.xattr_idx;
+            break :blk data;
+        },
+        else => unreachable,
+    };
+    defer data.deinit();
+
+    var wrt_buf: [1024]u8 = undefined;
+    var wrt = atomic.file.writer(io, &wrt_buf);
+    _ = try data.interface.streamRemaining(&wrt.interface);
+    try wrt.flush();
+
+    try atomic.link(io);
+
+    return ret;
+}
+fn extractSymlink(io: Io, path: []const u8, inode: Inode) Error!void {
+    const target = switch (inode.data) {
+        .symlink => |s| s.target,
+        .ext_symlink => |s| s.target,
+        else => unreachable,
+    };
+
+    try Io.Dir.cwd().symLink(io, target, path, .{});
+}
+fn extractNod(alloc: std.mem.Allocator, path: []const u8, inode: Inode) Error!FileReturn {
+    var ret: FileReturn = .{
+        .hdr = inode.hdr,
+        .path = path,
+    };
+
+    const DT = std.os.linux.DT;
+
+    var dev: u32 = 0;
+    var mode: u32 = undefined;
+
+    switch (inode.data) {
+        .char_dev => |d| {
+            dev = d.device;
+            mode = DT.CHR;
+        },
+        .ext_char_dev => |d| {
+            dev = d.device;
+            mode = DT.CHR;
+            if (d.xattr_idx != 0xFFFFFFFF)
+                ret.xattr_idx = d.xattr_idx;
+        },
+        .block_dev => |d| {
+            dev = d.device;
+            mode = DT.BLK;
+        },
+        .ext_block_dev => |d| {
+            dev = d.device;
+            mode = DT.BLK;
+            if (d.xattr_idx != 0xFFFFFFFF)
+                ret.xattr_idx = d.xattr_idx;
+        },
+        .fifo => mode = DT.FIFO,
+        .ext_fifo => |f| {
+            mode = DT.FIFO;
+            if (f.xattr_idx != 0xFFFFFFFF)
+                ret.xattr_idx = f.xattr_idx;
+        },
+        .socket => mode = DT.SOCK,
+        .ext_socket => |s| {
+            mode = DT.SOCK;
+            if (s.xattr_idx != 0xFFFFFFFF)
+                ret.xattr_idx = s.xattr_idx;
+        },
+        else => unreachable,
+    }
+
+    const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &.{path}, 0);
+    defer alloc.free(sentinel_path);
+
+    const res = std.os.linux.mknod(sentinel_path, mode, dev);
+    if (res != 0)
+        return Error.MknodError;
+
+    return ret;
+}
+
+fn setMetadata(alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions, hdr: Inode.Header, xattr_idx: ?u32, path: []const u8) Error!void {
+    if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == null)) return;
+
+    const file = try Io.Dir.cwd().openFile(io, path, .{});
+    defer file.close(io);
+
+    if (!options.ignore_permissions) {
+        try file.setTimestamps(io, .{ .modify_timestamp = .init(Io.Timestamp.fromNanoseconds(@as(i96, @intCast(hdr.mod_time)) * std.time.ns_per_s)) });
+        try file.setPermissions(io, @enumFromInt(hdr.permission));
+        try file.setOwner(io, try id_table.get(io, hdr.uid_idx), try id_table.get(io, hdr.gid_idx));
+    }
+
+    if (!options.ignore_permissions and xattr_idx != null) {
+        var xattrs = try xattr_table.get(alloc, io, xattr_idx.?);
+        defer xattrs.deinit(alloc);
+
+        for (xattrs.xattrs) |x| {
+            const res = std.os.linux.fsetxattr(file.handle, x.key, x.value.ptr, x.value.len, 0);
+            if (res != 0)
+                return Error.SetXattrError;
+        }
+    }
+}
+
+const Error = error{ Canceled, MknodError, SetXattrError } || Directory.Error || Io.Dir.CreateFileAtomicError || Io.File.Atomic.LinkError ||
+    Io.Dir.SymLinkError || Io.Dir.CreateDirPathError || Io.Reader.StreamError || Io.File.SetPermissionsError || Io.File.Writer.Error;
+
+const FileReturn = struct {
+    hdr: Inode.Header,
+    path: []const u8,
+
+    xattr_idx: ?u32 = null,
+
+    fn finish(self: FileReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void {
+        return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path);
+    }
+};
+const DirReturn = struct {
+    hdr: Inode.Header,
+    path: []const u8,
+
+    xattr_idx: ?u32 = null,
+
+    fn finish(self: DirReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void {
+        return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path);
+    }
+};

src/extract.zig

@@ -1,287 +1,33 @@
 const std = @import("std");
 const Io = std.Io;
-const Atomic = std.atomic.Value;
 
-const DataReader = @import("data/reader.zig");
 const DecompCache = @import("decomp_cache.zig");
-const Directory = @import("directory.zig");
 const ExtractionOptions = @import("options.zig");
 const Inode = @import("inode.zig");
-const Lookup = @import("lookup.zig");
+const MultiThreaded = @import("extract-multi.zig");
+const SingleThreaded = @import("extract-single.zig");
 const Superblock = @import("archive.zig").Superblock;
-const XattrTable = @import("xattr.zig");
 
-pub fn extractSingleThreaded(alloc: std.mem.Allocator, io: Io, inode: Inode, cache: *DecompCache, super: Superblock, ext_loc: []const u8, options: ExtractionOptions) !void {
+pub fn extractSingleThreaded(
-    const path = std.mem.trim(u8, ext_loc, "/");
-
-    var id_table: Lookup.Table(u16) = .init(alloc, cache, super.id_start, super.id_count);
-    defer id_table.deinit(io);
-
-    var xattr_table: XattrTable = try .init(alloc, cache, super.xattr_start);
-    defer xattr_table.deinit(io);
-
-    var frag_table: Lookup.Table(Lookup.FragmentEntry) = .init(alloc, cache, super.frag_start, super.frag_count);
-    defer frag_table.deinit(io);
-
-    try extractReal(alloc, io, cache, super, &frag_table, &id_table, &xattr_table, options, path, inode, true);
-}
-
-fn extractReal(
     alloc: std.mem.Allocator,
     io: Io,
+    inode: Inode,
     cache: *DecompCache,
     super: Superblock,
-    frag_table: *Lookup.Table(Lookup.FragmentEntry),
+    ext_loc: []const u8,
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
     options: ExtractionOptions,
-    path: []const u8,
+) !void {
-    inode: Inode,
+    return SingleThreaded.extractSingleThreaded(alloc, io, inode, cache, super, ext_loc, options);
-    origin: bool,
-) Error!void {
-    defer if (!origin) {
-        inode.deinit(alloc);
-        alloc.free(path);
-    };
-    switch (inode.data) {
-        .dir, .ext_dir => {
-            const res = try extractDir(alloc, io, cache, super, frag_table, id_table, xattr_table, options, path, inode);
-            try res.finish(alloc, io, id_table, xattr_table, options);
-        },
-        .file, .ext_file => {
-            const res = try extractFile(io, cache, super.block_size, frag_table, path, inode);
-            try res.finish(alloc, io, id_table, xattr_table, options);
-        },
-        .symlink, .ext_symlink => try extractSymlink(io, path, inode),
-        else => {
-            const res = try extractNod(alloc, path, inode);
-            try res.finish(alloc, io, id_table, xattr_table, options);
-        },
-    }
 }
 
-fn extractDir(
+pub fn extractAsync(
     alloc: std.mem.Allocator,
     io: Io,
+    inode: Inode,
     cache: *DecompCache,
     super: Superblock,
-    frag_table: *Lookup.Table(Lookup.FragmentEntry),
+    ext_loc: []const u8,
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
     options: ExtractionOptions,
-    path: []const u8,
+) !void {
-    inode: Inode,
+    return MultiThreaded.extractAsync(alloc, io, inode, cache, super, ext_loc, options);
-) Error!DirReturn {
-    try Io.Dir.cwd().createDirPath(io, path);
-
-    const dir = inode.directory(alloc, io, cache, super.dir_start) catch |err| switch (err) {
-        error.NotDirectory => unreachable,
-        else => |e| return e,
-    };
-    defer dir.deinit(alloc);
-
-    const ret: DirReturn = .{
-        .hdr = inode.hdr,
-        .path = path,
-
-        .xattr_idx = switch (inode.data) {
-            .ext_dir => |d| if (d.xattr_idx != 0xFFFFFFFF) d.xattr_idx else null,
-            else => null,
-        },
-    };
-
-    for (dir.entries) |entry| {
-        const new_inode: Inode = try .initDirEntry(alloc, io, cache, super.inode_start, super.block_size, entry);
-
-        const new_path = std.mem.concat(alloc, u8, &.{ path, "/", entry.name }) catch |err| {
-            new_inode.deinit(alloc);
-            return err;
-        };
-
-        try extractReal(
-            alloc,
-            io,
-            cache,
-            super,
-            frag_table,
-            id_table,
-            xattr_table,
-            options,
-            new_path,
-            new_inode,
-            false,
-        );
-    }
-    return ret;
 }
-fn extractFile(
-    io: Io,
-    cache: *DecompCache,
-    block_size: u32,
-    frag_table: *Lookup.Table(Lookup.FragmentEntry),
-    path: []const u8,
-    inode: Inode,
-) Error!FileReturn {
-    var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{});
-    defer atomic.deinit(io);
-
-    var ret: FileReturn = .{
-        .hdr = inode.hdr,
-        .path = path,
-    };
-
-    var data: DataReader = switch (inode.data) {
-        .file => |f| blk: {
-            var data: DataReader = .init(io, cache, block_size, f.size, f.data_start, f.blocks);
-            if (f.frag_idx != 0xFFFFFFFF) {
-                const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx);
-                if (entry.size.uncompressed) {
-                    data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset);
-                } else {
-                    const block = try cache.get(io, entry.start, entry.size.size, block_size);
-                    data.addFragment(block, f.frag_offset);
-                }
-            }
-            break :blk data;
-        },
-        .ext_file => |f| blk: {
-            var data: DataReader = .init(io, cache, block_size, f.size, f.data_start, f.blocks);
-            if (f.frag_idx != 0xFFFFFFFF) {
-                const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx);
-                if (entry.size.uncompressed) {
-                    data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset);
-                } else {
-                    const block = try cache.get(io, entry.start, entry.size.size, block_size);
-                    data.addFragment(block, f.frag_offset);
-                }
-            }
-            if (f.xattr_idx != 0xFFFFFFFF)
-                ret.xattr_idx = f.xattr_idx;
-            break :blk data;
-        },
-        else => unreachable,
-    };
-    defer data.deinit();
-
-    var wrt_buf: [1024]u8 = undefined;
-    var wrt = atomic.file.writer(io, &wrt_buf);
-    _ = try data.interface.streamRemaining(&wrt.interface);
-    try wrt.flush();
-
-    try atomic.link(io);
-
-    return ret;
-}
-fn extractSymlink(io: Io, path: []const u8, inode: Inode) Error!void {
-    const target = switch (inode.data) {
-        .symlink => |s| s.target,
-        .ext_symlink => |s| s.target,
-        else => unreachable,
-    };
-
-    try Io.Dir.cwd().symLink(io, target, path, .{});
-}
-fn extractNod(alloc: std.mem.Allocator, path: []const u8, inode: Inode) Error!FileReturn {
-    var ret: FileReturn = .{
-        .hdr = inode.hdr,
-        .path = path,
-    };
-
-    const DT = std.os.linux.DT;
-
-    var dev: u32 = 0;
-    var mode: u32 = undefined;
-
-    switch (inode.data) {
-        .char_dev => |d| {
-            dev = d.device;
-            mode = DT.CHR;
-        },
-        .ext_char_dev => |d| {
-            dev = d.device;
-            mode = DT.CHR;
-            if (d.xattr_idx != 0xFFFFFFFF)
-                ret.xattr_idx = d.xattr_idx;
-        },
-        .block_dev => |d| {
-            dev = d.device;
-            mode = DT.BLK;
-        },
-        .ext_block_dev => |d| {
-            dev = d.device;
-            mode = DT.BLK;
-            if (d.xattr_idx != 0xFFFFFFFF)
-                ret.xattr_idx = d.xattr_idx;
-        },
-        .fifo => mode = DT.FIFO,
-        .ext_fifo => |f| {
-            mode = DT.FIFO;
-            if (f.xattr_idx != 0xFFFFFFFF)
-                ret.xattr_idx = f.xattr_idx;
-        },
-        .socket => mode = DT.SOCK,
-        .ext_socket => |s| {
-            mode = DT.SOCK;
-            if (s.xattr_idx != 0xFFFFFFFF)
-                ret.xattr_idx = s.xattr_idx;
-        },
-        else => unreachable,
-    }
-
-    const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &.{path}, 0);
-    defer alloc.free(sentinel_path);
-
-    const res = std.os.linux.mknod(sentinel_path, mode, dev);
-    if (res != 0)
-        return Error.MknodError;
-
-    return ret;
-}
-
-fn setMetadata(alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions, hdr: Inode.Header, xattr_idx: ?u32, path: []const u8) Error!void {
-    if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == null)) return;
-
-    const file = try Io.Dir.cwd().openFile(io, path, .{});
-    defer file.close(io);
-
-    if (!options.ignore_permissions) {
-        try file.setTimestamps(io, .{ .modify_timestamp = .init(Io.Timestamp.fromNanoseconds(@as(i96, @intCast(hdr.mod_time)) * std.time.ns_per_s)) });
-        try file.setPermissions(io, @enumFromInt(hdr.permission));
-        try file.setOwner(io, try id_table.get(io, hdr.uid_idx), try id_table.get(io, hdr.gid_idx));
-    }
-
-    if (!options.ignore_permissions and xattr_idx != null) {
-        var xattrs = try xattr_table.get(alloc, io, xattr_idx.?);
-        defer xattrs.deinit(alloc);
-
-        for (xattrs.xattrs) |x| {
-            const res = std.os.linux.fsetxattr(file.handle, x.key, x.value.ptr, x.value.len, 0);
-            if (res != 0)
-                return Error.SetXattrError;
-        }
-    }
-}
-
-const Error = error{ Canceled, MknodError, SetXattrError } || Directory.Error || Io.Dir.CreateFileAtomicError || Io.File.Atomic.LinkError ||
-    Io.Dir.SymLinkError || Io.Dir.CreateDirPathError || Io.Reader.StreamError || Io.File.SetPermissionsError;
-
-const FileReturn = struct {
-    hdr: Inode.Header,
-    path: []const u8,
-
-    xattr_idx: ?u32 = null,
-
-    fn finish(self: FileReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void {
-        return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path);
-    }
-};
-const DirReturn = struct {
-    hdr: Inode.Header,
-    path: []const u8,
-
-    xattr_idx: ?u32 = null,
-
-    fn finish(self: DirReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void {
-        return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path);
-    }
-};

src/file.zig

@@ -102,6 +102,9 @@ pub fn open(self: *SfsFile, alloc: std.mem.Allocator, io: Io, filepath: []const
     return found_file.open(alloc, io, path[first_element.len..]);
 }
 
-pub fn extract(self: *SfsFile, alloc: std.mem.Allocator, io: Io, ext_dir: []const u8, options: ExtractionOptions) !void {
+pub fn extract(self: *SfsFile, alloc: std.mem.Allocator, io: Io, ext_loc: []const u8, options: ExtractionOptions) !void {
-    return Extract.extractSingleThreaded(alloc, io, self.inode, &self.archive.cache, self.archive.super, ext_dir, options);
+    return if (options.single_threaded)
+        Extract.extractSingleThreaded(alloc, io, self.inode, &self.archive.cache, self.archive.super, ext_loc, options)
+    else
+        Extract.extractAsync(alloc, io, self.inode, &self.archive.cache, self.archive.super, ext_loc, options);
 }

src/options.zig

@@ -5,6 +5,8 @@ const Writer = std.Io.Writer;
 
 const ExtractionOptions = @This();
 
+/// Force single-threaded extraction.
+single_threaded: bool = false,
 /// Don't set the file's owner & permissions after extraction
 ignore_permissions: bool = false,
 /// Don't set xattr values. Currently xattrs are never set anyway.
@@ -17,6 +19,7 @@ verbose: bool = false,
 verbose_writer: ?*Writer = null,
 
 pub const default: ExtractionOptions = .{};
+pub const default_single_threaded: ExtractionOptions = .{ .single_threaded = true };
 pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions {
     return .{
         .verbose = true,

src/test.zig

@@ -1,84 +0,0 @@
-const std = @import("std");
-const stuff = @import("builtin");
-
-const Archive = @import("archive.zig");
-const Superblock = @import("super.zig").Superblock;
-
-const TestArchive = "testing/LinuxPATest.sfs";
-
-test "Basics" {
-    var fil = try std.fs.cwd().openFile(TestArchive, .{});
-    defer fil.close();
-    var sfs: Archive = try .init(std.testing.allocator, fil);
-    defer sfs.deinit();
-    if (sfs.super != LinuxPATestCorrectSuperblock) {
-        std.debug.print("Superblock wrong\nShould be: {}\n\nis: {}\n", .{ LinuxPATestCorrectSuperblock, sfs.super });
-        return error.BadSuperblock;
-    }
-}
-
-const TestFile = "Start.exe";
-const TestFileExtractLocation = "testing/Start.exe";
-
-test "ExtractSingleFile" {
-    std.fs.cwd().deleteFile(TestFileExtractLocation) catch {};
-    var fil = try std.fs.cwd().openFile(TestArchive, .{});
-    defer fil.close();
-    var sfs: Archive = try .init(std.testing.allocator, fil);
-    defer sfs.deinit();
-    var test_fil = try sfs.open(TestFile);
-    defer test_fil.deinit();
-    try test_fil.extract(TestFileExtractLocation, .Default);
-    //TODO: validate extracted file.
-}
-
-const TestFullExtractLocation = "testing/TestExtract";
-
-test "ExtractCompleteArchive" {
-    std.fs.cwd().deleteTree(TestFullExtractLocation) catch {};
-    var fil = try std.fs.cwd().openFile(TestArchive, .{});
-    defer fil.close();
-    var sfs: Archive = try .init(std.testing.allocator, fil);
-    defer sfs.deinit();
-    try sfs.extract(TestFullExtractLocation, .Default);
-}
-
-const LinuxPATestCorrectSuperblock: Superblock = .{
-    .magic = std.mem.readInt(u32, "hsqs", .little),
-    .inode_count = 2974,
-    .mod_time = 1632696724,
-    .block_size = 131072,
-    .frag_count = 264,
-    .compression = .zstd,
-    .block_log = 17,
-    .flags = .{
-        .inode_uncompressed = false,
-        .data_uncompressed = false,
-        .check = false,
-        .frag_uncompressed = false,
-        .fragment_never = false,
-        .fragment_always = false,
-        .duplicates = true,
-        .exportable = true,
-        .xattr_uncompressed = false,
-        .xattr_never = false,
-        .compression_options = false,
-        .ids_uncompressed = false,
-        ._ = 0,
-    },
-    .id_count = 1,
-    .ver_maj = 4,
-    .ver_min = 0,
-    .root_ref = .{
-        .block_offset = 1363,
-        .block_start = 29237,
-        ._ = 0,
-    },
-    .size = 106841744,
-    .id_start = 106841632,
-    .xattr_start = 106841720,
-    .inode_start = 106778274,
-    .dir_start = 106807998,
-    .frag_start = 106837747,
-    .export_start = 106841602,
-};