Added dedicated single_threaded mode for extraction

Cleanup

README.md

@@ -8,13 +8,15 @@ A library and application to decompress or view squashfs archives.
 
 Overall works, but currently is missing some features ([see below](#capabilities)) and has significantly slow performance compared to `unsquashfs` ([see below](#performance)).
 
-Currently things are still in flux after Zig 0.16's Io changes and the documentation below *might* not be up to date.
-
 ## Build options
 
-> `-Duse_c_libs=true`
+> `-Duse_zig_decomp=true`
 
-Instead of using Zig's standard library for decompression, use the system's C libraries. Has the benefit of being much faster and enabling LZO and LZ4 decompression.
+Instead of using C libraries for decompression, use Zig's standard library for decompression. If using this option LZO and LZ4 decomrpession types are unsupported and decompression times will be significantly longer.
+
+> `-Ddynamic=true`
+
+Dynamicly link C libraries (if they're used) instead of statically linking them.
 
 > `-Dallow_lzo=true`
 
@@ -37,20 +39,20 @@ Most features are present except for the following:
 
 ## Performance
 
-This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive (which is fairly small and uses zstd compression) and with `-Doptimize=ReleaseFast`.
+This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive which is fairly small and uses zstd compression with `-Doptimize=ReleaseFast`.
 
 Currently, my only performance checks are checking execution time, nothing deeper.
 
-* Currently, using my test archive, performance matches `unsquashfs`.
+* Currently, using my test archive, performance aproximately matches `unsquashfs` when multi-threaded, but significantly slower when single-threaded.
 * Using Zig decompression libraries *significantly* increases decompression time by 5x. Under ideal circumstances.
 * Performance improvements/regressions will be common. I'm still learning Zig.
 
 Example Times:
 
-* *unsquashfs, multi-threaded*: .12s
+* *unsquashfs, multi-threaded*: .15s
-* *unsquashfs, single-threaded*: .13s
+* *unsquashfs, single-threaded*: .16s
-* *C-libs, single-threaded*: CURRENTLY UNTESTED
+* *C-libs, single-threaded*: .36s
-* *C-libs, multi-threaded*: .16s
+* *C-libs, multi-threaded*: .14s
 * *Zig-libs, single-threaded*: CURRENTLY UNTESTED
 * *Zig-libs, multi-threaded*: .76s
 

src/archive.zig

@@ -227,34 +227,24 @@ test "ExtractCompleteArchiveSingleThreaded" {
     std.debug.print("Starting test: ExtractCompleteArchive...\n", .{});
 
     const alloc = std.testing.allocator;
-    var threaded: Io.Evented = undefined;
+    const io = std.testing.io;
-    try threaded.init(alloc, .{});
-    defer threaded.deinit();
-    const io = threaded.io();
-    var signal: u32 = 0;
 
-    Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-
+    defer fil.close(io);
-    const tmp = struct {
+    {
-        fn singleThreadedExtract(sig: *u32) !void {
+        std.debug.print("First testing using Threaded.global_single_threaded...\n", .{});
-            var fil = try Io.Dir.cwd().openFile(Io.Threaded.global_single_threaded.io(), TestArchive, .{});
+        Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
-            defer fil.close(Io.Threaded.global_single_threaded.io());
+        var sfs: Archive = try .init(Io.Threaded.global_single_threaded.io(), fil, 0);
-            var sfs: Archive = try .init(Io.Threaded.global_single_threaded.io(), fil, 0);
+        defer sfs.deinit(Io.Threaded.global_single_threaded.io());
-            defer sfs.deinit(Io.Threaded.global_single_threaded.io());
+        try sfs.extract(alloc, Io.Threaded.global_single_threaded.io(), TestFullExtractLocation, .default);
-            try sfs.extract(std.testing.allocator, Io.Threaded.global_single_threaded.io(), TestFullExtractLocation, .default);
+    }
-            sig.* = 1;
+    {
-        }
+        std.debug.print("Next testing using ExtractionOptions.single_threaded...\n", .{});
-    };
+        Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
-    var ret = try io.concurrent(tmp.singleThreadedExtract, .{&signal});
+        var sfs: Archive = try .init(io, fil, 0);
-    try io.futexWaitTimeout(
+        defer sfs.deinit(io);
-        u32,
+        try sfs.extract(alloc, io, TestFullExtractLocation, .default_single_threaded);
-        &signal,
+    }
-        0,
-        .{ .deadline = .fromNow(io, .{ .raw = .fromSeconds(10), .clock = .awake }) },
-    );
-    if (ret.any_future == null) return ret.result;
-    try ret.cancel(io);
-    return error.TestTimeout;
 }
 
 const LinuxPATestCorrectSuperblock: Superblock = .{

src/bin/unsquashfs.zig

@@ -60,6 +60,7 @@ pub fn main(init: std.process.Init) !void {
 
     var arc: squashfs.Archive = try .init(io, fil, offset); //TODO: Handle error gracefully.
     const options: squashfs.ExtractionOptions = .{
+        .single_threaded = threads == 1,
         .verbose = verbose,
         .verbose_writer = if (verbose) &out.interface else null,
         .ignore_xattr = ignore_xattrs,
@@ -69,9 +70,12 @@ pub fn main(init: std.process.Init) !void {
     if (force)
         try Io.Dir.cwd().deleteTree(io, extLoc);
     if (threads != 0) {
-        if (threads == 1)
+        var limited_io = Io.Threaded.init(alloc, .{
-            return arc.extract(alloc, Io.Threaded.global_single_threaded.io(), extLoc, options); //TODO: Handle error gracefully.
+            .async_limit = .limited(threads - 1),
-        var limited_io = Io.Threaded.init(alloc, .{ .async_limit = .limited(threads - 1), .concurrent_limit = .limited(threads - 1) });
+            .concurrent_limit = .limited(threads - 1),
+            .argv0 = .init(init.minimal.args),
+            .environ = init.minimal.environ,
+        });
         return arc.extract(alloc, limited_io.io(), extLoc, options); //TODO: Handle error gracefully.
     }
     return arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully.

src/decomp/zig_zstd.zig

@@ -20,9 +20,9 @@ buf: [][]u8,
 buf_queue: Queue,
 
 pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self {
-    const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one.
+    const buf = try alloc.alloc([]u8, 5); // TODO: Choose a better number instead of a random one.
     var queue: Queue = .init(buf);
-    for (0..20) |_|
+    for (buf) |_|
         try queue.putOne(io, try alloc.alloc(u8, block_size + zstd.block_size_max));
 
     return .{

src/inode.zig

@@ -282,17 +282,20 @@ pub fn extract(
     var frag_mgr: FragManager = try .init(alloc, fil, decomp, super.frag_start, super.frag_count, super.block_size);
     defer frag_mgr.deinit(io);
 
+    if (options.single_threaded)
+        return self.extractSinglethreaded(alloc, io, fil, super, path, options, decomp, &frag_mgr);
+
     var sel_buf: [10]ExtractReturnUnion = undefined;
     var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf);
     defer sel.cancelDiscard();
 
     var loop = io.async(finishLoop, .{ alloc, io, fil, decomp, super, options, &sel });
 
-    sel.async(.path_ret, extractReal, .{ self, alloc, io, fil, super, decomp, &sel, &frag_mgr, path, true });
+    sel.async(.path_ret, extractRealAsync, .{ self, alloc, io, fil, super, decomp, &sel, &frag_mgr, path, true });
 
     try loop.await(io);
 }
-fn extractReal(
+fn extractRealAsync(
     self: Inode,
     alloc: std.mem.Allocator,
     io: Io,
@@ -335,7 +338,7 @@ fn extractReal(
                 const new_inode = try read(alloc, &meta.interface, super.block_size);
                 errdefer new_inode.deinit(alloc);
 
-                sel.async(.path_ret, extractReal, .{ new_inode, alloc, io, fil, super, decomp, sel, frag_mgr, new_path, false });
+                sel.async(.path_ret, extractRealAsync, .{ new_inode, alloc, io, fil, super, decomp, sel, frag_mgr, new_path, false });
             }
         },
         .file, .ext_file => {
@@ -404,7 +407,6 @@ fn extractReal(
         .origin = origin,
     };
 }
-
 fn finishLoop(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, super: Archive.Superblock, options: ExtractionOptions, sel: *Io.Select(ExtractReturnUnion)) !void {
     var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count);
     defer id_table.deinit(io);
@@ -433,8 +435,8 @@ fn finishLoop(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const
             try dir_queue.push(alloc, path_ret);
             continue;
         }
-
         defer path_ret.deinit(alloc);
+
         try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options);
     }
 
@@ -461,3 +463,173 @@ fn finishLoop(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const
         try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options);
     }
 }
+/// Extracts the given inode to the given path. If the inode not a directory, the given path must not exist.
+/// If the inode is a directory the path must not exist or be a directory.
+fn extractSinglethreaded(
+    self: Inode,
+    alloc: std.mem.Allocator,
+    io: Io,
+    fil: OffsetFile,
+    super: Archive.Superblock,
+    path: []const u8,
+    options: ExtractionOptions,
+    decomp: *const Decompressor,
+    frag: *FragManager,
+) !void {
+    var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count);
+    defer id_table.deinit(io);
+
+    var xattr_table: ?XattrTable = if (super.flags.xattr_never or options.ignore_xattr or !@hasField(std.os, "linux"))
+        null
+    else
+        try .init(alloc, fil, decomp, super.xattr_start);
+    defer if (xattr_table != null) xattr_table.?.deinit(io);
+
+    return self.extractReal(
+        alloc,
+        io,
+        fil,
+        super,
+        decomp,
+        frag,
+        &id_table,
+        if (xattr_table == null) null else &xattr_table.?,
+        path,
+        options,
+    );
+}
+fn extractReal(
+    self: Inode,
+    alloc: std.mem.Allocator,
+    io: Io,
+    fil: OffsetFile,
+    super: Archive.Superblock,
+    decomp: *const Decompressor,
+    frag_mgr: *FragManager,
+    id_table: *CachedTable(u16),
+    xattr_table: ?*XattrTable,
+    path: []const u8,
+    options: ExtractionOptions,
+) !void {
+    switch (self.hdr.inode_type) {
+        .dir, .ext_dir => {
+            try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777));
+
+            const entries = self.readDirectory(alloc, fil, decomp, super.dir_start) catch |err| switch (err) {
+                Error.NotDirectory, Error.NotExtended, Error.NotRegularFile, Error.NotSymlink => unreachable,
+                else => |e| return e,
+            };
+            defer {
+                for (entries) |e|
+                    e.deinit(alloc);
+                alloc.free(entries);
+            }
+
+            for (entries) |e| {
+                const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", e.name });
+                defer alloc.free(new_path);
+
+                var rdr = fil.readerAt(super.inode_start + e.block_start);
+                var meta: MetadataReader = .init(alloc, &rdr, decomp);
+                try meta.interface.discardAll(e.block_offset);
+
+                const new_inode = try read(alloc, &meta.interface, super.block_size);
+                defer new_inode.deinit(alloc);
+
+                try new_inode.extractReal(alloc, io, fil, super, decomp, frag_mgr, id_table, xattr_table, new_path, options);
+            }
+        },
+        .file, .ext_file => {
+            var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{ .make_path = true });
+            defer atomic.deinit(io);
+
+            var rdr: DataReader = switch (self.data) {
+                .file => |f| blk: {
+                    var ext: DataReader = try .init(alloc, io, fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
+                    if (f.frag_idx != 0xFFFFFFFF)
+                        ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
+                    break :blk ext;
+                },
+                .ext_file => |f| blk: {
+                    var ext: DataReader = try .init(alloc, io, fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
+                    if (f.frag_idx != 0xFFFFFFFF)
+                        ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
+                    break :blk ext;
+                },
+                else => unreachable,
+            };
+            defer rdr.deinit();
+
+            var buf: [512 * 1024]u8 = undefined;
+            var wrt = atomic.file.writer(io, &buf);
+
+            _ = try rdr.interface.streamRemaining(&wrt.interface);
+
+            try wrt.flush();
+
+            try atomic.link(io);
+        },
+        .symlink, .ext_symlink => try Io.Dir.cwd().symLink(io, self.symlinkTarget() catch unreachable, path, .{}),
+        else => {
+            var mode: u32 = undefined;
+            var dev: u32 = 0;
+
+            const DT = std.posix.DT;
+
+            switch (self.data) {
+                .char_dev => |d| {
+                    dev = d.dev;
+                    mode = DT.CHR;
+                },
+                .ext_char_dev => |d| {
+                    dev = d.dev;
+                    mode = DT.CHR;
+                },
+                .block_dev => |d| {
+                    dev = d.dev;
+                    mode = DT.BLK;
+                },
+                .ext_block_dev => |d| {
+                    dev = d.dev;
+                    mode = DT.BLK;
+                },
+                .fifo, .ext_fifo => mode = DT.FIFO,
+                .socket, .ext_socket => mode = DT.SOCK,
+                else => unreachable,
+            }
+
+            const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
+            const res = std.os.linux.mknod(sentinel_path, mode, dev);
+            alloc.free(sentinel_path);
+            if (res != 0)
+                return ExtractError.MknodFailed;
+        },
+    }
+    if (options.ignore_permissions and options.ignore_xattr) return;
+
+    var f = try Io.Dir.cwd().openFile(io, path, .{});
+    defer f.close(io);
+
+    if (!options.ignore_permissions) {
+        try f.setPermissions(io, @enumFromInt(self.hdr.permissions));
+        try f.setOwner(io, try id_table.get(io, self.hdr.uid_idx), try id_table.get(io, self.hdr.gid_idx));
+    }
+    if (xattr_table != null) {
+        const idx = self.xattrIndex() catch return;
+
+        const xattrs = try xattr_table.?.get(alloc, io, idx);
+        defer {
+            for (xattrs) |x|
+                x.deinit(alloc);
+            alloc.free(xattrs);
+        }
+
+        const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
+        defer alloc.free(sentinel_path);
+        for (xattrs) |x| {
+            const xattr_ret = std.os.linux.fsetxattr(f.handle, x.key, x.value.ptr, x.value.len, 0);
+            if (xattr_ret != 0)
+                return ExtractError.CannotSetXattr;
+        }
+    }
+}

src/options.zig

@@ -5,6 +5,10 @@ const Writer = std.Io.Writer;
 
 const ExtractionOptions = @This();
 
+/// Extract single-threaded only.
+/// Though not necessary if using Threaded.single_threaded,
+/// setting single_threaded is more efficient.
+single_threaded: bool = false,
 /// Don't set the file's owner & permissions after extraction
 ignore_permissions: bool = false,
 /// Don't set xattr values. Currently xattrs are never set anyway.
@@ -17,6 +21,7 @@ verbose: bool = false,
 verbose_writer: ?*Writer = null,
 
 pub const default: ExtractionOptions = .{};
+pub const default_single_threaded: ExtractionOptions = .{ .single_threaded = true };
 
 pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions {
     return .{