diff --git a/README.md b/README.md index 6194053..56c4075 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,15 @@ A library and application to decompress or view squashfs archives. Overall works, but currently is missing some features ([see below](#capabilities)) and has significantly slow performance compared to `unsquashfs` ([see below](#performance)). -Currently things are still in flux after Zig 0.16's Io changes and the documentation below *might* not be up to date. - ## Build options -> `-Duse_c_libs=true` +> `-Duse_zig_decomp=true` -Instead of using Zig's standard library for decompression, use the system's C libraries. Has the benefit of being much faster and enabling LZO and LZ4 decompression. +Instead of using C libraries for decompression, use Zig's standard library for decompression. If using this option LZO and LZ4 decomrpession types are unsupported and decompression times will be significantly longer. + +> `-Ddynamic=true` + +Dynamicly link C libraries (if they're used) instead of statically linking them. > `-Dallow_lzo=true` @@ -37,20 +39,20 @@ Most features are present except for the following: ## Performance -This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive (which is fairly small and uses zstd compression) and with `-Doptimize=ReleaseFast`. +This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive which is fairly small and uses zstd compression with `-Doptimize=ReleaseFast`. Currently, my only performance checks are checking execution time, nothing deeper. -* Currently, using my test archive, performance matches `unsquashfs`. +* Currently, using my test archive, performance aproximately matches `unsquashfs` when multi-threaded, but significantly slower when single-threaded. * Using Zig decompression libraries *significantly* increases decompression time by 5x. Under ideal circumstances. * Performance improvements/regressions will be common. I'm still learning Zig. Example Times: -* *unsquashfs, multi-threaded*: .12s -* *unsquashfs, single-threaded*: .13s -* *C-libs, single-threaded*: CURRENTLY UNTESTED -* *C-libs, multi-threaded*: .16s +* *unsquashfs, multi-threaded*: .15s +* *unsquashfs, single-threaded*: .16s +* *C-libs, single-threaded*: .36s +* *C-libs, multi-threaded*: .14s * *Zig-libs, single-threaded*: CURRENTLY UNTESTED * *Zig-libs, multi-threaded*: .76s diff --git a/src/archive.zig b/src/archive.zig index 49f945a..349aea0 100644 --- a/src/archive.zig +++ b/src/archive.zig @@ -227,34 +227,24 @@ test "ExtractCompleteArchiveSingleThreaded" { std.debug.print("Starting test: ExtractCompleteArchive...\n", .{}); const alloc = std.testing.allocator; - var threaded: Io.Evented = undefined; - try threaded.init(alloc, .{}); - defer threaded.deinit(); - const io = threaded.io(); - var signal: u32 = 0; + const io = std.testing.io; - Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {}; - - const tmp = struct { - fn singleThreadedExtract(sig: *u32) !void { - var fil = try Io.Dir.cwd().openFile(Io.Threaded.global_single_threaded.io(), TestArchive, .{}); - defer fil.close(Io.Threaded.global_single_threaded.io()); - var sfs: Archive = try .init(Io.Threaded.global_single_threaded.io(), fil, 0); - defer sfs.deinit(Io.Threaded.global_single_threaded.io()); - try sfs.extract(std.testing.allocator, Io.Threaded.global_single_threaded.io(), TestFullExtractLocation, .default); - sig.* = 1; - } - }; - var ret = try io.concurrent(tmp.singleThreadedExtract, .{&signal}); - try io.futexWaitTimeout( - u32, - &signal, - 0, - .{ .deadline = .fromNow(io, .{ .raw = .fromSeconds(10), .clock = .awake }) }, - ); - if (ret.any_future == null) return ret.result; - try ret.cancel(io); - return error.TestTimeout; + var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); + defer fil.close(io); + { + std.debug.print("First testing using Threaded.global_single_threaded...\n", .{}); + Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {}; + var sfs: Archive = try .init(Io.Threaded.global_single_threaded.io(), fil, 0); + defer sfs.deinit(Io.Threaded.global_single_threaded.io()); + try sfs.extract(alloc, Io.Threaded.global_single_threaded.io(), TestFullExtractLocation, .default); + } + { + std.debug.print("Next testing using ExtractionOptions.single_threaded...\n", .{}); + Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {}; + var sfs: Archive = try .init(io, fil, 0); + defer sfs.deinit(io); + try sfs.extract(alloc, io, TestFullExtractLocation, .default_single_threaded); + } } const LinuxPATestCorrectSuperblock: Superblock = .{ diff --git a/src/bin/unsquashfs.zig b/src/bin/unsquashfs.zig index e0bdcf5..34ac098 100644 --- a/src/bin/unsquashfs.zig +++ b/src/bin/unsquashfs.zig @@ -60,6 +60,7 @@ pub fn main(init: std.process.Init) !void { var arc: squashfs.Archive = try .init(io, fil, offset); //TODO: Handle error gracefully. const options: squashfs.ExtractionOptions = .{ + .single_threaded = threads == 1, .verbose = verbose, .verbose_writer = if (verbose) &out.interface else null, .ignore_xattr = ignore_xattrs, @@ -69,9 +70,12 @@ pub fn main(init: std.process.Init) !void { if (force) try Io.Dir.cwd().deleteTree(io, extLoc); if (threads != 0) { - if (threads == 1) - return arc.extract(alloc, Io.Threaded.global_single_threaded.io(), extLoc, options); //TODO: Handle error gracefully. - var limited_io = Io.Threaded.init(alloc, .{ .async_limit = .limited(threads - 1), .concurrent_limit = .limited(threads - 1) }); + var limited_io = Io.Threaded.init(alloc, .{ + .async_limit = .limited(threads - 1), + .concurrent_limit = .limited(threads - 1), + .argv0 = .init(init.minimal.args), + .environ = init.minimal.environ, + }); return arc.extract(alloc, limited_io.io(), extLoc, options); //TODO: Handle error gracefully. } return arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully. diff --git a/src/decomp/zig_zstd.zig b/src/decomp/zig_zstd.zig index 731a4cf..c4af466 100644 --- a/src/decomp/zig_zstd.zig +++ b/src/decomp/zig_zstd.zig @@ -20,9 +20,9 @@ buf: [][]u8, buf_queue: Queue, pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self { - const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one. + const buf = try alloc.alloc([]u8, 5); // TODO: Choose a better number instead of a random one. var queue: Queue = .init(buf); - for (0..20) |_| + for (buf) |_| try queue.putOne(io, try alloc.alloc(u8, block_size + zstd.block_size_max)); return .{ diff --git a/src/inode.zig b/src/inode.zig index 303d9a8..ecfda82 100644 --- a/src/inode.zig +++ b/src/inode.zig @@ -282,17 +282,20 @@ pub fn extract( var frag_mgr: FragManager = try .init(alloc, fil, decomp, super.frag_start, super.frag_count, super.block_size); defer frag_mgr.deinit(io); + if (options.single_threaded) + return self.extractSinglethreaded(alloc, io, fil, super, path, options, decomp, &frag_mgr); + var sel_buf: [10]ExtractReturnUnion = undefined; var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf); defer sel.cancelDiscard(); var loop = io.async(finishLoop, .{ alloc, io, fil, decomp, super, options, &sel }); - sel.async(.path_ret, extractReal, .{ self, alloc, io, fil, super, decomp, &sel, &frag_mgr, path, true }); + sel.async(.path_ret, extractRealAsync, .{ self, alloc, io, fil, super, decomp, &sel, &frag_mgr, path, true }); try loop.await(io); } -fn extractReal( +fn extractRealAsync( self: Inode, alloc: std.mem.Allocator, io: Io, @@ -335,7 +338,7 @@ fn extractReal( const new_inode = try read(alloc, &meta.interface, super.block_size); errdefer new_inode.deinit(alloc); - sel.async(.path_ret, extractReal, .{ new_inode, alloc, io, fil, super, decomp, sel, frag_mgr, new_path, false }); + sel.async(.path_ret, extractRealAsync, .{ new_inode, alloc, io, fil, super, decomp, sel, frag_mgr, new_path, false }); } }, .file, .ext_file => { @@ -404,7 +407,6 @@ fn extractReal( .origin = origin, }; } - fn finishLoop(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, super: Archive.Superblock, options: ExtractionOptions, sel: *Io.Select(ExtractReturnUnion)) !void { var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count); defer id_table.deinit(io); @@ -433,8 +435,8 @@ fn finishLoop(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const try dir_queue.push(alloc, path_ret); continue; } - defer path_ret.deinit(alloc); + try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options); } @@ -461,3 +463,173 @@ fn finishLoop(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options); } } +/// Extracts the given inode to the given path. If the inode not a directory, the given path must not exist. +/// If the inode is a directory the path must not exist or be a directory. +fn extractSinglethreaded( + self: Inode, + alloc: std.mem.Allocator, + io: Io, + fil: OffsetFile, + super: Archive.Superblock, + path: []const u8, + options: ExtractionOptions, + decomp: *const Decompressor, + frag: *FragManager, +) !void { + var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count); + defer id_table.deinit(io); + + var xattr_table: ?XattrTable = if (super.flags.xattr_never or options.ignore_xattr or !@hasField(std.os, "linux")) + null + else + try .init(alloc, fil, decomp, super.xattr_start); + defer if (xattr_table != null) xattr_table.?.deinit(io); + + return self.extractReal( + alloc, + io, + fil, + super, + decomp, + frag, + &id_table, + if (xattr_table == null) null else &xattr_table.?, + path, + options, + ); +} +fn extractReal( + self: Inode, + alloc: std.mem.Allocator, + io: Io, + fil: OffsetFile, + super: Archive.Superblock, + decomp: *const Decompressor, + frag_mgr: *FragManager, + id_table: *CachedTable(u16), + xattr_table: ?*XattrTable, + path: []const u8, + options: ExtractionOptions, +) !void { + switch (self.hdr.inode_type) { + .dir, .ext_dir => { + try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777)); + + const entries = self.readDirectory(alloc, fil, decomp, super.dir_start) catch |err| switch (err) { + Error.NotDirectory, Error.NotExtended, Error.NotRegularFile, Error.NotSymlink => unreachable, + else => |e| return e, + }; + defer { + for (entries) |e| + e.deinit(alloc); + alloc.free(entries); + } + + for (entries) |e| { + const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", e.name }); + defer alloc.free(new_path); + + var rdr = fil.readerAt(super.inode_start + e.block_start); + var meta: MetadataReader = .init(alloc, &rdr, decomp); + try meta.interface.discardAll(e.block_offset); + + const new_inode = try read(alloc, &meta.interface, super.block_size); + defer new_inode.deinit(alloc); + + try new_inode.extractReal(alloc, io, fil, super, decomp, frag_mgr, id_table, xattr_table, new_path, options); + } + }, + .file, .ext_file => { + var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{ .make_path = true }); + defer atomic.deinit(io); + + var rdr: DataReader = switch (self.data) { + .file => |f| blk: { + var ext: DataReader = try .init(alloc, io, fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes); + if (f.frag_idx != 0xFFFFFFFF) + ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx)); + break :blk ext; + }, + .ext_file => |f| blk: { + var ext: DataReader = try .init(alloc, io, fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes); + if (f.frag_idx != 0xFFFFFFFF) + ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx)); + break :blk ext; + }, + else => unreachable, + }; + defer rdr.deinit(); + + var buf: [512 * 1024]u8 = undefined; + var wrt = atomic.file.writer(io, &buf); + + _ = try rdr.interface.streamRemaining(&wrt.interface); + + try wrt.flush(); + + try atomic.link(io); + }, + .symlink, .ext_symlink => try Io.Dir.cwd().symLink(io, self.symlinkTarget() catch unreachable, path, .{}), + else => { + var mode: u32 = undefined; + var dev: u32 = 0; + + const DT = std.posix.DT; + + switch (self.data) { + .char_dev => |d| { + dev = d.dev; + mode = DT.CHR; + }, + .ext_char_dev => |d| { + dev = d.dev; + mode = DT.CHR; + }, + .block_dev => |d| { + dev = d.dev; + mode = DT.BLK; + }, + .ext_block_dev => |d| { + dev = d.dev; + mode = DT.BLK; + }, + .fifo, .ext_fifo => mode = DT.FIFO, + .socket, .ext_socket => mode = DT.SOCK, + else => unreachable, + } + + const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0); + const res = std.os.linux.mknod(sentinel_path, mode, dev); + alloc.free(sentinel_path); + if (res != 0) + return ExtractError.MknodFailed; + }, + } + if (options.ignore_permissions and options.ignore_xattr) return; + + var f = try Io.Dir.cwd().openFile(io, path, .{}); + defer f.close(io); + + if (!options.ignore_permissions) { + try f.setPermissions(io, @enumFromInt(self.hdr.permissions)); + try f.setOwner(io, try id_table.get(io, self.hdr.uid_idx), try id_table.get(io, self.hdr.gid_idx)); + } + if (xattr_table != null) { + const idx = self.xattrIndex() catch return; + + const xattrs = try xattr_table.?.get(alloc, io, idx); + defer { + for (xattrs) |x| + x.deinit(alloc); + alloc.free(xattrs); + } + + const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0); + defer alloc.free(sentinel_path); + for (xattrs) |x| { + const xattr_ret = std.os.linux.fsetxattr(f.handle, x.key, x.value.ptr, x.value.len, 0); + if (xattr_ret != 0) + return ExtractError.CannotSetXattr; + } + } +} diff --git a/src/options.zig b/src/options.zig index 64cdb45..83ab440 100644 --- a/src/options.zig +++ b/src/options.zig @@ -5,6 +5,10 @@ const Writer = std.Io.Writer; const ExtractionOptions = @This(); +/// Extract single-threaded only. +/// Though not necessary if using Threaded.single_threaded, +/// setting single_threaded is more efficient. +single_threaded: bool = false, /// Don't set the file's owner & permissions after extraction ignore_permissions: bool = false, /// Don't set xattr values. Currently xattrs are never set anyway. @@ -17,6 +21,7 @@ verbose: bool = false, verbose_writer: ?*Writer = null, pub const default: ExtractionOptions = .{}; +pub const default_single_threaded: ExtractionOptions = .{ .single_threaded = true }; pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions { return .{