From 039e517183496fa9b8d59b0908652941ee86e719 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sat, 6 Jun 2026 02:54:35 -0500 Subject: [PATCH] Added mutli-threaded extraction --- src/archive.zig | 7 +- src/bin/unsquashfs.zig | 16 +- src/extract-multi.zig | 404 +++++++++++++++++++++++++++++++++++++++++ src/extract-single.zig | 287 +++++++++++++++++++++++++++++ src/extract.zig | 278 ++-------------------------- src/file.zig | 7 +- src/options.zig | 3 + src/test.zig | 84 --------- 8 files changed, 729 insertions(+), 357 deletions(-) create mode 100644 src/extract-multi.zig create mode 100644 src/extract-single.zig delete mode 100644 src/test.zig diff --git a/src/archive.zig b/src/archive.zig index 58b2618..0b63eff 100644 --- a/src/archive.zig +++ b/src/archive.zig @@ -84,7 +84,10 @@ pub fn extract(self: *Archive, alloc: std.mem.Allocator, io: Io, ext_loc: []cons self.super.root_ref, ); - return Extract.extractSingleThreaded(alloc, io, root_inode, &self.cache, self.super, ext_loc, options); + return if (options.single_threaded) + Extract.extractSingleThreaded(alloc, io, root_inode, &self.cache, self.super, ext_loc, options) + else + Extract.extractAsync(alloc, io, root_inode, &self.cache, self.super, ext_loc, options); } // Superblock @@ -188,7 +191,7 @@ test "FullExtraction" { var arc: Archive = try .init(alloc, io, archive_file); defer arc.deinit(io); - try arc.extract(alloc, io, TestFullExtractLocation, .default); + try arc.extract(alloc, io, TestFullExtractLocation, .default_single_threaded); } const LinuxPATestCorrectSuperblock: Superblock = .{ diff --git a/src/bin/unsquashfs.zig b/src/bin/unsquashfs.zig index a43c0a9..f76021a 100644 --- a/src/bin/unsquashfs.zig +++ b/src/bin/unsquashfs.zig @@ -57,18 +57,28 @@ pub fn main(init: std.process.Init) !void { defer fil.close(io); var arc: squashfs.Archive = try .initAdvanced(alloc, io, fil, offset, 0); //TODO: Update when memory size matters. //TODO: Handle error gracefully. defer arc.deinit(io); + const options: squashfs.ExtractionOptions = .{ + .single_threaded = threads == 1, .verbose = verbose, .verbose_writer = if (verbose) &out.interface else null, .ignore_xattr = ignore_xattrs, .ignore_permissions = ignore_permissions, }; + if (force) try Io.Dir.cwd().deleteTree(io, ext_loc); - if (threads != 1) { - try arc.extract(alloc, io, ext_loc, options); //TODO: Handle error gracefully. + + if (threads != 0 and threads != 1) { + var limited_threaded: Io.Threaded = .init(alloc, .{ + .argv0 = .init(init.minimal.args), + .async_limit = @enumFromInt(threads), + .concurrent_limit = @enumFromInt(threads), + .environ = init.minimal.environ, + }); + try arc.extract(alloc, limited_threaded.io(), ext_loc, options); //TODO: Handle error gracefully. } else { - try arc.extract(alloc, Io.Threaded.global_single_threaded.io(), ext_loc, options); + try arc.extract(alloc, io, ext_loc, options); //TODO: Handle error gracefully. } } diff --git a/src/extract-multi.zig b/src/extract-multi.zig new file mode 100644 index 0000000..331849b --- /dev/null +++ b/src/extract-multi.zig @@ -0,0 +1,404 @@ +const std = @import("std"); +const Io = std.Io; +const Atomic = std.atomic.Value; + +const DataExtractor = @import("data/extractor.zig"); +const DecompCache = @import("decomp_cache.zig"); +const Directory = @import("directory.zig"); +const ExtractionOptions = @import("options.zig"); +const Inode = @import("inode.zig"); +const Lookup = @import("lookup.zig"); +const Superblock = @import("archive.zig").Superblock; +const XattrTable = @import("xattr.zig"); + +pub fn extractAsync( + alloc: std.mem.Allocator, + io: Io, + inode: Inode, + cache: *DecompCache, + super: Superblock, + ext_loc: []const u8, + options: ExtractionOptions, +) !void { + const path = std.mem.trim(u8, ext_loc, "/"); + + var id_table: Lookup.Table(u16) = .init(alloc, cache, super.id_start, super.id_count); + defer id_table.deinit(io); + + var xattr_table: XattrTable = try .init(alloc, cache, super.xattr_start); + defer xattr_table.deinit(io); + + var sel_buffer: [100]ReturnUnion = undefined; + var sel: Io.Select(ReturnUnion) = .init(io, &sel_buffer); + defer { + while (sel.cancel()) |ret| switch (ret) { + .file_ret => |f| { + const res: FileReturn = f catch continue; + if (res.parent != null) + alloc.free(res.path); + }, + .dir_ret => |d| { + const res: DirReturn = d catch continue; + if (res.parent != null) + alloc.free(res.path); + alloc.destroy(res.children); + }, + .num_ret => {}, + }; + } + + var concurrent = true; + + var loop = io.concurrent(returnLoop, .{ alloc, io, &sel, &id_table, &xattr_table, options, inode.hdr.num }) catch blk: { + concurrent = false; + break :blk io.async(returnLoop, .{ alloc, io, &sel, &id_table, &xattr_table, options, inode.hdr.num }); + }; + + var frag_table: Lookup.Table(Lookup.FragmentEntry) = .init(alloc, cache, super.frag_start, super.frag_count); + defer frag_table.deinit(io); + + try extractReal(alloc, io, cache, super, &frag_table, &sel, concurrent, null, path, inode); + + try loop.await(io); +} + +fn extractReal( + alloc: std.mem.Allocator, + io: Io, + cache: *DecompCache, + super: Superblock, + frag_table: *Lookup.Table(Lookup.FragmentEntry), + sel: *Io.Select(ReturnUnion), + concurrent: bool, + parent: ?*Atomic(u32), + path: []const u8, + inode: Inode, +) !void { + errdefer if (parent != null) { + alloc.free(path); + inode.deinit(alloc); + }; + + switch (inode.hdr.type) { + .dir, .ext_dir => if (concurrent) + try sel.concurrent(.dir_ret, extractDir, .{ alloc, io, cache, super, frag_table, sel, concurrent, parent, path, inode }) + else + sel.async(.dir_ret, extractDir, .{ alloc, io, cache, super, frag_table, sel, concurrent, parent, path, inode }), + .file, .ext_file => if (concurrent) + try sel.concurrent(.file_ret, extractFile, .{ alloc, io, cache, frag_table, super.block_size, parent, path, inode }) + else + sel.async(.file_ret, extractFile, .{ alloc, io, cache, frag_table, super.block_size, parent, path, inode }), + .symlink, .ext_symlink => if (concurrent) + try sel.concurrent(.num_ret, extractSymlink, .{ alloc, io, parent, path, inode }) + else + sel.async(.num_ret, extractSymlink, .{ alloc, io, parent, path, inode }), + else => if (concurrent) + try sel.concurrent(.file_ret, extractNod, .{ alloc, parent, path, inode }) + else + sel.async(.file_ret, extractNod, .{ alloc, parent, path, inode }), + } +} + +fn extractDir( + alloc: std.mem.Allocator, + io: Io, + cache: *DecompCache, + super: Superblock, + frag_table: *Lookup.Table(Lookup.FragmentEntry), + sel: *Io.Select(ReturnUnion), + concurrent: bool, + parent: ?*Atomic(u32), + path: []const u8, + inode: Inode, +) Error!DirReturn { + errdefer if (parent != null) alloc.free(path); + + var dir = inode.directory(alloc, io, cache, super.dir_start) catch |err| switch (err) { + error.NotDirectory => unreachable, + else => |e| return e, + }; + defer dir.deinit(alloc); + + const children = try alloc.create(Atomic(usize)); + errdefer alloc.destroy(children); + children.* = .init(dir.entries.len); + + for (dir.entries) |entry| { + const new_inode: Inode = try .initDirEntry(alloc, io, cache, super.inode_start, super.block_size, entry); + + const new_path = std.mem.concat(alloc, u8, &.{ path, "/", entry.name }) catch |err| { + new_inode.deinit(alloc); + return err; + }; + + try extractReal(alloc, io, cache, super, frag_table, sel, concurrent, parent, new_path, new_inode); + } + + return .{ + .hdr = inode.hdr, + .path = path, + .parent = parent, + .children = children, + + .xattr_idx = switch (inode.data) { + .ext_dir => |d| if (d.xattr_idx != 0xFFFFFFFF) + d.xattr_idx + else + null, + else => null, + }, + }; +} +fn extractFile( + alloc: std.mem.Allocator, + io: Io, + cache: *DecompCache, + frag_table: *Lookup.Table(Lookup.FragmentEntry), + block_size: u32, + parent: ?*Atomic(u32), + path: []const u8, + inode: Inode, +) Error!FileReturn { + defer if (parent != null) { + inode.deinit(alloc); + _ = parent.?.fetchSub(1, .acquire); + }; + errdefer alloc.free(path); + + var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{}); + defer atomic.deinit(io); + + var ret: FileReturn = .{ + .hdr = inode.hdr, + .path = path, + .parent = parent, + }; + + var data: DataExtractor = switch (inode.data) { + .file => |f| blk: { + var data: DataExtractor = .init(cache, block_size, f.size, f.data_start, f.blocks); + if (f.frag_idx != 0xFFFFFFFF) { + const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx); + if (entry.size.uncompressed) { + data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset); + } else { + const block = try cache.get(io, entry.start, entry.size.size, block_size); + data.addFragment(block, f.frag_offset); + } + } + break :blk data; + }, + .ext_file => |f| blk: { + var data: DataExtractor = .init(cache, block_size, f.size, f.data_start, f.blocks); + if (f.frag_idx != 0xFFFFFFFF) { + const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx); + if (entry.size.uncompressed) { + data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset); + } else { + const block = try cache.get(io, entry.start, entry.size.size, block_size); + data.addFragment(block, f.frag_offset); + } + } + if (f.xattr_idx != 0xFFFFFFFF) + ret.xattr_idx = f.xattr_idx; + break :blk data; + }, + else => unreachable, + }; + + try data.asyncExtract(io, atomic.file); + + try atomic.link(io); + + return ret; +} +fn extractSymlink(alloc: std.mem.Allocator, io: Io, parent: ?*Atomic(u32), path: []const u8, inode: Inode) Error!u32 { + defer { + alloc.free(path); + if (parent != null) { + inode.deinit(alloc); + _ = parent.?.fetchSub(1, .acquire); + } + } + + const target = switch (inode.data) { + .symlink => |s| s.target, + .ext_symlink => |s| s.target, + else => unreachable, + }; + + try Io.Dir.cwd().symLink(io, target, path, .{}); + + return inode.hdr.num; +} +fn extractNod(alloc: std.mem.Allocator, parent: ?*Atomic(u32), path: []const u8, inode: Inode) Error!FileReturn { + errdefer if (parent != null) alloc.free(path); + + var ret: FileReturn = .{ + .hdr = inode.hdr, + .path = path, + .parent = parent, + }; + + const DT = std.os.linux.DT; + + var dev: u32 = 0; + var mode: u32 = undefined; + + switch (inode.data) { + .char_dev => |d| { + dev = d.device; + mode = DT.CHR; + }, + .ext_char_dev => |d| { + dev = d.device; + mode = DT.CHR; + if (d.xattr_idx != 0xFFFFFFFF) + ret.xattr_idx = d.xattr_idx; + }, + .block_dev => |d| { + dev = d.device; + mode = DT.BLK; + }, + .ext_block_dev => |d| { + dev = d.device; + mode = DT.BLK; + if (d.xattr_idx != 0xFFFFFFFF) + ret.xattr_idx = d.xattr_idx; + }, + .fifo => mode = DT.FIFO, + .ext_fifo => |f| { + mode = DT.FIFO; + if (f.xattr_idx != 0xFFFFFFFF) + ret.xattr_idx = f.xattr_idx; + }, + .socket => mode = DT.SOCK, + .ext_socket => |s| { + mode = DT.SOCK; + if (s.xattr_idx != 0xFFFFFFFF) + ret.xattr_idx = s.xattr_idx; + }, + else => unreachable, + } + + const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &.{path}, 0); + defer alloc.free(sentinel_path); + + const res = std.os.linux.mknod(sentinel_path, mode, dev); + if (res != 0) + return Error.MknodError; + + return ret; +} + +fn returnLoop( + alloc: std.mem.Allocator, + io: Io, + sel: *Io.Select(ReturnUnion), + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + options: ExtractionOptions, + origin_num: u32, +) !void { + var origin_finished = false; + while (!origin_finished) { + const ret = try sel.await(); + + switch (ret) { + .file_ret => |f| { + const res: FileReturn = try f; + if (res.hdr.num == origin_num) + origin_finished = true; + try res.finish(alloc, io, id_table, xattr_table, options); + }, + .dir_ret => |d| { + const res: DirReturn = try d; + if (res.children.load(.unordered) > 0) { + try sel.queue.putOne(io, .{ .dir_ret = res }); + continue; + } + if (res.hdr.num == origin_num) + origin_finished = true; + try res.finish(alloc, io, id_table, xattr_table, options); + }, + .num_ret => |n| { + const res: u32 = try n; + if (res == origin_num) + origin_finished = true; + }, + } + } +} + +fn setMetadata(alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions, hdr: Inode.Header, xattr_idx: ?u32, path: []const u8) Error!void { + if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == null)) return; + + const file = try Io.Dir.cwd().openFile(io, path, .{}); + defer file.close(io); + + if (!options.ignore_permissions) { + try file.setTimestamps(io, .{ .modify_timestamp = .init(Io.Timestamp.fromNanoseconds(@as(i96, @intCast(hdr.mod_time)) * std.time.ns_per_s)) }); + try file.setPermissions(io, @enumFromInt(hdr.permission)); + try file.setOwner(io, try id_table.get(io, hdr.uid_idx), try id_table.get(io, hdr.gid_idx)); + } + + if (!options.ignore_permissions and xattr_idx != null) { + var xattrs = try xattr_table.get(alloc, io, xattr_idx.?); + defer xattrs.deinit(alloc); + + for (xattrs.xattrs) |x| { + const res = std.os.linux.fsetxattr(file.handle, x.key, x.value.ptr, x.value.len, 0); + if (res != 0) + return Error.SetXattrError; + } + } +} + +// Types + +const Error = error{ Canceled, MknodError, SetXattrError, ConcurrencyUnavailable } || Directory.Error || Io.Dir.CreateFileAtomicError || + Io.File.Atomic.LinkError || Io.Dir.SymLinkError || Io.Dir.CreateDirPathError || Io.Reader.StreamError || Io.File.SetPermissionsError || + Io.File.Writer.Error || DataExtractor.Error; + +const ReturnUnion = union(enum) { + file_ret: Error!FileReturn, + dir_ret: Error!DirReturn, + num_ret: Error!u32, +}; + +const FileReturn = struct { + hdr: Inode.Header, + path: []const u8, + parent: ?*Atomic(u32), + + xattr_idx: ?u32 = null, + + fn finish(self: FileReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void { + defer if (self.parent != null) { + alloc.free(self.path); + _ = self.parent.?.fetchSub(1, .acquire); + }; + return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path); + } +}; +const DirReturn = struct { + hdr: Inode.Header, + path: []const u8, + parent: ?*Atomic(u32), + children: *Atomic(usize), + + xattr_idx: ?u32 = null, + + fn finish(self: DirReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void { + std.debug.assert(self.children.load(.unordered) == 0); + + defer { + if (self.parent != null) { + alloc.free(self.path); + _ = self.parent.?.fetchSub(1, .acquire); + } + alloc.destroy(self.children); + } + return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path); + } +}; diff --git a/src/extract-single.zig b/src/extract-single.zig new file mode 100644 index 0000000..b94214e --- /dev/null +++ b/src/extract-single.zig @@ -0,0 +1,287 @@ +const std = @import("std"); +const Io = std.Io; +const Atomic = std.atomic.Value; + +const DataReader = @import("data/reader.zig"); +const DecompCache = @import("decomp_cache.zig"); +const Directory = @import("directory.zig"); +const ExtractionOptions = @import("options.zig"); +const Inode = @import("inode.zig"); +const Lookup = @import("lookup.zig"); +const Superblock = @import("archive.zig").Superblock; +const XattrTable = @import("xattr.zig"); + +pub fn extractSingleThreaded(alloc: std.mem.Allocator, io: Io, inode: Inode, cache: *DecompCache, super: Superblock, ext_loc: []const u8, options: ExtractionOptions) !void { + const path = std.mem.trim(u8, ext_loc, "/"); + + var id_table: Lookup.Table(u16) = .init(alloc, cache, super.id_start, super.id_count); + defer id_table.deinit(io); + + var xattr_table: XattrTable = try .init(alloc, cache, super.xattr_start); + defer xattr_table.deinit(io); + + var frag_table: Lookup.Table(Lookup.FragmentEntry) = .init(alloc, cache, super.frag_start, super.frag_count); + defer frag_table.deinit(io); + + try extractReal(alloc, io, cache, super, &frag_table, &id_table, &xattr_table, options, path, inode, true); +} + +fn extractReal( + alloc: std.mem.Allocator, + io: Io, + cache: *DecompCache, + super: Superblock, + frag_table: *Lookup.Table(Lookup.FragmentEntry), + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + options: ExtractionOptions, + path: []const u8, + inode: Inode, + origin: bool, +) Error!void { + defer if (!origin) { + inode.deinit(alloc); + alloc.free(path); + }; + switch (inode.data) { + .dir, .ext_dir => { + const res = try extractDir(alloc, io, cache, super, frag_table, id_table, xattr_table, options, path, inode); + try res.finish(alloc, io, id_table, xattr_table, options); + }, + .file, .ext_file => { + const res = try extractFile(io, cache, super.block_size, frag_table, path, inode); + try res.finish(alloc, io, id_table, xattr_table, options); + }, + .symlink, .ext_symlink => try extractSymlink(io, path, inode), + else => { + const res = try extractNod(alloc, path, inode); + try res.finish(alloc, io, id_table, xattr_table, options); + }, + } +} + +fn extractDir( + alloc: std.mem.Allocator, + io: Io, + cache: *DecompCache, + super: Superblock, + frag_table: *Lookup.Table(Lookup.FragmentEntry), + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + options: ExtractionOptions, + path: []const u8, + inode: Inode, +) Error!DirReturn { + try Io.Dir.cwd().createDirPath(io, path); + + const dir = inode.directory(alloc, io, cache, super.dir_start) catch |err| switch (err) { + error.NotDirectory => unreachable, + else => |e| return e, + }; + defer dir.deinit(alloc); + + const ret: DirReturn = .{ + .hdr = inode.hdr, + .path = path, + + .xattr_idx = switch (inode.data) { + .ext_dir => |d| if (d.xattr_idx != 0xFFFFFFFF) d.xattr_idx else null, + else => null, + }, + }; + + for (dir.entries) |entry| { + const new_inode: Inode = try .initDirEntry(alloc, io, cache, super.inode_start, super.block_size, entry); + + const new_path = std.mem.concat(alloc, u8, &.{ path, "/", entry.name }) catch |err| { + new_inode.deinit(alloc); + return err; + }; + + try extractReal( + alloc, + io, + cache, + super, + frag_table, + id_table, + xattr_table, + options, + new_path, + new_inode, + false, + ); + } + return ret; +} +fn extractFile( + io: Io, + cache: *DecompCache, + block_size: u32, + frag_table: *Lookup.Table(Lookup.FragmentEntry), + path: []const u8, + inode: Inode, +) Error!FileReturn { + var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{}); + defer atomic.deinit(io); + + var ret: FileReturn = .{ + .hdr = inode.hdr, + .path = path, + }; + + var data: DataReader = switch (inode.data) { + .file => |f| blk: { + var data: DataReader = .init(io, cache, block_size, f.size, f.data_start, f.blocks); + if (f.frag_idx != 0xFFFFFFFF) { + const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx); + if (entry.size.uncompressed) { + data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset); + } else { + const block = try cache.get(io, entry.start, entry.size.size, block_size); + data.addFragment(block, f.frag_offset); + } + } + break :blk data; + }, + .ext_file => |f| blk: { + var data: DataReader = .init(io, cache, block_size, f.size, f.data_start, f.blocks); + if (f.frag_idx != 0xFFFFFFFF) { + const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx); + if (entry.size.uncompressed) { + data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset); + } else { + const block = try cache.get(io, entry.start, entry.size.size, block_size); + data.addFragment(block, f.frag_offset); + } + } + if (f.xattr_idx != 0xFFFFFFFF) + ret.xattr_idx = f.xattr_idx; + break :blk data; + }, + else => unreachable, + }; + defer data.deinit(); + + var wrt_buf: [1024]u8 = undefined; + var wrt = atomic.file.writer(io, &wrt_buf); + _ = try data.interface.streamRemaining(&wrt.interface); + try wrt.flush(); + + try atomic.link(io); + + return ret; +} +fn extractSymlink(io: Io, path: []const u8, inode: Inode) Error!void { + const target = switch (inode.data) { + .symlink => |s| s.target, + .ext_symlink => |s| s.target, + else => unreachable, + }; + + try Io.Dir.cwd().symLink(io, target, path, .{}); +} +fn extractNod(alloc: std.mem.Allocator, path: []const u8, inode: Inode) Error!FileReturn { + var ret: FileReturn = .{ + .hdr = inode.hdr, + .path = path, + }; + + const DT = std.os.linux.DT; + + var dev: u32 = 0; + var mode: u32 = undefined; + + switch (inode.data) { + .char_dev => |d| { + dev = d.device; + mode = DT.CHR; + }, + .ext_char_dev => |d| { + dev = d.device; + mode = DT.CHR; + if (d.xattr_idx != 0xFFFFFFFF) + ret.xattr_idx = d.xattr_idx; + }, + .block_dev => |d| { + dev = d.device; + mode = DT.BLK; + }, + .ext_block_dev => |d| { + dev = d.device; + mode = DT.BLK; + if (d.xattr_idx != 0xFFFFFFFF) + ret.xattr_idx = d.xattr_idx; + }, + .fifo => mode = DT.FIFO, + .ext_fifo => |f| { + mode = DT.FIFO; + if (f.xattr_idx != 0xFFFFFFFF) + ret.xattr_idx = f.xattr_idx; + }, + .socket => mode = DT.SOCK, + .ext_socket => |s| { + mode = DT.SOCK; + if (s.xattr_idx != 0xFFFFFFFF) + ret.xattr_idx = s.xattr_idx; + }, + else => unreachable, + } + + const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &.{path}, 0); + defer alloc.free(sentinel_path); + + const res = std.os.linux.mknod(sentinel_path, mode, dev); + if (res != 0) + return Error.MknodError; + + return ret; +} + +fn setMetadata(alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions, hdr: Inode.Header, xattr_idx: ?u32, path: []const u8) Error!void { + if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == null)) return; + + const file = try Io.Dir.cwd().openFile(io, path, .{}); + defer file.close(io); + + if (!options.ignore_permissions) { + try file.setTimestamps(io, .{ .modify_timestamp = .init(Io.Timestamp.fromNanoseconds(@as(i96, @intCast(hdr.mod_time)) * std.time.ns_per_s)) }); + try file.setPermissions(io, @enumFromInt(hdr.permission)); + try file.setOwner(io, try id_table.get(io, hdr.uid_idx), try id_table.get(io, hdr.gid_idx)); + } + + if (!options.ignore_permissions and xattr_idx != null) { + var xattrs = try xattr_table.get(alloc, io, xattr_idx.?); + defer xattrs.deinit(alloc); + + for (xattrs.xattrs) |x| { + const res = std.os.linux.fsetxattr(file.handle, x.key, x.value.ptr, x.value.len, 0); + if (res != 0) + return Error.SetXattrError; + } + } +} + +const Error = error{ Canceled, MknodError, SetXattrError } || Directory.Error || Io.Dir.CreateFileAtomicError || Io.File.Atomic.LinkError || + Io.Dir.SymLinkError || Io.Dir.CreateDirPathError || Io.Reader.StreamError || Io.File.SetPermissionsError || Io.File.Writer.Error; + +const FileReturn = struct { + hdr: Inode.Header, + path: []const u8, + + xattr_idx: ?u32 = null, + + fn finish(self: FileReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void { + return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path); + } +}; +const DirReturn = struct { + hdr: Inode.Header, + path: []const u8, + + xattr_idx: ?u32 = null, + + fn finish(self: DirReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void { + return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path); + } +}; diff --git a/src/extract.zig b/src/extract.zig index 1d32d24..1d41b11 100644 --- a/src/extract.zig +++ b/src/extract.zig @@ -1,287 +1,33 @@ const std = @import("std"); const Io = std.Io; -const Atomic = std.atomic.Value; -const DataReader = @import("data/reader.zig"); const DecompCache = @import("decomp_cache.zig"); -const Directory = @import("directory.zig"); const ExtractionOptions = @import("options.zig"); const Inode = @import("inode.zig"); -const Lookup = @import("lookup.zig"); +const MultiThreaded = @import("extract-multi.zig"); +const SingleThreaded = @import("extract-single.zig"); const Superblock = @import("archive.zig").Superblock; -const XattrTable = @import("xattr.zig"); -pub fn extractSingleThreaded(alloc: std.mem.Allocator, io: Io, inode: Inode, cache: *DecompCache, super: Superblock, ext_loc: []const u8, options: ExtractionOptions) !void { - const path = std.mem.trim(u8, ext_loc, "/"); - - var id_table: Lookup.Table(u16) = .init(alloc, cache, super.id_start, super.id_count); - defer id_table.deinit(io); - - var xattr_table: XattrTable = try .init(alloc, cache, super.xattr_start); - defer xattr_table.deinit(io); - - var frag_table: Lookup.Table(Lookup.FragmentEntry) = .init(alloc, cache, super.frag_start, super.frag_count); - defer frag_table.deinit(io); - - try extractReal(alloc, io, cache, super, &frag_table, &id_table, &xattr_table, options, path, inode, true); -} - -fn extractReal( +pub fn extractSingleThreaded( alloc: std.mem.Allocator, io: Io, + inode: Inode, cache: *DecompCache, super: Superblock, - frag_table: *Lookup.Table(Lookup.FragmentEntry), - id_table: *Lookup.Table(u16), - xattr_table: *XattrTable, + ext_loc: []const u8, options: ExtractionOptions, - path: []const u8, - inode: Inode, - origin: bool, -) Error!void { - defer if (!origin) { - inode.deinit(alloc); - alloc.free(path); - }; - switch (inode.data) { - .dir, .ext_dir => { - const res = try extractDir(alloc, io, cache, super, frag_table, id_table, xattr_table, options, path, inode); - try res.finish(alloc, io, id_table, xattr_table, options); - }, - .file, .ext_file => { - const res = try extractFile(io, cache, super.block_size, frag_table, path, inode); - try res.finish(alloc, io, id_table, xattr_table, options); - }, - .symlink, .ext_symlink => try extractSymlink(io, path, inode), - else => { - const res = try extractNod(alloc, path, inode); - try res.finish(alloc, io, id_table, xattr_table, options); - }, - } +) !void { + return SingleThreaded.extractSingleThreaded(alloc, io, inode, cache, super, ext_loc, options); } -fn extractDir( +pub fn extractAsync( alloc: std.mem.Allocator, io: Io, + inode: Inode, cache: *DecompCache, super: Superblock, - frag_table: *Lookup.Table(Lookup.FragmentEntry), - id_table: *Lookup.Table(u16), - xattr_table: *XattrTable, + ext_loc: []const u8, options: ExtractionOptions, - path: []const u8, - inode: Inode, -) Error!DirReturn { - try Io.Dir.cwd().createDirPath(io, path); - - const dir = inode.directory(alloc, io, cache, super.dir_start) catch |err| switch (err) { - error.NotDirectory => unreachable, - else => |e| return e, - }; - defer dir.deinit(alloc); - - const ret: DirReturn = .{ - .hdr = inode.hdr, - .path = path, - - .xattr_idx = switch (inode.data) { - .ext_dir => |d| if (d.xattr_idx != 0xFFFFFFFF) d.xattr_idx else null, - else => null, - }, - }; - - for (dir.entries) |entry| { - const new_inode: Inode = try .initDirEntry(alloc, io, cache, super.inode_start, super.block_size, entry); - - const new_path = std.mem.concat(alloc, u8, &.{ path, "/", entry.name }) catch |err| { - new_inode.deinit(alloc); - return err; - }; - - try extractReal( - alloc, - io, - cache, - super, - frag_table, - id_table, - xattr_table, - options, - new_path, - new_inode, - false, - ); - } - return ret; +) !void { + return MultiThreaded.extractAsync(alloc, io, inode, cache, super, ext_loc, options); } -fn extractFile( - io: Io, - cache: *DecompCache, - block_size: u32, - frag_table: *Lookup.Table(Lookup.FragmentEntry), - path: []const u8, - inode: Inode, -) Error!FileReturn { - var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{}); - defer atomic.deinit(io); - - var ret: FileReturn = .{ - .hdr = inode.hdr, - .path = path, - }; - - var data: DataReader = switch (inode.data) { - .file => |f| blk: { - var data: DataReader = .init(io, cache, block_size, f.size, f.data_start, f.blocks); - if (f.frag_idx != 0xFFFFFFFF) { - const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx); - if (entry.size.uncompressed) { - data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset); - } else { - const block = try cache.get(io, entry.start, entry.size.size, block_size); - data.addFragment(block, f.frag_offset); - } - } - break :blk data; - }, - .ext_file => |f| blk: { - var data: DataReader = .init(io, cache, block_size, f.size, f.data_start, f.blocks); - if (f.frag_idx != 0xFFFFFFFF) { - const entry: Lookup.FragmentEntry = try frag_table.get(io, f.frag_idx); - if (entry.size.uncompressed) { - data.addFragment(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset); - } else { - const block = try cache.get(io, entry.start, entry.size.size, block_size); - data.addFragment(block, f.frag_offset); - } - } - if (f.xattr_idx != 0xFFFFFFFF) - ret.xattr_idx = f.xattr_idx; - break :blk data; - }, - else => unreachable, - }; - defer data.deinit(); - - var wrt_buf: [1024]u8 = undefined; - var wrt = atomic.file.writer(io, &wrt_buf); - _ = try data.interface.streamRemaining(&wrt.interface); - try wrt.flush(); - - try atomic.link(io); - - return ret; -} -fn extractSymlink(io: Io, path: []const u8, inode: Inode) Error!void { - const target = switch (inode.data) { - .symlink => |s| s.target, - .ext_symlink => |s| s.target, - else => unreachable, - }; - - try Io.Dir.cwd().symLink(io, target, path, .{}); -} -fn extractNod(alloc: std.mem.Allocator, path: []const u8, inode: Inode) Error!FileReturn { - var ret: FileReturn = .{ - .hdr = inode.hdr, - .path = path, - }; - - const DT = std.os.linux.DT; - - var dev: u32 = 0; - var mode: u32 = undefined; - - switch (inode.data) { - .char_dev => |d| { - dev = d.device; - mode = DT.CHR; - }, - .ext_char_dev => |d| { - dev = d.device; - mode = DT.CHR; - if (d.xattr_idx != 0xFFFFFFFF) - ret.xattr_idx = d.xattr_idx; - }, - .block_dev => |d| { - dev = d.device; - mode = DT.BLK; - }, - .ext_block_dev => |d| { - dev = d.device; - mode = DT.BLK; - if (d.xattr_idx != 0xFFFFFFFF) - ret.xattr_idx = d.xattr_idx; - }, - .fifo => mode = DT.FIFO, - .ext_fifo => |f| { - mode = DT.FIFO; - if (f.xattr_idx != 0xFFFFFFFF) - ret.xattr_idx = f.xattr_idx; - }, - .socket => mode = DT.SOCK, - .ext_socket => |s| { - mode = DT.SOCK; - if (s.xattr_idx != 0xFFFFFFFF) - ret.xattr_idx = s.xattr_idx; - }, - else => unreachable, - } - - const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &.{path}, 0); - defer alloc.free(sentinel_path); - - const res = std.os.linux.mknod(sentinel_path, mode, dev); - if (res != 0) - return Error.MknodError; - - return ret; -} - -fn setMetadata(alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions, hdr: Inode.Header, xattr_idx: ?u32, path: []const u8) Error!void { - if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == null)) return; - - const file = try Io.Dir.cwd().openFile(io, path, .{}); - defer file.close(io); - - if (!options.ignore_permissions) { - try file.setTimestamps(io, .{ .modify_timestamp = .init(Io.Timestamp.fromNanoseconds(@as(i96, @intCast(hdr.mod_time)) * std.time.ns_per_s)) }); - try file.setPermissions(io, @enumFromInt(hdr.permission)); - try file.setOwner(io, try id_table.get(io, hdr.uid_idx), try id_table.get(io, hdr.gid_idx)); - } - - if (!options.ignore_permissions and xattr_idx != null) { - var xattrs = try xattr_table.get(alloc, io, xattr_idx.?); - defer xattrs.deinit(alloc); - - for (xattrs.xattrs) |x| { - const res = std.os.linux.fsetxattr(file.handle, x.key, x.value.ptr, x.value.len, 0); - if (res != 0) - return Error.SetXattrError; - } - } -} - -const Error = error{ Canceled, MknodError, SetXattrError } || Directory.Error || Io.Dir.CreateFileAtomicError || Io.File.Atomic.LinkError || - Io.Dir.SymLinkError || Io.Dir.CreateDirPathError || Io.Reader.StreamError || Io.File.SetPermissionsError; - -const FileReturn = struct { - hdr: Inode.Header, - path: []const u8, - - xattr_idx: ?u32 = null, - - fn finish(self: FileReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void { - return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path); - } -}; -const DirReturn = struct { - hdr: Inode.Header, - path: []const u8, - - xattr_idx: ?u32 = null, - - fn finish(self: DirReturn, alloc: std.mem.Allocator, io: Io, id_table: *Lookup.Table(u16), xattr_table: *XattrTable, options: ExtractionOptions) Error!void { - return setMetadata(alloc, io, id_table, xattr_table, options, self.hdr, self.xattr_idx, self.path); - } -}; diff --git a/src/file.zig b/src/file.zig index 21009b7..d887cd0 100644 --- a/src/file.zig +++ b/src/file.zig @@ -102,6 +102,9 @@ pub fn open(self: *SfsFile, alloc: std.mem.Allocator, io: Io, filepath: []const return found_file.open(alloc, io, path[first_element.len..]); } -pub fn extract(self: *SfsFile, alloc: std.mem.Allocator, io: Io, ext_dir: []const u8, options: ExtractionOptions) !void { - return Extract.extractSingleThreaded(alloc, io, self.inode, &self.archive.cache, self.archive.super, ext_dir, options); +pub fn extract(self: *SfsFile, alloc: std.mem.Allocator, io: Io, ext_loc: []const u8, options: ExtractionOptions) !void { + return if (options.single_threaded) + Extract.extractSingleThreaded(alloc, io, self.inode, &self.archive.cache, self.archive.super, ext_loc, options) + else + Extract.extractAsync(alloc, io, self.inode, &self.archive.cache, self.archive.super, ext_loc, options); } diff --git a/src/options.zig b/src/options.zig index 85c5ea7..aec330a 100644 --- a/src/options.zig +++ b/src/options.zig @@ -5,6 +5,8 @@ const Writer = std.Io.Writer; const ExtractionOptions = @This(); +/// Force single-threaded extraction. +single_threaded: bool = false, /// Don't set the file's owner & permissions after extraction ignore_permissions: bool = false, /// Don't set xattr values. Currently xattrs are never set anyway. @@ -17,6 +19,7 @@ verbose: bool = false, verbose_writer: ?*Writer = null, pub const default: ExtractionOptions = .{}; +pub const default_single_threaded: ExtractionOptions = .{ .single_threaded = true }; pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions { return .{ .verbose = true, diff --git a/src/test.zig b/src/test.zig deleted file mode 100644 index 4bb58cb..0000000 --- a/src/test.zig +++ /dev/null @@ -1,84 +0,0 @@ -const std = @import("std"); -const stuff = @import("builtin"); - -const Archive = @import("archive.zig"); -const Superblock = @import("super.zig").Superblock; - -const TestArchive = "testing/LinuxPATest.sfs"; - -test "Basics" { - var fil = try std.fs.cwd().openFile(TestArchive, .{}); - defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - if (sfs.super != LinuxPATestCorrectSuperblock) { - std.debug.print("Superblock wrong\nShould be: {}\n\nis: {}\n", .{ LinuxPATestCorrectSuperblock, sfs.super }); - return error.BadSuperblock; - } -} - -const TestFile = "Start.exe"; -const TestFileExtractLocation = "testing/Start.exe"; - -test "ExtractSingleFile" { - std.fs.cwd().deleteFile(TestFileExtractLocation) catch {}; - var fil = try std.fs.cwd().openFile(TestArchive, .{}); - defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - var test_fil = try sfs.open(TestFile); - defer test_fil.deinit(); - try test_fil.extract(TestFileExtractLocation, .Default); - //TODO: validate extracted file. -} - -const TestFullExtractLocation = "testing/TestExtract"; - -test "ExtractCompleteArchive" { - std.fs.cwd().deleteTree(TestFullExtractLocation) catch {}; - var fil = try std.fs.cwd().openFile(TestArchive, .{}); - defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - try sfs.extract(TestFullExtractLocation, .Default); -} - -const LinuxPATestCorrectSuperblock: Superblock = .{ - .magic = std.mem.readInt(u32, "hsqs", .little), - .inode_count = 2974, - .mod_time = 1632696724, - .block_size = 131072, - .frag_count = 264, - .compression = .zstd, - .block_log = 17, - .flags = .{ - .inode_uncompressed = false, - .data_uncompressed = false, - .check = false, - .frag_uncompressed = false, - .fragment_never = false, - .fragment_always = false, - .duplicates = true, - .exportable = true, - .xattr_uncompressed = false, - .xattr_never = false, - .compression_options = false, - .ids_uncompressed = false, - ._ = 0, - }, - .id_count = 1, - .ver_maj = 4, - .ver_min = 0, - .root_ref = .{ - .block_offset = 1363, - .block_start = 29237, - ._ = 0, - }, - .size = 106841744, - .id_start = 106841632, - .xattr_start = 106841720, - .inode_start = 106778274, - .dir_start = 106807998, - .frag_start = 106837747, - .export_start = 106841602, -};