From b28508a9c29eb9473e8891cdf8726b7e435d9e9a Mon Sep 17 00:00:00 2001 From: "Caleb J. Gardner" Date: Sat, 25 Jul 2026 03:16:06 -0500 Subject: [PATCH] Work on mutli-threaded extraction --- src/data/extractor.zig | 6 +- src/extract-multi.zig | 445 +++++++++++++++++++++++++---------------- 2 files changed, 281 insertions(+), 170 deletions(-) diff --git a/src/data/extractor.zig b/src/data/extractor.zig index 9012520..5dd0fc7 100644 --- a/src/data/extractor.zig +++ b/src/data/extractor.zig @@ -43,16 +43,16 @@ pub fn addCache(self: *Extractor, cache: *Cache) void { self.cache = cache; } -pub fn extractAsync(self: Extractor, alloc: std.mem.Allocator, io: Io, select: *Io.Select(Multi.SelectUnion), finish: *FileFinish) void { +pub fn extractAsync(self: Extractor, alloc: std.mem.Allocator, io: Io, select: *Io.Select(Multi.SelectUnion), finish: *FileFinish) !void { if (self.size == 0) return; var read_offset: u64 = self.start; for (0.., self.blocks) |i, block| { - select.async(.reg, blockThread, .{ self, alloc, io, finish.file.file, read_offset, @truncate(i), finish }); + try select.concurrent(.reg, blockThread, .{ self, alloc, io, finish.file.file, read_offset, @truncate(i), finish }); read_offset += block.size; } if (self.frag_data != null) - select.async(.reg, fragThread, .{ self, io, finish.file.file, finish }); + try select.concurrent(.reg, fragThread, .{ self, io, finish.file.file, finish }); } fn blockThread( diff --git a/src/extract-multi.zig b/src/extract-multi.zig index ca1ec3b..3efd94e 100644 --- a/src/extract-multi.zig +++ b/src/extract-multi.zig @@ -16,87 +16,158 @@ const XattrTable = @import("xattr.zig"); pub fn extract(alloc: std.mem.Allocator, io: Io, super: Superblock, data: []u8, decomp: Decomp.Fn, inode: Inode, filepath: []const u8, options: ExtractionOption) !void { const path = std.mem.trim(u8, filepath, "/"); - var common: Common = try .init(alloc, io, super, data, decomp, options); - defer common.deinit(); + var frag_table: Lookup.Table(Lookup.FragEntry) = .init(alloc, data, decomp, super.frag_start, super.frag_count); + defer frag_table.deinit(); - common.start(io, inode, path, null); + var id_table: Lookup.Table(u16) = .init(alloc, data, decomp, super.id_start, super.id_count); + defer id_table.deinit(); - var buf: [5]SelectUnion = undefined; + var xattr_table: XattrTable = .init(alloc, data, decomp, super.xattr_start); + defer xattr_table.deinit(); - while (common.select.group.token.load(.unordered)) |_| { - const num = try common.select.awaitMany(&buf, 1); - for (buf[0..num]) |res| - try res.reg; - } + var cache: Cache = .init(alloc, data, decomp); + defer cache.deinit(); - if (common.err != null) - return common.err.?; + var err: ?Error = null; + var group: Io.Group = .init; + + group.async( + io, + groupExtract, + .{ alloc, io, data, decomp, super, &group, &frag_table, &id_table, &xattr_table, &cache, &err, inode, path, options, null }, + ); } -fn extractDir(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) Error!void { + +fn groupExtract( + alloc: std.mem.Allocator, + io: Io, + data: []u8, + decomp: Decomp.Fn, + super: Superblock, + group: *Io.Group, + frag_table: *Lookup.Table(Lookup.FragEntry), + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + cache: *Cache, + err: *?Error, + inode: Inode, + path: []const u8, + options: ExtractionOption, + parent: ?*Parent, +) error{Canceled}!void { + if (err != null) { + if (parent != null) { + alloc.free(path); + inode.deinit(alloc); + parent.?.finish(alloc, io); + } + return; + } + switch (inode.hdr.type) { + .dir, .ext_dir => group.async(io, extractDir, .{ alloc, io, data, decomp, super, group, frag_table, id_table, xattr_table, cache, inode, path, options, parent }), + .file, .ext_file => group.async(io, extractReg, .{ alloc, io, data, decomp, super.block_size, group, frag_table, cache, inode, path, options, parent }), + .symlink, .ext_symlink => group.async(io, extractSymlink, .{ alloc, io, inode, path, parent }), + else => group.async(io, extractNod, .{ alloc, io, id_table, xattr_table, inode, path, options, parent }), + } +} + +fn extractDir( + alloc: std.mem.Allocator, + io: Io, + data: []u8, + decomp: Decomp.Fn, + super: Superblock, + group: *Io.Group, + frag_table: *Lookup.Table(Lookup.FragEntry), + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + cache: *Cache, + err: *?Error, + inode: Inode, + path: []const u8, + options: ExtractionOption, + parent: ?*Parent, +) Error!void { var xattr_idx: u32 = 0xFFFFFFFF; const dir: Directory = switch (inode.data) { .dir => |d| blk: { - var meta: MetadataReader = .init(common.alloc, common.data, common.decomp, common.dir_start + d.block_start); + var meta: MetadataReader = .init(alloc, data, decomp, super.dir_start + d.block_start); try meta.interface.discardAll(d.block_offset); - break :blk try .init(common.alloc, &meta.interface, d.size); + break :blk try .init(alloc, &meta.interface, d.size); }, .ext_dir => |d| blk: { xattr_idx = d.xattr_idx; - var meta: MetadataReader = .init(common.alloc, common.data, common.decomp, common.dir_start + d.block_start); + var meta: MetadataReader = .init(alloc, data, decomp, super.dir_start + d.block_start); try meta.interface.discardAll(d.block_offset); - break :blk try .init(common.alloc, &meta.interface, d.size); + break :blk try .init(alloc, &meta.interface, d.size); }, else => unreachable, }; - defer dir.deinit(common.alloc); + defer dir.deinit(alloc); try Io.Dir.cwd().createDirPath(io, path); if (dir.entries.len == 0) { - try setMetadataPath(common, io, inode.hdr, path, xattr_idx); + try setMetadataPath(alloc, io, inode.hdr, id_table, xattr_table, path, xattr_idx, options); if (parent != null) { try parent.?.finish(io); - common.alloc.free(path); } return; } - const cur_dir: *Parent = try .init(common, dir.entries.len, inode.hdr, path, xattr_idx, parent); + const cur_dir = try Parent.create(alloc, id_table, xattr_table, err, inode.hdr, path, options, parent, xattr_idx, dir.entries); for (dir.entries) |entry| { - var new_inode: Inode = try .initEntry(common.alloc, common.data, common.decomp, common.inode_start, common.block_size, entry); + const new_inode: Inode = try .initEntry(alloc, data, decomp, super.inode_start, super.block_size, entry); - const new_path = std.mem.concat(common.alloc, u8, &.{ path, "/", entry.name }) catch |err| { - new_inode.deinit(common.alloc); - return err; + const new_path = std.mem.concat(alloc, u8, &.{ path, "/", entry.name }) catch |e| { + new_inode.deinit(alloc); + return e; }; - common.start(io, new_inode, new_path, cur_dir); + group.async( + io, + groupExtract, + .{ alloc, io, data, decomp, super, group, frag_table, id_table, xattr_table, cache, err, new_inode, new_path, options, cur_dir }, + ); } } -fn extractReg(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) Error!void { - defer if (parent != null) - common.alloc.free(path); - +fn extractReg( + alloc: std.mem.Allocator, + io: Io, + data: []u8, + decomp: Decomp.Fn, + block_size: u32, + group: *Io.Group, + frag_table: *Lookup.Table(Lookup.FragEntry), + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + cache: *Cache, + err: *?Error, + inode: Inode, + path: []const u8, + options: ExtractionOption, + parent: ?*Parent, +) Error!void { var blocks: usize = 0; var xattr_idx: u32 = 0xFFFFFFFF; var ext: DataExtractor = switch (inode.data) { .file => |f| blk: { - var ext: DataExtractor = .init(common.data, common.decomp, common.block_size, f.blocks, f.block_start, f.size); + var ext: DataExtractor = .init(data, decomp, block_size, f.blocks, f.block_start, f.size); blocks = f.blocks.len; if (f.frag_idx != 0xFFFFFFFF) { blocks += 1; - const frag: Lookup.FragEntry = try common.frag_table.get(io, f.frag_idx); + const frag: Lookup.FragEntry = try frag_table.get(io, f.frag_idx); if (frag.size.uncompressed) { - ext.addFrag(common.data[frag.block_start..][0..frag.size.size], f.frag_offset); + ext.addFrag(data[frag.block_start..][0..frag.size.size], f.frag_offset); } else { - const frag_block = try common.cache.get(io, frag.block_start, frag.size.size); + const frag_block = try cache.get(io, frag.block_start, frag.size.size); ext.addFrag(frag_block, f.frag_offset); } } @@ -105,16 +176,16 @@ fn extractReg(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ? .ext_file => |f| blk: { xattr_idx = f.xattr_idx; - var ext: DataExtractor = .init(common.data, common.decomp, common.block_size, f.blocks, f.block_start, f.size); + var ext: DataExtractor = .init(data, decomp, block_size, f.blocks, f.block_start, f.size); blocks = f.blocks.len; if (f.frag_idx != 0xFFFFFFFF) { blocks += 1; - const frag: Lookup.FragEntry = try common.frag_table.get(io, f.frag_idx); + const frag: Lookup.FragEntry = try frag_table.get(io, f.frag_idx); if (frag.size.uncompressed) { - ext.addFrag(common.data[frag.block_start..][0..frag.size.size], f.frag_offset); + ext.addFrag(data[frag.block_start..][0..frag.size.size], f.frag_offset); } else { - const frag_block = try common.cache.get(io, frag.block_start, frag.size.size); + const frag_block = try cache.get(io, frag.block_start, frag.size.size); ext.addFrag(frag_block, f.frag_offset); } } @@ -123,14 +194,14 @@ fn extractReg(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ? else => unreachable, }; - const fin: *FileFinish = try .init(common, io, blocks, inode, path, xattr_idx, parent); + const fin = try FileFinish.create(alloc, io, id_table, xattr_table, err, inode.hdr, path, options, parent, xattr_idx, blocks); - ext.extractAsync(common.alloc, io, &common.select, fin); + try ext.extractAsync(alloc, io, group, fin); } -fn extractSymlink(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) Error!void { +fn extractSymlink(alloc: std.mem.Allocator, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) Error!void { defer if (parent != null) { - inode.deinit(common.alloc); - common.alloc.free(path); + inode.deinit(alloc); + alloc.free(path); }; const target = switch (inode.data) { @@ -142,11 +213,18 @@ fn extractSymlink(common: *Common, io: Io, inode: Inode, path: []const u8, paren try Io.Dir.cwd().symLink(io, target, path, .{}); if (parent != null) - try parent.?.finish(io); + parent.?.finish(io); } -fn extractNod(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) Error!void { - defer if (parent != null) - common.alloc.free(path); +fn extractNod( + alloc: std.mem.Allocator, + io: Io, + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + inode: Inode, + path: []const u8, + parent: ?*Parent, + options: ExtractionOption, +) Error!void { var xattr_idx: u32 = 0xFFFFFFFF; var device: u32 = 0; var mode: u32 = undefined; @@ -189,33 +267,51 @@ fn extractNod(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ? else => unreachable, } - const sentinel_path = try common.alloc.dupeSentinel(u8, path, 0); - defer common.alloc.free(sentinel_path); + const sentinel_path = try alloc.dupeSentinel(u8, path, 0); + defer alloc.free(sentinel_path); const res = std.os.linux.mknod(sentinel_path, mode, device); if (res != 0) return Error.Mknod; - try setMetadataPath(common, io, inode.hdr, path, xattr_idx); + try setMetadataPath(alloc, io, inode.hdr, id_table, xattr_table, path, xattr_idx, options); if (parent != null) try parent.?.finish(io); } -fn setMetadataPath(common: *Common, io: Io, hdr: Inode.Header, path: []const u8, xattr_idx: u32) Error!void { - if (common.options.ignore_permissions and (common.options.ignore_xattr or xattr_idx == 0xFFFFFFFF)) return; +fn setMetadataPath( + alloc: std.mem.Allocator, + io: Io, + hdr: Inode.Header, + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + path: []const u8, + xattr_idx: u32, + options: ExtractionOption, +) Error!void { + if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == 0xFFFFFFFF)) return; var file = try Io.Dir.cwd().openFile(io, path, .{}); defer file.close(io); - return setMetadata(common, io, hdr, file, xattr_idx); + return setMetadata(alloc, io, hdr, id_table, xattr_table, file, xattr_idx, options); } -fn setMetadata(common: *Common, io: Io, hdr: Inode.Header, file: Io.File, xattr_idx: u32) Error!void { - if (common.options.ignore_permissions and (common.options.ignore_xattr or xattr_idx == 0xFFFFFFFF)) return; +fn setMetadata( + alloc: std.mem.Allocator, + io: Io, + hdr: Inode.Header, + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + file: Io.File, + xattr_idx: u32, + options: ExtractionOption, +) Error!void { + if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == 0xFFFFFFFF)) return; - if (!common.options.ignore_xattr and xattr_idx != 0xFFFFFFFF) { - var xattr = try common.xattr_table.get(common.alloc, io, xattr_idx); - defer xattr.deinit(common.alloc); + if (!options.ignore_xattr and xattr_idx != 0xFFFFFFFF) { + var xattr = try xattr_table.get(alloc, io, xattr_idx); + defer xattr.deinit(alloc); for (xattr.kvs) |kv| { const res = std.os.linux.fsetxattr(file.handle, kv.key, kv.value.ptr, kv.value.len, 0); @@ -224,169 +320,184 @@ fn setMetadata(common: *Common, io: Io, hdr: Inode.Header, file: Io.File, xattr_ } } - if (common.options.ignore_permissions) return; + if (options.ignore_permissions) return; try file.setTimestamps(io, .{ - .modify_timestamp = .init(Io.Timestamp.fromNanoseconds(@as(i96, @intCast(hdr.mod_time)) * std.time.ns_per_s)), + .modify_timestamp = .init( + Io.Timestamp.fromNanoseconds(@as(i96, @intCast(hdr.mod_time)) * std.time.ns_per_s), + ), }); try file.setPermissions(io, @enumFromInt(hdr.permissions)); - try file.setOwner(io, try common.id_table.get(io, hdr.uid_idx), try common.id_table.get(io, hdr.gid_idx)); + try file.setOwner(io, try id_table.get(io, hdr.uid_idx), try id_table.get(io, hdr.gid_idx)); } // Types pub const Error = error{ Mknod, SetXattr } || std.mem.Allocator.Error || Io.Cancelable || Io.Reader.Error || Io.Dir.CreateDirPathError || Cache.Error || - Io.File.SetPermissionsError || Io.Dir.SymLinkError || Io.File.SeekError || Io.Writer.Error || Io.File.Atomic.LinkError; + Io.File.SetPermissionsError || Io.Dir.SymLinkError || Io.File.SeekError || Io.Writer.Error || Io.File.Atomic.LinkError || Io.ConcurrentError; -pub const Parent = struct { - common: *Common, +const FileFinish = struct { + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + err: *?Error, - atomic: Atomic(usize), - hdr: Inode.Header, - path: []const u8, + inode: Inode, + options: ExtractionOption, + parent: ?*Parent, xattr_idx: u32, - parent: ?*Parent, + file: Io.File.Atomic, - fn init(common: *Common, len: usize, hdr: Inode.Header, path: []const u8, xattr_idx: u32, parent: ?*Parent) !*Parent { - const out = try common.arenaAlloc().create(Parent); + atomic: Atomic(usize), + has_err: bool = false, + fn create( + alloc: std.mem.Allocator, + io: Io, + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + err: *?Error, + hdr: Inode.Header, + path: []const u8, + options: ExtractionOption, + parent: ?*Parent, + xattr_idx: u32, + blocks: usize, + ) Error!*FileFinish { + const out = try alloc.create(FileFinish); out.* = .{ - .common = common, + .id_table = id_table, + .xattr_table = xattr_table, + .err = err, - .atomic = .init(len), .hdr = hdr, .path = path, + .options = options, + .parent = parent, .xattr_idx = xattr_idx, - .parent = parent, + .file = Io.Dir.cwd().createFileAtomic(io, path, .{}), + + .atomic = .init(blocks), }; return out; } - pub fn finish(self: *Parent, io: Io) Error!void { + fn finish(self: *FileFinish, alloc: std.mem.Allocator, io: Io) void { if (self.atomic.fetchSub(1, .release) > 0) return; - try setMetadataPath(self.common, io, self.hdr, self.path, self.xattr_idx); + defer { + if (self.parent != null) { + alloc.free(self.path); + if (self.has_err) + self.parent.?.errFinish(alloc, io) + else + self.parent.?.finish(alloc, io); + } + alloc.destroy(self); + } + + if (self.has_err) return; + + setMetadata(alloc, io, self.hdr, self.id_table, self.xattr_table, self.file, self.xattr_idx, self.options) catch |err| { + self.has_err = true; + self.err.* = err; + return; + }; + self.file.link(io) catch |err| { + self.has_err = true; + self.err.* = err; + return; + }; + + self.file.deinit(io); + } + fn errFinish(self: *FileFinish, alloc: std.mem.Allocator, io: Io) void { + self.has_err = true; + + if (self.atomic.fetchSub(1, .release) > 0) return; if (self.parent != null) { - self.common.alloc.free(self.path); - try self.parent.?.finish(io); + alloc.free(self.path); + self.parent.?.errFinish(alloc, io); } + self.file.deinit(io); + alloc.destroy(self); } }; -pub const FileFinish = struct { - common: *Common, +const Parent = struct { + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + err: *?Error, - atomic: Atomic(usize), - inode: Inode, - file: Io.File.Atomic, + hdr: Inode.Header, + path: []const u8, + options: ExtractionOption, + parent: ?*Parent, xattr_idx: u32, - parent: ?*Parent, - - fn init(common: *Common, io: Io, len: usize, inode: Inode, path: []const u8, xattr_idx: u32, parent: ?*Parent) !*FileFinish { - const out = try common.arenaAlloc().create(FileFinish); + atomic: Atomic(usize), + has_err: bool = false, + fn create( + alloc: std.mem.Allocator, + id_table: *Lookup.Table(u16), + xattr_table: *XattrTable, + err: *?Error, + hdr: Inode.Header, + path: []const u8, + options: ExtractionOption, + parent: ?*Parent, + xattr_idx: u32, + num: usize, + ) Error!*Parent { + const out = try alloc.create(Parent); out.* = .{ - .common = common, + .id_table = id_table, + .xattr_table = xattr_table, + .err = err, - .atomic = .init(len), - .inode = inode, - .file = try Io.Dir.cwd().createFileAtomic(io, path, .{}), + .hdr = hdr, + .path = path, + .options = options, + .parent = parent, .xattr_idx = xattr_idx, - .parent = parent, + .atomic = .init(num), }; return out; } - pub fn finish(self: *FileFinish, io: Io) Error!void { + fn finish(self: *Parent, alloc: std.mem.Allocator, io: Io) void { if (self.atomic.fetchSub(1, .release) > 0) return; - defer self.file.deinit(io); - - try setMetadata(self.common, io, self.inode.hdr, self.file.file, self.xattr_idx); - - try self.file.link(io); - - if (self.parent != null) { - self.inode.deinit(self.common.alloc); - try self.parent.?.finish(io); + defer { + if (self.parent != null) { + alloc.free(self.path); + if (self.has_err) + self.parent.?.errFinish(alloc, io) + else + self.parent.?.finish(alloc, io); + } + alloc.destroy(self); } - } -}; -pub const SelectUnion = union { reg: Error!void }; + if (self.has_err) return; -const Common = struct { - alloc: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - - sel_buf: []SelectUnion, - select: Io.Select(SelectUnion), - err: ?Error = null, - - data: []u8, - decomp: Decomp.Fn, - - inode_start: u64, - dir_start: u64, - block_size: u32, - - id_table: Lookup.Table(u16), - frag_table: Lookup.Table(Lookup.FragEntry), - xattr_table: XattrTable, - cache: Cache, - - options: ExtractionOption, - - fn init(alloc: std.mem.Allocator, io: Io, super: Superblock, data: []u8, decomp: Decomp.Fn, options: ExtractionOption) !Common { - const sel_buf = try alloc.alloc(SelectUnion, 50); - return .{ - .alloc = alloc, - .arena = .init(alloc), - - .sel_buf = sel_buf, - .select = .init(io, sel_buf), - - .data = data, - .decomp = decomp, - - .inode_start = super.inode_start, - .dir_start = super.dir_start, - .block_size = super.block_size, - - .id_table = .init(alloc, data, decomp, super.id_start, super.id_count), - .frag_table = .init(alloc, data, decomp, super.frag_start, super.frag_count), - .xattr_table = .init(alloc, data, decomp, super.xattr_start), - .cache = .init(alloc, data, decomp), - - .options = options, + setMetadataPath(alloc, io, self.hdr, self.id_table, self.xattr_table, self.path, self.xattr_idx, self.options) catch |err| { + self.has_err = true; + self.err.* = err; }; } - fn deinit(self: *Common) void { - self.select.cancelDiscard(); - self.arena.deinit(); + fn errFinish(self: *Parent, alloc: std.mem.Allocator, io: Io) void { + self.has_err = true; - self.alloc.free(self.sel_buf); + if (self.atomic.fetchSub(1, .release) > 0) return; - self.id_table.deinit(); - self.frag_table.deinit(); - self.xattr_table.deinit(); - self.cache.deinit(); - } - - fn arenaAlloc(self: *Common) std.mem.Allocator { - return self.arena.allocator(); - } - - fn start(self: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) void { - switch (inode.hdr.type) { - .dir, .ext_dir => self.select.async(.reg, extractDir, .{ self, io, inode, path, parent }), - .file, .ext_file => self.select.async(.reg, extractReg, .{ self, io, inode, path, parent }), - .symlink, .ext_symlink => self.select.async(.reg, extractSymlink, .{ self, io, inode, path, parent }), - else => self.select.async(.reg, extractNod, .{ self, io, inode, path, parent }), + if (self.parent != null) { + alloc.free(self.path); + self.parent.?.errFinish(alloc, io); } + alloc.destroy(self); } };