From 28f7d1fe28804343c7a2fff6fc68f4bc73d39e0f Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sat, 11 Jul 2026 00:47:39 -0500 Subject: [PATCH] Finished (probably not actually) mutli-threaded extraction --- src/data/extractor.zig | 17 +-- src/extract-multi.zig | 234 ++++++++++++++++++++++++++++++------- src/extract-single.zig | 3 +- src/extract.zig | 6 +- src/util/protected_map.zig | 2 +- 5 files changed, 210 insertions(+), 52 deletions(-) diff --git a/src/data/extractor.zig b/src/data/extractor.zig index 7d86bec..abbf4f7 100644 --- a/src/data/extractor.zig +++ b/src/data/extractor.zig @@ -2,7 +2,8 @@ const std = @import("std"); const Io = std.Io; const Decomp = @import("../decomp.zig"); -const Finish = @import("../extract-multi.zig").Finish; +const FileFinish = @import("../extract-multi.zig").FileFinish; +const ExtractError = @import("../extract-multi.zig").Error; const DataBlock = @import("../inode.zig").DataBlock; const Cache = @import("../util/cache.zig"); @@ -42,16 +43,16 @@ pub fn addCache(self: *Extractor, cache: *Cache) void { self.cache = cache; } -pub fn extractAsync(self: Extractor, alloc: std.mem.Allocator, io: Io, file: Io.File, group: *Io.Group, err: *?Error, finish: *Finish) Error!void { +pub fn extractAsync(self: Extractor, alloc: std.mem.Allocator, io: Io, group: *Io.Group, err: *?ExtractError, finish: *FileFinish) void { if (self.size == 0) return; var read_offset: u64 = self.start; for (0.., self.blocks) |i, block| { - group.async(io, blockThread, .{ self, alloc, io, file, read_offset, @truncate(i), err, finish }); + group.async(io, blockThread, .{ self, alloc, io, finish.file.file, read_offset, @truncate(i), err, finish }); read_offset += block.size; } if (self.frag_data != null) - group.async(io, fragThread, .{ self, io, file, err, finish }); + group.async(io, fragThread, .{ self, io, finish.file.file, err, finish }); } fn blockThread( @@ -61,8 +62,8 @@ fn blockThread( file: Io.File, read_offset: u64, block_idx: u32, - err: *?Error, - finish: *Finish, + err: *?ExtractError, + finish: *FileFinish, ) error{Canceled}!void { const size = if (self.frag_data == null and block_idx == self.blocks.len - 1) self.size % self.block_size @@ -135,8 +136,8 @@ fn fragThread( self: Extractor, io: Io, file: Io.File, - err: *?Error, - finish: *Finish, + err: *?ExtractError, + finish: *FileFinish, ) error{Canceled}!void { const size = self.size % self.block_size; diff --git a/src/extract-multi.zig b/src/extract-multi.zig index 82d43b2..54b71cf 100644 --- a/src/extract-multi.zig +++ b/src/extract-multi.zig @@ -2,7 +2,6 @@ const std = @import("std"); const Io = std.Io; const Atomic = std.atomic.Value; -const Archive = @import("archive.zig"); const DataExtractor = @import("data/extractor.zig"); const Decomp = @import("decomp.zig"); const Directory = @import("directory.zig"); @@ -10,46 +9,57 @@ const ExtractionOption = @import("options.zig"); const Inode = @import("inode.zig"); const Lookup = @import("lookup.zig"); const MetadataReader = @import("meta_rdr.zig"); +const Superblock = @import("archive.zig").Superblock; +const Cache = @import("util/cache.zig"); const XattrTable = @import("xattr.zig"); -pub fn extract(alloc: std.mem.Allocator, io: Io, arc: Archive, options: ExtractionOption) !void { - var common: Common = .init(alloc, arc, options); +pub fn extract(alloc: std.mem.Allocator, io: Io, super: Superblock, data: []u8, decomp: Decomp.Fn, inode: Inode, filepath: []const u8, options: ExtractionOption) !void { + const path = std.mem.trim(u8, filepath, "/"); + + var common: Common = .init(alloc, super, data, decomp, options); defer common.deinit(); + + common.group.async(io, extractAsync, .{ &common, io, inode, path, null }); + + try common.group.await(io); + + if (common.err != null) + return common.err.?; } -fn extractAsync(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Finish) error{Canceled}!void { +fn extractAsync(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) error{Canceled}!void { switch (inode.hdr.type) { .dir, .ext_dir => extractDir(common, io, inode, path, parent) catch |err| switch (err) { error.Canceled => { - io.recancel(io); + io.recancel(); return error.Canceled; }, else => common.err = err, }, .file, .ext_file => extractReg(common, io, inode, path, parent) catch |err| switch (err) { error.Canceled => { - io.recancel(io); + io.recancel(); return error.Canceled; }, else => common.err = err, }, .symlink, .ext_symlink => extractReg(common, io, inode, path, parent) catch |err| switch (err) { error.Canceled => { - io.recancel(io); + io.recancel(); return error.Canceled; }, else => common.err = err, }, else => extractNod(common, io, inode, path, parent) catch |err| switch (err) { error.Canceled => { - io.recancel(io); + io.recancel(); return error.Canceled; }, else => common.err = err, }, } } -fn extractDir(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Finish) Error!void { +fn extractDir(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) Error!void { var xattr_idx: u32 = 0xFFFFFFFF; const dir: Directory = switch (inode.data) { @@ -74,15 +84,15 @@ fn extractDir(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ? try Io.Dir.cwd().createDirPath(io, path); if (dir.entries.len == 0) { - try setMetadataPath(io, inode.hdr, path, xattr_idx); + try setMetadataPath(common, io, inode.hdr, path, xattr_idx); if (parent != null) { - try parent.?.finish(); + try parent.?.finish(io); common.alloc.free(path); } return; } - const cur_dir: *Finish = try .init(common, dir.entries.len, path, xattr_idx, parent); + const cur_dir: *Parent = try .init(common, dir.entries.len, inode.hdr, path, xattr_idx, parent); for (dir.entries) |entry| { var new_inode: Inode = try .initEntry(common.alloc, common.data, common.decomp, common.inode_start, common.block_size, entry); @@ -95,7 +105,7 @@ fn extractDir(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ? common.group.async(io, extractAsync, .{ common, io, new_inode, new_path, cur_dir }); } } -fn extractReg(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Finish) Error!void { +fn extractReg(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) Error!void { var blocks: usize = 0; var xattr_idx: u32 = 0xFFFFFFFF; @@ -104,22 +114,122 @@ fn extractReg(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ? var ext: DataExtractor = .init(common.data, common.decomp, common.block_size, f.blocks, f.block_start, f.size); blocks = f.blocks.len; - if (f.frag_idx != 0xFFFFFFFF) {} + if (f.frag_idx != 0xFFFFFFFF) { + blocks += 1; + const frag: Lookup.FragEntry = try common.frag_table.get(io, f.frag_idx); + if (frag.size.uncompressed) { + ext.addFrag(common.data[frag.block_start..][0 .. f.size % common.block_size], f.frag_offset); + } else { + const frag_block = try common.cache.get(io, frag.block_start, frag.size.size); + ext.addFrag(frag_block, f.frag_offset); + } + } + break :blk ext; }, .ext_file => |f| blk: { xattr_idx = f.xattr_idx; + + var ext: DataExtractor = .init(common.data, common.decomp, common.block_size, f.blocks, f.block_start, f.size); + blocks = f.blocks.len; + + if (f.frag_idx != 0xFFFFFFFF) { + blocks += 1; + const frag: Lookup.FragEntry = try common.frag_table.get(io, f.frag_idx); + if (frag.size.uncompressed) { + ext.addFrag(common.data[frag.block_start..][0 .. f.size % common.block_size], f.frag_offset); + } else { + const frag_block = try common.cache.get(io, frag.block_start, frag.size.size); + ext.addFrag(frag_block, f.frag_offset); + } + } + break :blk ext; }, else => unreachable, }; - var fin: *Finish = try .init(common, blocks, inode.hdr, path, xattr_idx, parent); + const fin: *FileFinish = try .init(common, io, blocks, inode.hdr, path, xattr_idx, parent); + + ext.extractAsync(common.alloc, io, &common.group, @ptrCast(&common.err), fin); +} +fn extractSymlink(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) Error!void { + defer if (parent != null) { + inode.deinit(common.alloc); + common.alloc.free(path); + }; + + const target = switch (inode.data) { + .symlink => |s| s.target, + .ext_symlink => |s| s.target, + else => unreachable, + }; + + try Io.Dir.cwd().symLink(io, target, path, .{}); + + if (parent != null) + try parent.?.finish(io); +} +fn extractNod(common: *Common, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) Error!void { + defer if (parent != null) {}; + var xattr_idx: u32 = 0xFFFFFFFF; + var device: u32 = 0; + var mode: u32 = undefined; + + const DT = std.os.linux.DT; + + switch (inode.data) { + .block_dev => |d| { + mode = DT.BLK; + device = d.device; + }, + .ext_block_dev => |d| { + mode = DT.BLK; + device = d.device; + xattr_idx = d.xattr_idx; + }, + .char_dev => |d| { + mode = DT.CHR; + device = d.device; + }, + .ext_char_dev => |d| { + mode = DT.CHR; + device = d.device; + xattr_idx = d.xattr_idx; + }, + .fifo => { + mode = DT.FIFO; + }, + .ext_fifo => |f| { + mode = DT.FIFO; + xattr_idx = f.xattr_idx; + }, + .socket => { + mode = DT.SOCK; + }, + .ext_socket => |s| { + mode = DT.SOCK; + xattr_idx = s.xattr_idx; + }, + else => unreachable, + } + + const sentinel_path = try common.alloc.dupeSentinel(u8, path, 0); + defer common.alloc.free(sentinel_path); + + const res = std.os.linux.mknod(sentinel_path, mode, device); + if (res != 0) + return Error.Mknod; + + try setMetadataPath(common, io, inode.hdr, path, xattr_idx); + + if (parent != null) + try parent.?.finish(io); } fn setMetadataPath(common: *Common, io: Io, hdr: Inode.Header, path: []const u8, xattr_idx: u32) Error!void { if (common.options.ignore_permissions and (common.options.ignore_xattr or xattr_idx == 0xFFFFFFFF)) return; var file = try Io.Dir.cwd().openFile(io, path, .{}); - defer file.deinit(); + defer file.close(io); return setMetadata(common, io, hdr, file, xattr_idx); } @@ -131,7 +241,7 @@ fn setMetadata(common: *Common, io: Io, hdr: Inode.Header, file: Io.File, xattr_ defer xattr.deinit(common.alloc); for (xattr.kvs) |kv| { - const res = std.os.linux.fsetxattr(file.handle, kv.key, kv.value, kv.value.len, 0); + const res = std.os.linux.fsetxattr(file.handle, kv.key, kv.value.ptr, kv.value.len, 0); if (res != 0) return Error.SetXattr; } @@ -143,14 +253,15 @@ fn setMetadata(common: *Common, io: Io, hdr: Inode.Header, file: Io.File, xattr_ .modify_timestamp = .init(Io.Timestamp.fromNanoseconds(hdr.mod_time * std.time.ns_per_s)), }); try file.setPermissions(io, @enumFromInt(hdr.permissions)); - try file.setOwner(io, try common.id_table.get(hdr.uid_idx), try common.id_table.get(hdr.gid_idx)); + try file.setOwner(io, try common.id_table.get(io, hdr.uid_idx), try common.id_table.get(io, hdr.gid_idx)); } // Types -const Error = error{ Mknod, SetXattr } || std.mem.Allocator.Error; +pub const Error = error{ Mknod, SetXattr } || std.mem.Allocator.Error || Io.Cancelable || Io.Reader.Error || Io.Dir.CreateDirPathError || Cache.Error || + Io.File.SetPermissionsError || DataExtractor.Error; -pub const Finish = struct { +pub const Parent = struct { common: *Common, atomic: Atomic(usize), @@ -158,10 +269,10 @@ pub const Finish = struct { path: []const u8, xattr_idx: u32, - parent: ?*Finish, + parent: ?*Parent, - fn init(common: *Common, len: usize, hdr: Inode.Header, path: []const u8, xattr_idx: u32, parent: ?*Finish) !*Finish { - const out = try common.dirAlloc().create(Finish); + fn init(common: *Common, len: usize, hdr: Inode.Header, path: []const u8, xattr_idx: u32, parent: ?*Parent) !*Parent { + const out = try common.arenaAlloc().create(Parent); out.* = .{ .common = common, @@ -176,8 +287,8 @@ pub const Finish = struct { return out; } - pub fn finish(self: *Finish, io: Io) Error!void { - if (self.atomic.fetchSub(1, .unordered) >= 0) return; + pub fn finish(self: *Parent, io: Io) Error!void { + if (self.atomic.fetchSub(1, .release) >= 0) return; try setMetadataPath(self.common, io, self.hdr, self.path, self.xattr_idx); @@ -188,10 +299,52 @@ pub const Finish = struct { } } }; +pub const FileFinish = struct { + common: *Common, + + atomic: Atomic(usize), + hdr: Inode.Header, + file: Io.File.Atomic, + xattr_idx: u32, + + parent: ?*Parent, + + fn init(common: *Common, io: Io, len: usize, hdr: Inode.Header, path: []const u8, xattr_idx: u32, parent: ?*Parent) !*FileFinish { + const out = try common.arenaAlloc().create(FileFinish); + + out.* = .{ + .common = common, + + .atomic = .init(len), + .hdr = hdr, + .file = try Io.Dir.cwd().createFileAtomic(io, path, .{}), + .xattr_idx = xattr_idx, + + .parent = parent, + }; + return out; + } + + pub fn finish(self: *FileFinish, io: Io) Error!void { + if (self.atomic.fetchSub(1, .release) >= 0) return; + + defer self.file.deinit(io); + + try setMetadata(self.common, io, self.hdr, self.file.file, self.xattr_idx); + + try self.file.link(io); + + if (self.parent != null) { + if (self.hdr.type != .dir and self.hdr.type != .ext_dir) + self.common.alloc.free(self.path); + try self.parent.?.finish(io); + } + } +}; const Common = struct { alloc: std.mem.Allocator, - dir_alloc: std.heap.ArenaAllocator, + arena: std.heap.ArenaAllocator, group: Io.Group = .init, err: ?Error = null, @@ -206,37 +359,40 @@ const Common = struct { id_table: Lookup.Table(u16), frag_table: Lookup.Table(Lookup.FragEntry), xattr_table: XattrTable, + cache: Cache, options: ExtractionOption, - fn init(alloc: std.mem.Allocator, arc: Archive, options: ExtractionOption) Common { + fn init(alloc: std.mem.Allocator, super: Superblock, data: []u8, decomp: Decomp.Fn, options: ExtractionOption) Common { return .{ .alloc = alloc, - .dir_alloc = .init(alloc), + .arena = .init(alloc), - .data = arc.map.memory, - .decomp = arc.decomp, + .data = data, + .decomp = decomp, - .inode_start = arc.super.inode_start, - .dir_start = arc.super.dir_start, - .block_size = arc.super.block_size, + .inode_start = super.inode_start, + .dir_start = super.dir_start, + .block_size = super.block_size, - .id_table = .init(alloc, arc.map.memory, arc.decomp, arc.super.id_start, arc.super.id_count), - .frag_table = .init(alloc, arc.map.memory, arc.decomp, arc.super.frag_start, arc.super.frag_count), - .xattr_table = .init(alloc, arc.map.memory, arc.decomp, arc.super.xattr_start), + .id_table = .init(alloc, data, decomp, super.id_start, super.id_count), + .frag_table = .init(alloc, data, decomp, super.frag_start, super.frag_count), + .xattr_table = .init(alloc, data, decomp, super.xattr_start), + .cache = .init(alloc, data, decomp), .options = options, }; } fn deinit(self: *Common) void { - self.dir_alloc.deinit(); + self.arena.deinit(); self.id_table.deinit(); self.frag_table.deinit(); self.xattr_table.deinit(); + self.cache.deinit(); } - fn dirAlloc(self: *Common) std.mem.Allocator { - return self.dir_alloc.allocator(); + fn arenaAlloc(self: *Common) std.mem.Allocator { + return self.arena.allocator(); } }; diff --git a/src/extract-single.zig b/src/extract-single.zig index b3fe1b4..1aa8a0d 100644 --- a/src/extract-single.zig +++ b/src/extract-single.zig @@ -1,6 +1,8 @@ const std = @import("std"); const Io = std.Io; +const Archive = @import("archive.zig"); +const Superblock = Archive.Superblock; const DataReader = @import("data/reader.zig"); const Decomp = @import("decomp.zig"); const Directory = @import("directory.zig"); @@ -8,7 +10,6 @@ const ExtractionOptions = @import("options.zig"); const Inode = @import("inode.zig"); const Lookup = @import("lookup.zig"); const MetadaReader = @import("meta_rdr.zig"); -const Superblock = @import("archive.zig").Superblock; const Cache = @import("util/cache.zig"); const XattrTable = @import("xattr.zig"); diff --git a/src/extract.zig b/src/extract.zig index 06f6ad8..9be04ca 100644 --- a/src/extract.zig +++ b/src/extract.zig @@ -1,12 +1,12 @@ const std = @import("std"); const Io = std.Io; -const Superblock = @import("archive.zig").Superblock; const Decomp = @import("decomp.zig"); -const Inode = @import("inode.zig"); const ExtractionOptions = @import("options.zig"); -const Single = @import("extract-single.zig"); +const Inode = @import("inode.zig"); const Multi = @import("extract-multi.zig"); +const Single = @import("extract-single.zig"); +const Superblock = @import("archive.zig").Superblock; pub fn extract( alloc: std.mem.Allocator, diff --git a/src/util/protected_map.zig b/src/util/protected_map.zig index 1c75ab7..80837a9 100644 --- a/src/util/protected_map.zig +++ b/src/util/protected_map.zig @@ -38,7 +38,7 @@ pub fn ProtectedMap(comptime K: anytype, comptime T: anytype, comptime create_fn if (value != null) { if (!value.?.filled.isSet()) { self.mut.unlockShared(io); - defer self.mut.lockShared(io); + defer self.mut.lockSharedUncancelable(io); try value.?.filled.wait(io); }