diff --git a/build.zig b/build.zig index 1b6900b..385e369 100644 --- a/build.zig +++ b/build.zig @@ -58,6 +58,7 @@ pub fn build(b: *std.Build) !void { b.installArtifact(lib); b.installArtifact(exe); + const run_step = b.step("run", "Run the app"); const run_cmd = b.addRunArtifact(exe); run_step.dependOn(&run_cmd.step); @@ -76,4 +77,12 @@ pub fn build(b: *std.Build) !void { const test_step = b.step("test", "Run tests"); test_step.dependOn(&run_mod_tests.step); test_step.dependOn(&run_exe_tests.step); + + // Wanted by ZLS for better detection. + const exe_check = b.addExecutable(.{ + .name = "unsquashfs", + .root_module = mod, + }); + const check = b.step("check", "Check if unsquashfs compiles"); + check.dependOn(&exe_check.step); } diff --git a/src/archive.zig b/src/archive.zig index b0bc21f..9a9cc75 100644 --- a/src/archive.zig +++ b/src/archive.zig @@ -31,121 +31,66 @@ pub const FragEntry = packed struct { const Archive = @This(); -// 4 Gigs -const DEFAULT_MEM_SIZE = 4 * 1024 * 1024 * 1024; - -parent_alloc: std.mem.Allocator, -alloc: std.heap.ThreadSafeAllocator, -// alloc: std.heap.FixedBufferAllocator, -// fixed_buf: []u8, -thread_count: usize, - fil: OffsetFile, super: Superblock, -setup: bool = false, - decomp: Decomp.DecompFn, -frag_table: Table(FragEntry) = undefined, -id_table: Table(u16) = undefined, -export_table: Table(InodeRef) = undefined, +frag_table: Table(FragEntry), +id_table: Table(u16), +export_table: Table(InodeRef), -/// Default settings using std.Thread.getCpuCount() threads and the minimum of 4gb or half of system memory for memory usage. -pub fn init(alloc: std.mem.Allocator, fil: File) !Archive { - return initAdvanced( - alloc, - fil, - 0, - try std.Thread.getCpuCount(), - ); -} -/// Create the Archive dictating the amount of threads used for extraction. -/// If you're planning on only interacting with a small number of files, it should be fine to use few (or one) threads. -pub fn initAdvanced(alloc: std.mem.Allocator, fil: File, offset: u64, threads: usize) !Archive { +/// Begin reading a squashfs archive from the given File at the given offset. +pub fn init(fil: File, offset: u64) !Archive { var super: Superblock = undefined; - const red = try fil.pread(@ptrCast(&super), offset); - std.debug.assert(red == @sizeOf(Superblock)); + _ = try fil.pread(@ptrCast(&super), offset); try super.validate(); - // const fixed_buf = try alloc.alloc(u8, mem); + const decomp: Decomp.DecompFn = switch (super.compression) { + .gzip => Decomp.gzipDecompress, + .lzma => Decomp.lzmaDecompress, + .xz => Decomp.xzDecompress, + .zstd => Decomp.zstdDecompress, + .lz4 => if (config.use_c_libs) Decomp.cLz4 else return error.Lz4Unsupported, + .lzo => if (config.use_c_libs and config.allow_lzo) Decomp.lzoDecompress else return error.LzoUnsupported, + }; + const offset_fil: OffsetFile = .init(fil, offset); return .{ - .parent_alloc = alloc, - .alloc = .{ .child_allocator = alloc }, - // .fixed_buf = fixed_buf, - .thread_count = if (threads > 0) threads else try std.Thread.getCpuCount(), - .fil = .init(fil, offset), - .decomp = switch (super.compression) { - .gzip => Decomp.gzipDecompress, - .lzma => Decomp.lzmaDecompress, - .xz => Decomp.xzDecompress, - .zstd => Decomp.zstdDecompress, - .lz4 => if (config.use_c_libs) Decomp.cLz4 else return error.Lz4Unsupported, - .lzo => if (config.use_c_libs and config.allow_lzo) Decomp.lzoDecompress else return error.LzoUnsupported, - }, - + .fil = offset_fil, .super = super, + + .decomp = decomp, + + .frag_table = .init(offset_fil, decomp, super.frag_start, super.frag_count), + .id_table = .init(offset_fil, decomp, super.id_start, super.id_count), + .export_table = .init(offset_fil, decomp, super.export_start, super.inode_count), }; } -pub fn deinit(self: *Archive) void { - // self.parent_alloc.free(self.fixed_buf); - if (self.setup) { - self.frag_table.deinit(); - self.export_table.deinit(); - self.id_table.deinit(); - } -} -pub fn allocator(self: *Archive) std.mem.Allocator { - return self.alloc.allocator(); -} - -fn setupValues(self: *Archive) !void { - const alloc = self.allocator(); - self.frag_table = try .init(alloc, self.fil, self.decomp, self.super.frag_start, self.super.frag_count); - self.id_table = try .init(alloc, self.fil, self.decomp, self.super.id_start, self.super.id_count); - self.export_table = try .init(alloc, self.fil, self.decomp, self.super.export_start, self.super.inode_count); - self.setup = true; -} - -pub fn id(self: *Archive, idx: u32) !u16 { - if (!self.setup) try self.setupValues(); - return self.id_table.get(idx); -} - -pub fn frag(self: *Archive, idx: u32) !FragEntry { - if (!self.setup) try self.setupValues(); - return self.frag_table.get(idx); -} - -pub fn inode(self: *Archive, num: u32) !Inode { +pub fn inode(self: Archive, num: u32) !Inode { if (!self.setup) try self.setupValues(); const ref = try self.export_table.get(num - 1); var rdr = try self.fil.readerAt(ref.block_start + self.super.inode_start, &[0]u8{}); - var meta: MetadataReader = .init(self.allocator(), &rdr.interface, &self.decomp); + var meta: MetadataReader = .init(self.alloc, &rdr.interface, &self.decomp); try meta.interface.discardAll(ref.block_offset); - return try .read(self.allocator(), &meta.interface, self.super.block_size); + return try .read(self.alloc, &meta.interface, self.super.block_size); } -pub fn root(self: *Archive) !SfsFile { - if (!self.setup) try self.setupValues(); +pub fn root(self: Archive, alloc: std.mem.Allocator) !SfsFile { var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{}); - var meta: MetadataReader = .init(self.allocator(), &rdr.interface, self.decomp); + var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp); try meta.interface.discardAll(self.super.root_ref.block_offset); - const in: Inode = try .read(self.allocator(), &meta.interface, self.super.block_size); - return .init(self, in, ""); + const in: Inode = try .read(alloc, &meta.interface, self.super.block_size); + return .init(alloc, self, in, ""); } -pub fn open(self: *Archive, path: []const u8) !SfsFile { - if (!self.setup) try self.setupValues(); - var root_fil = try self.root(); +pub fn open(self: Archive, alloc: std.mem.Allocator, path: []const u8) !SfsFile { + var root_fil = try self.root(alloc); defer if (!SfsFile.pathIsSelf(path)) root_fil.deinit(); - return root_fil.open(path); + return root_fil.open(alloc, path); } -pub fn extract(self: *Archive, path: []const u8, options: ExtractionOptions) !void { - if (!self.setup) try self.setupValues(); - var alloc = self.allocator(); +pub fn extract(self: Archive, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void { var ext_path: []u8 = undefined; if (std.fs.cwd().statFile(path)) |stat| { if (stat.kind == .directory) { @@ -161,8 +106,8 @@ pub fn extract(self: *Archive, path: []const u8, options: ExtractionOptions) !vo } defer if (ext_path.len > path.len) alloc.free(ext_path); var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{}); - var meta: MetadataReader = .init(self.allocator(), &rdr.interface, self.decomp); + var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp); try meta.interface.discardAll(self.super.root_ref.block_offset); - const in: Inode = try .read(self.allocator(), &meta.interface, self.super.block_size); - try in.extractToThreaded(self, ext_path, options, self.thread_count); + const in: Inode = try .read(alloc, &meta.interface, self.super.block_size); + try in.extractTo(alloc, self, ext_path, options); } diff --git a/src/bin/unsquashfs.zig b/src/bin/unsquashfs.zig index e1cd2f5..0268178 100644 --- a/src/bin/unsquashfs.zig +++ b/src/bin/unsquashfs.zig @@ -44,9 +44,8 @@ pub fn main() !void { } var fil: std.fs.File = try std.fs.cwd().openFile(archive, .{}); //TODO: Handle error gracefully. defer fil.close(); - var arc: squashfs.Archive = try .initAdvanced(alloc, fil, offset, threads); //TODO: Update when memory size matters. //TODO: Handle error gracefully. - defer arc.deinit(); - try arc.extract(extLoc, if (verbose) .VerboseDefault(&out.interface) else .Default); //TODO: Handle error gracefully. + var arc: squashfs.Archive = try .init(fil, offset); //TODO: Update when memory size matters. //TODO: Handle error gracefully. + try arc.extract(alloc, extLoc, if (verbose) try .VerboseDefault(&out.interface) else try .Default()); //TODO: Handle error gracefully. } fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void { diff --git a/src/decomp.zig b/src/decomp.zig index 76ac5b9..7004b81 100644 --- a/src/decomp.zig +++ b/src/decomp.zig @@ -29,6 +29,7 @@ pub const CompressionType = enum(u16) { zstd, }; +/// A generic decompression function. alloc is only used for internal use and any allocations made will be freed. pub const DecompFn = *const fn (alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize; // TODO: replace anyerror to definitive error types. pub const gzipDecompress = if (config.use_c_libs) cGzip else zigGzip; diff --git a/src/file.zig b/src/file.zig index 7e012a5..c7bcd75 100644 --- a/src/file.zig +++ b/src/file.zig @@ -24,39 +24,40 @@ const FileError = error{ const SfsFile = @This(); -archive: *Archive, +alloc: std.mem.Allocator, +archive: Archive, inode: Inode, name: []const u8, /// Initialize a new File. /// name is copied to the File so can be safely freed afterwards. -pub fn init(archive: *Archive, inode: Inode, name: []const u8) !SfsFile { - const new_name = try archive.allocator().alloc(u8, name.len); +pub fn init(alloc: std.mem.Allocator, archive: Archive, inode: Inode, name: []const u8) !SfsFile { + const new_name = try alloc.alloc(u8, name.len); @memcpy(new_name, name); return .{ + .alloc = alloc, .archive = archive, .inode = inode, .name = new_name, }; } -pub fn fromEntry(archive: *Archive, entry: DirEntry) !SfsFile { +pub fn fromEntry(alloc: std.mem.Allocator, archive: Archive, entry: DirEntry) !SfsFile { var rdr = try archive.fil.readerAt(entry.block_start + archive.super.inode_start, &[0]u8{}); - var meta: MetadataReader = .init(archive.allocator(), &rdr.interface, archive.decomp); + var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp); try meta.interface.discardAll(entry.block_offset); - const inode: Inode = try .read(archive.allocator(), &meta.interface, archive.super.block_size); - errdefer inode.deinit(archive.allocator()); - return .init(archive, inode, entry.name); + const inode: Inode = try .read(alloc, &meta.interface, archive.super.block_size); + errdefer inode.deinit(alloc); + return .init(alloc, archive, inode, entry.name); } pub fn deinit(self: SfsFile) void { - var alloc = self.archive.allocator(); - alloc.free(self.name); - self.inode.deinit(alloc); + self.alloc.free(self.name); + self.inode.deinit(self.alloc); } -fn getEntries(self: SfsFile) ![]DirEntry { - return self.inode.dirEntries(self.archive.allocator(), self.archive.*); +fn getEntries(self: SfsFile, alloc: std.mem.Allocator) ![]DirEntry { + return self.inode.dirEntries(alloc, self.archive); } pub fn ownerUid(self: SfsFile) !u16 { @@ -96,20 +97,19 @@ pub fn iterate(self: SfsFile) !Iterator { } /// Open a sub-file/folder within a directory at the given path. /// If path is "", ".", "/", or "./", this File is returned. -pub fn open(self: SfsFile, path: []const u8) !SfsFile { +pub fn open(self: SfsFile, alloc: std.mem.Allocator, path: []const u8) !SfsFile { if (!self.isDir()) return FileError.NotDirectory; if (pathIsSelf(path)) return self; // Recursively stip ending & leading path separators. - if (path[0] == '/') return self.open(path[1..]); - if (path[path.len - 1] == '/') return self.open(path[0 .. path.len - 1]); + if (path[0] == '/') return self.open(alloc, path[1..]); + if (path[path.len - 1] == '/') return self.open(alloc, path[0 .. path.len - 1]); const idx = std.mem.indexOf(u8, path, "/") orelse path.len; const first_element = path[0..idx]; - if (std.mem.eql(u8, first_element, ".")) return self.open(path[idx + 1 ..]); - const entries = try self.getEntries(); + if (std.mem.eql(u8, first_element, ".")) return self.open(alloc, path[idx + 1 ..]); + const entries = try self.getEntries(alloc); defer { - var alloc = self.archive.allocator(); for (entries) |e| { e.deinit(alloc); } @@ -122,12 +122,12 @@ pub fn open(self: SfsFile, path: []const u8) !SfsFile { const comp = std.mem.order(u8, first_element, cur_slice[split].name); switch (comp) { .eq => { - var fil: SfsFile = try .fromEntry(self.archive, cur_slice[split]); + var fil: SfsFile = try .fromEntry(alloc, self.archive, cur_slice[split]); if (idx == path.len) { return fil; } defer fil.deinit(); - return fil.open(path[idx + 1 ..]); + return fil.open(alloc, path[idx + 1 ..]); }, .lt => cur_slice = cur_slice[0..split], .gt => cur_slice = cur_slice[split + 1 ..], @@ -170,35 +170,8 @@ pub fn devNum(self: SfsFile) !u32 { /// Extract the given File to the path. If File is a regular file, the path must be a directory or not exist. /// If the gievn path is a folder, the File's contents will be extracted within. -pub fn extract(self: *SfsFile, path: []const u8, options: ExtractionOptions) !void { - var alloc = self.archive.allocator(); - var ext_path: []u8 = undefined; - if (std.fs.cwd().statFile(path)) |stat| { - if (stat.kind == .directory) { - if (!self.isDir()) { - const has_end_sep = path[path.len - 1] == '/'; - const alloc_size = if (has_end_sep) - path.len + self.name.len - else - path.len + self.name.len + 1; - ext_path = try alloc.alloc(u8, alloc_size); - @memcpy(ext_path[0..path.len], path); - @memcpy(ext_path[ext_path.len - self.name.len ..], self.name); - if (!has_end_sep) ext_path[path.len] = '/'; - } else { - ext_path = @constCast(path); - } - } else return FileError.ExtractionPathExists; - } else |err| { - if (err == error.FileNotFound) { - ext_path = @constCast(path); - } else { - std.log.err("Error stat-ing extraction path {s}: {}\n", .{ path, err }); - return err; - } - } - defer if (ext_path.len > path.len) alloc.free(ext_path); - return self.inode.extractToThreaded(self.archive, path, options, self.archive.thread_count); +pub fn extract(self: SfsFile, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void { + return self.inode.extractTo(alloc, self.archive, path, options); } /// Utility function. @@ -210,8 +183,10 @@ pub fn pathIsSelf(path: []const u8) bool { } pub const Iterator = struct { + alloc: std.mem.Allocator, + entries: []DirEntry, - archive: *Archive, + archive: Archive, idx: u32 = 0, @@ -221,10 +196,9 @@ pub const Iterator = struct { return try SfsFile.fromEntry(self.archive, self.entries[self.idx]); } pub fn deinit(self: Iterator) void { - var alloc = self.archive.allocator(); for (self.entries) |e| { - e.deinit(alloc); + e.deinit(self.alloc); } - alloc.free(self.entries); + self.alloc.free(self.entries); } }; diff --git a/src/inode.zig b/src/inode.zig index 243b016..13cd5f6 100644 --- a/src/inode.zig +++ b/src/inode.zig @@ -93,7 +93,7 @@ pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode { }, }; } -pub fn readFromEntry(alloc: std.mem.Allocator, archive: *Archive, entry: DirEntry) !Inode { +pub fn readFromEntry(alloc: std.mem.Allocator, archive: Archive, entry: DirEntry) !Inode { var rdr = try archive.fil.readerAt(archive.super.inode_start + entry.block_start, &[0]u8{}); var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp); try meta.interface.discardAll(entry.block_offset); @@ -111,32 +111,42 @@ pub fn deinit(self: Inode, alloc: std.mem.Allocator) void { } /// Get the data reader for a file inode. -pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: *Archive) !DataReader { +pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive) !DataReader { return switch (self.hdr.inode_type) { .file => readerFromData(alloc, archive, self.data.file), .ext_file => readerFromData(alloc, archive, self.data.ext_file), else => error.NotRegularFile, }; } -fn readerFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !DataReader { - var out: DataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size); - if (data.frag_idx != 0xFFFFFFFF) - out.addFragment(try archive.frag(data.frag_idx), data.frag_block_offset); - return out; +fn readerFromData(alloc: std.mem.Allocator, archive: Archive, data: anytype) !DataReader { + return .init( + alloc, + archive, + data.block_sizes, + data.block_start, + data.size, + data.frag_block_offset, + if (data.frag_idx == 0xFFFFFFFF) null else try archive.frag_table.get(alloc, data.frag_idx), + ); } /// Get a threaded data reader for a file inode. -pub fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: *Archive) !ThreadedDataReader { +pub fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive) !ThreadedDataReader { return switch (self.hdr.inode_type) { .file => threadedReaderFromData(alloc, archive, self.data.file), .ext_file => threadedReaderFromData(alloc, archive, self.data.ext_file), else => error.NotRegularFile, }; } -fn threadedReaderFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !ThreadedDataReader { - var out: ThreadedDataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size); - if (data.frag_idx != 0xFFFFFFFF) - out.addFragment(try archive.frag(data.frag_idx), data.frag_block_offset); - return out; +fn threadedReaderFromData(alloc: std.mem.Allocator, archive: Archive, data: anytype) !ThreadedDataReader { + return .init( + alloc, + archive, + data.block_sizes, + data.block_start, + data.size, + data.frag_block_offset, + if (data.frag_idx == 0xFFFFFFFF) null else try archive.frag_table.get(alloc, data.frag_idx), + ); } /// Get the directory entries for a directory inode. @@ -154,17 +164,17 @@ fn entriesFromData(alloc: std.mem.Allocator, archive: Archive, data: anytype) ![ return DirEntry.readDir(alloc, &meta.interface, data.size); } -/// Extract the inode to the given path. Single threaded. -pub fn extractTo(self: Inode, archive: *Archive, path: []const u8, options: ExtractionOptions) !void { +/// Extract the inode to the given path. +pub fn extractTo(self: Inode, alloc: std.mem.Allocator, archive: Archive, path: []const u8, options: ExtractionOptions) !void { + if (options.threads > 1) return self.extractToThreaded(alloc, archive, path, options); // We go to a dedicated function for mutli-threaded switch (self.hdr.inode_type) { .dir, .ext_dir => { // Removing any trailing separators since that's the easiest path forward. - if (path[path.len - 1] == '/') return self.extractTo(archive, path[0 .. path.len - 1], options); + if (path[path.len - 1] == '/') return self.extractTo(alloc, archive, path[0 .. path.len - 1], options); std.fs.cwd().makeDir(path) catch |err| { if (err != std.fs.Dir.MakeError.PathAlreadyExists) return err; }; - var alloc = archive.allocator(); - const entries = try self.dirEntries(alloc, archive.*); + const entries = try self.dirEntries(alloc, archive); defer { for (entries) |entry| entry.deinit(alloc); alloc.free(entries); @@ -178,12 +188,12 @@ pub fn extractTo(self: Inode, archive: *Archive, path: []const u8, options: Extr var inode: Inode = try readFromEntry(alloc, archive, entry); defer inode.deinit(alloc); - try inode.extractTo(archive, new_path, options); + try inode.extractTo(alloc, archive, new_path, options); } }, - .file, .ext_file => try self.extractRegFile(archive.allocator(), archive, path, options), + .file, .ext_file => try self.extractRegFile(alloc, archive, path, options), .symlink, .ext_symlink => try self.extractSymlink(path), - else => try self.extractDevice(archive, path, options), + else => try self.extractDevice(alloc, archive, path, options), } } @@ -202,23 +212,24 @@ const Parent = struct { wg: WaitGroup, mut: Mutex = .{}, - fn create(alloc: std.mem.Allocator, hdr: Header, archive: *Archive, path: []const u8, options: ExtractionOptions, dir_size: usize) !*Parent { + fn create(alloc: std.mem.Allocator, hdr: Header, archive: Archive, path: []const u8, options: ExtractionOptions, dir_size: usize) !*Parent { const out = try alloc.create(Parent); errdefer alloc.destroy(out); out.* = .{ .alloc = alloc, .path = path, - .uid = try archive.id(hdr.uid_idx), - .gid = try archive.id(hdr.gid_idx), + .uid = try archive.id_table.get(alloc, hdr.uid_idx), + .gid = try archive.id_table.get(alloc, hdr.gid_idx), .perm = hdr.permissions, .mod_time = hdr.mod_time, .ignore_permissions = options.ignore_permissions, .ignore_xattr = options.ignore_xattr, - .wg = .{ .state = .init(dir_size) }, + .wg = .{}, }; + out.wg.startMany(dir_size); return out; } @@ -232,7 +243,7 @@ const Parent = struct { defer p.alloc.destroy(p); var fil = try std.fs.cwd().openFile(p.path, .{}); defer fil.close(); - const time = p.mod_time * 1000000000; + const time = @as(i128, p.mod_time) * 1000000000; try fil.updateTimes(time, time); if (p.ignore_permissions) { try fil.chmod(p.perm); @@ -243,36 +254,26 @@ const Parent = struct { /// Extract the inode to the given path. Multi-threaded. /// Functions identically to extractTo on all but regular files and directories. -/// -/// If threads <= 1, then this just calls extractTo. -pub fn extractToThreaded(self: Inode, archive: *Archive, path: []const u8, options: ExtractionOptions, threads: usize) !void { - if (threads <= 1) return self.extractTo(archive, path, options); +fn extractToThreaded(self: Inode, alloc: std.mem.Allocator, archive: Archive, path: []const u8, options: ExtractionOptions) !void { switch (self.hdr.inode_type) { .dir, .ext_dir => { // Removing any trailing separators since that's the easiest path forward. - if (path[path.len - 1] == '/') return self.extractToThreaded(archive, path[0 .. path.len - 1], options, threads); - - // Fixed Allocator - // const mem_buf = archive.allocator().alloc(u8, 2 * 1024 * 1024 * 1024); - // defer archive.allocator().free(mem_buf); - // var fixed_alloc: std.heap.FixedBufferAllocator = .init(mem_buf); - // const alloc = fixed_alloc.threadSafeAllocator(); + if (path[path.len - 1] == '/') return self.extractToThreaded(alloc, archive, path[0 .. path.len - 1], options); // Arena Allocator - var arena_alloc: std.heap.ArenaAllocator = .init(archive.allocator()); + var arena_alloc: std.heap.ArenaAllocator = .init(alloc); defer arena_alloc.deinit(); var thread_alloc: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena_alloc.allocator() }; - const alloc = thread_alloc.allocator(); var wg: WaitGroup = .{}; // defer if(!options.ignore_permissions) perms.?.deinit(alloc); We don't need to do this due to ArenaAllocator var pool: Pool = undefined; - try pool.init(.{ .allocator = alloc, .n_jobs = threads - 1 }); + try pool.init(.{ .allocator = alloc, .n_jobs = options.threads - 1 }); defer pool.deinit(); var out_err: ?anyerror = null; wg.start(); - self.extractThread(alloc, archive, path, options, &wg, &pool, &out_err, null); + self.extractThread(thread_alloc.allocator(), archive, path, options, &wg, &pool, &out_err, null); pool.waitAndWork(&wg); if (out_err != null) return out_err.?; @@ -282,17 +283,17 @@ pub fn extractToThreaded(self: Inode, archive: *Archive, path: []const u8, optio try fil.updateTimes(time, time); if (options.ignore_permissions) { try fil.chmod(self.hdr.permissions); - try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx)); + try fil.chown(try archive.id_table.get(alloc, self.hdr.uid_idx), try archive.id_table.get(alloc, self.hdr.gid_idx)); } }, .file, .ext_file => { - const alloc = archive.allocator(); - var pool: Pool = undefined; - try pool.init(.{ .allocator = alloc, .n_jobs = threads }); + try pool.init(.{ .allocator = alloc, .n_jobs = options.threads - 1 }); defer pool.deinit(); - try self.extractRegFileThreaded(alloc, archive, path, options, &pool); + var thread_alloc: std.heap.ThreadSafeAllocator = .{ .child_allocator = alloc }; + + try self.extractRegFileThreaded(thread_alloc.allocator(), archive, path, options, &pool); var fil = try std.fs.cwd().openFile(path, .{}); defer fil.close(); @@ -300,18 +301,18 @@ pub fn extractToThreaded(self: Inode, archive: *Archive, path: []const u8, optio try fil.updateTimes(time, time); if (!options.ignore_permissions) { try fil.chmod(self.hdr.permissions); - try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx)); + try fil.chown(try archive.id_table.get(alloc, self.hdr.uid_idx), try archive.id_table.get(alloc, self.hdr.gid_idx)); } }, .symlink, .ext_symlink => try self.extractSymlink(path), - else => try self.extractDevice(archive, path, options), + else => try self.extractDevice(alloc, archive, path, options), } } fn extractThreadEntry( entry: DirEntry, alloc: std.mem.Allocator, - archive: *Archive, + archive: Archive, path: []const u8, options: ExtractionOptions, wg: *WaitGroup, @@ -339,7 +340,7 @@ fn extractThreadEntry( fn extractThread( self: Inode, alloc: std.mem.Allocator, - archive: *Archive, + archive: Archive, path: []const u8, options: ExtractionOptions, wg: *WaitGroup, @@ -352,7 +353,7 @@ fn extractThread( defer { if (parent != null) parent.?.finish() catch |err| { if (options.verbose) - options.verbose_writer.?.print("Error setting folder permission to {s}: {}\n", .{ path, err }) catch {}; + options.verbose_writer.?.print("Error setting folder permission to {s}: {}\n", .{ parent.?.path, err }) catch {}; out_err.* = err; }; wg.finish(); @@ -367,7 +368,7 @@ fn extractThread( return; }; - const entries = self.dirEntries(alloc, archive.*) catch |err| { + const entries = self.dirEntries(alloc, archive) catch |err| { if (options.verbose) options.verbose_writer.?.print("Error getting directory entries for inode #{} (extracting to {s}): {}\n", .{ self.hdr.num, path, err }) catch {}; out_err.* = err; @@ -399,6 +400,10 @@ fn extractThread( }, ) catch |err| { wg.finish(); + p.finish() catch |e| { + if (options.verbose) + options.verbose_writer.?.print("Error setting folder permission to {s}: {}\n", .{ p.path, e }) catch {}; + }; if (options.verbose) options.verbose_writer.?.print("Error starting extraction thread: {}\n", .{err}) catch {}; out_err.* = err; @@ -421,7 +426,7 @@ fn extractThread( }; }, else => { - self.extractDevice(archive, path, options) catch |err| { + self.extractDevice(alloc, archive, path, options) catch |err| { if (options.verbose) options.verbose_writer.?.print("Error extracting device/IPC inode #{} to {s}: {}\n", .{ self.hdr.num, path, err }) catch {}; out_err.* = err; @@ -433,7 +438,7 @@ fn extractThread( /// Optionally set owner & permissions. /// /// Assumes the inode is a file or ext_file type. -fn extractRegFile(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void { +fn extractRegFile(self: Inode, alloc: std.mem.Allocator, archive: Archive, path: []const u8, options: ExtractionOptions) !void { var fil = try std.fs.cwd().createFile(path, .{ .exclusive = true }); defer fil.close(); var wrt = fil.writer(&[0]u8{}); @@ -446,14 +451,14 @@ fn extractRegFile(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path try fil.updateTimes(time, time); if (!options.ignore_permissions) { try fil.chmod(self.hdr.permissions); - try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx)); + try fil.chown(try archive.id_table.get(alloc, self.hdr.uid_idx), try archive.id_table.get(alloc, self.hdr.gid_idx)); } } /// Extract the inode file contents to the given path threadedly. /// pool is used to spawn threads. /// /// Assumes the inode is a file or ext_file type. -fn extractRegFileThreaded(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions, pool: *Pool) !void { +fn extractRegFileThreaded(self: Inode, alloc: std.mem.Allocator, archive: Archive, path: []const u8, options: ExtractionOptions, pool: *Pool) !void { var fil = try std.fs.cwd().createFile(path, .{}); defer fil.close(); var data = try self.threadedDataReader(alloc, archive); @@ -463,7 +468,7 @@ fn extractRegFileThreaded(self: Inode, alloc: std.mem.Allocator, archive: *Archi try fil.updateTimes(time, time); if (!options.ignore_permissions) { try fil.chmod(self.hdr.permissions); - try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx)); + try fil.chown(try archive.id_table.get(alloc, self.hdr.uid_idx), try archive.id_table.get(alloc, self.hdr.gid_idx)); } } /// Creates the symlink described by the inode. @@ -481,7 +486,7 @@ fn extractSymlink(self: Inode, path: []const u8) !void { /// /// Optionally set owner & permissions. /// Assumes the inode is a char_dev, block_dev, fifo, socket, or their extended counterparts. -fn extractDevice(self: Inode, archive: *Archive, path: []const u8, options: ExtractionOptions) !void { +fn extractDevice(self: Inode, alloc: std.mem.Allocator, archive: Archive, path: []const u8, options: ExtractionOptions) !void { var mode: u32 = undefined; var dev: u32 = 0; switch (self.data) { @@ -530,7 +535,7 @@ fn extractDevice(self: Inode, archive: *Archive, path: []const u8, options: Extr try fil.updateTimes(time, time); if (!options.ignore_permissions) { try fil.chmod(self.hdr.permissions); - try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx)); + try fil.chown(try archive.id_table.get(alloc, self.hdr.uid_idx), try archive.id_table.get(alloc, self.hdr.gid_idx)); } if (!options.ignore_xattr) { // TODO diff --git a/src/options.zig b/src/options.zig index 94ae525..54acd37 100644 --- a/src/options.zig +++ b/src/options.zig @@ -5,6 +5,8 @@ const Writer = std.Io.Writer; const ExtractionOptions = @This(); +/// The amount of threads to use for extraction. +threads: usize, /// Don't set the file's owner & permissions after extraction ignore_permissions: bool = false, /// Don't set xattr values. Currently xattrs are never set anyway. @@ -16,9 +18,16 @@ verbose: bool = false, /// Where to print verbose log. verbose_writer: ?*Writer = null, -pub const Default: ExtractionOptions = .{}; -pub fn VerboseDefault(wrt: *Writer) ExtractionOptions { +pub const SingleThreaded: ExtractionOptions = .{ .threads = 1 }; +/// Use std.Thread.getCpuCount threads for extraction. +pub fn Default() !ExtractionOptions { return .{ + .threads = try std.Thread.getCpuCount(), + }; +} +pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions { + return .{ + .threads = try std.Thread.getCpuCount(), .verbose = true, .verbose_writer = wrt, }; diff --git a/src/table.zig b/src/table.zig index c98e85d..a351e70 100644 --- a/src/table.zig +++ b/src/table.zig @@ -16,61 +16,34 @@ pub fn Table(T: anytype) type { const VALS_PER_BLOCK = 8192 / @sizeOf(T); - alloc: std.mem.Allocator, fil: OffsetFile, decomp: DecompFn, - tab_start: u64, - tab: std.AutoHashMap(u32, []T), + tab_start: u64, values: u32, - mut: Mutex = .{}, - - pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: DecompFn, tab_start: u64, values: u32) !Self { + pub fn init(fil: OffsetFile, decomp: DecompFn, tab_start: u64, values: u32) Self { return .{ - .alloc = alloc, .fil = fil, .decomp = decomp, - .tab_start = tab_start, - .tab = .init(alloc), + .tab_start = tab_start, .values = values, }; } - pub fn deinit(self: *Self) void { - var iter = self.tab.valueIterator(); - while (iter.next()) |s| { - self.alloc.free(s.*); - } - self.tab.deinit(); - } - - pub fn get(self: *Self, idx: u32) !T { - if (idx >= self.values) return TableError.InvalidIndex; + pub fn get(self: Self, alloc: std.mem.Allocator, idx: u32) !T { const block_num = idx / VALS_PER_BLOCK; const idx_offset = idx - (block_num * VALS_PER_BLOCK); - if (self.tab.contains(block_num)) { - const block = self.tab.get(block_num).?; - return block[idx_offset]; - } - self.mut.lock(); - defer self.mut.unlock(); - // Double check in case of race condition.. - if (self.tab.contains(block_num)) { - const block = self.tab.get(block_num).?; - return block[idx_offset]; - } const is_last = (self.values - 1) / VALS_PER_BLOCK == block_num; const slice_size = if (is_last) self.values - (block_num * VALS_PER_BLOCK) else VALS_PER_BLOCK; - const slice = try self.alloc.alloc(T, slice_size); + var slice: [VALS_PER_BLOCK]T = undefined; var rdr = try self.fil.readerAt(self.tab_start + (8 * block_num), &[0]u8{}); var offset: u64 = 0; try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little); rdr = try self.fil.readerAt(offset, &[0]u8{}); - var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp); - try meta.interface.readSliceEndian(T, @ptrCast(slice), .little); - try self.tab.put(block_num, slice); + var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp); + try meta.interface.readSliceEndian(T, slice[0..slice_size], .little); return slice[idx_offset]; } }; diff --git a/src/test.zig b/src/test.zig index 4bb58cb..a92dde9 100644 --- a/src/test.zig +++ b/src/test.zig @@ -9,8 +9,7 @@ const TestArchive = "testing/LinuxPATest.sfs"; test "Basics" { var fil = try std.fs.cwd().openFile(TestArchive, .{}); defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); + const sfs: Archive = try .init(fil, 0); if (sfs.super != LinuxPATestCorrectSuperblock) { std.debug.print("Superblock wrong\nShould be: {}\n\nis: {}\n", .{ LinuxPATestCorrectSuperblock, sfs.super }); return error.BadSuperblock; @@ -24,11 +23,10 @@ test "ExtractSingleFile" { std.fs.cwd().deleteFile(TestFileExtractLocation) catch {}; var fil = try std.fs.cwd().openFile(TestArchive, .{}); defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - var test_fil = try sfs.open(TestFile); + var sfs: Archive = try .init(fil, 0); + var test_fil = try sfs.open(std.testing.allocator, TestFile); defer test_fil.deinit(); - try test_fil.extract(TestFileExtractLocation, .Default); + try test_fil.extract(std.testing.allocator, TestFileExtractLocation, try .Default()); //TODO: validate extracted file. } @@ -38,9 +36,8 @@ test "ExtractCompleteArchive" { std.fs.cwd().deleteTree(TestFullExtractLocation) catch {}; var fil = try std.fs.cwd().openFile(TestArchive, .{}); defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - try sfs.extract(TestFullExtractLocation, .Default); + var sfs: Archive = try .init(fil, 0); + try sfs.extract(std.testing.allocator, TestFullExtractLocation, try .Default()); } const LinuxPATestCorrectSuperblock: Superblock = .{ diff --git a/src/util/data.zig b/src/util/data.zig index 177e7c1..c75d918 100644 --- a/src/util/data.zig +++ b/src/util/data.zig @@ -29,13 +29,16 @@ interface: Reader, cur_offset: u64, block_idx: u32 = 0, -pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64) DataReader { +pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64, frag_offset: u32, frag_entry: ?FragEntry) DataReader { return .{ .alloc = alloc, .fil = archive.fil, .decomp = archive.decomp, .block_size = archive.super.block_size, .blocks = blocks, + + .frag = frag_entry, + .frag_offset = frag_offset, .size = size, .cur_offset = start, .interface = .{ @@ -56,11 +59,6 @@ pub fn deinit(self: *DataReader) void { self.interface.seek = 0; } -pub fn addFragment(self: *DataReader, entry: FragEntry, frag_offset: u32) void { - self.frag = entry; - self.frag_offset = frag_offset; -} - fn numBlocks(self: DataReader) usize { var res = self.blocks.len; if (self.frag != null) res += 1; diff --git a/src/util/data_threaded.zig b/src/util/data_threaded.zig index 68ec14f..1ac2c44 100644 --- a/src/util/data_threaded.zig +++ b/src/util/data_threaded.zig @@ -28,7 +28,7 @@ size: u64, start_offset: u64, -pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64) ThreadedDataReader { +pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64, frag_offset: u32, frag_entry: ?FragEntry) ThreadedDataReader { return .{ .alloc = alloc, .fil = archive.fil, @@ -37,12 +37,10 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, sta .blocks = blocks, .size = size, .start_offset = start, - }; -} -pub fn addFragment(self: *ThreadedDataReader, entry: FragEntry, frag_offset: u32) void { - self.frag = entry; - self.frag_offset = frag_offset; + .frag_offset = frag_offset, + .frag = frag_entry, + }; } fn numBlocks(self: ThreadedDataReader) usize {