diff --git a/.zed/debug.json b/.zed/debug.json new file mode 100644 index 0000000..198e0f6 --- /dev/null +++ b/.zed/debug.json @@ -0,0 +1,30 @@ +// Project-local debug tasks +// +// For more documentation on how to configure debug tasks, +// see: https://zed.dev/docs/debugger +[ + { + "label": "Build & Run", + + "adapter": "CodeLLDB", + "request": "launch", + + "build": { + "command": "zig", + "args": [ + "build", + "-Doptimize=Debug", + "-Duse_c_libs=true", + "-Dvalgrind=true", + ], + }, + + "program": "zig-out/bin/unsquashfs", + "args": [ + "--force", + "-d", + "testing/TestExtractUnsquashfs", + "testing/LinuxPATest.sfs", + ], + }, +] diff --git a/README.md b/README.md index 171d3a4..53c64bb 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,6 @@ Sets the version of `unsquashfs` shown when `--version` is passed. Most features are present except for the following: -* xattrs are not applied on extraction * When using Zig decompression libraries then lzo and lz4 compression types are unavailable. I don't _currently_ plan on spending the time to find and validate a library since neither is popular. * When using C decompression libraries, lzo is not supported by default due to [some issues](#build-considerations). If it's needed it's trivial to fix, but it's easiest to just leave it disabled. @@ -38,16 +37,16 @@ Most features are present except for the following: This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive (which is fairly small and uses zstd compression) and with `--release=fast`. -* Under ideal circumstances, my library is ~60% slower (.12s vs .19s). +* Under ideal circumstances, my library is ~70% slower (.12s vs .20s). * Using Zig decompression libraries *significantly* increases decompression time by ~600%. Under ideal circumstances. * Performance improvements/regressions will be common. I'm still learning Zig. -Times: +Example Times: * *unsquashfs, multi-threaded*: .12s * *unsquashfs, single-threaded*: .13s -* *C-libs, single-threaded*: .56s -* *C-libs, multi-threaded*: .19s +* *C-libs, single-threaded*: .45s +* *C-libs, multi-threaded*: .20s * *Zig-libs, single-threaded*: 5.78s * *Zig-libs, multi-threaded*: 1.08s diff --git a/build.zig b/build.zig index ca8f18d..55da7bd 100644 --- a/build.zig +++ b/build.zig @@ -21,12 +21,12 @@ pub fn build(b: *std.Build) !void { }); mod.addOptions("config", zig_squashfs_options); if (use_c_libs_option == true) { - mod.linkSystemLibrary("zlib", .{}); - mod.linkSystemLibrary("lzma", .{}); + mod.linkSystemLibrary("zlib-ng", .{ .preferred_link_mode = .static }); + mod.linkSystemLibrary("lzma", .{ .preferred_link_mode = .static }); if (allow_lzo == true) - mod.linkSystemLibrary("minilzo", .{}); - mod.linkSystemLibrary("lz4", .{}); - mod.linkSystemLibrary("zstd", .{}); + mod.linkSystemLibrary("minilzo", .{ .preferred_link_mode = .static }); + mod.linkSystemLibrary("lz4", .{ .preferred_link_mode = .static }); + mod.linkSystemLibrary("zstd", .{ .preferred_link_mode = .static }); } var version = version_string_option orelse "0.0.0-testing"; @@ -52,6 +52,7 @@ pub fn build(b: *std.Build) !void { const exe = b.addExecutable(.{ .name = "unsquashfs", .root_module = exe_mod, + // .use_llvm = true, This can be needed to properly debug }); const lib = b.addLibrary(.{ diff --git a/src/archive.zig b/src/archive.zig index c19c635..f053e0b 100644 --- a/src/archive.zig +++ b/src/archive.zig @@ -12,7 +12,7 @@ const InodeRef = Inode.Ref; const BlockSize = @import("inode_data/file.zig").BlockSize; const SfsFile = @import("file.zig"); const Superblock = @import("super.zig").Superblock; -const Table = @import("table.zig").Table; +const Tables = @import("tables.zig"); const MetadataReader = @import("util/metadata.zig"); const OffsetFile = @import("util/offset_file.zig"); const XattrTable = @import("xattr.zig"); @@ -22,14 +22,6 @@ const config = if (builtin.is_test) .{ .allow_lzo = false, } else @import("config"); -/// Information about a fragment section. Multiple fragments are contained in the block described by a single FragEntry. -/// The offset into the block and fragment size is stored in the file's inode. -pub const FragEntry = packed struct { - start: u64, - size: BlockSize, - _: u32, -}; - const Archive = @This(); alloc: std.mem.Allocator, @@ -39,10 +31,7 @@ decomp: Decomp.DecompFn, super: Superblock, -frag_table: Table(FragEntry), -id_table: Table(u16), -export_table: Table(InodeRef), -xattr_table: XattrTable, +tables: ?Tables = null, /// Default settings using std.Thread.getCpuCount() threads and the minimum of 4gb or half of system memory for memory usage. pub fn init(alloc: std.mem.Allocator, fil: File, offset: u64) !Archive { @@ -64,19 +53,16 @@ pub fn init(alloc: std.mem.Allocator, fil: File, offset: u64) !Archive { .fil = off_fil, .decomp = decomp, .super = super, - .frag_table = try .init(alloc, off_fil, decomp, super.frag_start, super.frag_count), - .id_table = try .init(alloc, off_fil, decomp, super.id_start, super.id_count), - .export_table = try .init(alloc, off_fil, decomp, super.export_start, super.inode_count), - .xattr_table = try .init(alloc, off_fil, decomp, super.xattr_start), }; } pub fn deinit(self: *Archive) void { - self.frag_table.deinit(); - self.export_table.deinit(); - self.id_table.deinit(); + if (self.tables != null) + self.tables.?.deinit(); } pub fn inode(self: *Archive, alloc: std.mem.Allocator, num: u32) !Inode { + if (self.tables == null) + self.tables = try .init(alloc, self); const ref = try self.export_table.get(num - 1); var rdr = try self.fil.readerAt(ref.block_start + self.super.inode_start, &[0]u8{}); var meta: MetadataReader = .init(alloc, &rdr.interface, &self.decomp); @@ -85,6 +71,8 @@ pub fn inode(self: *Archive, alloc: std.mem.Allocator, num: u32) !Inode { } pub fn root(self: *Archive, alloc: std.mem.Allocator) !SfsFile { + if (self.tables == null) + self.tables = try .init(alloc, self); var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{}); var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp); try meta.interface.discardAll(self.super.root_ref.block_offset); @@ -93,12 +81,14 @@ pub fn root(self: *Archive, alloc: std.mem.Allocator) !SfsFile { } pub fn open(self: *Archive, alloc: std.mem.Allocator, path: []const u8) !SfsFile { + if (self.tables == null) + self.tables = try .init(alloc, self); var root_fil = try self.root(alloc); defer if (!SfsFile.pathIsSelf(path)) root_fil.deinit(); return root_fil.open(path); } -pub fn extract(self: *Archive, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void { +pub fn extract(self: Archive, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void { var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{}); var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp); try meta.interface.discardAll(self.super.root_ref.block_offset); diff --git a/src/bin/unsquashfs.zig b/src/bin/unsquashfs.zig index 1db474f..a945c01 100644 --- a/src/bin/unsquashfs.zig +++ b/src/bin/unsquashfs.zig @@ -20,6 +20,8 @@ const help_mgs = \\ -p Specify how many threads to use. If no present or zero, the system's logical cores count is used. \\ -v Verbose \\ + \\ --force Force extraction. If the destination already exists, it will be deleted. + \\ \\ --help Display this messages \\ --version Display the version \\ @@ -34,6 +36,7 @@ var threads: u32 = 0; var verbose: bool = false; var ignore_xattrs: bool = false; var ignore_permissions: bool = false; +var force: bool = false; pub fn main() !void { const alloc = std.heap.smp_allocator; @@ -57,6 +60,8 @@ pub fn main() !void { .ignore_xattr = ignore_xattrs, .ignore_permissions = ignore_permissions, }; + if (force) + try std.fs.cwd().deleteTree(extLoc); try arc.extract(alloc, extLoc, options); //TODO: Handle error gracefully. } @@ -104,6 +109,9 @@ fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void { } else if (std.mem.eql(u8, arg, "-dp")) { ignore_permissions = true; continue; + } else if (std.mem.eql(u8, arg, "--force")) { + force = true; + continue; } else if (std.mem.eql(u8, arg, "--version")) { try out.print("zig-unsquashfs v", .{}); try config.version.format(out); diff --git a/src/decomp.zig b/src/decomp.zig index 76ac5b9..0928e09 100644 --- a/src/decomp.zig +++ b/src/decomp.zig @@ -11,7 +11,7 @@ const config = if (builtin.is_test) .{ } else @import("config"); const c = @cImport({ - @cInclude("zlib.h"); + @cInclude("zlib-ng.h"); @cInclude("lzma.h"); @cInclude("lz4.h"); @cInclude("zstd.h"); @@ -43,7 +43,7 @@ fn zigGzip(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize { fn cGzip(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize { _ = alloc; var out_len: usize = out.len; - const res = c.uncompress(out.ptr, &out_len, in.ptr, in.len); + const res = c.zng_uncompress2(out.ptr, &out_len, in.ptr, in.len); return switch (res) { c.Z_OK => out_len, c.Z_MEM_ERROR => error.NotEnoughMemory, @@ -137,6 +137,7 @@ fn cXz(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize { .avail_in = in.len, .next_out = out.ptr, .avail_out = out.len, + // .allocator = _, TODO: create a custom allocator based on alloc, }; var res = c.lzma_stream_decoder(&stream, in.len * 2, 0); switch (res) { diff --git a/src/inode.zig b/src/inode.zig index 58d46bf..48f1f4b 100644 --- a/src/inode.zig +++ b/src/inode.zig @@ -12,10 +12,13 @@ const ExtractionOptions = @import("options.zig"); const dir = @import("inode_data/dir.zig"); const file = @import("inode_data/file.zig"); const misc = @import("inode_data/misc.zig"); +const Tables = @import("tables.zig"); const DataReader = @import("util/data.zig"); const ThreadedDataReader = @import("util/data_threaded.zig"); +const InodeExtract = @import("util/extract.zig"); +const InodeFinish = @import("util/inode_finish.zig"); +const FinishUnion = InodeFinish.FinishUnion; const MetadataReader = @import("util/metadata.zig"); -const XattrTable = @import("xattr.zig"); pub const Ref = packed struct { block_offset: u16, @@ -94,7 +97,7 @@ pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode { }, }; } -pub fn readFromEntry(alloc: std.mem.Allocator, archive: *Archive, entry: DirEntry) !Inode { +pub fn readFromEntry(alloc: std.mem.Allocator, archive: Archive, entry: DirEntry) !Inode { var rdr = try archive.fil.readerAt(archive.super.inode_start + entry.block_start, &[0]u8{}); var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp); try meta.interface.discardAll(entry.block_offset); @@ -112,31 +115,17 @@ pub fn deinit(self: Inode, alloc: std.mem.Allocator) void { } /// Get the data reader for a file inode. -pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: *Archive) !DataReader { +pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive, tables: *Tables) !DataReader { return switch (self.hdr.inode_type) { - .file => readerFromData(alloc, archive, self.data.file), - .ext_file => readerFromData(alloc, archive, self.data.ext_file), + .file => readerFromData(alloc, archive, tables, self.data.file), + .ext_file => readerFromData(alloc, archive, tables, self.data.ext_file), else => error.NotRegularFile, }; } -fn readerFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !DataReader { - var out: DataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size); +fn readerFromData(alloc: std.mem.Allocator, archive: Archive, tables: *Tables, data: anytype) !DataReader { + var out: DataReader = .init(alloc, archive, data.block_sizes, data.block_start, data.size); if (data.frag_idx != 0xFFFFFFFF) - out.addFragment(try archive.frag_table.get(data.frag_idx), data.frag_block_offset); - return out; -} -/// Get a threaded data reader for a file inode. -pub fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: *Archive) !ThreadedDataReader { - return switch (self.hdr.inode_type) { - .file => threadedReaderFromData(alloc, archive, self.data.file), - .ext_file => threadedReaderFromData(alloc, archive, self.data.ext_file), - else => error.NotRegularFile, - }; -} -fn threadedReaderFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !ThreadedDataReader { - var out: ThreadedDataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size); - if (data.frag_idx != 0xFFFFFFFF) - out.addFragment(try archive.frag_table.get(data.frag_idx), data.frag_block_offset); + out.addFragment(try tables.frag_table.get(data.frag_idx), data.frag_block_offset); return out; } @@ -155,7 +144,7 @@ fn entriesFromData(alloc: std.mem.Allocator, archive: Archive, data: anytype) ![ return DirEntry.readDir(alloc, &meta.interface, data.size); } -/// Returns the xattr index for the given inode. If the inode isn't an extended variant or doesn't have any, the returned value is the max u32 value (0xFFFFFFFF) +/// Returns the xattr index for the given inode. If the inode isn't an extended variant or doesn't have any, the u32 max is returned (0xFFFFFFFF). pub fn xattrIdx(self: Inode) u32 { return switch (self.data) { .ext_dir => |d| d.xattr_id, @@ -167,17 +156,19 @@ pub fn xattrIdx(self: Inode) u32 { }; } -inline fn setPermissionAndXattr(self: Inode, alloc: std.mem.Allocator, archive: *Archive, fil: std.fs.File, options: ExtractionOptions) !void { +/// Applies the Inode's metadata to the given File. +/// Mod time is always set, but permissions and xattrs are set based on the given ExtractionOptions. +pub fn setMetadata(self: Inode, alloc: std.mem.Allocator, tables: *Tables, fil: std.fs.File, options: ExtractionOptions) !void { const time = @as(i128, self.hdr.mod_time) * 1000000000; try fil.updateTimes(time, time); if (!options.ignore_permissions) { try fil.chmod(self.hdr.permissions); - try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx)); + try fil.chown(try tables.id_table.get(self.hdr.uid_idx), try tables.id_table.get(self.hdr.gid_idx)); } if (!options.ignore_xattr) { const idx = self.xattrIdx(); if (idx == 0xFFFFFFFF) return; - const xattrs = try archive.xattr_table.get(alloc, idx); + const xattrs = try tables.xattr_table.get(alloc, idx); defer alloc.free(xattrs); for (xattrs) |kv| { const res = std.os.linux.fsetxattr(fil.handle, @ptrCast(kv.key), @ptrCast(kv.value), kv.value.len, 0); @@ -194,343 +185,6 @@ inline fn setPermissionAndXattr(self: Inode, alloc: std.mem.Allocator, archive: } /// Extract the inode to the given path. -pub fn extractTo(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void { - if (options.threads > 1) return self.extractToThreaded(alloc, archive, path, options); - switch (self.hdr.inode_type) { - .dir, .ext_dir => { - // Removing any trailing separators since that's the easiest path forward. - if (path[path.len - 1] == '/') return self.extractTo(alloc, archive, path[0 .. path.len - 1], options); - std.fs.cwd().makeDir(path) catch |err| { - if (err != std.fs.Dir.MakeError.PathAlreadyExists) return err; - }; - const entries = try self.dirEntries(alloc, archive.*); - defer { - for (entries) |entry| entry.deinit(alloc); - alloc.free(entries); - } - for (entries) |entry| { - var new_path = try alloc.alloc(u8, path.len + 1 + entry.name.len); - @memcpy(new_path[0..path.len], path); - @memcpy(new_path[path.len + 1 ..], entry.name); - new_path[path.len] = '/'; - defer alloc.free(new_path); - - var inode: Inode = try readFromEntry(alloc, archive, entry); - defer inode.deinit(alloc); - try inode.extractTo(alloc, archive, new_path, options); - } - - var fil = try std.fs.cwd().openFile(path, .{}); - defer fil.close(); - try self.setPermissionAndXattr(alloc, archive, fil, options); - }, - .file, .ext_file => try self.extractRegFile(alloc, archive, path, options), - .symlink, .ext_symlink => try self.extractSymlink(path), - else => try self.extractDevice(alloc, archive, path, options), - } -} - -const Parent = struct { - alloc: std.mem.Allocator, - - inode: Inode, - path: []const u8, - archive: *Archive, - options: ExtractionOptions, - - wg: WaitGroup = .{}, - mut: Mutex = .{}, - - fn create(alloc: std.mem.Allocator, inode: Inode, path: []const u8, archive: *Archive, options: ExtractionOptions, dir_size: usize) !*Parent { - const out = try alloc.create(Parent); - errdefer alloc.destroy(out); - out.* = .{ - .alloc = alloc, - - .inode = inode, - .path = path, - .archive = archive, - .options = options, - }; - out.wg.startMany(dir_size); - return out; - } - - fn finish(p: *Parent) !void { - p.mut.lock(); - { - defer p.mut.unlock(); - p.wg.finish(); - if (!p.wg.isDone()) return; - } - defer p.alloc.destroy(p); - var fil = try std.fs.cwd().openFile(p.path, .{}); - defer fil.close(); - try p.inode.setPermissionAndXattr(p.alloc, p.archive, fil, p.options); - } -}; - -/// Extract the inode to the given path. Multi-threaded. -/// Functions identically to extractTo on all but regular files and directories. -fn extractToThreaded(self: Inode, allocator: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void { - switch (self.hdr.inode_type) { - .dir, .ext_dir => { - // Removing any trailing separators since that's the easiest path forward. - if (path[path.len - 1] == '/') return self.extractToThreaded(allocator, archive, path[0 .. path.len - 1], options); - - // Arena Allocator - var stack_alloc = std.heap.stackFallback(1024 * 1024, allocator); - var arena_alloc: std.heap.ArenaAllocator = .init(stack_alloc.get()); - defer arena_alloc.deinit(); - var thread_alloc: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena_alloc.allocator() }; - const alloc = thread_alloc.allocator(); - - var wg: WaitGroup = .{}; - // defer if(!options.ignore_permissions) perms.?.deinit(alloc); We don't need to do this due to ArenaAllocator - var pool: Pool = undefined; - try pool.init(.{ .allocator = alloc, .n_jobs = options.threads - 1 }); - defer pool.deinit(); - var out_err: ?anyerror = null; - - wg.start(); - self.extractThread(alloc, archive, path, options, &wg, &pool, &out_err, null); - pool.waitAndWork(&wg); - if (out_err != null) return out_err.?; - - var fil = try std.fs.cwd().openFile(path, .{}); - defer fil.close(); - try self.setPermissionAndXattr(alloc, archive, fil, options); - }, - .file, .ext_file => { - var pool: Pool = undefined; - try pool.init(.{ .allocator = allocator, .n_jobs = options.threads - 1 }); - defer pool.deinit(); - - // Arena Allocator - var stack_alloc = std.heap.stackFallback(1024 * 1024, allocator); - var arena_alloc: std.heap.ArenaAllocator = .init(stack_alloc.get()); - defer arena_alloc.deinit(); - var thread_alloc: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena_alloc.allocator() }; - const alloc = thread_alloc.allocator(); - - try self.extractRegFileThreaded(alloc, archive, path, options, &pool); - - var fil = try std.fs.cwd().openFile(path, .{}); - defer fil.close(); - try self.setPermissionAndXattr(alloc, archive, fil, options); - }, - .symlink, .ext_symlink => try self.extractSymlink(path), - else => try self.extractDevice(allocator, archive, path, options), - } -} - -fn extractThreadEntry( - entry: DirEntry, - alloc: std.mem.Allocator, - archive: *Archive, - path: []const u8, - options: ExtractionOptions, - wg: *WaitGroup, - pool: *Pool, - out_err: *?anyerror, - parent: ?*Parent, -) void { - var new_path = alloc.alloc(u8, path.len + entry.name.len + 1) catch |err| { - wg.finish(); - out_err.* = err; - return; - }; - @memcpy(new_path[0..path.len], path); - @memcpy(new_path[path.len + 1 ..], entry.name); - new_path[path.len] = '/'; - var inode = readFromEntry(alloc, archive, entry) catch |err| { - out_err.* = err; - wg.finish(); - return; - }; - inode.extractThread(alloc, archive, new_path, options, wg, pool, out_err, parent); -} - -/// Extract threadedly the inode to the path. -fn extractThread( - self: Inode, - alloc: std.mem.Allocator, - archive: *Archive, - path: []const u8, - options: ExtractionOptions, - wg: *WaitGroup, - pool: *Pool, - out_err: *?anyerror, - parent: ?*Parent, -) void { - if (options.verbose) - options.verbose_writer.?.print("Extracting inode #{} to {s}\n", .{ self.hdr.num, path }) catch {}; - defer { - if (parent != null) parent.?.finish() catch |err| { - if (options.verbose) - options.verbose_writer.?.print("Error setting folder permission to {s}: {}\n", .{ path, err }) catch {}; - out_err.* = err; - }; - wg.finish(); - } - if (out_err.* != null) return; - switch (self.hdr.inode_type) { - .dir, .ext_dir => { - _ = std.fs.cwd().makePathStatus(path) catch |err| { - if (options.verbose) - options.verbose_writer.?.print("Error creating {s}: {}\n", .{ path, err }) catch {}; - out_err.* = err; - return; - }; - - const entries = self.dirEntries(alloc, archive.*) catch |err| { - if (options.verbose) - options.verbose_writer.?.print("Error getting directory entries for inode #{} (extracting to {s}): {}\n", .{ self.hdr.num, path, err }) catch {}; - out_err.* = err; - return; - }; - const p = Parent.create(alloc, self, path, archive, options, entries.len) catch |err| { - out_err.* = err; - return; - }; - wg.startMany(entries.len); - // defer files.deinit(alloc); We don't need to do this due to ArenaAllocator - for (entries) |entry| { - if (entry.inode_type == .dir) { - extractThreadEntry(entry, alloc, archive, path, options, wg, pool, out_err, p); - continue; - } - pool.spawn( - extractThreadEntry, - .{ - entry, - alloc, - archive, - path, - options, - wg, - pool, - out_err, - p, - }, - ) catch |err| { - wg.finish(); - if (options.verbose) - options.verbose_writer.?.print("Error starting extraction thread: {}\n", .{err}) catch {}; - out_err.* = err; - continue; - }; - } - }, - .file, .ext_file => { - self.extractRegFileThreaded(alloc, archive, path, options, pool) catch |err| { - if (options.verbose) - options.verbose_writer.?.print("Error extracting file inode #{} to {s}: {}\n", .{ self.hdr.num, path, err }) catch {}; - out_err.* = err; - }; - }, - .symlink, .ext_symlink => { - self.extractSymlink(path) catch |err| { - if (options.verbose) - options.verbose_writer.?.print("Error extracting symlink inode #{} to {s}: {}\n", .{ self.hdr.num, path, err }) catch {}; - out_err.* = err; - }; - }, - else => { - self.extractDevice(alloc, archive, path, options) catch |err| { - if (options.verbose) - options.verbose_writer.?.print("Error extracting device/IPC inode #{} to {s}: {}\n", .{ self.hdr.num, path, err }) catch {}; - out_err.* = err; - }; - }, - } -} -/// Creates and writes the inode file contents to the given path. -/// Optionally set owner & permissions. -/// -/// Assumes the inode is a file or ext_file type. -fn extractRegFile(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void { - var fil = try std.fs.cwd().createFile(path, .{ .exclusive = true }); - defer fil.close(); - var wrt = fil.writer(&[0]u8{}); - var dat_rdr = try self.dataReader(alloc, archive); - defer dat_rdr.deinit(); - _ = try dat_rdr.interface.streamRemaining(&wrt.interface); - try wrt.interface.flush(); - - try self.setPermissionAndXattr(alloc, archive, fil, options); -} -/// Extract the inode file contents to the given path threadedly. -/// pool is used to spawn threads. -/// -/// Assumes the inode is a file or ext_file type. -fn extractRegFileThreaded(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions, pool: *Pool) !void { - var fil = try std.fs.cwd().createFile(path, .{}); - defer fil.close(); - var data = try self.threadedDataReader(alloc, archive); - try data.extractThreaded(fil, pool); - - try self.setPermissionAndXattr(alloc, archive, fil, options); -} -/// Creates the symlink described by the inode. -/// -/// Assumes the inode is a symlink or ext_symlink type. -fn extractSymlink(self: Inode, path: []const u8) !void { - const target = switch (self.data) { - .symlink => |s| s.target, - .ext_symlink => |s| s.target, - else => unreachable, - }; - try std.fs.cwd().symLink(target, path, .{}); -} -/// Creates the device described by the inode. -/// -/// Optionally set owner & permissions. -/// Assumes the inode is a char_dev, block_dev, fifo, socket, or their extended counterparts. -fn extractDevice(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void { - var mode: u32 = undefined; - var dev: u32 = 0; - switch (self.data) { - .char_dev => |d| { - mode = std.posix.S.IFCHR; - dev = d.dev; - }, - .ext_char_dev => |d| { - mode = std.posix.S.IFCHR; - dev = d.dev; - }, - .block_dev => |d| { - mode = std.posix.S.IFBLK; - dev = d.dev; - }, - .ext_block_dev => |d| { - mode = std.posix.S.IFBLK; - dev = d.dev; - }, - .fifo, .ext_fifo => mode = std.posix.S.IFIFO, - .socket, .ext_socket => mode = std.posix.S.IFSOCK, - else => unreachable, - } - const res: std.os.linux.E = @enumFromInt(std.os.linux.mknod(@ptrCast(path), mode, dev)); - switch (res) { - .SUCCESS => {}, - .ACCES => return std.fs.Dir.MakeError.AccessDenied, - .DQUOT => return std.fs.Dir.MakeError.DiskQuota, - .EXIST => return std.fs.Dir.MakeError.PathAlreadyExists, - .FAULT, .NOENT => return std.fs.Dir.MakeError.BadPathName, - .LOOP => return std.fs.Dir.MakeError.SymLinkLoop, - .NAMETOOLONG => return std.fs.Dir.MakeError.NameTooLong, - .NOMEM => return std.fs.Dir.MakeError.SystemResources, - .NOSPC => return std.fs.Dir.MakeError.NoSpaceLeft, - .NOTDIR => return std.fs.Dir.MakeError.NotDir, - .PERM => return std.fs.Dir.MakeError.PermissionDenied, - .ROFS => return std.fs.Dir.MakeError.ReadOnlyFileSystem, - else => return blk: { - std.debug.print("unhandled mknod result: {}\n", .{res}); - break :blk std.fs.Dir.MakeError.Unexpected; - }, - } - var fil = try std.fs.cwd().openFile(path, .{}); - defer fil.close(); - try self.setPermissionAndXattr(alloc, archive, fil, options); +pub fn extractTo(self: Inode, alloc: std.mem.Allocator, archive: Archive, path: []const u8, options: ExtractionOptions) !void { + return InodeExtract.extractTo(alloc, self, archive, path, options); } diff --git a/src/table.zig b/src/tables.zig similarity index 64% rename from src/table.zig rename to src/tables.zig index c98e85d..700d3b6 100644 --- a/src/table.zig +++ b/src/tables.zig @@ -1,14 +1,45 @@ const std = @import("std"); const Mutex = std.Thread.Mutex; +const Archive = @import("archive.zig"); const DecompFn = @import("decomp.zig").DecompFn; +const BlockSize = @import("inode_data/file.zig").BlockSize; +const InodeRef = @import("inode.zig").Ref; +const Superblock = @import("super.zig").Superblock; const MetadataReader = @import("util/metadata.zig"); const OffsetFile = @import("util/offset_file.zig"); +const XattrTable = @import("xattr.zig"); -const TableError = error{ - InvalidIndex, +/// Information about a fragment section. Multiple fragments are contained in the block described by a single FragEntry. +/// The offset into the block and fragment size is stored in the file's inode. +pub const FragEntry = packed struct { + start: u64, + size: BlockSize, + _: u32, }; +const Tables = @This(); + +frag_table: Table(FragEntry), +id_table: Table(u16), +export_table: Table(InodeRef), +xattr_table: XattrTable, + +pub fn init(alloc: std.mem.Allocator, archive: Archive) !Tables { + return .{ + .frag_table = try .init(alloc, archive.fil, archive.decomp, archive.super.frag_start, archive.super.frag_count), + .id_table = try .init(alloc, archive.fil, archive.decomp, archive.super.id_start, archive.super.id_count), + .export_table = try .init(alloc, archive.fil, archive.decomp, archive.super.export_start, archive.super.inode_count), + .xattr_table = try .init(alloc, archive.fil, archive.decomp, archive.super.xattr_start), + }; +} +pub fn deinit(self: *Tables) void { + self.frag_table.deinit(); + self.id_table.deinit(); + self.export_table.deinit(); + self.xattr_table.deinit(); +} + /// A two-layer metadata table. pub fn Table(T: anytype) type { return struct { @@ -47,7 +78,7 @@ pub fn Table(T: anytype) type { } pub fn get(self: *Self, idx: u32) !T { - if (idx >= self.values) return TableError.InvalidIndex; + if (idx >= self.values) return error.InvalidIndex; const block_num = idx / VALS_PER_BLOCK; const idx_offset = idx - (block_num * VALS_PER_BLOCK); if (self.tab.contains(block_num)) { diff --git a/src/util/data.zig b/src/util/data.zig index 177e7c1..0c1d43e 100644 --- a/src/util/data.zig +++ b/src/util/data.zig @@ -6,9 +6,9 @@ const Writer = std.Io.Writer; const Limit = std.Io.Limit; const Archive = @import("../archive.zig"); -const FragEntry = Archive.FragEntry; const DecompFn = @import("../decomp.zig").DecompFn; const BlockSize = @import("../inode_data/file.zig").BlockSize; +const FragEntry = @import("../tables.zig").FragEntry; const OffsetFile = @import("offset_file.zig"); const DataReader = @This(); diff --git a/src/util/data_threaded.zig b/src/util/data_threaded.zig index 68ec14f..280586a 100644 --- a/src/util/data_threaded.zig +++ b/src/util/data_threaded.zig @@ -8,9 +8,10 @@ const WaitGroup = std.Thread.WaitGroup; const Pool = std.Thread.Pool; const Archive = @import("../archive.zig"); -const FragEntry = Archive.FragEntry; const DecompFn = @import("../decomp.zig").DecompFn; const BlockSize = @import("../inode_data/file.zig").BlockSize; +const FragEntry = @import("../tables.zig").FragEntry; +const InodeFinish = @import("inode_finish.zig"); const OffsetFile = @import("offset_file.zig"); const ThreadedDataReader = @This(); @@ -25,6 +26,7 @@ blocks: []BlockSize, frag: ?FragEntry = null, // TODO: do something better? frag_offset: u32 = 0, size: u64, +num_blocks: usize, start_offset: u64, @@ -34,8 +36,12 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, sta .fil = archive.fil, .decomp = archive.decomp, .block_size = archive.super.block_size, + .blocks = blocks, + .size = size, + .num_blocks = blocks.len, + .start_offset = start, }; } @@ -43,141 +49,159 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, sta pub fn addFragment(self: *ThreadedDataReader, entry: FragEntry, frag_offset: u32) void { self.frag = entry; self.frag_offset = frag_offset; -} - -fn numBlocks(self: ThreadedDataReader) usize { - var res = self.blocks.len; - if (self.frag != null) res += 1; - return res; + self.num_blocks = self.blocks.len + 1; } /// Extract the data to the file threadedly, using pool to spawn threads. -/// If multiple errors occur, thread spawning errors will have, then the last decompression error that occurs; -/// -/// The function must be called from an unused DataReader. The DataReader is still usable afterwards. -/// If only extractThreaded is used, there is no need to call deinit() afterwards. -/// -/// The file will always be written to starting at 0. -pub fn extractThreaded(self: ThreadedDataReader, file: std.fs.File, pool: *Pool) !void { - var wg: WaitGroup = .{}; - wg.startMany(self.numBlocks()); - var out_err: ?anyerror = null; - +/// If errors occur, they are set to finish.out_err. +pub fn extractThreaded(self: ThreadedDataReader, file: std.fs.File, pool: *Pool, finish: *InodeFinish) void { var cur_write_offset: u64 = 0; var cur_read_offset: u64 = self.start_offset; for (0..self.blocks.len) |i| { - const cur_block_size = if (i == self.numBlocks() - 1) self.size % self.block_size else self.block_size; - try pool.spawn(workThreadBlocks, .{ self, file, cur_write_offset, cur_read_offset, self.blocks[i], cur_block_size, &wg, &out_err }); + const cur_block_size = if (i == self.num_blocks - 1) self.size % self.block_size else self.block_size; + pool.spawn(workThreadBlocks, .{ self, file, cur_write_offset, cur_read_offset, self.blocks[i], cur_block_size, finish }) catch |res_err| { + finish.logError("Can't spawn pool task: {}", .{res_err}); + finish.out_err.* = res_err; + finish.finish(); + }; cur_write_offset += cur_block_size; cur_read_offset += self.blocks[i].size; } - if (self.frag != null) { - try pool.spawn(workThreadFragment, .{ self, file, cur_write_offset, &wg, &out_err }); - } - pool.waitAndWork(&wg); - if (out_err != null) return out_err.?; + if (self.frag != null) + pool.spawn(workThreadFragment, .{ self, file, cur_write_offset, finish }) catch |res_err| { + finish.logError("Can't spawn pool task: {}", .{res_err}); + finish.out_err.* = res_err; + finish.finish(); + }; } -fn workThreadBlocks(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, read_offset: u64, block: BlockSize, cur_block_size: u64, wg: *WaitGroup, out_err: *?anyerror) void { - defer wg.finish(); +fn workThreadBlocks( + self: ThreadedDataReader, + fil: std.fs.File, + write_offset: u64, + read_offset: u64, + block: BlockSize, + cur_block_size: u64, + finish: *InodeFinish, +) void { + defer finish.finish(); var wrt = fil.writer(&[0]u8{}); wrt.seekTo(write_offset) catch |err| { - out_err.* = err; + finish.logError("Error seeking file writer: {}", .{err}); + finish.out_err.* = err; return; }; defer wrt.interface.flush() catch |err| { - out_err.* = err; + finish.logError("Error flushing file writer: {}", .{err}); + finish.out_err.* = err; }; if (block.size == 0) { wrt.interface.splatByteAll(0, cur_block_size) catch |err| { - out_err.* = err; + finish.logError("Error writing zeroes: {}", .{err}); + finish.out_err.* = err; return; }; return; } var rdr = self.fil.readerAt(read_offset, &[0]u8{}) catch |err| { - out_err.* = err; + finish.logError("Error creating file reader: {}", .{err}); + finish.out_err.* = err; return; }; if (block.uncompressed) { rdr.interface.streamExact(&wrt.interface, block.size) catch |err| { - out_err.* = err; + finish.logError("Error streaming data: {}", .{err}); + finish.out_err.* = err; return; }; return; } // TODO: shared buffers const read_buf = self.alloc.alloc(u8, block.size) catch |err| { - out_err.* = err; + finish.logError("Error creating reader buffer: {}", .{err}); + finish.out_err.* = err; return; }; defer self.alloc.free(read_buf); rdr.interface.readSliceAll(read_buf) catch |err| { - out_err.* = err; + finish.logError("Error reading data into reader buffer: {}", .{err}); + finish.out_err.* = err; return; }; // TODO: shared buffers const res_buf = self.alloc.alloc(u8, cur_block_size) catch |err| { - out_err.* = err; + finish.logError("Error creating result buffer: {}", .{err}); + finish.out_err.* = err; return; }; defer self.alloc.free(res_buf); _ = self.decomp(self.alloc, read_buf, res_buf) catch |err| { - out_err.* = err; + finish.logError("Error decompressing data block: {}", .{err}); + finish.out_err.* = err; return; }; wrt.interface.writeAll(res_buf) catch |err| { - out_err.* = err; + finish.logError("Error writing to file: {}", .{err}); + finish.out_err.* = err; return; }; } -fn workThreadFragment(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, wg: *WaitGroup, out_err: *?anyerror) void { - defer wg.finish(); +fn workThreadFragment(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, finish: *InodeFinish) void { + defer finish.finish(); var wrt = fil.writer(&[0]u8{}); wrt.seekTo(write_offset) catch |err| { - out_err.* = err; + finish.logError("Error seeking file writer for file fragment: {}", .{err}); + finish.out_err.* = err; return; }; defer wrt.interface.flush() catch |err| { - out_err.* = err; + finish.out_err.* = err; }; var rdr = self.fil.readerAt(self.frag.?.start, &[0]u8{}) catch |err| { - out_err.* = err; + finish.logError("Error creating file reader for file fragment: {}", .{err}); + finish.out_err.* = err; return; }; if (self.frag.?.size.uncompressed) { rdr.interface.discardAll(self.frag_offset) catch |err| { - out_err.* = err; + finish.logError("Error discarding useless fragment data: {}", .{err}); + finish.out_err.* = err; return; }; rdr.interface.streamExact(&wrt.interface, self.size % self.block_size) catch |err| { - out_err.* = err; + finish.logError("Error streaming fragment data: {}", .{err}); + finish.out_err.* = err; return; }; return; } const tmp_buf = self.alloc.alloc(u8, self.frag.?.size.size) catch |err| { - out_err.* = err; + finish.logError("Error creating a temporary buffer for a file fragment: {}", .{err}); + finish.out_err.* = err; return; }; defer self.alloc.free(tmp_buf); rdr.interface.readSliceAll(tmp_buf) catch |err| { - out_err.* = err; + finish.logError("Error reading data into fragment buffer: {}", .{err}); + finish.out_err.* = err; return; }; const needed_block = self.alloc.alloc(u8, self.block_size) catch |err| { - out_err.* = err; + finish.logError("Error allocating fragment decompression results: {}", .{err}); + finish.out_err.* = err; return; }; defer self.alloc.free(needed_block); _ = self.decomp(self.alloc, tmp_buf, needed_block) catch |err| { - out_err.* = err; + finish.logError("Error decompressing fragment: {}", .{err}); + finish.out_err.* = err; return; }; wrt.interface.writeAll(needed_block[self.frag_offset .. self.frag_offset + (self.size % self.block_size)]) catch |err| { - out_err.* = err; + finish.logError("Error writing fragment: {}", .{err}); + finish.out_err.* = err; return; }; } diff --git a/src/util/extract.zig b/src/util/extract.zig new file mode 100644 index 0000000..285997c --- /dev/null +++ b/src/util/extract.zig @@ -0,0 +1,353 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Pool = std.Thread.Pool; +const WaitGroup = std.Thread.WaitGroup; + +const Archive = @import("../archive.zig"); +const DirEntry = @import("../dir_entry.zig"); +const Inode = @import("../inode.zig"); +const ExtractionOptions = @import("../options.zig"); +const Tables = @import("../tables.zig"); +const InodeFinish = @import("inode_finish.zig"); +const FinishUnion = InodeFinish.FinishUnion; +const ThreadedDataReader = @import("data_threaded.zig"); + +// 1 MB +const STACK_ALLOC_SIZE = 1024 * 1024; + +pub fn extractTo( + allocator: Allocator, + inode: Inode, + archive: Archive, + path: []const u8, + options: ExtractionOptions, +) !void { + if (path[path.len - 1] == '/') + return extractTo(allocator, inode, archive, path[0 .. path.len - 2], options); + + var stack_alloc = std.heap.stackFallback(STACK_ALLOC_SIZE, allocator); + var arena: std.heap.ArenaAllocator = .init(stack_alloc.get()); + defer arena.deinit(); + if (options.threads <= 1) { + const alloc = arena.allocator(); + var tables: Tables = try .init(alloc, archive); + return extractSingleThread(arena.allocator(), inode, archive, &tables, path, options); + } + + var thread_alloc = std.heap.ThreadSafeAllocator{ .child_allocator = arena.allocator() }; + const alloc = thread_alloc.allocator(); + var tables: Tables = try .init(alloc, archive); + + var pool_alloc = std.heap.stackFallback(10 * 1024, alloc); + var pool: Pool = undefined; + try pool.init(.{ .allocator = pool_alloc.get(), .n_jobs = options.threads - 1 }); + + var wg: WaitGroup = .{}; + var err: ?anyerror = null; + wg.start(); + try pool.spawn(extractMultiThread, .{ + alloc, + inode, + archive, + &tables, + path, + options, + &pool, + FinishUnion{ .wg = &wg }, + &err, + }); + pool.waitAndWork(&wg); + if (err != null) return err.?; +} + +fn extractSingleThread( + alloc: Allocator, + inode: Inode, + archive: Archive, + tables: *Tables, + path: []const u8, + options: ExtractionOptions, +) !void { + switch (inode.hdr.inode_type) { + .dir, .ext_dir => { + _ = std.fs.cwd().makeDir(path) catch |err| switch (err) { + std.fs.Dir.MakeError.PathAlreadyExists => {}, + else => return err, + }; + + // Currently we are ignoring any deinit or free calls since we know we are under an ArenaAllocator. + // Possibly in the future, do some simple math to see if it would be safe to ONLY deinit via Arena, + // otherwise be more conscientious about freeing memory. + // For now, this is good enough. + + const entries = try inode.dirEntries(alloc, archive); + for (entries) |ent| { + const sub_inode: Inode = try .readFromEntry(alloc, archive, ent); + const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", ent.name }); + try extractSingleThread(alloc, sub_inode, archive, tables, new_path, options); + } + + const fil = try std.fs.cwd().openFile(path, .{}); + defer fil.close(); + try inode.setMetadata(alloc, tables, fil, options); + }, + .file, .ext_file => { + var fil = try std.fs.cwd().createFile(path, .{ .exclusive = true }); + defer fil.close(); + var wrt = fil.writer(&[0]u8{}); + var dat_rdr = try inode.dataReader(alloc, archive, tables); + defer dat_rdr.deinit(); + _ = try dat_rdr.interface.streamRemaining(&wrt.interface); + try wrt.interface.flush(); + + try inode.setMetadata(alloc, tables, fil, options); + }, + .symlink, .ext_symlink => return extractSymlink(inode, path), + else => return extractDeviceAndIPC(inode, alloc, tables, path, options), + } +} + +fn extractMultiThread( + alloc: Allocator, + inode: Inode, + archive: Archive, + tables: *Tables, + path: []const u8, + options: ExtractionOptions, + pool: *Pool, + fin: FinishUnion, + err: *?anyerror, +) void { + if (err.* != null) { + fin.finish(); + return; + } + switch (inode.hdr.inode_type) { + .dir, .ext_dir => { + _ = std.fs.cwd().makeDir(path) catch |res_err| switch (res_err) { + std.fs.Dir.MakeError.PathAlreadyExists => {}, + else => { + err.* = res_err; + fin.finish(); + return; + }, + }; + + // Currently we are ignoring any deinit or free calls since we know we are under an ArenaAllocator. + // Possibly in the future, do some simple math to see if it would be safe to ONLY deinit via Arena, + // otherwise be more conscientious about freeing memory. + // For now, this is good enough. + + const entries = inode.dirEntries(alloc, archive) catch |res_err| { + err.* = res_err; + fin.finish(); + return; + }; + + if (entries.len == 0) { + fin.finish(); + return; + } + + var dir_fin = InodeFinish.create( + alloc, + inode, + path, + tables, + options, + fin, + err, + null, + entries.len, + ) catch |res_err| { + err.* = res_err; + fin.finish(); + return; + }; + + for (entries) |ent| { + if (ent.inode_type == .dir) { + extractEntry( + alloc, + ent, + archive, + tables, + path, + options, + pool, + .{ .fin = dir_fin }, + err, + ); + continue; + } + + pool.spawn( + extractEntry, + .{ alloc, ent, archive, tables, path, options, pool, FinishUnion{ .fin = dir_fin }, err }, + ) catch |res_err| { + err.* = res_err; + dir_fin.finish(); + return; + }; + } + }, + .file, .ext_file => { + const fil = std.fs.cwd().createFile(path, .{ .exclusive = true }) catch |res_err| { + if (options.verbose) + options.verbose_writer.?.print("Can't create file at {s}: {}\n", .{ path, res_err }) catch {}; + err.* = res_err; + fin.finish(); + return; + }; + + var data_rdr = threadedDataReader(inode, alloc, archive, tables) catch |res_err| { + if (options.verbose) + options.verbose_writer.?.print("Can't create data reader for inode #{} (extracting to {s}): {}\n", .{ inode.hdr.num, path, res_err }) catch {}; + err.* = res_err; + fin.finish(); + return; + }; + if (data_rdr == null) { + inode.setMetadata(alloc, tables, fil, options) catch |res_err| { + if (options.verbose) + options.verbose_writer.?.print("Can't set metadata to {s}: {}\n", .{ path, res_err }) catch {}; + err.* = res_err; + }; + fin.finish(); + return; + } + const file_fin = InodeFinish.create( + alloc, + inode, + path, + tables, + options, + fin, + err, + fil, + data_rdr.?.num_blocks, + ) catch |res_err| { + if (options.verbose) + options.verbose_writer.?.print("Can't create callback for inode #{} (extracting to {s}): {}\n", .{ inode.hdr.num, path, res_err }) catch {}; + err.* = res_err; + fin.finish(); + return; + }; + + data_rdr.?.extractThreaded(fil, pool, file_fin); + }, + .symlink, .ext_symlink => { + extractSymlink(inode, path) catch |res_err| { + err.* = res_err; + }; + fin.finish(); + }, + else => { + extractDeviceAndIPC(inode, alloc, tables, path, options) catch |res_err| { + err.* = res_err; + }; + fin.finish(); + }, + } +} + +fn extractEntry( + alloc: Allocator, + ent: DirEntry, + archive: Archive, + tables: *Tables, + path: []const u8, + options: ExtractionOptions, + pool: *Pool, + fin: FinishUnion, + err: *?anyerror, +) void { + const new_path = std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", ent.name }) catch |res_err| { + err.* = res_err; + fin.finish(); + return; + }; + + const inode = Inode.readFromEntry(alloc, archive, ent) catch |res_err| { + err.* = res_err; + fin.finish(); + return; + }; + extractMultiThread(alloc, inode, archive, tables, new_path, options, pool, fin, err); +} + +/// Get a threaded data reader for a file inode. +fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive, tables: *Tables) !?ThreadedDataReader { + return switch (self.hdr.inode_type) { + .file => threadedReaderFromData(alloc, archive, tables, self.data.file), + .ext_file => threadedReaderFromData(alloc, archive, tables, self.data.ext_file), + else => error.NotRegularFile, + }; +} +fn threadedReaderFromData(alloc: std.mem.Allocator, archive: Archive, tables: *Tables, data: anytype) !?ThreadedDataReader { + if (data.block_sizes.len == 0 and data.frag_idx == 0xFFFFFFFF) return null; + var out: ThreadedDataReader = .init(alloc, archive, data.block_sizes, data.block_start, data.size); + if (data.frag_idx != 0xFFFFFFFF) + out.addFragment(try tables.frag_table.get(data.frag_idx), data.frag_block_offset); + return out; +} + +/// Creates the symlink described by the inode. +/// Sets metadata. +fn extractSymlink(self: Inode, path: []const u8) !void { + const target = switch (self.data) { + .symlink => |s| s.target, + .ext_symlink => |s| s.target, + else => unreachable, + }; + try std.fs.cwd().symLink(target, path, .{}); +} +/// Creates the device described by the inode. +/// Sets metadata. +fn extractDeviceAndIPC(self: Inode, alloc: std.mem.Allocator, tables: *Tables, path: []const u8, options: ExtractionOptions) !void { + var mode: u32 = undefined; + var dev: u32 = 0; + switch (self.data) { + .char_dev => |d| { + mode = std.posix.S.IFCHR; + dev = d.dev; + }, + .ext_char_dev => |d| { + mode = std.posix.S.IFCHR; + dev = d.dev; + }, + .block_dev => |d| { + mode = std.posix.S.IFBLK; + dev = d.dev; + }, + .ext_block_dev => |d| { + mode = std.posix.S.IFBLK; + dev = d.dev; + }, + .fifo, .ext_fifo => mode = std.posix.S.IFIFO, + .socket, .ext_socket => mode = std.posix.S.IFSOCK, + else => unreachable, + } + const res: std.os.linux.E = @enumFromInt(std.os.linux.mknod(@ptrCast(path), mode, dev)); + switch (res) { + .SUCCESS => {}, + .ACCES => return std.fs.Dir.MakeError.AccessDenied, + .DQUOT => return std.fs.Dir.MakeError.DiskQuota, + .EXIST => return std.fs.Dir.MakeError.PathAlreadyExists, + .FAULT, .NOENT => return std.fs.Dir.MakeError.BadPathName, + .LOOP => return std.fs.Dir.MakeError.SymLinkLoop, + .NAMETOOLONG => return std.fs.Dir.MakeError.NameTooLong, + .NOMEM => return std.fs.Dir.MakeError.SystemResources, + .NOSPC => return std.fs.Dir.MakeError.NoSpaceLeft, + .NOTDIR => return std.fs.Dir.MakeError.NotDir, + .PERM => return std.fs.Dir.MakeError.PermissionDenied, + .ROFS => return std.fs.Dir.MakeError.ReadOnlyFileSystem, + else => return blk: { + std.debug.print("unhandled mknod result: {}\n", .{res}); + break :blk std.fs.Dir.MakeError.Unexpected; + }, + } + var fil = try std.fs.cwd().openFile(path, .{}); + defer fil.close(); + try self.setMetadata(alloc, tables, fil, options); +} diff --git a/src/util/inode_finish.zig b/src/util/inode_finish.zig new file mode 100644 index 0000000..c16dfd2 --- /dev/null +++ b/src/util/inode_finish.zig @@ -0,0 +1,101 @@ +const std = @import("std"); +const WaitGroup = std.Thread.WaitGroup; +const Mutex = std.Thread.Mutex; + +const Archive = @import("../archive.zig"); +const Inode = @import("../inode.zig"); +const ExtractionOptions = @import("../options.zig"); +const Tables = @import("../tables.zig"); + +const InodeFinish = @This(); + +const FinishEnum = enum { + wg, + fin, +}; +pub const FinishUnion = union(FinishEnum) { + wg: *WaitGroup, + fin: *InodeFinish, + + pub fn finish(self: FinishUnion) void { + switch (self) { + .wg => |wg| wg.finish(), + .fin => |fin| fin.finish(), + } + } +}; + +alloc: std.mem.Allocator, + +inode: Inode, +path: []const u8, +tables: *Tables, +options: ExtractionOptions, +parent_finish: FinishUnion, +fil: ?std.fs.File, +out_err: *?anyerror, + +wg: WaitGroup = .{}, +mut: Mutex = .{}, + +pub fn create( + alloc: std.mem.Allocator, + inode: Inode, + path: []const u8, + tables: *Tables, + options: ExtractionOptions, + parent_finish: FinishUnion, + out_err: *?anyerror, + fil: ?std.fs.File, + work_size: usize, +) !*InodeFinish { + if (work_size == 0) + return error.InvalidWorkSize; + const out = try alloc.create(InodeFinish); + errdefer alloc.destroy(out); + out.* = .{ + .alloc = alloc, + + .inode = inode, + .path = path, + .tables = tables, + .options = options, + .parent_finish = parent_finish, + .out_err = out_err, + .fil = fil, + }; + out.wg.startMany(work_size); + return out; +} + +pub fn logError(self: *InodeFinish, comptime fmt: []const u8, args: anytype) void { + if (self.options.verbose) + self.options.verbose_writer.?.print(fmt, args) catch {}; +} + +pub fn finish(self: *InodeFinish) void { + self.mut.lock(); + { + defer self.mut.unlock(); + self.wg.finish(); + if (!self.wg.isDone()) return; + } + defer { + self.parent_finish.finish(); + self.alloc.destroy(self); + } + if (self.fil == null) + self.fil = std.fs.cwd().openFile(self.path, .{}) catch |err| { + if (self.options.verbose) + self.options.verbose_writer.?.print("Error opening {s} to set metadata: {}\n", .{ self.path, err }) catch {}; + self.out_err.* = err; + return; + }; + defer self.fil.?.close(); + self.inode.setMetadata(self.alloc, self.tables, self.fil.?, self.options) catch |err| { + if (self.options.verbose) + self.options.verbose_writer.?.print("Error setting metadata to {s}: {}\n", .{ self.path, err }) catch {}; + self.out_err.* = err; + return; + }; +} diff --git a/src/xattr.zig b/src/xattr.zig index 1bffd00..e5203b7 100644 --- a/src/xattr.zig +++ b/src/xattr.zig @@ -1,7 +1,7 @@ const std = @import("std"); const DecompFn = @import("decomp.zig").DecompFn; -const Table = @import("table.zig").Table; +const Table = @import("tables.zig").Table; const MetadataReader = @import("util/metadata.zig"); const OffsetFile = @import("util/offset_file.zig"); @@ -30,7 +30,7 @@ const KeyRaw = packed struct { }; pub const KeyValue = struct { - key: []u8, + key: [:0]u8, value: []u8, }; @@ -62,7 +62,7 @@ pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: DecompFn, table_s .table = try .init(alloc, fil, decomp, table_start + 16, info.count), }; } -pub fn deinit(self: XattrTable) void { +pub fn deinit(self: *XattrTable) void { self.table.deinit(); } @@ -80,19 +80,22 @@ pub fn get(self: *XattrTable, alloc: std.mem.Allocator, idx: u32) ![]KeyValue { switch (key_raw.type.prefix) { .user => { - kv.key = try alloc.alloc(u8, key_raw.name_size + 5); + kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 5 + 1)); @memcpy(kv.key[0..5], "user."); - try meta.interface.readSliceAll(kv.key[5..]); + try meta.interface.readSliceAll(kv.key[5 .. kv.key.len - 1]); + kv.key[kv.key.len - 1] = 0; }, .security => { - kv.key = try alloc.alloc(u8, key_raw.name_size + 9); + kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 9 + 1)); @memcpy(kv.key[0..9], "security."); - try meta.interface.readSliceAll(kv.key[9..]); + try meta.interface.readSliceAll(kv.key[9 .. kv.key.len - 1]); + kv.key[kv.key.len - 1] = 0; }, .trusted => { - kv.key = try alloc.alloc(u8, key_raw.name_size + 8); + kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 8 + 1)); @memcpy(kv.key[0..8], "trusted."); - try meta.interface.readSliceAll(kv.key[8..]); + try meta.interface.readSliceAll(kv.key[8 .. kv.key.len - 1]); + kv.key[kv.key.len - 1] = 0; }, } if (key_raw.type.out_of_line) {