From b67d02074d3d7eb1a0915861de13e5e2b98e69fe Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Wed, 29 Apr 2026 03:48:34 -0500 Subject: [PATCH] Started re-write (once again) Main reason for this re-write is to be compatible with zig 0.16.0 --- .gitignore | 1 + build.zig | 111 +++++------- src/archive.zig | 157 ++++++++--------- src/bin/unsquashfs.zig | 39 ++-- src/c.zig | 8 - src/decomp.zig | 25 --- src/decomp/c/lz4.zig | 14 -- src/decomp/c/lzma.zig | 127 ------------- src/decomp/c/lzo.zig | 67 ------- src/decomp/c/xz.zig | 127 ------------- src/decomp/c/zlib.zig | 112 ------------ src/decomp/c/zstd.zig | 165 ----------------- src/decomp/types.zig | 43 ----- src/decomp/zig/lzma.zig | 19 -- src/decomp/zig/xz.zig | 25 --- src/decomp/zig/zlib.zig | 18 -- src/decomp/zig/zstd.zig | 18 -- src/dir_entry.zig | 60 ------- src/file.zig | 202 --------------------- src/inode.zig | 171 ++++-------------- src/lookup_table.zig | 65 +++++++ src/super.zig | 69 -------- src/tables.zig | 108 ------------ src/test.zig | 84 --------- src/util/data.zig | 174 ------------------ src/util/data_threaded.zig | 207 ---------------------- src/util/extract.zig | 353 ------------------------------------- src/util/inode_finish.zig | 101 ----------- src/util/meta_cache.zig | 41 ----- src/util/offset_file.zig | 8 +- src/xattr.zig | 117 ------------ 31 files changed, 238 insertions(+), 2598 deletions(-) delete mode 100644 src/c.zig delete mode 100644 src/decomp.zig delete mode 100644 src/decomp/c/lz4.zig delete mode 100644 src/decomp/c/lzma.zig delete mode 100644 src/decomp/c/lzo.zig delete mode 100644 src/decomp/c/xz.zig delete mode 100644 src/decomp/c/zlib.zig delete mode 100644 src/decomp/c/zstd.zig delete mode 100644 src/decomp/types.zig delete mode 100644 src/decomp/zig/lzma.zig delete mode 100644 src/decomp/zig/xz.zig delete mode 100644 src/decomp/zig/zlib.zig delete mode 100644 src/decomp/zig/zstd.zig delete mode 100644 src/dir_entry.zig delete mode 100644 src/file.zig create mode 100644 src/lookup_table.zig delete mode 100644 src/super.zig delete mode 100644 src/tables.zig delete mode 100644 src/util/data.zig delete mode 100644 src/util/data_threaded.zig delete mode 100644 src/util/extract.zig delete mode 100644 src/util/inode_finish.zig delete mode 100644 src/util/meta_cache.zig delete mode 100644 src/xattr.zig diff --git a/.gitignore b/.gitignore index 3400e46..194b300 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ testing/ .zig-cache/ zig-out/ +zig-pkg/ diff --git a/build.zig b/build.zig index 65cd0ad..1a17b2c 100644 --- a/build.zig +++ b/build.zig @@ -1,54 +1,28 @@ const std = @import("std"); pub fn build(b: *std.Build) !void { - const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false; + // const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false; // const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false; - const debug = b.option(bool, "debug", "Enable options to make debugging easier.") orelse false; + const debug = b.option(bool, "debug", "Enable options to make debugging easier."); const version_string_option = b.option([]const u8, "version", "Version of the library/binary"); - const zig_squashfs_options = b.addOptions(); - zig_squashfs_options.addOption(bool, "use_zig_decomp", use_zig_decomp); + // const zig_squashfs_options = b.addOptions(); + // zig_squashfs_options.addOption(bool, "use_zig_decomp", use_zig_decomp); // zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo); const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); - const mod = b.addModule("zig_squashfs", .{ - .root_source_file = b.path("src/root.zig"), - .target = target, - .optimize = if (debug == true) .Debug else optimize, - .link_libc = !use_zig_decomp, - .valgrind = debug, - .error_tracing = debug, - .strip = if (debug == true) false else null, + + const lib = b.addLibrary(.{ + .name = "squashfs", + .root_module = b.createModule(.{ + .optimize = if (debug == true) .Debug else optimize, + .target = target, + .valgrind = debug, + .root_source_file = b.path("src/root.zig"), + }), + .use_llvm = debug, }); - mod.addOptions("config", zig_squashfs_options); - if (!use_zig_decomp) { - var zlib_ng = b.dependency("zlib_ng", .{ - .target = target, - .optimize = optimize, - }); - mod.linkLibrary(zlib_ng.artifact("zng")); - - mod.linkSystemLibrary("lzma", .{ .preferred_link_mode = .static }); - - var minilzo = b.dependency("minilzo", .{ - .target = target, - .optimize = optimize, - }); - mod.linkLibrary(minilzo.artifact("minilzo")); - - var lz4 = b.dependency("lz4", .{ - .target = target, - .optimize = optimize, - }); - mod.linkLibrary(lz4.artifact("lz4")); - - var zstd = b.dependency("zstd", .{ - .target = target, - .optimize = optimize, - }); - mod.linkLibrary(zstd.artifact("zstd")); - } var version = version_string_option orelse "0.0.0-testing"; if (version[0] == 'v') version = version[1..]; @@ -58,29 +32,18 @@ pub fn build(b: *std.Build) !void { "version", try std.SemanticVersion.parse(version), ); - - var exe_mod = b.createModule(.{ - .root_source_file = b.path("src/bin/unsquashfs.zig"), - .target = target, - .optimize = if (debug == true) .Debug else optimize, - .link_libc = !use_zig_decomp, - .imports = &.{ - .{ .name = "zig_squashfs", .module = mod }, - }, - .valgrind = debug, - .error_tracing = debug, - .strip = if (debug == true) false else null, - }); - exe_mod.addOptions("config", unsquashfs_options); const exe = b.addExecutable(.{ .name = "unsquashfs", - .root_module = exe_mod, - .use_llvm = debug, - }); - - const lib = b.addLibrary(.{ - .name = "squashfs", - .root_module = mod, + .root_module = b.createModule(.{ + .optimize = if (debug == true) .Debug else optimize, + .target = target, + .valgrind = debug, + .root_source_file = b.path("src/bin/unsquashfs.zig"), + .imports = &.{ + .{ .name = "zig_squashfs", .module = lib.root_module }, + .{ .name = "config", .module = unsquashfs_options.createModule() }, + }, + }), .use_llvm = debug, }); @@ -88,7 +51,11 @@ pub fn build(b: *std.Build) !void { b.installArtifact(exe); const mod_tests = b.addTest(.{ - .root_module = mod, + .root_module = b.createModule(.{ + .optimize = optimize, + .target = target, + .root_source_file = b.path("src/root.zig"), + }), .test_runner = .{ .mode = .simple, .path = b.path("src/test.zig"), @@ -101,13 +68,21 @@ pub fn build(b: *std.Build) !void { // zls build check steps const lib_check = b.addLibrary(.{ .name = "squashfs", - .root_module = mod, - }); - const exe_check = b.addExecutable(.{ - .name = "unsquashfs", - .root_module = exe_mod, + .root_module = b.createModule(.{ + .optimize = optimize, + .target = target, + .root_source_file = b.path("src/root.zig"), + }), }); + // const exe_check = b.addExecutable(.{ + // .name = "unsquashfs", + // .root_module = b.createModule(.{ + // .optimize = optimize, + // .target = target, + // .root_source_file = b.path("src/bin/unsquashfs.zig"), + // }), + // }); const check = b.step("check", "Check if unsquashfs compiles"); check.dependOn(&lib_check.step); - check.dependOn(&exe_check.step); + // check.dependOn(&exe_check.step); } diff --git a/src/archive.zig b/src/archive.zig index bce92e8..2c5b1cc 100644 --- a/src/archive.zig +++ b/src/archive.zig @@ -1,105 +1,88 @@ -//! A squashfs archive read from a file. -//! Can be used to directly access File's contents or extract to the filesystem. - const std = @import("std"); -const File = std.fs.File; -const builtin = @import("builtin"); +const Io = std.Io; -const config = @import("config"); - -const cDecomp = @import("decomp/misc_c.zig"); -const Decomp = @import("decomp/zig_decomp.zig"); -const ExtractionOptions = @import("options.zig"); const Inode = @import("inode.zig"); -const InodeRef = Inode.Ref; -const BlockSize = @import("inode_data/file.zig").BlockSize; -const SfsFile = @import("file.zig"); -const Superblock = @import("super.zig").Superblock; -const Tables = @import("tables.zig"); -const MetadataReader = @import("util/metadata.zig"); const OffsetFile = @import("util/offset_file.zig"); -const XattrTable = @import("xattr.zig"); const Archive = @This(); -alloc: std.mem.Allocator, - -fil: OffsetFile, - +file: OffsetFile, super: Superblock, -tables: ?Tables = null, - -decomp: @import("decomp/types.zig").Decomp, - -/// Default settings using std.Thread.getCpuCount() threads and the minimum of 4gb or half of system memory for memory usage. -pub fn init(alloc: std.mem.Allocator, fil: File, offset: u64) !Archive { +pub fn init(io: Io, file: std.Io.File, offset: u64) !Archive { + var rdr = file.reader(io, &[0]u8{}); + try rdr.seekTo(offset); var super: Superblock = undefined; - const red = try fil.pread(@ptrCast(&super), offset); - std.debug.assert(red == @sizeOf(Superblock)); - try super.validate(); - const off_fil: OffsetFile = .init(fil, offset); + try rdr.interface.readSliceEndian(Superblock, @ptrCast(&super), .little); return .{ - .alloc = alloc, - .fil = off_fil, - .decomp = if (config.use_zig_decomp) - switch (super.compression) { - .lz4 => return error.Lz4Unsupported, - .lzo => return error.LzoUnsupported, - .gzip => .{ .gzip = .{} }, - .lzma => .{ .lzma = .{} }, - .xz => .{ .xz = .{} }, - .zstd => .{ .zstd = .{} }, - } - else switch (super.compression) { - .gzip => .{ .gzip = .init(alloc) }, - .zstd => .{ .zstd = .init(alloc) }, - .xz => .{ .xz = .init(alloc) }, - .lzma => .{ .lzma = .init(alloc) }, - .lzo => .{ .lzo = .{} }, - .lz4 => .{ .lz4 = .{} }, - }, + .file = .init(file, offset), .super = super, }; } -pub fn deinit(self: *Archive) void { - if (self.tables != null) - self.tables.?.deinit(); - self.decomp.deinit(); -} -pub fn inode(self: *Archive, alloc: std.mem.Allocator, num: u32) !Inode { - if (self.tables == null) - self.tables = try .init(alloc, self); - const ref = try self.export_table.get(num - 1); - var rdr = try self.fil.readerAt(ref.block_start + self.super.inode_start, &[0]u8{}); - var meta: MetadataReader = .init(alloc, &rdr.interface, &self.decomp); - try meta.interface.discardAll(ref.block_offset); - return try .read(alloc, &meta.interface, self.super.block_size); -} +// Superblock -pub fn root(self: *Archive, alloc: std.mem.Allocator) !SfsFile { - if (self.tables == null) - self.tables = try .init(alloc, self); - var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{}); - var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp); - try meta.interface.discardAll(self.super.root_ref.block_offset); - const in: Inode = try .read(alloc, &meta.interface, self.super.block_size); - return .init(self, in, ""); -} +const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little); -pub fn open(self: *Archive, alloc: std.mem.Allocator, path: []const u8) !SfsFile { - if (self.tables == null) - self.tables = try .init(alloc, self); - var root_fil = try self.root(alloc); - defer if (!SfsFile.pathIsSelf(path)) root_fil.deinit(); - return root_fil.open(path); -} +const SuperblockError = error{ + InvalidMagic, + InvalidBlockLog, + InvalidVersion, + InvalidCheck, +}; -pub fn extract(self: Archive, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void { - var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{}); - var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp); - try meta.interface.discardAll(self.super.root_ref.block_offset); - const in: Inode = try .read(self.alloc, &meta.interface, self.super.block_size); - try in.extractTo(alloc, self, path, options); -} +/// A squashfs Superblock +pub const Superblock = packed struct { + magic: u32, + inode_count: u32, + mod_time: u32, + block_size: u32, + frag_count: u32, + compression: enum(u16) { + gzip = 1, + lzma, + lzo, + xz, + lz4, + zstd, + }, + block_log: u16, + flags: packed struct { + inode_uncompressed: bool, + data_uncompressed: bool, + check: bool, + frag_uncompressed: bool, + fragment_never: bool, + fragment_always: bool, + duplicates: bool, + exportable: bool, + xattr_uncompressed: bool, + xattr_never: bool, + compression_options: bool, + ids_uncompressed: bool, + _: u4, + }, + id_count: u16, + ver_maj: u16, + ver_min: u16, + root_ref: Inode.Ref, + size: u64, + id_start: u64, + xattr_start: u64, + inode_start: u64, + dir_start: u64, + frag_start: u64, + export_start: u64, + + /// Validate the Superblock. If an error is returned, it's likely the archive is corrupted or not a squashfs archive. + pub fn validate(self: Superblock) !void { + if (self.magic != SQUASHFS_MAGIC) + return SuperblockError.InvalidMagic; + if (self.flags.check) + return SuperblockError.InvalidCheck; + if (self.ver_maj != 4 or self.ver_min != 0) + return SuperblockError.InvalidVersion; + if (std.math.log2(self.block_size) != self.block_log) + return SuperblockError.InvalidBlockLog; + } +}; diff --git a/src/bin/unsquashfs.zig b/src/bin/unsquashfs.zig index a945c01..a3591fb 100644 --- a/src/bin/unsquashfs.zig +++ b/src/bin/unsquashfs.zig @@ -1,5 +1,6 @@ const std = @import("std"); -const Writer = std.Io.Writer; +const Io = std.Io; +const Writer = Io.Writer; const builtin = @import("builtin"); const config = @import("config"); @@ -38,21 +39,21 @@ var ignore_xattrs: bool = false; var ignore_permissions: bool = false; var force: bool = false; -pub fn main() !void { - const alloc = std.heap.smp_allocator; - var stdout = std.fs.File.stdout(); - var out = stdout.writer(&[0]u8{}); +pub fn main(init: std.process.Init) !void { + const alloc = init.gpa; + const io = init.io; + var stdout = std.Io.File.stdout(); + var out = stdout.writer(io, &[0]u8{}); defer out.interface.flush() catch {}; - try handleArgs(alloc, &out.interface); + try handleArgs(init.minimal.args, &out.interface); if (archive.len == 0) { try out.interface.print("You must provide a squashfs archive\n", .{}); try out.interface.print(help_mgs, .{}); return; } - var fil: std.fs.File = try std.fs.cwd().openFile(archive, .{}); //TODO: Handle error gracefully. - defer fil.close(); - var arc: squashfs.Archive = try .init(alloc, fil, offset); //TODO: Update when memory size matters. //TODO: Handle error gracefully. - defer arc.deinit(); + var fil = try Io.Dir.cwd().openFile(io, archive, .{}); //TODO: Handle error gracefully. + defer fil.close(io); + var arc: squashfs.Archive = try .init(io, fil, offset); //TODO: Update when memory size matters. //TODO: Handle error gracefully. const options: squashfs.ExtractionOptions = .{ .threads = if (threads == 0) try std.Thread.getCpuCount() else threads, .verbose = verbose, @@ -61,17 +62,17 @@ pub fn main() !void { .ignore_permissions = ignore_permissions, }; if (force) - try std.fs.cwd().deleteTree(extLoc); + try Io.Dir.cwd().deleteTree(io, extLoc); try arc.extract(alloc, extLoc, options); //TODO: Handle error gracefully. } -fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void { - var args = try std.process.argsWithAllocator(alloc); - defer args.deinit(); - _ = args.next(); // args[0] is the application launch command. - while (args.next()) |arg| { +fn handleArgs(args: std.process.Args, out: *Writer) !void { + var arg_iter = args.iterate(); + defer arg_iter.deinit(); + _ = arg_iter.next(); // args[0] is the application launch command. + while (arg_iter.next()) |arg| { if (std.mem.eql(u8, arg, "-o")) { - const nxt = args.next(); + const nxt = arg_iter.next(); if (nxt == null or nxt.?.len == 0) { try out.print("-o must be followed by a number\n", .{}); return errors.InvalidArguments; @@ -82,7 +83,7 @@ fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void { }; continue; } else if (std.mem.eql(u8, arg, "-d")) { - const nxt = args.next(); + const nxt = arg_iter.next(); if (nxt == null or nxt.?.len == 0) { try out.print("-d must be followed by a location\n", .{}); return errors.InvalidArguments; @@ -90,7 +91,7 @@ fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void { extLoc = nxt.?; continue; } else if (std.mem.eql(u8, arg, "-p")) { - const nxt = args.next(); + const nxt = arg_iter.next(); if (nxt == null or nxt.?.len == 0) { try out.print("-p must be followed by a number\n", .{}); return errors.InvalidArguments; diff --git a/src/c.zig b/src/c.zig deleted file mode 100644 index 42f23c9..0000000 --- a/src/c.zig +++ /dev/null @@ -1,8 +0,0 @@ -pub const c = @cImport({ - @cInclude("zlib-ng.h"); - @cInclude("lzma.h"); - @cInclude("lz4.h"); - @cInclude("zstd.h"); - @cInclude("zstd_errors.h"); - @cInclude("lzo/minilzo.h"); -}); diff --git a/src/decomp.zig b/src/decomp.zig deleted file mode 100644 index e8cfcd3..0000000 --- a/src/decomp.zig +++ /dev/null @@ -1,25 +0,0 @@ -const std = @import("std"); - -const Decompressor = @This(); - -pub const Error = error{ - OutOfMemory, - BadInput, - OutputTooSmall, - ReadFailed, - WriteFailed, - EndOfStream, -}; - -vtable: *const struct { - decompress: *const fn (*Decompressor, []u8, []u8) Error!usize = DefaultDecompress, - stateless: *const fn (std.mem.Allocator, []u8, []u8) Error!usize, -}, - -pub fn decompress(self: *Decompressor, in: []u8, out: []u8) Error!usize { - return self.vtable.decompress(self, in, out); -} - -fn DefaultDecompress(self: *Decompressor, in: []u8, out: []u8) Error!usize { - return self.vtable.stateless(std.heap.smp_allocator, in, out); -} diff --git a/src/decomp/c/lz4.zig b/src/decomp/c/lz4.zig deleted file mode 100644 index d230e62..0000000 --- a/src/decomp/c/lz4.zig +++ /dev/null @@ -1,14 +0,0 @@ -const std = @import("std"); - -const c = @import("../../c.zig").c; -const Decompressor = @import("../../decomp.zig"); - -const Self = @This(); - -interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } }, - -fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize { - const res = c.LZ4_decompress_fast(in.ptr, out.ptr, @intCast(out.len)); - if (res < 0) return Decompressor.Error.ReadFailed; - return @abs(res); -} diff --git a/src/decomp/c/lzma.zig b/src/decomp/c/lzma.zig deleted file mode 100644 index e09e0bf..0000000 --- a/src/decomp/c/lzma.zig +++ /dev/null @@ -1,127 +0,0 @@ -const std = @import("std"); - -const c = @import("../../c.zig").c; -const Decompressor = @import("../../decomp.zig"); - -const Self = @This(); - -alloc: std.mem.Allocator, -streams: std.AutoHashMap(std.Thread.Id, c.lzma_stream), - -interface: Decompressor, - -pub fn init(alloc: std.mem.Allocator) Self { - return .{ - .alloc = alloc, - .streams = .init(alloc), - .interface = .{ - .vtable = &.{ - .decompress = decompress, - .stateless = stateless, - }, - }, - }; -} -pub fn deinit(self: *Self) void { - self.streams.deinit(); -} - -fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize { - var self: *Self = @fieldParentPtr("interface", decomp); - - var strm = try self.getOrCreate(); - strm.next_in = in.ptr; - strm.avail_in = in.len; - strm.next_out = out.ptr; - strm.avail_out = out.len; - var res = c.lzma_alone_decoder(strm, out.len * 2); - decodeResult(res) catch |err| return lzmaErrToDecompErr(err); - while (res == c.LZMA_OK) - res = c.lzma_code(strm, c.LZMA_RUN); - decodeResult(res) catch |err| return lzmaErrToDecompErr(err); - return strm.total_out; -} -fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize { - var strm: c.lzma_stream = .{ - .allocator = &.{ - .alloc = lzmaAlloc, - .free = lzmaFree, - .@"opaque" = @ptrCast(@constCast(&alloc)), - }, - .next_in = in.ptr, - .avail_in = in.len, - .next_out = out.ptr, - .avail_out = out.len, - }; - var res = c.lzma_alone_decoder(&strm, out.len * 2); - decodeResult(res) catch |err| return lzmaErrToDecompErr(err); - while (res == c.LZMA_OK) - res = c.lzma_code(&strm, c.LZMA_RUN); - decodeResult(res) catch |err| return lzmaErrToDecompErr(err); - return strm.total_out; -} - -inline fn getOrCreate(self: *Self) !*c.lzma_stream { - const res = try self.streams.getOrPut(std.Thread.getCurrentId()); - if (res.found_existing) return res.value_ptr; - res.value_ptr.* = .{ .allocator = &.{ - .alloc = lzmaAlloc, - .free = lzmaFree, - .@"opaque" = @ptrCast(&self.alloc), - } }; - return res.value_ptr; -} -inline fn decodeResult(res: usize) Error!void { - return switch (res) { - c.LZMA_OK, c.LZMA_STREAM_END => {}, - c.LZMA_NO_CHECK => Error.NoCheck, - c.LZMA_UNSUPPORTED_CHECK => Error.UnsupportedCheck, - c.LZMA_GET_CHECK => Error.GetCheck, - c.LZMA_MEM_ERROR, c.LZMA_MEMLIMIT_ERROR => Error.OutOfMemory, - c.LZMA_FORMAT_ERROR => Error.Format, - c.LZMA_OPTIONS_ERROR => Error.Options, - c.LZMA_DATA_ERROR => Error.Data, - c.LZMA_BUF_ERROR => Error.Buffer, - c.LZMA_PROG_ERROR => Error.Program, - c.LZMA_SEEK_NEEDED => Error.SeekNeeded, - else => Error.Unknown, - }; -} -inline fn lzmaErrToDecompErr(err: Error) Decompressor.Error { - return switch (err) { - Error.OutOfMemory => Decompressor.Error.OutOfMemory, - Error.NoCheck => Decompressor.Error.ReadFailed, - Error.UnsupportedCheck => Decompressor.Error.ReadFailed, - Error.GetCheck => Decompressor.Error.ReadFailed, - Error.Format => Decompressor.Error.ReadFailed, - Error.Options => Decompressor.Error.ReadFailed, - Error.Data => Decompressor.Error.ReadFailed, - Error.Buffer => Decompressor.Error.WriteFailed, - Error.Program => Decompressor.Error.ReadFailed, - Error.SeekNeeded => Decompressor.Error.ReadFailed, - else => Decompressor.Error.ReadFailed, - }; -} - -fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - return alloc.rawAlloc(size, .@"1", 0); -} -fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); -} - -const Error = error{ - OutOfMemory, - NoCheck, - UnsupportedCheck, - GetCheck, - Format, - Options, - Data, - Buffer, - Program, - SeekNeeded, - Unknown, -}; diff --git a/src/decomp/c/lzo.zig b/src/decomp/c/lzo.zig deleted file mode 100644 index 4e4977f..0000000 --- a/src/decomp/c/lzo.zig +++ /dev/null @@ -1,67 +0,0 @@ -const std = @import("std"); - -const c = @import("../../c.zig").c; -const Decompressor = @import("../../decomp.zig"); - -const Self = @This(); - -interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } }, - -fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize { - var out_len: usize = out.len; - const res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &out_len, null); - decodeError(res) catch |err| return lzoErrToDecompErr(err); - return out_len; -} - -inline fn decodeError(res: c_int) Error!void { - return switch (res) { - c.LZO_E_OK => {}, - c.LZO_E_EOF_NOT_FOUND => Error.EofNotFound, - c.LZO_E_INPUT_NOT_CONSUMED => Error.InputNotConsumed, - c.LZO_E_INPUT_OVERRUN => Error.InputOverrun, - c.LZO_E_INTERNAL_ERROR => Error.InternalError, - c.LZO_E_INVALID_ALIGNMENT => Error.InvalidAlignment, - c.LZO_E_INVALID_ARGUMENT => Error.InvalidArgument, - c.LZO_E_LOOKBEHIND_OVERRUN => Error.LookbehindOverrun, - c.LZO_E_NOT_COMPRESSIBLE => Error.NotCompressible, - c.LZO_E_NOT_YET_IMPLEMENTED => Error.NotYetImplemented, - c.LZO_E_OUTPUT_NOT_CONSUMED => Error.OutputNotConsumed, - c.LZO_E_OUTPUT_OVERRUN => Error.OutputOverrun, - c.LZO_E_OUT_OF_MEMORY => Error.OutOfMemory, - else => Error.Unknown, - }; -} -inline fn lzoErrToDecompErr(err: Error) Decompressor.Error { - return switch (err) { - Error.EofNotFound => Decompressor.Error.ReadFailed, - Error.InputNotConsumed => Decompressor.Error.ReadFailed, - Error.InputOverrun => Decompressor.Error.ReadFailed, - Error.InternalError => Decompressor.Error.ReadFailed, - Error.InvalidAlignment => Decompressor.Error.ReadFailed, - Error.InvalidArgument => Decompressor.Error.ReadFailed, - Error.LookbehindOverrun => Decompressor.Error.ReadFailed, - Error.NotCompressible => Decompressor.Error.ReadFailed, - Error.NotYetImplemented => Decompressor.Error.ReadFailed, - Error.OutputNotConsumed => Decompressor.Error.WriteFailed, - Error.OutputOverrun => Decompressor.Error.WriteFailed, - Error.OutOfMemory => Decompressor.Error.OutOfMemory, - else => Decompressor.Error.ReadFailed, - }; -} - -const Error = error{ - EofNotFound, - InputNotConsumed, - InputOverrun, - InternalError, - InvalidAlignment, - InvalidArgument, - LookbehindOverrun, - NotCompressible, - NotYetImplemented, - OutputNotConsumed, - OutputOverrun, - OutOfMemory, - Unknown, -}; diff --git a/src/decomp/c/xz.zig b/src/decomp/c/xz.zig deleted file mode 100644 index e1c3d21..0000000 --- a/src/decomp/c/xz.zig +++ /dev/null @@ -1,127 +0,0 @@ -const std = @import("std"); - -const c = @import("../../c.zig").c; -const Decompressor = @import("../../decomp.zig"); - -const Self = @This(); - -alloc: std.mem.Allocator, -streams: std.AutoHashMap(std.Thread.Id, c.lzma_stream), - -interface: Decompressor, - -pub fn init(alloc: std.mem.Allocator) Self { - return .{ - .alloc = alloc, - .streams = .init(alloc), - .interface = .{ - .vtable = &.{ - .decompress = decompress, - .stateless = stateless, - }, - }, - }; -} -pub fn deinit(self: *Self) void { - self.streams.deinit(); -} - -fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize { - var self: *Self = @fieldParentPtr("interface", decomp); - - var strm = try self.getOrCreate(); - strm.next_in = in.ptr; - strm.avail_in = in.len; - strm.next_out = out.ptr; - strm.avail_out = out.len; - var res = c.lzma_stream_decoder(strm, out.len * 2, 0); - decodeResult(res) catch |err| return lzmaErrToDecompErr(err); - while (res == c.LZMA_OK) - res = c.lzma_code(strm, c.LZMA_RUN); - decodeResult(res) catch |err| return lzmaErrToDecompErr(err); - return strm.total_out; -} -fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize { - var strm: c.lzma_stream = .{ - .allocator = &.{ - .alloc = lzmaAlloc, - .free = lzmaFree, - .@"opaque" = @ptrCast(@constCast(&alloc)), - }, - .next_in = in.ptr, - .avail_in = in.len, - .next_out = out.ptr, - .avail_out = out.len, - }; - var res = c.lzma_stream_decoder(&strm, out.len * 2, 0); - decodeResult(res) catch |err| return lzmaErrToDecompErr(err); - while (res == c.LZMA_OK) - res = c.lzma_code(&strm, c.LZMA_RUN); - decodeResult(res) catch |err| return lzmaErrToDecompErr(err); - return strm.total_out; -} - -inline fn getOrCreate(self: *Self) !*c.lzma_stream { - const res = try self.streams.getOrPut(std.Thread.getCurrentId()); - if (res.found_existing) return res.value_ptr; - res.value_ptr.* = .{ .allocator = &.{ - .alloc = lzmaAlloc, - .free = lzmaFree, - .@"opaque" = @ptrCast(&self.alloc), - } }; - return res.value_ptr; -} -inline fn decodeResult(res: usize) Error!void { - return switch (res) { - c.LZMA_OK, c.LZMA_STREAM_END => {}, - c.LZMA_NO_CHECK => Error.NoCheck, - c.LZMA_UNSUPPORTED_CHECK => Error.UnsupportedCheck, - c.LZMA_GET_CHECK => Error.GetCheck, - c.LZMA_MEM_ERROR, c.LZMA_MEMLIMIT_ERROR => Error.OutOfMemory, - c.LZMA_FORMAT_ERROR => Error.Format, - c.LZMA_OPTIONS_ERROR => Error.Options, - c.LZMA_DATA_ERROR => Error.Data, - c.LZMA_BUF_ERROR => Error.Buffer, - c.LZMA_PROG_ERROR => Error.Program, - c.LZMA_SEEK_NEEDED => Error.SeekNeeded, - else => Error.Unknown, - }; -} -inline fn lzmaErrToDecompErr(err: Error) Decompressor.Error { - return switch (err) { - Error.OutOfMemory => Decompressor.Error.OutOfMemory, - Error.NoCheck => Decompressor.Error.ReadFailed, - Error.UnsupportedCheck => Decompressor.Error.ReadFailed, - Error.GetCheck => Decompressor.Error.ReadFailed, - Error.Format => Decompressor.Error.ReadFailed, - Error.Options => Decompressor.Error.ReadFailed, - Error.Data => Decompressor.Error.ReadFailed, - Error.Buffer => Decompressor.Error.WriteFailed, - Error.Program => Decompressor.Error.ReadFailed, - Error.SeekNeeded => Decompressor.Error.ReadFailed, - else => Decompressor.Error.ReadFailed, - }; -} - -fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - return alloc.rawAlloc(size, .@"1", 0); -} -fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); -} - -const Error = error{ - OutOfMemory, - NoCheck, - UnsupportedCheck, - GetCheck, - Format, - Options, - Data, - Buffer, - Program, - SeekNeeded, - Unknown, -}; diff --git a/src/decomp/c/zlib.zig b/src/decomp/c/zlib.zig deleted file mode 100644 index 52edc10..0000000 --- a/src/decomp/c/zlib.zig +++ /dev/null @@ -1,112 +0,0 @@ -const std = @import("std"); - -const c = @import("../../c.zig").c; -const Decompressor = @import("../../decomp.zig"); - -const Self = @This(); - -alloc: std.mem.Allocator, -streams: std.AutoHashMap(std.Thread.Id, c.zng_stream), - -interface: Decompressor, - -pub fn init(alloc: std.mem.Allocator) Self { - return .{ - .alloc = alloc, - .streams = .init(alloc), - .interface = .{ - .vtable = &.{ - .decompress = decompress, - .stateless = stateless, - }, - }, - }; -} -pub fn deinit(self: *Self) void { - self.streams.deinit(); -} - -fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize { - const self: *Self = @fieldParentPtr("interface", decomp); - - var strm = try self.getOrCreate(); - strm.next_in = in.ptr; - strm.avail_in = @truncate(in.len); - strm.next_out = out.ptr; - strm.total_out = out.len; - var res = c.zng_inflateReset(strm); - decodeError(res) catch |err| return zlibErrToDecompErr(err); - - res = c.zng_inflate(strm, c.Z_FULL_FLUSH); - decodeError(res) catch |err| return zlibErrToDecompErr(err); - return strm.total_out; -} -fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize { - var strm: c.zng_stream = .{ - .zalloc = zalloc, - .zfree = zfree, - .@"opaque" = @constCast(&alloc), - - .next_in = in.ptr, - .avail_in = @truncate(in.len), - .next_out = out.ptr, - .total_out = out.len, - }; - var res = c.zng_inflateInit(&strm); - decodeError(res) catch |err| return zlibErrToDecompErr(err); - - res = c.zng_inflate(&strm, c.Z_FULL_FLUSH); - decodeError(res) catch |err| return zlibErrToDecompErr(err); - return strm.total_out; -} - -fn getOrCreate(self: *Self) !*c.zng_stream { - const res = try self.streams.getOrPut(std.Thread.getCurrentId()); - if (res.found_existing) return res.value_ptr; - res.value_ptr.* = .{ - .zalloc = zalloc, - .zfree = zfree, - .@"opaque" = &self.alloc, - }; - return res.value_ptr; -} - -fn zalloc(ptr: ?*anyopaque, size: c_uint, len: c_uint) callconv(.c) ?*anyopaque { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - return alloc.rawAlloc(size * len, .@"1", 0); -} -fn zfree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); -} - -inline fn decodeError(res: i32) Error!void { - if (res >= 0) return; - return switch (res) { - c.Z_STREAM_ERROR => Error.Stream, - c.Z_DATA_ERROR => Error.Data, - c.Z_MEM_ERROR => Error.OutOfMemory, - c.Z_BUF_ERROR => Error.Buffer, - c.Z_VERSION_ERROR => Error.Version, - else => Error.Misc, - }; -} -inline fn zlibErrToDecompErr(err: Error) Decompressor.Error { - return switch (err) { - Error.OutOfMemory => Decompressor.Error.OutOfMemory, - Error.Misc => Decompressor.Error.ReadFailed, - Error.Stream => Decompressor.Error.ReadFailed, - Error.Data => Decompressor.Error.ReadFailed, - Error.Buffer => Decompressor.Error.WriteFailed, - Error.Version => Decompressor.Error.ReadFailed, - }; -} - -const Error = error{ - OutOfMemory, - Misc, - Stream, - Data, - Buffer, - Version, -}; diff --git a/src/decomp/c/zstd.zig b/src/decomp/c/zstd.zig deleted file mode 100644 index 3d3c119..0000000 --- a/src/decomp/c/zstd.zig +++ /dev/null @@ -1,165 +0,0 @@ -const std = @import("std"); - -const c = @import("../../c.zig").c; -const Decompressor = @import("../../decomp.zig"); - -const Self = @This(); - -alloc: std.mem.Allocator, -ctx: std.AutoHashMap(std.Thread.Id, ?*c.ZSTD_DCtx), - -interface: Decompressor, - -pub fn init(alloc: std.mem.Allocator) Self { - return .{ - .alloc = alloc, - .ctx = .init(alloc), - .interface = .{ - .vtable = &.{ - .decompress = decompress, - .stateless = stateless, - }, - }, - }; -} -pub fn deinit(self: *Self) void { - self.ctx.deinit(); -} - -fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize { - var self: *Self = @fieldParentPtr("interface", decomp); - - const ctx = try self.getOrCreate(); - const res = c.ZSTD_decompressDCtx(ctx, out.ptr, out.len, in.ptr, in.len); - decodeError(res) catch |err| return zstdErrToDecompErr(err); - return res; -} -fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize { - const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len); - decodeError(res) catch |err| return zstdErrToDecompErr(err); - return res; -} - -inline fn getOrCreate(self: *Self) !?*c.ZSTD_DCtx { - const res = try self.ctx.getOrPut(std.Thread.getCurrentId()); - if (res.found_existing) return res.value_ptr.*; - res.value_ptr.* = c.ZSTD_createDCtx(); - return res.value_ptr.*; -} - -inline fn decodeError(res: usize) Error!void { - if (c.ZSTD_isError(res) == 0) return; - return switch (c.ZSTD_getErrorCode(res)) { - c.ZSTD_error_prefix_unknown => Error.PrefixUnknown, - c.ZSTD_error_version_unsupported => Error.VersionUnsupported, - c.ZSTD_error_frameParameter_unsupported => Error.FrameParameterUnsupported, - c.ZSTD_error_frameParameter_windowTooLarge => Error.FrameParameterWindowTooLarge, - c.ZSTD_error_corruption_detected => Error.CorruptionDetected, - c.ZSTD_error_checksum_wrong => Error.ChecksumWrong, - c.ZSTD_error_literals_headerWrong => Error.LiteralsHeaderWrong, - c.ZSTD_error_dictionary_corrupted => Error.DictionaryCorrupted, - c.ZSTD_error_dictionary_wrong => Error.DictionaryWrong, - c.ZSTD_error_dictionaryCreation_failed => Error.DictionaryCreationFailed, - c.ZSTD_error_parameter_unsupported => Error.ParameterUnsupported, - c.ZSTD_error_parameter_combination_unsupported => Error.ParameterCombinationUnsupported, - c.ZSTD_error_parameter_outOfBound => Error.ParameterOutOfBound, - c.ZSTD_error_tableLog_tooLarge => Error.TableLogTooLarge, - c.ZSTD_error_maxSymbolValue_tooLarge => Error.MaxSymbolValueTooLarge, - c.ZSTD_error_maxSymbolValue_tooSmall => Error.MaxSymbolValueTooSmall, - c.ZSTD_error_cannotProduce_uncompressedBlock => Error.CannotProduceUncompressedBlock, - c.ZSTD_error_stabilityCondition_notRespected => Error.StabilityConditionNotRespected, - c.ZSTD_error_stage_wrong => Error.StageWrong, - c.ZSTD_error_init_missing => Error.InitMissing, - c.ZSTD_error_memory_allocation => Error.MemoryAllocation, - c.ZSTD_error_workSpace_tooSmall => Error.WorkSpaceTooSmall, - c.ZSTD_error_dstSize_tooSmall => Error.DstSizeTooSmall, - c.ZSTD_error_srcSize_wrong => Error.SrcSizeWrong, - c.ZSTD_error_dstBuffer_null => Error.DstBufferNull, - c.ZSTD_error_noForwardProgress_destFull => Error.NoForwardProgressDestFull, - c.ZSTD_error_noForwardProgress_inputEmpty => Error.NoForwardProgressInputEmpty, - c.ZSTD_error_frameIndex_tooLarge => Error.FrameIndexTooLarge, - c.ZSTD_error_seekableIO => Error.SeekableIo, - c.ZSTD_error_dstBuffer_wrong => Error.DstBufferWrong, - c.ZSTD_error_srcBuffer_wrong => Error.SrcBufferWrong, - c.ZSTD_error_sequenceProducer_failed => Error.SequenceProducerFailed, - c.ZSTD_error_externalSequences_invalid => Error.ExternalSequencesInvalid, - else => Error.Generic, - }; -} -inline fn zstdErrToDecompErr(err: Error) Decompressor.Error { - return switch (err) { - Error.OutOfMemory => Decompressor.Error.OutOfMemory, - Error.Generic => Decompressor.Error.ReadFailed, - Error.PrefixUnknown => Decompressor.Error.ReadFailed, - Error.VersionUnsupported => Decompressor.Error.ReadFailed, - Error.FrameParameterUnsupported => Decompressor.Error.ReadFailed, - Error.FrameParameterWindowTooLarge => Decompressor.Error.ReadFailed, - Error.CorruptionDetected => Decompressor.Error.ReadFailed, - Error.ChecksumWrong => Decompressor.Error.ReadFailed, - Error.LiteralsHeaderWrong => Decompressor.Error.ReadFailed, - Error.DictionaryCorrupted => Decompressor.Error.ReadFailed, - Error.DictionaryWrong => Decompressor.Error.ReadFailed, - Error.DictionaryCreationFailed => Decompressor.Error.ReadFailed, - Error.ParameterUnsupported => Decompressor.Error.ReadFailed, - Error.ParameterCombinationUnsupported => Decompressor.Error.ReadFailed, - Error.ParameterOutOfBound => Decompressor.Error.ReadFailed, - Error.TableLogTooLarge => Decompressor.Error.ReadFailed, - Error.MaxSymbolValueTooLarge => Decompressor.Error.ReadFailed, - Error.MaxSymbolValueTooSmall => Decompressor.Error.ReadFailed, - Error.CannotProduceUncompressedBlock => Decompressor.Error.ReadFailed, - Error.StabilityConditionNotRespected => Decompressor.Error.ReadFailed, - Error.StageWrong => Decompressor.Error.ReadFailed, - Error.InitMissing => Decompressor.Error.ReadFailed, - Error.MemoryAllocation => Decompressor.Error.OutOfMemory, - Error.WorkSpaceTooSmall => Decompressor.Error.WriteFailed, - Error.DstSizeTooSmall => Decompressor.Error.WriteFailed, - Error.SrcSizeWrong => Decompressor.Error.ReadFailed, - Error.DstBufferNull => Decompressor.Error.WriteFailed, - Error.NoForwardProgressDestFull => Decompressor.Error.WriteFailed, - Error.NoForwardProgressInputEmpty => Decompressor.Error.ReadFailed, - Error.FrameIndexTooLarge => Decompressor.Error.ReadFailed, - Error.SeekableIo => Decompressor.Error.ReadFailed, - Error.DstBufferWrong => Decompressor.Error.WriteFailed, - Error.SrcBufferWrong => Decompressor.Error.ReadFailed, - Error.SequenceProducerFailed => Decompressor.Error.ReadFailed, - Error.ExternalSequencesInvalid => Decompressor.Error.ReadFailed, - }; -} - -const Error = error{ - OutOfMemory, - Generic, - PrefixUnknown, - VersionUnsupported, - FrameParameterUnsupported, - FrameParameterWindowTooLarge, - CorruptionDetected, - ChecksumWrong, - LiteralsHeaderWrong, - DictionaryCorrupted, - DictionaryWrong, - DictionaryCreationFailed, - ParameterUnsupported, - ParameterCombinationUnsupported, - ParameterOutOfBound, - TableLogTooLarge, - MaxSymbolValueTooLarge, - MaxSymbolValueTooSmall, - CannotProduceUncompressedBlock, - StabilityConditionNotRespected, - StageWrong, - InitMissing, - MemoryAllocation, - WorkSpaceTooSmall, - DstSizeTooSmall, - SrcSizeWrong, - DstBufferNull, - NoForwardProgressDestFull, - NoForwardProgressInputEmpty, - FrameIndexTooLarge, - SeekableIo, - DstBufferWrong, - SrcBufferWrong, - SequenceProducerFailed, - ExternalSequencesInvalid, -}; diff --git a/src/decomp/types.zig b/src/decomp/types.zig deleted file mode 100644 index aa1e94a..0000000 --- a/src/decomp/types.zig +++ /dev/null @@ -1,43 +0,0 @@ -const config = @import("config"); - -const Decompressor = @import("../decomp.zig"); -const cLz4 = @import("c/lz4.zig"); -const cLzma = @import("c/lzma.zig"); -const cLzo = @import("c/lzo.zig"); -const cXz = @import("c/xz.zig"); -const cZlib = @import("c/zlib.zig"); -const cZstd = @import("c/zstd.zig"); -const zigLzma = @import("zig/lzma.zig"); -const zigXz = @import("zig/xz.zig"); -const zigZlib = @import("zig/zstd.zig"); -const zigZstd = @import("zig/zstd.zig"); - -pub const Decomp = union(enum) { - gzip: if (config.use_zig_decomp) zigZlib else cZlib, - lzma: if (config.use_zig_decomp) zigLzma else cLzma, - lzo: if (config.use_zig_decomp) void else cLzo, - xz: if (config.use_zig_decomp) zigXz else cXz, - lz4: if (config.use_zig_decomp) void else cLz4, - zstd: if (config.use_zig_decomp) zigZstd else cZstd, - - pub fn deinit(self: *Decomp) void { - switch (self) { - .gzip => self.gzip.deinit(), - .lzma => self.lzma.deinit(), - .xz => self.xz.deinit(), - .zstd => self.zstd.deinit(), - else => {}, - } - } - - pub fn decompressor(self: *Decomp) *Decompressor { - return switch (self) { - .gzip => &self.gzip.interface, - .lzma => &self.lzma.interface, - .lzo => &self.lzo.interface, - .xz => &self.xz.interface, - .lz4 => &self.lz4.interface, - .zstd => &self.zstd.interface, - }; - } -}; diff --git a/src/decomp/zig/lzma.zig b/src/decomp/zig/lzma.zig deleted file mode 100644 index f8c8bdb..0000000 --- a/src/decomp/zig/lzma.zig +++ /dev/null @@ -1,19 +0,0 @@ -const std = @import("std"); -const lzma = std.compress.lzma; -const Reader = std.Io.Reader; - -const Decompressor = @import("../../decomp.zig"); - -const Self = @This(); - -interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } }, - -fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize { - var rdr: Reader = .fixed(in); - var decomp = try lzma.decompress(alloc, rdr.adaptToOldInterface()); - defer decomp.deinit(); - return decomp.read(out) catch |err| switch (err) { - error.CorruptInput, error.EndOfStream, error.Overflow => return Decompressor.Error.ReadFailed, - else => return err, - }; -} diff --git a/src/decomp/zig/xz.zig b/src/decomp/zig/xz.zig deleted file mode 100644 index 57e2e5d..0000000 --- a/src/decomp/zig/xz.zig +++ /dev/null @@ -1,25 +0,0 @@ -const std = @import("std"); -const xz = std.compress.xz; -const Reader = std.Io.Reader; - -const Decompressor = @import("../../decomp.zig"); - -const Self = @This(); - -interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } }, - -fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize { - var rdr: Reader = .fixed(in); - var decomp = try xz.decompress(alloc, rdr.adaptToOldInterface()); - defer decomp.deinit(); - return decomp.read(out) catch |err| switch (err) { - error.CorruptInput, - error.EndOfStream, - error.EndOfStreamWithNoError, - error.WrongChecksum, - error.Unsupported, - error.Overflow, - => Decompressor.Error.ReadFailed, - else => return err, - }; -} diff --git a/src/decomp/zig/zlib.zig b/src/decomp/zig/zlib.zig deleted file mode 100644 index 9f3fec5..0000000 --- a/src/decomp/zig/zlib.zig +++ /dev/null @@ -1,18 +0,0 @@ -const std = @import("std"); -const Reader = std.Io.Reader; -const flate = std.compress.flate; - -const Decompressor = @import("../../decomp.zig"); - -const Self = @This(); - -interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } }, - -fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize { - const buf = try alloc.alloc(u8, out.len); - defer alloc.free(buf); - var rdr: Reader = .fixed(in); - - var decomp = flate.Decompress.init(&rdr, .zlib, buf); - return decomp.reader.readSliceShort(out); -} diff --git a/src/decomp/zig/zstd.zig b/src/decomp/zig/zstd.zig deleted file mode 100644 index 17d803d..0000000 --- a/src/decomp/zig/zstd.zig +++ /dev/null @@ -1,18 +0,0 @@ -const std = @import("std"); -const Reader = std.Io.Reader; -const zstd = std.compress.zstd; - -const Decompressor = @import("../../decomp.zig"); - -const Self = @This(); - -interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } }, - -fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize { - const buf = try alloc.alloc(u8, out.len * 2); - defer alloc.free(buf); - var rdr: Reader = .fixed(in); - - var decomp = zstd.Decompress.init(&rdr, buf, .{ .window_len = @min(out.len, zstd.default_window_len) }); - return decomp.reader.readSliceShort(out); -} diff --git a/src/dir_entry.zig b/src/dir_entry.zig deleted file mode 100644 index 191da47..0000000 --- a/src/dir_entry.zig +++ /dev/null @@ -1,60 +0,0 @@ -//! Directory entry from the directory table. - -const std = @import("std"); -const Reader = std.Io.Reader; - -const InodeType = @import("inode.zig").InodeType; - -const Entry = @This(); - -const Header = extern struct { // use extern due to bad alignment with packed. - count: u32, - block_start: u32, - num: u32, -}; - -const RawEntry = packed struct { - offset: u16, - inode_offset: i16, - inode_type: InodeType, - name_size: u16, -}; - -block_start: u32, -block_offset: u16, -num: u32, -inode_type: InodeType, -name: []const u8, - -pub fn readDir(alloc: std.mem.Allocator, rdr: *Reader, size: u32) ![]Entry { - var cur_red: u32 = 3; // start at 3 due to "." & ".." being counted in the dir size. - var hdr: Header = undefined; - var raw: RawEntry = undefined; - var out: std.ArrayList(Entry) = try .initCapacity(alloc, 100); // Start out with a decent capacity instead of needing to allocate per header. - errdefer out.deinit(alloc); - while (cur_red < size) { - try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little); - cur_red += @sizeOf(Header); - try out.ensureUnusedCapacity(alloc, hdr.count + 1); - for (0..hdr.count + 1) |_| { - try rdr.readSliceEndian(RawEntry, @ptrCast(&raw), .little); - const name = try alloc.alloc(u8, raw.name_size + 1); - errdefer alloc.free(name); - try rdr.readSliceEndian(u8, name, .little); - const val = out.addOneAssumeCapacity(); - val.* = .{ - .block_start = hdr.block_start, - .block_offset = raw.offset, - .num = @abs(hdr.num + raw.offset), - .inode_type = raw.inode_type, - .name = name, - }; - cur_red += @sizeOf(RawEntry) + raw.name_size + 1; - } - } - return out.toOwnedSlice(alloc); -} - -pub fn deinit(self: Entry, alloc: std.mem.Allocator) void { - alloc.free(self.name); -} diff --git a/src/file.zig b/src/file.zig deleted file mode 100644 index 915e101..0000000 --- a/src/file.zig +++ /dev/null @@ -1,202 +0,0 @@ -//! A file/directory within the squashfs archive. - -const std = @import("std"); -const File = std.fs.File; -const WaitGroup = std.Thread.WaitGroup; -const Mutex = std.Thread.Mutex; - -const Archive = @import("archive.zig"); -const DirEntry = @import("dir_entry.zig"); -const ExtractionOptions = @import("options.zig"); -const Inode = @import("inode.zig"); -const BlockSize = @import("inode_data/file.zig").BlockSize; -const DataReader = @import("util/data.zig"); -const MetadataReader = @import("util/metadata.zig"); - -const FileError = error{ - NotDirectory, - NotRegularFile, - NotSymlink, - NotDevice, - NotFound, - ExtractionPathExists, -}; - -const SfsFile = @This(); - -archive: *Archive, - -inode: Inode, -name: []const u8, - -/// Initialize a new File. -/// name is copied to the File so can be safely freed afterwards. -pub fn init(archive: *Archive, inode: Inode, name: []const u8) !SfsFile { - const new_name = try archive.allocator().alloc(u8, name.len); - @memcpy(new_name, name); - return .{ - .archive = archive, - .inode = inode, - .name = new_name, - }; -} -pub fn fromEntry(archive: *Archive, entry: DirEntry) !SfsFile { - var rdr = try archive.fil.readerAt(entry.block_start + archive.super.inode_start, &[0]u8{}); - var meta: MetadataReader = .init(archive.allocator(), &rdr.interface, archive.decomp); - try meta.interface.discardAll(entry.block_offset); - const inode: Inode = try .read(archive.allocator(), &meta.interface, archive.super.block_size); - errdefer inode.deinit(archive.allocator()); - return .init(archive, inode, entry.name); -} - -pub fn deinit(self: SfsFile) void { - var alloc = self.archive.allocator(); - alloc.free(self.name); - self.inode.deinit(alloc); -} - -fn getEntries(self: SfsFile) ![]DirEntry { - return self.inode.dirEntries(self.archive.allocator(), self.archive.*); -} - -pub fn ownerUid(self: SfsFile) !u16 { - return self.archive.id(self.inode.hdr.uid_idx); -} -pub fn ownerGid(self: SfsFile) !u16 { - return self.archive.id(self.inode.hdr.gid_idx); -} -pub fn permissions(self: SfsFile) u16 { - return self.inode.hdr.permissions; -} - -pub fn isRegular(self: SfsFile) bool { - return switch (self.inode.hdr.inode_type) { - .file, .ext_file => true, - else => false, - }; -} -/// The returned DataReader will no longer work if the File's deinit function is called -/// or, more specifically, it's inode's deinit function is called. -pub fn dataReader(self: SfsFile) !DataReader { - return self.inode.dataReader(self.archive); -} - -pub fn isDir(self: SfsFile) bool { - return switch (self.inode.hdr.inode_type) { - .dir, .ext_dir => true, - else => false, - }; -} -pub fn iterate(self: SfsFile) !Iterator { - if (!self.isDir()) return FileError.NotDirectory; - return .{ - .entries = try self.getEntries(), - .archive = self.archive, - }; -} -/// Open a sub-file/folder within a directory at the given path. -/// If path is "", ".", "/", or "./", this File is returned. -pub fn open(self: SfsFile, alloc: std.mem.Allocator, path: []const u8) !SfsFile { - if (!self.isDir()) return FileError.NotDirectory; - if (pathIsSelf(path)) return self; - - // Recursively stip ending & leading path separators. - if (path[0] == '/') return self.open(path[1..]); - if (path[path.len - 1] == '/') return self.open(path[0 .. path.len - 1]); - - const idx = std.mem.indexOf(u8, path, "/") orelse path.len; - const first_element = path[0..idx]; - if (std.mem.eql(u8, first_element, ".")) return self.open(path[idx + 1 ..]); - const entries = try self.getEntries(); - defer { - for (entries) |e| { - e.deinit(alloc); - } - alloc.free(entries); - } - var cur_slice = entries; - var split = cur_slice.len / 2; - while (cur_slice.len > 0) { - split = cur_slice.len / 2; - const comp = std.mem.order(u8, first_element, cur_slice[split].name); - switch (comp) { - .eq => { - var fil: SfsFile = try .fromEntry(self.archive, cur_slice[split]); - if (idx == path.len) { - return fil; - } - defer fil.deinit(); - return fil.open(alloc, path[idx + 1 ..]); - }, - .lt => cur_slice = cur_slice[0..split], - .gt => cur_slice = cur_slice[split + 1 ..], - } - } - return FileError.NotFound; -} - -pub fn isSymlink(self: SfsFile) bool { - return switch (self.inode.hdr.inode_type) { - .symlink, .ext_symlink => true, - else => false, - }; -} -pub fn symlinkPath(self: SfsFile) ![]const u8 { - if (!self.isSymlink()) return FileError.NotSymlink; - return switch (self.inode.data) { - .symlink => |s| s.target, - .ext_symlink => |s| s.target, - else => unreachable, - }; -} - -/// Check if the File is a block or character device. -pub fn isDevice(self: SfsFile) bool { - return switch (self.inode.hdr.inode_type) { - .block_dev, .char_dev, .ext_block_dev, .ext_char_dev => true, - else => false, - }; -} -/// If the File is a block or character device, get's it's device number. -pub fn devNum(self: SfsFile) !u32 { - if (!self.isDevice()) return FileError.NotDevice; - return switch (self.inode.data) { - .block_dev, .char_dev => |d| d.dev, - .ext_block_dev, .ext_char_dev => |d| d.dev, - else => unreachable, - }; -} - -/// Extract the given File to the path. If File is a regular file, the path must be a directory or not exist. -/// If the gievn path is a folder, the File's contents will be extracted within. -pub fn extract(self: *SfsFile, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void { - return self.inode.extractToThreaded(alloc, self.archive, path, options); -} - -/// Utility function. -pub fn pathIsSelf(path: []const u8) bool { - if (path.len == 0) return true; - if (path.len == 1 and (path[0] == '/' or path[0] == '.')) return true; - if (path.len == 2 and (path[0] == '.' and path[1] == '/')) return true; - return false; -} - -pub const Iterator = struct { - entries: []DirEntry, - archive: *Archive, - - idx: u32 = 0, - - pub fn next(self: *Iterator) !?SfsFile { - if (self.idx >= self.entries.len) return null; - defer self.idx += 1; - return try SfsFile.fromEntry(self.archive, self.entries[self.idx]); - } - pub fn deinit(self: Iterator) void { - var alloc = self.archive.allocator(); - for (self.entries) |e| { - e.deinit(alloc); - } - alloc.free(self.entries); - } -}; diff --git a/src/inode.zig b/src/inode.zig index 39e7738..759cb6c 100644 --- a/src/inode.zig +++ b/src/inode.zig @@ -2,31 +2,49 @@ const std = @import("std"); const Reader = std.Io.Reader; -const WaitGroup = std.Thread.WaitGroup; -const Pool = std.Thread.Pool; -const Mutex = std.Thread.Mutex; -const Archive = @import("archive.zig"); -const DirEntry = @import("dir_entry.zig"); -const ExtractionOptions = @import("options.zig"); const dir = @import("inode_data/dir.zig"); const file = @import("inode_data/file.zig"); const misc = @import("inode_data/misc.zig"); -const Tables = @import("tables.zig"); -const DataReader = @import("util/data.zig"); -const ThreadedDataReader = @import("util/data_threaded.zig"); -const InodeExtract = @import("util/extract.zig"); -const InodeFinish = @import("util/inode_finish.zig"); -const FinishUnion = InodeFinish.FinishUnion; -const MetadataReader = @import("util/metadata.zig"); -pub const Ref = packed struct { +const Inode = @This(); + +hdr: Header, +data: Data, + +pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u16) !Inode { + var hdr: Header = undefined; + try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little); + return .{ + .hdr = hdr, + .data = switch (hdr.inode_type) { + .dir => .{ .dir = .read(rdr) }, + .file => .{ .file = .read(alloc, rdr, block_size) }, + .symlink => .{ .symlink = .read(alloc, rdr) }, + .block_dev => .{ .block_dev = .read(rdr) }, + .char_dev => .{ .char_dev = .read(rdr) }, + .fifo => .{ .fifo = .read(rdr) }, + .socket => .{ .socket = .read(rdr) }, + .ext_dir => .{ .ext_dir = .read(rdr) }, + .ext_file => .{ .ext_file = .read(alloc, rdr, block_size) }, + .ext_symlink => .{ .ext_symlink = .read(alloc, rdr) }, + .ext_block_dev => .{ .ext_block_dev = .read(rdr) }, + .ext_char_dev => .{ .ext_char_dev = .read(rdr) }, + .ext_fifo => .{ .ext_fifo = .read(rdr) }, + .ext_socket => .{ .ext_socket = .read(rdr) }, + }, + }; +} + +// Types + +pub const Ref = packed struct(u64) { block_offset: u16, block_start: u32, _: u16, }; -pub const InodeType = enum(u16) { +pub const Type = enum(u16) { dir = 1, file, symlink, @@ -43,7 +61,7 @@ pub const InodeType = enum(u16) { ext_socket, }; -pub const InodeData = union(InodeType) { +pub const Data = union(Type) { dir: dir.Dir, file: file.File, symlink: misc.Symlink, @@ -61,129 +79,10 @@ pub const InodeData = union(InodeType) { }; pub const Header = packed struct { - inode_type: InodeType, + inode_type: Type, permissions: u16, uid_idx: u16, gid_idx: u16, mod_time: u32, num: u32, }; - -const Inode = @This(); - -hdr: Header, -data: InodeData, - -pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode { - var hdr: Header = undefined; - try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little); - return .{ - .hdr = hdr, - .data = switch (hdr.inode_type) { - .dir => .{ .dir = try .read(rdr) }, - .file => .{ .file = try .read(alloc, rdr, block_size) }, - .symlink => .{ .symlink = try .read(alloc, rdr) }, - .block_dev => .{ .block_dev = try .read(rdr) }, - .char_dev => .{ .char_dev = try .read(rdr) }, - .fifo => .{ .fifo = try .read(rdr) }, - .socket => .{ .socket = try .read(rdr) }, - .ext_dir => .{ .ext_dir = try .read(rdr) }, - .ext_file => .{ .ext_file = try .read(alloc, rdr, block_size) }, - .ext_symlink => .{ .ext_symlink = try .read(alloc, rdr) }, - .ext_block_dev => .{ .ext_block_dev = try .read(rdr) }, - .ext_char_dev => .{ .ext_char_dev = try .read(rdr) }, - .ext_fifo => .{ .ext_fifo = try .read(rdr) }, - .ext_socket => .{ .ext_socket = try .read(rdr) }, - }, - }; -} -pub fn readFromEntry(alloc: std.mem.Allocator, archive: Archive, entry: DirEntry) !Inode { - var rdr = try archive.fil.readerAt(archive.super.inode_start + entry.block_start, &[0]u8{}); - var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp); - try meta.interface.discardAll(entry.block_offset); - return read(alloc, &meta.interface, archive.super.block_size); -} - -pub fn deinit(self: Inode, alloc: std.mem.Allocator) void { - switch (self.data) { - .file => |f| alloc.free(f.block_sizes), - .ext_file => |f| alloc.free(f.block_sizes), - .symlink => |s| alloc.free(s.target), - .ext_symlink => |s| alloc.free(s.target), - else => {}, - } -} - -/// Get the data reader for a file inode. -pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive, tables: *Tables) !DataReader { - return switch (self.hdr.inode_type) { - .file => readerFromData(alloc, archive, tables, self.data.file), - .ext_file => readerFromData(alloc, archive, tables, self.data.ext_file), - else => error.NotRegularFile, - }; -} -fn readerFromData(alloc: std.mem.Allocator, archive: Archive, tables: *Tables, data: anytype) !DataReader { - var out: DataReader = .init(alloc, archive, data.block_sizes, data.block_start, data.size); - if (data.frag_idx != 0xFFFFFFFF) - out.addFragment(try tables.frag_table.get(data.frag_idx), data.frag_block_offset); - return out; -} - -/// Get the directory entries for a directory inode. -pub fn dirEntries(self: Inode, alloc: std.mem.Allocator, archive: Archive) ![]DirEntry { - return switch (self.hdr.inode_type) { - .dir => entriesFromData(alloc, archive, self.data.dir), - .ext_dir => entriesFromData(alloc, archive, self.data.ext_dir), - else => error.NotDirectory, - }; -} -fn entriesFromData(alloc: std.mem.Allocator, archive: Archive, data: anytype) ![]DirEntry { - var rdr = try archive.fil.readerAt(archive.super.dir_start + data.block_start, &[0]u8{}); - var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp); - try meta.interface.discardAll(data.block_offset); - return DirEntry.readDir(alloc, &meta.interface, data.size); -} - -/// Returns the xattr index for the given inode. If the inode isn't an extended variant or doesn't have any, the u32 max is returned (0xFFFFFFFF). -pub fn xattrIdx(self: Inode) u32 { - return switch (self.data) { - .ext_dir => |d| d.xattr_id, - .ext_file => |f| f.xattr_idx, - .ext_symlink => |s| s.xattr_idx, - .ext_block_dev, .ext_char_dev => |d| d.xattr_idx, - .ext_fifo, .ext_socket => |i| i.xattr_idx, - else => 0xFFFFFFFF, - }; -} - -/// Applies the Inode's metadata to the given File. -/// Mod time is always set, but permissions and xattrs are set based on the given ExtractionOptions. -pub fn setMetadata(self: Inode, alloc: std.mem.Allocator, tables: *Tables, fil: std.fs.File, options: ExtractionOptions) !void { - const time = @as(i128, self.hdr.mod_time) * 1000000000; - try fil.updateTimes(time, time); - if (!options.ignore_permissions) { - try fil.chmod(self.hdr.permissions); - try fil.chown(try tables.id_table.get(self.hdr.uid_idx), try tables.id_table.get(self.hdr.gid_idx)); - } - if (!options.ignore_xattr) { - const idx = self.xattrIdx(); - if (idx == 0xFFFFFFFF) return; - const xattrs = try tables.xattr_table.get(alloc, idx); - defer alloc.free(xattrs); - for (xattrs) |kv| { - const res = std.os.linux.fsetxattr(fil.handle, kv.key, kv.value.ptr, kv.value.len, 0); - alloc.free(kv.key); - alloc.free(kv.value); - if (res != 0) { - if (options.verbose) - options.verbose_writer.?.print("fsetxattr has result of: {}\n", .{res}) catch {}; - return error.SetXattr; - } - } - } -} - -/// Extract the inode to the given path. -pub fn extractTo(self: Inode, alloc: std.mem.Allocator, archive: Archive, path: []const u8, options: ExtractionOptions) !void { - return InodeExtract.extractTo(alloc, self, archive, path, options); -} diff --git a/src/lookup_table.zig b/src/lookup_table.zig new file mode 100644 index 0000000..efcb225 --- /dev/null +++ b/src/lookup_table.zig @@ -0,0 +1,65 @@ +const std = @import("std"); +const Io = std.Io; + +const OffsetFile = @import("util/offset_file.zig"); +const MetadataReader = @import("util/metadata.zig"); + +pub fn CachedTable(comptime T: anytype) type { + return struct { + const T_PER_BLOCK: u16 = 8192 / @sizeOf(T); + + const Table = @This(); + + alloc: std.mem.Allocator, + io: Io, + fil: OffsetFile, + table_start: u64, + total_num: u32, + + table: std.AutoHashMap(u32, []T), + + mut: Io.Mutex = .init, + + pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, offset: u64, total_num: u32) Table { + return .{ + .alloc = alloc, + .io = io, + .fil = fil, + .table_start = offset, + .total_num = total_num, + + .table = .init(alloc), + }; + } + + pub fn get(self: *Table, idx: u32) !T { + const block = idx / T_PER_BLOCK; + const block_offset = idx % T_PER_BLOCK; + if (self.table.contains(block)) + return self.table.get(block).?[block_offset]; + + try self.mut.lock(self.io); + defer self.mut.unlock(self.io); + + if (self.table.contains(block)) + return self.table.get(block).?[block_offset]; + + var rdr = try self.fil.readerAt(self.io, self.table_start + (8 * block), &[0]u8{}); + var offset: u64 = undefined; + try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little); + + const arr_num: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK) + self.total_num % T_PER_BLOCK + else + T_PER_BLOCK; + + rdr = try self.fil.readerAt(self.io, offset, &[0]u8{}); + var meta: MetadataReader = .init(self.alloc, &rdr, self.decomp); + + try self.table.put( + block, + try meta.interface.readSliceEndianAlloc(self.alloc, T, arr_num, .little), + ); + } + }; +} diff --git a/src/super.zig b/src/super.zig deleted file mode 100644 index 29ec002..0000000 --- a/src/super.zig +++ /dev/null @@ -1,69 +0,0 @@ -const std = @import("std"); -const math = std.math; - -const InodeRef = @import("inode.zig").Ref; - -const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little); - -const SuperblockError = error{ - InvalidMagic, - InvalidBlockLog, - InvalidVersion, - InvalidCheck, -}; - -/// A squashfs Superblock -pub const Superblock = packed struct { - magic: u32, - inode_count: u32, - mod_time: u32, - block_size: u32, - frag_count: u32, - compression: enum(u16) { - gzip = 1, - lzma, - lzo, - xz, - lz4, - zstd, - }, - block_log: u16, - flags: packed struct { - inode_uncompressed: bool, - data_uncompressed: bool, - check: bool, - frag_uncompressed: bool, - fragment_never: bool, - fragment_always: bool, - duplicates: bool, - exportable: bool, - xattr_uncompressed: bool, - xattr_never: bool, - compression_options: bool, - ids_uncompressed: bool, - _: u4, - }, - id_count: u16, - ver_maj: u16, - ver_min: u16, - root_ref: InodeRef, - size: u64, - id_start: u64, - xattr_start: u64, - inode_start: u64, - dir_start: u64, - frag_start: u64, - export_start: u64, - - /// Validate the Superblock. If an error is returned, it's likely the archive is corrupted or not a squashfs archive. - pub fn validate(self: Superblock) !void { - if (self.magic != SQUASHFS_MAGIC) - return SuperblockError.InvalidMagic; - if (self.flags.check) - return SuperblockError.InvalidCheck; - if (self.ver_maj != 4 or self.ver_min != 0) - return SuperblockError.InvalidVersion; - if (math.log2(self.block_size) != self.block_log) - return SuperblockError.InvalidBlockLog; - } -}; diff --git a/src/tables.zig b/src/tables.zig deleted file mode 100644 index 5341cca..0000000 --- a/src/tables.zig +++ /dev/null @@ -1,108 +0,0 @@ -const std = @import("std"); -const Mutex = std.Thread.Mutex; - -const Archive = @import("archive.zig"); -const BlockSize = @import("inode_data/file.zig").BlockSize; -const InodeRef = @import("inode.zig").Ref; -const Superblock = @import("super.zig").Superblock; -const MetadataReader = @import("util/metadata.zig"); -const OffsetFile = @import("util/offset_file.zig"); -const XattrTable = @import("xattr.zig"); -const Decompressor = @import("decomp.zig"); - -/// Information about a fragment section. Multiple fragments are contained in the block described by a single FragEntry. -/// The offset into the block and fragment size is stored in the file's inode. -pub const FragEntry = packed struct { - start: u64, - size: BlockSize, - _: u32, -}; - -const Tables = @This(); - -frag_table: Table(FragEntry), -id_table: Table(u16), -export_table: Table(InodeRef), -xattr_table: XattrTable, - -pub fn init(alloc: std.mem.Allocator, archive: Archive) !Tables { - return .{ - .frag_table = try .init(alloc, archive.fil, archive.decomp, archive.super.frag_start, archive.super.frag_count), - .id_table = try .init(alloc, archive.fil, archive.decomp, archive.super.id_start, archive.super.id_count), - .export_table = try .init(alloc, archive.fil, archive.decomp, archive.super.export_start, archive.super.inode_count), - .xattr_table = try .init(alloc, archive.fil, archive.decomp, archive.super.xattr_start), - }; -} -pub fn deinit(self: *Tables) void { - self.frag_table.deinit(); - self.id_table.deinit(); - self.export_table.deinit(); - self.xattr_table.deinit(); -} - -/// A two-layer metadata table. -pub fn Table(T: anytype) type { - return struct { - const Self = @This(); - - const VALS_PER_BLOCK = 8192 / @sizeOf(T); - - alloc: std.mem.Allocator, - fil: OffsetFile, - decomp: *Decompressor, - tab_start: u64, - - tab: std.AutoHashMap(u32, []T), - values: u32, - - mut: Mutex = .{}, - - pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, tab_start: u64, values: u32) !Self { - return .{ - .alloc = alloc, - .fil = fil, - .decomp = decomp, - .tab_start = tab_start, - - .tab = .init(alloc), - .values = values, - }; - } - - pub fn deinit(self: *Self) void { - var iter = self.tab.valueIterator(); - while (iter.next()) |s| { - self.alloc.free(s.*); - } - self.tab.deinit(); - } - - pub fn get(self: *Self, idx: u32) !T { - if (idx >= self.values) return error.InvalidIndex; - const block_num = idx / VALS_PER_BLOCK; - const idx_offset = idx - (block_num * VALS_PER_BLOCK); - if (self.tab.contains(block_num)) { - const block = self.tab.get(block_num).?; - return block[idx_offset]; - } - self.mut.lock(); - defer self.mut.unlock(); - // Double check in case of race condition.. - if (self.tab.contains(block_num)) { - const block = self.tab.get(block_num).?; - return block[idx_offset]; - } - const is_last = (self.values - 1) / VALS_PER_BLOCK == block_num; - const slice_size = if (is_last) self.values - (block_num * VALS_PER_BLOCK) else VALS_PER_BLOCK; - const slice = try self.alloc.alloc(T, slice_size); - var rdr = try self.fil.readerAt(self.tab_start + (8 * block_num), &[0]u8{}); - var offset: u64 = 0; - try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little); - rdr = try self.fil.readerAt(offset, &[0]u8{}); - var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp); - try meta.interface.readSliceEndian(T, @ptrCast(slice), .little); - try self.tab.put(block_num, slice); - return slice[idx_offset]; - } - }; -} diff --git a/src/test.zig b/src/test.zig index 4bb58cb..e69de29 100644 --- a/src/test.zig +++ b/src/test.zig @@ -1,84 +0,0 @@ -const std = @import("std"); -const stuff = @import("builtin"); - -const Archive = @import("archive.zig"); -const Superblock = @import("super.zig").Superblock; - -const TestArchive = "testing/LinuxPATest.sfs"; - -test "Basics" { - var fil = try std.fs.cwd().openFile(TestArchive, .{}); - defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - if (sfs.super != LinuxPATestCorrectSuperblock) { - std.debug.print("Superblock wrong\nShould be: {}\n\nis: {}\n", .{ LinuxPATestCorrectSuperblock, sfs.super }); - return error.BadSuperblock; - } -} - -const TestFile = "Start.exe"; -const TestFileExtractLocation = "testing/Start.exe"; - -test "ExtractSingleFile" { - std.fs.cwd().deleteFile(TestFileExtractLocation) catch {}; - var fil = try std.fs.cwd().openFile(TestArchive, .{}); - defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - var test_fil = try sfs.open(TestFile); - defer test_fil.deinit(); - try test_fil.extract(TestFileExtractLocation, .Default); - //TODO: validate extracted file. -} - -const TestFullExtractLocation = "testing/TestExtract"; - -test "ExtractCompleteArchive" { - std.fs.cwd().deleteTree(TestFullExtractLocation) catch {}; - var fil = try std.fs.cwd().openFile(TestArchive, .{}); - defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - try sfs.extract(TestFullExtractLocation, .Default); -} - -const LinuxPATestCorrectSuperblock: Superblock = .{ - .magic = std.mem.readInt(u32, "hsqs", .little), - .inode_count = 2974, - .mod_time = 1632696724, - .block_size = 131072, - .frag_count = 264, - .compression = .zstd, - .block_log = 17, - .flags = .{ - .inode_uncompressed = false, - .data_uncompressed = false, - .check = false, - .frag_uncompressed = false, - .fragment_never = false, - .fragment_always = false, - .duplicates = true, - .exportable = true, - .xattr_uncompressed = false, - .xattr_never = false, - .compression_options = false, - .ids_uncompressed = false, - ._ = 0, - }, - .id_count = 1, - .ver_maj = 4, - .ver_min = 0, - .root_ref = .{ - .block_offset = 1363, - .block_start = 29237, - ._ = 0, - }, - .size = 106841744, - .id_start = 106841632, - .xattr_start = 106841720, - .inode_start = 106778274, - .dir_start = 106807998, - .frag_start = 106837747, - .export_start = 106841602, -}; diff --git a/src/util/data.zig b/src/util/data.zig deleted file mode 100644 index 0c1d43e..0000000 --- a/src/util/data.zig +++ /dev/null @@ -1,174 +0,0 @@ -//! A reader for a regular file. - -const std = @import("std"); -const Reader = std.Io.Reader; -const Writer = std.Io.Writer; -const Limit = std.Io.Limit; - -const Archive = @import("../archive.zig"); -const DecompFn = @import("../decomp.zig").DecompFn; -const BlockSize = @import("../inode_data/file.zig").BlockSize; -const FragEntry = @import("../tables.zig").FragEntry; -const OffsetFile = @import("offset_file.zig"); - -const DataReader = @This(); - -alloc: std.mem.Allocator, -fil: OffsetFile, -decomp: DecompFn, -block_size: u32, - -blocks: []BlockSize, - -frag: ?FragEntry = null, // TODO: do something better? -frag_offset: u32 = 0, -size: u64, - -interface: Reader, - -cur_offset: u64, -block_idx: u32 = 0, - -pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64) DataReader { - return .{ - .alloc = alloc, - .fil = archive.fil, - .decomp = archive.decomp, - .block_size = archive.super.block_size, - .blocks = blocks, - .size = size, - .cur_offset = start, - .interface = .{ - .end = 0, - .seek = 0, - .buffer = &[0]u8{}, - .vtable = &.{ - .stream = stream, - .discard = discard, - .readVec = readVec, - }, - }, - }; -} -pub fn deinit(self: *DataReader) void { - self.alloc.free(self.interface.buffer); - self.interface.end = 0; - self.interface.seek = 0; -} - -pub fn addFragment(self: *DataReader, entry: FragEntry, frag_offset: u32) void { - self.frag = entry; - self.frag_offset = frag_offset; -} - -fn numBlocks(self: DataReader) usize { - var res = self.blocks.len; - if (self.frag != null) res += 1; - return res; -} - -fn advance(self: *DataReader) !void { - if (self.block_idx > self.blocks.len or (self.block_idx == self.blocks.len and self.frag == null)) { - if (self.interface.buffer.len > 0) { - self.alloc.free(self.interface.buffer); - self.interface.buffer = &[0]u8{}; - self.interface.end = 0; - self.interface.seek = 0; - } - return Reader.Error.EndOfStream; - } - defer self.block_idx += 1; - const cur_block_size = if (self.block_idx == self.numBlocks() - 1) self.size % self.block_size else self.block_size; - try self.resizeBuffer(cur_block_size); - self.interface.seek = 0; - self.interface.end = cur_block_size; - if (self.block_idx == self.blocks.len) { // fragment - var rdr = try self.fil.readerAt(self.frag.?.start, &[0]u8{}); - if (self.frag.?.size.uncompressed) { - try rdr.interface.discardAll(self.frag_offset); - try rdr.interface.readSliceAll(self.interface.buffer); - return; - } - const tmp_buf = try self.alloc.alloc(u8, self.frag.?.size.size); - defer self.alloc.free(tmp_buf); - try rdr.interface.readSliceAll(tmp_buf); - const needed_block = try self.alloc.alloc(u8, self.block_size); - defer self.alloc.free(needed_block); - _ = try self.decomp(self.alloc, tmp_buf, needed_block); - @memcpy(self.interface.buffer, needed_block[self.frag_offset .. self.frag_offset + cur_block_size]); - return; - } - const block = self.blocks[self.block_idx]; - if (block.size == 0) { - @memset(self.interface.buffer, 0); - return; - } - var rdr = try self.fil.readerAt(self.cur_offset, &[0]u8{}); - self.cur_offset += block.size; - if (block.uncompressed) { - try rdr.interface.readSliceAll(self.interface.buffer); - return; - } - const tmp_buf = try self.alloc.alloc(u8, block.size); - defer self.alloc.free(tmp_buf); - try rdr.interface.readSliceAll(tmp_buf); - _ = try self.decomp(self.alloc, tmp_buf, self.interface.buffer); -} -/// Does not guarentee that data currently in the buffer is retained. -fn resizeBuffer(self: *DataReader, size: usize) !void { - if (self.interface.buffer.len == size) return; - if (!self.alloc.resize(self.interface.buffer, size)) { - self.alloc.free(self.interface.buffer); - self.interface.buffer = self.alloc.alloc(u8, size) catch |err| { - self.interface.buffer = &[0]u8{}; - return err; - }; - } else { - self.interface.buffer.len = size; - } -} - -fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) Reader.StreamError!usize { - var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr)); - if (rdr.seek >= rdr.end) self.advance() catch |err| { - if (err == error.EndOfStream) return error.EndOfStream; - std.log.err("Error advancing data reader: {}\n", .{err}); - return Reader.Error.ReadFailed; - }; - if (limit == .nothing) return 0; - const to_read = @min(rdr.end - rdr.seek, @intFromEnum(limit)); - const res = try wrt.write(rdr.buffer[rdr.seek .. rdr.seek + to_read]); - rdr.seek += res; - return res; -} - -fn discard(rdr: *Reader, limit: Limit) Reader.Error!usize { - var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr)); - if (rdr.seek >= rdr.end) self.advance() catch |err| { - if (err == error.EndOfStream) return error.EndOfStream; - std.log.err("Error advancing data reader: {}\n", .{err}); - return Reader.Error.ReadFailed; - }; - if (limit == .nothing) return 0; - const to_adv = @min(rdr.end - rdr.seek, @intFromEnum(limit)); - rdr.seek += to_adv; - return to_adv; -} - -fn readVec(rdr: *Reader, vec: [][]u8) Reader.Error!usize { - var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr)); - if (rdr.seek >= rdr.end) self.advance() catch |err| { - if (err == error.EndOfStream) return error.EndOfStream; - std.log.err("Error advancing data reader: {}\n", .{err}); - return Reader.Error.ReadFailed; - }; - var cur_red: usize = 0; - for (vec) |s| { - const to_copy: usize = @min(rdr.end - rdr.seek, s.len); - @memcpy(s[0..to_copy], rdr.buffer[rdr.seek .. rdr.seek + to_copy]); - rdr.seek += to_copy; - cur_red += to_copy; - if (rdr.end == rdr.seek) break; - } - return cur_red; -} diff --git a/src/util/data_threaded.zig b/src/util/data_threaded.zig deleted file mode 100644 index 280586a..0000000 --- a/src/util/data_threaded.zig +++ /dev/null @@ -1,207 +0,0 @@ -//! Similiar to DataReader, but set-up for threaded writing to files. - -const std = @import("std"); -const Reader = std.Io.Reader; -const Writer = std.Io.Writer; -const Limit = std.Io.Limit; -const WaitGroup = std.Thread.WaitGroup; -const Pool = std.Thread.Pool; - -const Archive = @import("../archive.zig"); -const DecompFn = @import("../decomp.zig").DecompFn; -const BlockSize = @import("../inode_data/file.zig").BlockSize; -const FragEntry = @import("../tables.zig").FragEntry; -const InodeFinish = @import("inode_finish.zig"); -const OffsetFile = @import("offset_file.zig"); - -const ThreadedDataReader = @This(); - -alloc: std.mem.Allocator, -fil: OffsetFile, -decomp: DecompFn, -block_size: u32, - -blocks: []BlockSize, - -frag: ?FragEntry = null, // TODO: do something better? -frag_offset: u32 = 0, -size: u64, -num_blocks: usize, - -start_offset: u64, - -pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64) ThreadedDataReader { - return .{ - .alloc = alloc, - .fil = archive.fil, - .decomp = archive.decomp, - .block_size = archive.super.block_size, - - .blocks = blocks, - - .size = size, - .num_blocks = blocks.len, - - .start_offset = start, - }; -} - -pub fn addFragment(self: *ThreadedDataReader, entry: FragEntry, frag_offset: u32) void { - self.frag = entry; - self.frag_offset = frag_offset; - self.num_blocks = self.blocks.len + 1; -} - -/// Extract the data to the file threadedly, using pool to spawn threads. -/// If errors occur, they are set to finish.out_err. -pub fn extractThreaded(self: ThreadedDataReader, file: std.fs.File, pool: *Pool, finish: *InodeFinish) void { - var cur_write_offset: u64 = 0; - var cur_read_offset: u64 = self.start_offset; - for (0..self.blocks.len) |i| { - const cur_block_size = if (i == self.num_blocks - 1) self.size % self.block_size else self.block_size; - pool.spawn(workThreadBlocks, .{ self, file, cur_write_offset, cur_read_offset, self.blocks[i], cur_block_size, finish }) catch |res_err| { - finish.logError("Can't spawn pool task: {}", .{res_err}); - finish.out_err.* = res_err; - finish.finish(); - }; - cur_write_offset += cur_block_size; - cur_read_offset += self.blocks[i].size; - } - if (self.frag != null) - pool.spawn(workThreadFragment, .{ self, file, cur_write_offset, finish }) catch |res_err| { - finish.logError("Can't spawn pool task: {}", .{res_err}); - finish.out_err.* = res_err; - finish.finish(); - }; -} - -fn workThreadBlocks( - self: ThreadedDataReader, - fil: std.fs.File, - write_offset: u64, - read_offset: u64, - block: BlockSize, - cur_block_size: u64, - finish: *InodeFinish, -) void { - defer finish.finish(); - var wrt = fil.writer(&[0]u8{}); - wrt.seekTo(write_offset) catch |err| { - finish.logError("Error seeking file writer: {}", .{err}); - finish.out_err.* = err; - return; - }; - defer wrt.interface.flush() catch |err| { - finish.logError("Error flushing file writer: {}", .{err}); - finish.out_err.* = err; - }; - if (block.size == 0) { - wrt.interface.splatByteAll(0, cur_block_size) catch |err| { - finish.logError("Error writing zeroes: {}", .{err}); - finish.out_err.* = err; - return; - }; - return; - } - var rdr = self.fil.readerAt(read_offset, &[0]u8{}) catch |err| { - finish.logError("Error creating file reader: {}", .{err}); - finish.out_err.* = err; - return; - }; - if (block.uncompressed) { - rdr.interface.streamExact(&wrt.interface, block.size) catch |err| { - finish.logError("Error streaming data: {}", .{err}); - finish.out_err.* = err; - return; - }; - return; - } - // TODO: shared buffers - const read_buf = self.alloc.alloc(u8, block.size) catch |err| { - finish.logError("Error creating reader buffer: {}", .{err}); - finish.out_err.* = err; - return; - }; - defer self.alloc.free(read_buf); - rdr.interface.readSliceAll(read_buf) catch |err| { - finish.logError("Error reading data into reader buffer: {}", .{err}); - finish.out_err.* = err; - return; - }; - // TODO: shared buffers - const res_buf = self.alloc.alloc(u8, cur_block_size) catch |err| { - finish.logError("Error creating result buffer: {}", .{err}); - finish.out_err.* = err; - return; - }; - defer self.alloc.free(res_buf); - _ = self.decomp(self.alloc, read_buf, res_buf) catch |err| { - finish.logError("Error decompressing data block: {}", .{err}); - finish.out_err.* = err; - return; - }; - wrt.interface.writeAll(res_buf) catch |err| { - finish.logError("Error writing to file: {}", .{err}); - finish.out_err.* = err; - return; - }; -} -fn workThreadFragment(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, finish: *InodeFinish) void { - defer finish.finish(); - - var wrt = fil.writer(&[0]u8{}); - wrt.seekTo(write_offset) catch |err| { - finish.logError("Error seeking file writer for file fragment: {}", .{err}); - finish.out_err.* = err; - return; - }; - defer wrt.interface.flush() catch |err| { - finish.out_err.* = err; - }; - - var rdr = self.fil.readerAt(self.frag.?.start, &[0]u8{}) catch |err| { - finish.logError("Error creating file reader for file fragment: {}", .{err}); - finish.out_err.* = err; - return; - }; - if (self.frag.?.size.uncompressed) { - rdr.interface.discardAll(self.frag_offset) catch |err| { - finish.logError("Error discarding useless fragment data: {}", .{err}); - finish.out_err.* = err; - return; - }; - rdr.interface.streamExact(&wrt.interface, self.size % self.block_size) catch |err| { - finish.logError("Error streaming fragment data: {}", .{err}); - finish.out_err.* = err; - return; - }; - return; - } - const tmp_buf = self.alloc.alloc(u8, self.frag.?.size.size) catch |err| { - finish.logError("Error creating a temporary buffer for a file fragment: {}", .{err}); - finish.out_err.* = err; - return; - }; - defer self.alloc.free(tmp_buf); - rdr.interface.readSliceAll(tmp_buf) catch |err| { - finish.logError("Error reading data into fragment buffer: {}", .{err}); - finish.out_err.* = err; - return; - }; - const needed_block = self.alloc.alloc(u8, self.block_size) catch |err| { - finish.logError("Error allocating fragment decompression results: {}", .{err}); - finish.out_err.* = err; - return; - }; - defer self.alloc.free(needed_block); - _ = self.decomp(self.alloc, tmp_buf, needed_block) catch |err| { - finish.logError("Error decompressing fragment: {}", .{err}); - finish.out_err.* = err; - return; - }; - wrt.interface.writeAll(needed_block[self.frag_offset .. self.frag_offset + (self.size % self.block_size)]) catch |err| { - finish.logError("Error writing fragment: {}", .{err}); - finish.out_err.* = err; - return; - }; -} diff --git a/src/util/extract.zig b/src/util/extract.zig deleted file mode 100644 index 285997c..0000000 --- a/src/util/extract.zig +++ /dev/null @@ -1,353 +0,0 @@ -const std = @import("std"); -const Allocator = std.mem.Allocator; -const Pool = std.Thread.Pool; -const WaitGroup = std.Thread.WaitGroup; - -const Archive = @import("../archive.zig"); -const DirEntry = @import("../dir_entry.zig"); -const Inode = @import("../inode.zig"); -const ExtractionOptions = @import("../options.zig"); -const Tables = @import("../tables.zig"); -const InodeFinish = @import("inode_finish.zig"); -const FinishUnion = InodeFinish.FinishUnion; -const ThreadedDataReader = @import("data_threaded.zig"); - -// 1 MB -const STACK_ALLOC_SIZE = 1024 * 1024; - -pub fn extractTo( - allocator: Allocator, - inode: Inode, - archive: Archive, - path: []const u8, - options: ExtractionOptions, -) !void { - if (path[path.len - 1] == '/') - return extractTo(allocator, inode, archive, path[0 .. path.len - 2], options); - - var stack_alloc = std.heap.stackFallback(STACK_ALLOC_SIZE, allocator); - var arena: std.heap.ArenaAllocator = .init(stack_alloc.get()); - defer arena.deinit(); - if (options.threads <= 1) { - const alloc = arena.allocator(); - var tables: Tables = try .init(alloc, archive); - return extractSingleThread(arena.allocator(), inode, archive, &tables, path, options); - } - - var thread_alloc = std.heap.ThreadSafeAllocator{ .child_allocator = arena.allocator() }; - const alloc = thread_alloc.allocator(); - var tables: Tables = try .init(alloc, archive); - - var pool_alloc = std.heap.stackFallback(10 * 1024, alloc); - var pool: Pool = undefined; - try pool.init(.{ .allocator = pool_alloc.get(), .n_jobs = options.threads - 1 }); - - var wg: WaitGroup = .{}; - var err: ?anyerror = null; - wg.start(); - try pool.spawn(extractMultiThread, .{ - alloc, - inode, - archive, - &tables, - path, - options, - &pool, - FinishUnion{ .wg = &wg }, - &err, - }); - pool.waitAndWork(&wg); - if (err != null) return err.?; -} - -fn extractSingleThread( - alloc: Allocator, - inode: Inode, - archive: Archive, - tables: *Tables, - path: []const u8, - options: ExtractionOptions, -) !void { - switch (inode.hdr.inode_type) { - .dir, .ext_dir => { - _ = std.fs.cwd().makeDir(path) catch |err| switch (err) { - std.fs.Dir.MakeError.PathAlreadyExists => {}, - else => return err, - }; - - // Currently we are ignoring any deinit or free calls since we know we are under an ArenaAllocator. - // Possibly in the future, do some simple math to see if it would be safe to ONLY deinit via Arena, - // otherwise be more conscientious about freeing memory. - // For now, this is good enough. - - const entries = try inode.dirEntries(alloc, archive); - for (entries) |ent| { - const sub_inode: Inode = try .readFromEntry(alloc, archive, ent); - const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", ent.name }); - try extractSingleThread(alloc, sub_inode, archive, tables, new_path, options); - } - - const fil = try std.fs.cwd().openFile(path, .{}); - defer fil.close(); - try inode.setMetadata(alloc, tables, fil, options); - }, - .file, .ext_file => { - var fil = try std.fs.cwd().createFile(path, .{ .exclusive = true }); - defer fil.close(); - var wrt = fil.writer(&[0]u8{}); - var dat_rdr = try inode.dataReader(alloc, archive, tables); - defer dat_rdr.deinit(); - _ = try dat_rdr.interface.streamRemaining(&wrt.interface); - try wrt.interface.flush(); - - try inode.setMetadata(alloc, tables, fil, options); - }, - .symlink, .ext_symlink => return extractSymlink(inode, path), - else => return extractDeviceAndIPC(inode, alloc, tables, path, options), - } -} - -fn extractMultiThread( - alloc: Allocator, - inode: Inode, - archive: Archive, - tables: *Tables, - path: []const u8, - options: ExtractionOptions, - pool: *Pool, - fin: FinishUnion, - err: *?anyerror, -) void { - if (err.* != null) { - fin.finish(); - return; - } - switch (inode.hdr.inode_type) { - .dir, .ext_dir => { - _ = std.fs.cwd().makeDir(path) catch |res_err| switch (res_err) { - std.fs.Dir.MakeError.PathAlreadyExists => {}, - else => { - err.* = res_err; - fin.finish(); - return; - }, - }; - - // Currently we are ignoring any deinit or free calls since we know we are under an ArenaAllocator. - // Possibly in the future, do some simple math to see if it would be safe to ONLY deinit via Arena, - // otherwise be more conscientious about freeing memory. - // For now, this is good enough. - - const entries = inode.dirEntries(alloc, archive) catch |res_err| { - err.* = res_err; - fin.finish(); - return; - }; - - if (entries.len == 0) { - fin.finish(); - return; - } - - var dir_fin = InodeFinish.create( - alloc, - inode, - path, - tables, - options, - fin, - err, - null, - entries.len, - ) catch |res_err| { - err.* = res_err; - fin.finish(); - return; - }; - - for (entries) |ent| { - if (ent.inode_type == .dir) { - extractEntry( - alloc, - ent, - archive, - tables, - path, - options, - pool, - .{ .fin = dir_fin }, - err, - ); - continue; - } - - pool.spawn( - extractEntry, - .{ alloc, ent, archive, tables, path, options, pool, FinishUnion{ .fin = dir_fin }, err }, - ) catch |res_err| { - err.* = res_err; - dir_fin.finish(); - return; - }; - } - }, - .file, .ext_file => { - const fil = std.fs.cwd().createFile(path, .{ .exclusive = true }) catch |res_err| { - if (options.verbose) - options.verbose_writer.?.print("Can't create file at {s}: {}\n", .{ path, res_err }) catch {}; - err.* = res_err; - fin.finish(); - return; - }; - - var data_rdr = threadedDataReader(inode, alloc, archive, tables) catch |res_err| { - if (options.verbose) - options.verbose_writer.?.print("Can't create data reader for inode #{} (extracting to {s}): {}\n", .{ inode.hdr.num, path, res_err }) catch {}; - err.* = res_err; - fin.finish(); - return; - }; - if (data_rdr == null) { - inode.setMetadata(alloc, tables, fil, options) catch |res_err| { - if (options.verbose) - options.verbose_writer.?.print("Can't set metadata to {s}: {}\n", .{ path, res_err }) catch {}; - err.* = res_err; - }; - fin.finish(); - return; - } - const file_fin = InodeFinish.create( - alloc, - inode, - path, - tables, - options, - fin, - err, - fil, - data_rdr.?.num_blocks, - ) catch |res_err| { - if (options.verbose) - options.verbose_writer.?.print("Can't create callback for inode #{} (extracting to {s}): {}\n", .{ inode.hdr.num, path, res_err }) catch {}; - err.* = res_err; - fin.finish(); - return; - }; - - data_rdr.?.extractThreaded(fil, pool, file_fin); - }, - .symlink, .ext_symlink => { - extractSymlink(inode, path) catch |res_err| { - err.* = res_err; - }; - fin.finish(); - }, - else => { - extractDeviceAndIPC(inode, alloc, tables, path, options) catch |res_err| { - err.* = res_err; - }; - fin.finish(); - }, - } -} - -fn extractEntry( - alloc: Allocator, - ent: DirEntry, - archive: Archive, - tables: *Tables, - path: []const u8, - options: ExtractionOptions, - pool: *Pool, - fin: FinishUnion, - err: *?anyerror, -) void { - const new_path = std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", ent.name }) catch |res_err| { - err.* = res_err; - fin.finish(); - return; - }; - - const inode = Inode.readFromEntry(alloc, archive, ent) catch |res_err| { - err.* = res_err; - fin.finish(); - return; - }; - extractMultiThread(alloc, inode, archive, tables, new_path, options, pool, fin, err); -} - -/// Get a threaded data reader for a file inode. -fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive, tables: *Tables) !?ThreadedDataReader { - return switch (self.hdr.inode_type) { - .file => threadedReaderFromData(alloc, archive, tables, self.data.file), - .ext_file => threadedReaderFromData(alloc, archive, tables, self.data.ext_file), - else => error.NotRegularFile, - }; -} -fn threadedReaderFromData(alloc: std.mem.Allocator, archive: Archive, tables: *Tables, data: anytype) !?ThreadedDataReader { - if (data.block_sizes.len == 0 and data.frag_idx == 0xFFFFFFFF) return null; - var out: ThreadedDataReader = .init(alloc, archive, data.block_sizes, data.block_start, data.size); - if (data.frag_idx != 0xFFFFFFFF) - out.addFragment(try tables.frag_table.get(data.frag_idx), data.frag_block_offset); - return out; -} - -/// Creates the symlink described by the inode. -/// Sets metadata. -fn extractSymlink(self: Inode, path: []const u8) !void { - const target = switch (self.data) { - .symlink => |s| s.target, - .ext_symlink => |s| s.target, - else => unreachable, - }; - try std.fs.cwd().symLink(target, path, .{}); -} -/// Creates the device described by the inode. -/// Sets metadata. -fn extractDeviceAndIPC(self: Inode, alloc: std.mem.Allocator, tables: *Tables, path: []const u8, options: ExtractionOptions) !void { - var mode: u32 = undefined; - var dev: u32 = 0; - switch (self.data) { - .char_dev => |d| { - mode = std.posix.S.IFCHR; - dev = d.dev; - }, - .ext_char_dev => |d| { - mode = std.posix.S.IFCHR; - dev = d.dev; - }, - .block_dev => |d| { - mode = std.posix.S.IFBLK; - dev = d.dev; - }, - .ext_block_dev => |d| { - mode = std.posix.S.IFBLK; - dev = d.dev; - }, - .fifo, .ext_fifo => mode = std.posix.S.IFIFO, - .socket, .ext_socket => mode = std.posix.S.IFSOCK, - else => unreachable, - } - const res: std.os.linux.E = @enumFromInt(std.os.linux.mknod(@ptrCast(path), mode, dev)); - switch (res) { - .SUCCESS => {}, - .ACCES => return std.fs.Dir.MakeError.AccessDenied, - .DQUOT => return std.fs.Dir.MakeError.DiskQuota, - .EXIST => return std.fs.Dir.MakeError.PathAlreadyExists, - .FAULT, .NOENT => return std.fs.Dir.MakeError.BadPathName, - .LOOP => return std.fs.Dir.MakeError.SymLinkLoop, - .NAMETOOLONG => return std.fs.Dir.MakeError.NameTooLong, - .NOMEM => return std.fs.Dir.MakeError.SystemResources, - .NOSPC => return std.fs.Dir.MakeError.NoSpaceLeft, - .NOTDIR => return std.fs.Dir.MakeError.NotDir, - .PERM => return std.fs.Dir.MakeError.PermissionDenied, - .ROFS => return std.fs.Dir.MakeError.ReadOnlyFileSystem, - else => return blk: { - std.debug.print("unhandled mknod result: {}\n", .{res}); - break :blk std.fs.Dir.MakeError.Unexpected; - }, - } - var fil = try std.fs.cwd().openFile(path, .{}); - defer fil.close(); - try self.setMetadata(alloc, tables, fil, options); -} diff --git a/src/util/inode_finish.zig b/src/util/inode_finish.zig deleted file mode 100644 index c16dfd2..0000000 --- a/src/util/inode_finish.zig +++ /dev/null @@ -1,101 +0,0 @@ -const std = @import("std"); -const WaitGroup = std.Thread.WaitGroup; -const Mutex = std.Thread.Mutex; - -const Archive = @import("../archive.zig"); -const Inode = @import("../inode.zig"); -const ExtractionOptions = @import("../options.zig"); -const Tables = @import("../tables.zig"); - -const InodeFinish = @This(); - -const FinishEnum = enum { - wg, - fin, -}; -pub const FinishUnion = union(FinishEnum) { - wg: *WaitGroup, - fin: *InodeFinish, - - pub fn finish(self: FinishUnion) void { - switch (self) { - .wg => |wg| wg.finish(), - .fin => |fin| fin.finish(), - } - } -}; - -alloc: std.mem.Allocator, - -inode: Inode, -path: []const u8, -tables: *Tables, -options: ExtractionOptions, -parent_finish: FinishUnion, -fil: ?std.fs.File, -out_err: *?anyerror, - -wg: WaitGroup = .{}, -mut: Mutex = .{}, - -pub fn create( - alloc: std.mem.Allocator, - inode: Inode, - path: []const u8, - tables: *Tables, - options: ExtractionOptions, - parent_finish: FinishUnion, - out_err: *?anyerror, - fil: ?std.fs.File, - work_size: usize, -) !*InodeFinish { - if (work_size == 0) - return error.InvalidWorkSize; - const out = try alloc.create(InodeFinish); - errdefer alloc.destroy(out); - out.* = .{ - .alloc = alloc, - - .inode = inode, - .path = path, - .tables = tables, - .options = options, - .parent_finish = parent_finish, - .out_err = out_err, - .fil = fil, - }; - out.wg.startMany(work_size); - return out; -} - -pub fn logError(self: *InodeFinish, comptime fmt: []const u8, args: anytype) void { - if (self.options.verbose) - self.options.verbose_writer.?.print(fmt, args) catch {}; -} - -pub fn finish(self: *InodeFinish) void { - self.mut.lock(); - { - defer self.mut.unlock(); - self.wg.finish(); - if (!self.wg.isDone()) return; - } - defer { - self.parent_finish.finish(); - self.alloc.destroy(self); - } - if (self.fil == null) - self.fil = std.fs.cwd().openFile(self.path, .{}) catch |err| { - if (self.options.verbose) - self.options.verbose_writer.?.print("Error opening {s} to set metadata: {}\n", .{ self.path, err }) catch {}; - self.out_err.* = err; - return; - }; - defer self.fil.?.close(); - self.inode.setMetadata(self.alloc, self.tables, self.fil.?, self.options) catch |err| { - if (self.options.verbose) - self.options.verbose_writer.?.print("Error setting metadata to {s}: {}\n", .{ self.path, err }) catch {}; - self.out_err.* = err; - return; - }; -} diff --git a/src/util/meta_cache.zig b/src/util/meta_cache.zig deleted file mode 100644 index e60288a..0000000 --- a/src/util/meta_cache.zig +++ /dev/null @@ -1,41 +0,0 @@ -const std = @import("std"); - -const OffsetFile = @import("offset_file.zig"); - -const MetadataCache = @This(); - -alloc: std.mem.Allocator, - -buf: []u8, -fixed_alloc: std.heap.FixedBufferAllocator, - -cache: std.AutoArrayHashMap(u64, [8192]u8), - -mut: std.Thread.Mutex = .{}, -cache_mut: std.AutoArrayHashMap(u64, std.Thread.Mutex), - -fil: OffsetFile, - -pub fn init(alloc: std.mem.Allocator, cache_size: u64) !MetadataCache {} -pub fn deinit(self: *MetadataCache) void { - self.mut.lock(); - defer self.mut.unlock(); - self.cache.deinit(); - self.cache_mut.deinit(); - self.alloc.free(self.buf); -} - -pub fn getChunk(self: *MetadataCache, offset: u64) ![8192]u8 { - var res = self.cache.get(offset); - if (res != null) return res.?; - var offset_mut = blk: { - self.mut.lock(); - defer self.mut.unlock(); - const mut = try self.cache_mut.getOrPut(offset); - if (!mut.found_existing) - mut.value_ptr.* = .{}; - break :blk mut.value_ptr; - }; - offset_mut.lock(); - defer offset_mut.unlock(); -} diff --git a/src/util/offset_file.zig b/src/util/offset_file.zig index b03c1b2..2935a47 100644 --- a/src/util/offset_file.zig +++ b/src/util/offset_file.zig @@ -1,8 +1,8 @@ //! A File where it's meaningful (to us) content starts at a given offset. const std = @import("std"); -const File = std.fs.File; -const Reader = std.fs.File.Reader; +const File = std.Io.File; +const Reader = File.Reader; const OffsetFile = @This(); @@ -13,8 +13,8 @@ pub fn init(fil: File, init_offset: u64) OffsetFile { return .{ .fil = fil, .offset = init_offset }; } -pub fn readerAt(self: OffsetFile, offset: u64, buffer: []u8) !Reader { - var rdr = self.fil.reader(buffer); +pub fn readerAt(self: OffsetFile, io: std.Io, offset: u64, buffer: []u8) !Reader { + var rdr = self.fil.reader(io, buffer); try rdr.seekTo(self.offset + offset); return rdr; } diff --git a/src/xattr.zig b/src/xattr.zig deleted file mode 100644 index 3f0cc0d..0000000 --- a/src/xattr.zig +++ /dev/null @@ -1,117 +0,0 @@ -const std = @import("std"); - -const Decompressor = @import("decomp.zig"); -const Table = @import("tables.zig").Table; -const MetadataReader = @import("util/metadata.zig"); -const OffsetFile = @import("util/offset_file.zig"); - -const Ref = packed struct { - block_offset: u16, - block_start: u32, - _: u16, -}; -const Entry = packed struct { - ref: Ref, - count: u32, - size: u32, -}; -const KeyPrefix = enum(u8) { - user, - trusted, - security, -}; -const KeyRaw = packed struct { - type: packed struct { - prefix: KeyPrefix, - out_of_line: bool, - _: u7, - }, - name_size: u16, -}; - -pub const KeyValue = struct { - key: [:0]u8, - value: []u8, -}; - -const XattrTable = @This(); - -alloc: std.mem.Allocator, -fil: OffsetFile, -decomp: *Decompressor, - -count: u32, -start: u64, - -table: Table(Entry), - -pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, table_start: u64) !XattrTable { - var info = packed struct { - start: u64 = undefined, - count: u32 = undefined, - _: u32 = undefined, - }{}; - var rdr = try fil.readerAt(table_start, &[0]u8{}); - try rdr.interface.readSliceEndian(@TypeOf(info), @ptrCast(&info), .little); - return .{ - .alloc = alloc, - .fil = fil, - .decomp = decomp, - .count = info.count, - .start = info.start, - .table = try .init(alloc, fil, decomp, table_start + 16, info.count), - }; -} -pub fn deinit(self: *XattrTable) void { - self.table.deinit(); -} - -pub fn get(self: *XattrTable, alloc: std.mem.Allocator, idx: u32) ![]KeyValue { - const entry: Entry = try self.table.get(idx); - const out = try alloc.alloc(KeyValue, entry.count); - - for (out) |*kv| { - var rdr = try self.fil.readerAt(self.start + entry.ref.block_start, &[0]u8{}); - var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp); - try meta.interface.discardAll(entry.ref.block_offset); - - var key_raw: KeyRaw = undefined; - try meta.interface.readSliceEndian(KeyRaw, @ptrCast(&key_raw), .little); - - switch (key_raw.type.prefix) { - .user => { - kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 5 + 1)); - @memcpy(kv.key[0..5], "user."); - try meta.interface.readSliceAll(kv.key[5 .. kv.key.len - 1]); - kv.key[kv.key.len - 1] = 0; - }, - .security => { - kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 9 + 1)); - @memcpy(kv.key[0..9], "security."); - try meta.interface.readSliceAll(kv.key[9 .. kv.key.len - 1]); - kv.key[kv.key.len - 1] = 0; - }, - .trusted => { - kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 8 + 1)); - @memcpy(kv.key[0..8], "trusted."); - try meta.interface.readSliceAll(kv.key[8 .. kv.key.len - 1]); - kv.key[kv.key.len - 1] = 0; - }, - } - if (key_raw.type.out_of_line) { - try meta.interface.discardAll(4); - var ref: Ref = undefined; - try meta.interface.readSliceEndian(Ref, @ptrCast(&ref), .little); - - rdr = try self.fil.readerAt(self.start + ref.block_start, &[0]u8{}); - meta = .init(alloc, &rdr.interface, self.decomp); - try meta.interface.discardAll(ref.block_offset); - } - var value_size: u32 = undefined; - try meta.interface.readSliceEndian(u32, @ptrCast(&value_size), .little); - kv.value = try alloc.alloc(u8, value_size); - try meta.interface.readSliceAll(kv.value); - } - - return out; -}