Fixed some bugs. Caused some bugs

Added DataExtract
Added LookupTable

.github/workflows/release-build.yml

@@ -13,13 +13,13 @@ jobs:
         uses: actions/checkout@v6
       - uses: mlugg/setup-zig@v2
       - name: Install deps
-        run: sudo apt update && sudo apt install -y zlib1g-dev libzstd-dev liblzma-dev liblz4-dev liblzo2-dev
+        run: sudo apt update && sudo apt install -y liblzma-dev liblzo2-dev
       - name: Build normal version
-        run: zig build --release=fast -Dversion=${{ github.ref_name }}
-      - name: Move normal build out
-        run: mv zig-out/bin/unsquashfs ./unsquashfs-x86_64
+        run: zig build --release=fast -Duse_zig_decomp=true -Dversion=${{ github.ref_name }}
+      - name: Move zig build out
+        run: mv zig-out/bin/unsquashfs ./unsquashfs-x86_64-zig-libs
       - name: Rebuild with C libraries
-        run: zig build --release=fast -Duse_c_libs=true -Dversion="${{ github.ref_name }}"
+        run: zig build --release=fast -Dversion="${{ github.ref_name }}"
       - name: Move C build out
         run: mv zig-out/bin/unsquashfs ./unsquashfs-x86_64-c-libs
       - name: Release
@@ -27,5 +27,5 @@ jobs:
         with:
           prerelease: true
           files: |
-            unsquashfs-x86_64
+            unsquashfs-x86_64-zig-libs
             unsquashfs-x86_64-c-libs

.gitignore

@@ -2,3 +2,4 @@ testing/
 
 .zig-cache/
 zig-out/
+zig-pkg/

.zed/debug.json

@@ -0,0 +1,25 @@
+// Project-local debug tasks
+//
+// For more documentation on how to configure debug tasks,
+// see: https://zed.dev/docs/debugger
+[
+	{
+		"label": "Build & Run",
+
+		"adapter": "CodeLLDB",
+		"request": "launch",
+
+		"build": {
+			"command": "zig",
+			"args": ["build", "-Duse_c_libs=true", "-Ddebug=true"],
+		},
+
+		"program": "zig-out/bin/unsquashfs",
+		"args": [
+			"--force",
+			"-d",
+			"testing/TestExtractUnsquashfs",
+			"testing/LinuxPATest.sfs",
+		],
+	},
+]

.zed/settings.json

@@ -0,0 +1,18 @@
+// Folder-specific settings
+//
+// For a full list of overridable settings, and general information on folder-specific settings,
+// see the documentation: https://zed.dev/docs/configuring-zed#settings-files
+{
+	"lsp": {
+		"zls": {
+			"initialization_options": {
+				"usePlaceholders": false,
+			},
+			"settings": {
+				"build_on_save": true,
+				"use_placeholders": false,
+				"build_on_save_args": ["-fincremental", "-Dallow_lzo=true"],
+			},
+		},
+	},
+}

README.md

@@ -18,9 +18,9 @@ Instead of using Zig's standard library for decompression, use the system's C li
 
 Enable compiling with LZO decompression support. The LZO library currently has some issues with Zig when imported so it's easier to just disable it by default. Only has an effect when using `-Duse_c_libs=true`.
 
-> `-Dvalgrind=true`
+> `-Ddebug=true`
 
-Just sets the valgrind build option.
+Sets various build options that make debugging easier. Specifically, debug optimization is forced, valgrind support is enabled, error tracing is enabled, stipping is disabled, and copmilation uses LLVM (this is due to some linking issues when on Debug optimization and is required for debugging tools such as `lldb`. In the future this may be removed from the debug flag).
 
 > `-Dversion=0.0.0`
 
@@ -30,7 +30,6 @@ Sets the version of `unsquashfs` shown when `--version` is passed.
 
 Most features are present except for the following:
 
-* xattrs are not applied on extraction
 * When using Zig decompression libraries then lzo and lz4 compression types are unavailable. I don't _currently_ plan on spending the time to find and validate a library since neither is popular.
 * When using C decompression libraries, lzo is not supported by default due to [some issues](#build-considerations). If it's needed it's trivial to fix, but it's easiest to just leave it disabled.
 
@@ -38,17 +37,20 @@ Most features are present except for the following:
 
 This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive (which is fairly small and uses zstd compression) and with `--release=fast`.
 
-* Under ideal circumstances, my library is ~70% slower (.11s vs .18s)
-* Mutli-threading on small archives noticably increases extraction times (when using C libraries) (.18s vs .57s). This should theoretically reverse on larger archives with many inodes, but I haven't tested that yet.
-* Using Zig libraries *significantly* increases decompression time by ~600% under ideal circumstances.
+Currently, my only performance checks are checking execution time, nothing deeper.
 
-Times:
+* Under ideal circumstances, my library is ~70% slower (.12s vs .20s).
+* Using Zig decompression libraries *significantly* increases decompression time by ~600%. Under ideal circumstances.
+* Performance improvements/regressions will be common. I'm still learning Zig.
 
-* *unsquashfs*: .11s
-* *C-libs, single-threaded*: .18s
-* *C-libs, multi-threaded*: .57s
-* *Zig-libs, single-threaded*: 5.87s
-* *Zig-libs, multi-threaded*: 1.10s
+Example Times:
+
+* *unsquashfs, multi-threaded*: .12s
+* *unsquashfs, single-threaded*: .13s
+* *C-libs, single-threaded*: .45s
+* *C-libs, multi-threaded*: .20s
+* *Zig-libs, single-threaded*: 5.78s
+* *Zig-libs, multi-threaded*: 1.08s
 
 ## Build considerations
 

build.zig

@@ -1,89 +1,173 @@
 const std = @import("std");
+const Build = std.Build;
+const Step = Build.Step;
 
 pub fn build(b: *std.Build) !void {
-    const use_c_libs_option = b.option(bool, "use_c_libs", "Use C versions of decompression libraries instead of the Zig standard library ones");
-    const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support");
-    const valgrind = b.option(bool, "valgrind", "Compile with valgrind integration");
-    const version_string_option = b.option([]const u8, "version", "Version of the library/binary");
-
-    const zig_squashfs_options = b.addOptions();
-    zig_squashfs_options.addOption(bool, "use_c_libs", use_c_libs_option orelse false);
-    zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo orelse false);
+    const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
+    const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
+    var debug = b.option(bool, "debug", "Enable options to make debugging easier.") orelse false;
+    const dynamic = b.option(bool, "dynamic", "Use dynamic linking for C libraries (if used).") orelse false;
+    var version_string = b.option([]const u8, "version", "Version of the library/binary") orelse "0.0.0-testing";
 
     const target = b.standardTargetOptions(.{});
-    const optimize = b.standardOptimizeOption(.{});
-    const mod = b.addModule("zig_squashfs", .{
-        .root_source_file = b.path("src/root.zig"),
-        .target = target,
-        .optimize = optimize,
-        .link_libc = use_c_libs_option,
-        .valgrind = valgrind,
-    });
-    mod.addOptions("config", zig_squashfs_options);
-    if (use_c_libs_option == true) {
-        mod.linkSystemLibrary("zlib", .{});
-        mod.linkSystemLibrary("lzma", .{});
-        if (allow_lzo == true)
-            mod.linkSystemLibrary("minilzo", .{});
-        mod.linkSystemLibrary("lz4", .{});
-        mod.linkSystemLibrary("zstd", .{});
-    }
+    var optimize = b.standardOptimizeOption(.{});
 
-    var version = version_string_option orelse "0.0.0-testing";
-    if (version[0] == 'v') version = version[1..];
+    const zig_squashfs_options = b.addOptions();
+    zig_squashfs_options.addOption(bool, "use_zig_decomp", use_zig_decomp);
+    zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo);
+
+    version_string = std.mem.trimStart(u8, version_string, "v");
+    const version = try std.SemanticVersion.parse(version_string);
     const unsquashfs_options = b.addOptions();
     unsquashfs_options.addOption(
         std.SemanticVersion,
         "version",
-        try std.SemanticVersion.parse(version),
+        version,
     );
 
-    var exe_mod = b.createModule(.{
+    if (debug) optimize = .Debug;
+    if (optimize == .Debug) debug = true;
+
+    const c_import = b.addTranslateC(.{
+        .root_source_file = b.path("src/c.h"),
+        .target = target,
+        .optimize = optimize,
+    });
+    if (allow_lzo) c_import.defineCMacro("ALLOW_LZO", null);
+    if (dynamic) {
+        c_import.linkSystemLibrary("zlib-ng", .{});
+        c_import.linkSystemLibrary("lzma", .{});
+        if (allow_lzo)
+            c_import.linkSystemLibrary("minilzo", .{});
+        c_import.linkSystemLibrary("lz4", .{});
+        c_import.linkSystemLibrary("zstd", .{});
+    }
+
+    var lib = b.addLibrary(.{
+        .name = "squashfs",
+        .root_module = b.addModule("squashfs", .{
+            .root_source_file = b.path("src/root.zig"),
+            .target = target,
+            .optimize = optimize,
+            .valgrind = debug,
+            .error_tracing = debug,
+            .strip = !debug,
+            .imports = &.{
+                .{ .name = "config", .module = zig_squashfs_options.createModule() },
+                .{ .name = "c", .module = c_import.createModule() },
+            },
+        }),
+        .use_llvm = debug,
+        .version = version,
+    });
+
+    const deps = try getDependencies(b, target, optimize, allow_lzo, dynamic);
+
+    for (deps) |d|
+        lib.root_module.linkLibrary(d);
+
+    const exe = b.addExecutable(.{
+        .name = "unsquashfs",
+        .root_module = b.createModule(.{
             .root_source_file = b.path("src/bin/unsquashfs.zig"),
             .target = target,
             .optimize = optimize,
-        .link_libc = use_c_libs_option,
             .imports = &.{
-            .{ .name = "zig_squashfs", .module = mod },
+                .{ .name = "config", .module = unsquashfs_options.createModule() },
+                .{ .name = "squashfs", .module = lib.root_module },
             },
-        .valgrind = valgrind,
-    });
-    exe_mod.addOptions("config", unsquashfs_options);
-    const exe = b.addExecutable(.{
-        .name = "unsquashfs",
-        .root_module = exe_mod,
-    });
-
-    const lib = b.addLibrary(.{
-        .name = "squashfs",
-        .root_module = mod,
+            .valgrind = debug,
+            .error_tracing = debug,
+            .strip = !debug,
+        }),
+        .use_llvm = debug,
+        .version = version,
     });
 
     b.installArtifact(lib);
     b.installArtifact(exe);
 
     const mod_tests = b.addTest(.{
-        .root_module = mod,
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("src/root.zig"),
+            .target = target,
+            .optimize = optimize,
+            .valgrind = true,
+            .error_tracing = true,
+            .strip = false,
+            .imports = &.{
+                .{ .name = "config", .module = zig_squashfs_options.createModule() },
+                .{ .name = "c", .module = c_import.createModule() },
+            },
+        }),
+        .use_llvm = debug, // Helps with lldb degugging
     });
+
+    for (deps) |d|
+        mod_tests.root_module.linkLibrary(d);
+
+    if (dynamic) {
+        mod_tests.root_module.linkSystemLibrary("zlib-ng", .{});
+        mod_tests.root_module.linkSystemLibrary("lzma", .{});
+        mod_tests.root_module.linkSystemLibrary("minilzo", .{});
+        mod_tests.root_module.linkSystemLibrary("lz4", .{});
+    }
+
     const run_mod_tests = b.addRunArtifact(mod_tests);
-    const exe_tests = b.addTest(.{
-        .root_module = exe.root_module,
-    });
-    const run_exe_tests = b.addRunArtifact(exe_tests);
     const test_step = b.step("test", "Run tests");
     test_step.dependOn(&run_mod_tests.step);
-    test_step.dependOn(&run_exe_tests.step);
 
     // zls build check steps
     const lib_check = b.addLibrary(.{
         .name = "squashfs",
-        .root_module = mod,
+        .root_module = lib.root_module,
     });
     const exe_check = b.addExecutable(.{
         .name = "unsquashfs",
-        .root_module = exe_mod,
+        .root_module = exe.root_module,
     });
     const check = b.step("check", "Check if unsquashfs compiles");
     check.dependOn(&lib_check.step);
     check.dependOn(&exe_check.step);
 }
+
+fn getDependencies(b: *Build, target: Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, allow_lzo: bool, dynamic: bool) ![]*Step.Compile {
+    if (dynamic) return &.{};
+
+    var list: std.ArrayList(*Step.Compile) = .empty;
+    errdefer list.clearAndFree(b.allocator);
+
+    var zlib_ng = b.dependency("zlib_ng", .{
+        .target = target,
+        .optimize = optimize,
+    });
+    try list.append(b.allocator, zlib_ng.artifact("zng"));
+
+    var xz = b.dependency("xz", .{
+        .target = target,
+        .optimize = optimize,
+    });
+    try list.append(b.allocator, xz.artifact("lzma"));
+
+    if (allow_lzo) {
+        var minilzo = b.dependency("minilzo", .{
+            .target = target,
+            .optimize = optimize,
+        });
+        try list.append(b.allocator, minilzo.artifact("minilzo"));
+    }
+
+    var lz4 = b.dependency("lz4", .{
+        .target = target,
+        .optimize = optimize,
+    });
+    try list.append(b.allocator, lz4.artifact("lz4"));
+
+    var zstd = b.dependency("zstd", .{
+        .target = target,
+        .optimize = optimize,
+    });
+    try list.append(b.allocator, zstd.artifact("zstd"));
+
+    return list.toOwnedSlice(b.allocator);
+}

build.zig.zon

@@ -1,43 +1,29 @@
 .{
     .name = .squashfs,
-    .version = "0.0.1",
+    .version = "0.0.6",
     .fingerprint = 0x37ba29474b87f145, // Changing this has security and trust implications.
-    .minimum_zig_version = "0.15.2",
-    // This field is optional.
-    // Each dependency must either provide a `url` and `hash`, or a `path`.
-    // `zig build --fetch` can be used to fetch all dependencies of a package, recursively.
-    // Once all dependencies are fetched, `zig build` no longer requires
-    // internet connectivity.
+    .minimum_zig_version = "0.16.1",
     .dependencies = .{
-        // See `zig fetch --save <url>` for a command-line interface for adding dependencies.
-        //.example = .{
-        //    // When updating this field to a new URL, be sure to delete the corresponding
-        //    // `hash`, otherwise you are communicating that you expect to find the old hash at
-        //    // the new URL. If the contents of a URL change this will result in a hash mismatch
-        //    // which will prevent zig from using it.
-        //    .url = "https://example.com/foo.tar.gz",
-        //
-        //    // This is computed from the file contents of the directory of files that is
-        //    // obtained after fetching `url` and applying the inclusion rules given by
-        //    // `paths`.
-        //    //
-        //    // This field is the source of truth; packages do not come from a `url`; they
-        //    // come from a `hash`. `url` is just one of many possible mirrors for how to
-        //    // obtain a package matching this `hash`.
-        //    //
-        //    // Uses the [multihash](https://multiformats.io/multihash/) format.
-        //    .hash = "...",
-        //
-        //    // When this is provided, the package is found in a directory relative to the
-        //    // build root. In this case the package's hash is irrelevant and therefore not
-        //    // computed. This field and `url` are mutually exclusive.
-        //    .path = "foo",
-        //
-        //    // When this is set to `true`, a package is declared to be lazily
-        //    // fetched. This makes the dependency only get fetched if it is
-        //    // actually used.
-        //    .lazy = false,
-        //},
+        .zlib_ng = .{
+            .url = "git+https://github.com/CalebQ42/zig-zlib-ng#5f2f02dfb28acca2517dacbbd09e9b987f57b133",
+            .hash = "zlib_ng-2.3.3-pre1-2HYS4ClFAABW8KlHMyBHtlNKE3V7kCS8wqfxawG7xeaa",
+        },
+        .zstd = .{
+            .url = "git+https://github.com/allyourcodebase/zstd.git?ref=1.5.7-1#e1a501be57f42c541e8a5597e4b59a074dfd09a3",
+            .hash = "zstd-1.5.7-1-KEItkAMwAAD6OKY3m0OOmXG7aL-aLUfrDqbP5J5oYapU",
+        },
+        .lz4 = .{
+            .url = "git+https://github.com/allyourcodebase/lz4.git?ref=1.10.0-6#41f52ab227caf9d48cf88c89a4d2946caa12b102",
+            .hash = "lz4-1.10.0-6-ewyzw-4NAAAWDpY4xpiqr4LQhZQAC0x_rGnW2iPh6jk2",
+        },
+        .minilzo = .{
+            .url = "git+https://github.com/CalebQ42/zig-minilzo.git#7cbae997b91a44d74b7cd6c073584dc9562a6c90",
+            .hash = "minilzo-2.10.0-Ij7BO8wLAADeWI4Pe4jp8XTDsDaquZR14oZ7_9yKKDWP",
+        },
+        .xz = .{
+            .url = "git+https://github.com/akunaakwei/zig-xz.git#e2d389262c8291907e3e4c6fb119819141c16c0f",
+            .hash = "xz-5.8.2-6v47_JYeAABSL-jonprpL5-E_YaaGc4B5xrbe93WsJ3G",
+        },
     },
     .paths = .{
         "build.zig",

mise.toml

@@ -0,0 +1,2 @@
+[tools]
+zig = "0.16.0"

src/archive.zig

@@ -1,107 +1,100 @@
-//! A squashfs archive read from a file.
-//! Can be used to directly access File's contents or extract to the filesystem.
-
 const std = @import("std");
-const File = std.fs.File;
-const builtin = @import("builtin");
+const Io = std.Io;
+const MemoryMap = Io.File.MemoryMap;
+
+const c = @import("c");
+const config = @import("config");
 
-const Decomp = @import("decomp.zig");
 const ExtractionOptions = @import("options.zig");
+const File = @import("file.zig");
 const Inode = @import("inode.zig");
-const InodeRef = Inode.Ref;
-const BlockSize = @import("inode_data/file.zig").BlockSize;
-const SfsFile = @import("file.zig");
 const Superblock = @import("super.zig").Superblock;
-const Table = @import("table.zig").Table;
-const MetadataReader = @import("util/metadata.zig");
-const OffsetFile = @import("util/offset_file.zig");
-
-const config = if (builtin.is_test) .{
-    .use_c_libs = true,
-    .allow_lzo = false,
-} else @import("config");
-
-/// Information about a fragment section. Multiple fragments are contained in the block described by a single FragEntry.
-/// The offset into the block and fragment size is stored in the file's inode.
-pub const FragEntry = packed struct {
-    start: u64,
-    size: BlockSize,
-    _: u32,
-};
+const DecompCache = @import("util/decomp_cache.zig");
+const CompressionType = @import("util/decompress.zig").CompressionType;
 
 const Archive = @This();
 
-alloc: std.mem.Allocator,
+const CACHE_MIN = 16 * 1024 * 1024;
+const CACHE_MAX = 1 * 1024 * 1024 * 1024;
 
-fil: OffsetFile,
-decomp: Decomp.DecompFn,
+cache: DecompCache,
 
 super: Superblock,
 
-frag_table: Table(FragEntry) = undefined,
-id_table: Table(u16) = undefined,
-export_table: Table(InodeRef) = undefined,
+/// Open a squashfs archive from an Io.File.
+pub fn init(alloc: std.mem.Allocator, io: Io, fil: Io.File) !Archive {
+    return initAdvanced(alloc, io, fil, 0, 0);
+}
+/// If max_cache_size is zero, a size is selected based on system ram, up to 1GB with a minimum of 16MB.
+pub fn initAdvanced(alloc: std.mem.Allocator, io: Io, file: Io.File, offset: u64, max_cache_size: u64) !Archive {
+    var rdr = file.reader(io, &[0]u8{});
+    try rdr.seekTo(offset);
 
-/// Default settings using std.Thread.getCpuCount() threads and the minimum of 4gb or half of system memory for memory usage.
-pub fn init(alloc: std.mem.Allocator, fil: File, offset: u64) !Archive {
     var super: Superblock = undefined;
-    const red = try fil.pread(@ptrCast(&super), offset);
-    std.debug.assert(red == @sizeOf(Superblock));
+    try rdr.interface.readSliceEndian(Superblock, @ptrCast(&super), .little);
     try super.validate();
-    const off_fil: OffsetFile = .init(fil, offset);
-    const decomp: Decomp.DecompFn = switch (super.compression) {
-        .gzip => Decomp.gzipDecompress,
-        .lzma => Decomp.lzmaDecompress,
-        .xz => Decomp.xzDecompress,
-        .zstd => Decomp.zstdDecompress,
-        .lz4 => if (config.use_c_libs) Decomp.cLz4 else return error.Lz4Unsupported,
-        .lzo => if (config.use_c_libs and config.allow_lzo) Decomp.lzoDecompress else return error.LzoUnsupported,
+
+    if (!config.use_zig_decomp and config.allow_lzo)
+        _ = c.lzo_init();
+
+    const cache_size = blk: {
+        if (max_cache_size > CACHE_MIN) break :blk CACHE_MIN;
+        const sys_mem = std.process.totalSystemMemory() catch break :blk CACHE_MIN;
+        var min = @min(CACHE_MAX, sys_mem / 4);
+        if (min < CACHE_MIN and sys_mem > CACHE_MIN)
+            min = CACHE_MIN;
+        break :blk min;
     };
     return .{
-        .alloc = alloc,
-
-        .fil = off_fil,
-        .decomp = decomp,
+        .cache = try .init(
+            alloc,
+            try file.createMemoryMap(
+                io,
+                .{
+                    .offset = offset,
+                    .len = super.size,
+                    .protection = .{ .read = true },
+                },
+            ),
+            super.compression,
+            cache_size,
+        ),
 
         .super = super,
-
-        .frag_table = try .init(alloc, off_fil, decomp, super.frag_start, super.frag_count),
-        .id_table = try .init(alloc, off_fil, decomp, super.id_start, super.id_count),
-        .export_table = try .init(alloc, off_fil, decomp, super.export_start, super.inode_count),
     };
 }
-pub fn deinit(self: *Archive) void {
-    self.frag_table.deinit();
-    self.export_table.deinit();
-    self.id_table.deinit();
+pub fn deinit(self: *Archive, io: Io) void {
+    self.cache.deinit(io);
 }
 
-pub fn inode(self: *Archive, alloc: std.mem.Allocator, num: u32) !Inode {
-    const ref = try self.export_table.get(num - 1);
-    var rdr = try self.fil.readerAt(ref.block_start + self.super.inode_start, &[0]u8{});
-    var meta: MetadataReader = .init(alloc, &rdr.interface, &self.decomp);
-    try meta.interface.discardAll(ref.block_offset);
-    return try .read(alloc, &meta.interface, self.super.block_size);
+pub fn root(self: *Archive, alloc: std.mem.Allocator, io: Io) !File {
+    return .fromRef(alloc, io, self, "", self.super.root_ref);
 }
 
-pub fn root(self: *Archive, alloc: std.mem.Allocator) !SfsFile {
-    var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{});
-    var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
-    try meta.interface.discardAll(self.super.root_ref.block_offset);
-    const in: Inode = try .read(alloc, &meta.interface, self.super.block_size);
-    return .init(self, in, "");
+pub fn open(self: *Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
+    const path = std.mem.trim(u8, filepath, "/");
+
+    var root_file = try self.root(alloc, io);
+
+    if (path.len == 0 or std.mem.eql(u8, path, ".")) return root_file;
+    defer root_file.deinit();
+
+    return root_file.open(alloc, io, path);
 }
 
-pub fn open(self: *Archive, alloc: std.mem.Allocator, path: []const u8) !SfsFile {
-    var root_fil = try self.root(alloc);
-    defer if (!SfsFile.pathIsSelf(path)) root_fil.deinit();
-    return root_fil.open(path);
-}
-
-pub fn extract(self: *Archive, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void {
-    var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{});
-    var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
-    try meta.interface.discardAll(self.super.root_ref.block_offset);
-    const in: Inode = try .read(self.alloc, &meta.interface, self.super.block_size);
-    try in.extractTo(alloc, self, path, options);
+pub fn extract(self: *Archive, alloc: std.mem.Allocator, io: Io, ext_dir: []const u8, options: ExtractionOptions) !void {
+    const root_inode: Inode = try .fromRef(alloc, io, &self.cache, self.super.inode_start, self.super.block_size, self.super.root_ref);
+    return root_inode.extract(
+        alloc,
+        io,
+        &self.cache,
+        self.super.dir_start,
+        self.super.inode_start,
+        self.super.frag_start,
+        self.super.block_size,
+        self.super.id_start,
+        self.super.xattr_start,
+        ext_dir,
+        options,
+    );
 }

src/bin/unsquashfs.zig

@@ -1,9 +1,10 @@
 const std = @import("std");
+const Io = std.Io;
 const Writer = std.Io.Writer;
 const builtin = @import("builtin");
 
 const config = @import("config");
-const squashfs = @import("zig_squashfs");
+const squashfs = @import("squashfs");
 
 //TODO: Add more options
 const help_mgs =
@@ -14,10 +15,14 @@ const help_mgs =
     \\  -d <location>   Extract to the given location instead of "squashfs-root"
     \\
     \\  -o <offset>     Start reading the archive at the given offset.
+    \\  -dx             Don't set xattr values
+    \\  -dp             Don't set permissions (includes setting uid & gid owner)
     \\
     \\  -p <threads>    Specify how many threads to use. If no present or zero, the system's logical cores count is used.
     \\  -v              Verbose
     \\
+    \\  --force         Force extraction. If the destination already exists, it will be deleted.
+    \\
     \\  --help          Display this messages
     \\  --version       Display the version
     \\
@@ -30,37 +35,47 @@ var extLoc: []const u8 = "squashfs-root";
 var offset: u64 = 0;
 var threads: u32 = 0;
 var verbose: bool = false;
+var ignore_xattrs: bool = false;
+var ignore_permissions: bool = false;
+var force: bool = false;
 
-pub fn main() !void {
-    const alloc = std.heap.smp_allocator;
-    var stdout = std.fs.File.stdout();
-    var out = stdout.writer(&[0]u8{});
+pub fn main(init: std.process.Init) !void {
+    const alloc = init.gpa;
+    const io = init.io;
+
+    var stdout = Io.File.stdout();
+    var out = stdout.writer(io, &[0]u8{});
     defer out.interface.flush() catch {};
-    try handleArgs(alloc, &out.interface);
+
+    try handleArgs(&out.interface, init.minimal.args);
     if (archive.len == 0) {
         try out.interface.print("You must provide a squashfs archive\n", .{});
         try out.interface.print(help_mgs, .{});
         return;
     }
-    var fil: std.fs.File = try std.fs.cwd().openFile(archive, .{}); //TODO: Handle error gracefully.
-    defer fil.close();
-    var arc: squashfs.Archive = try .init(alloc, fil, offset); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
-    defer arc.deinit();
+
+    var fil: std.Io.File = try Io.Dir.cwd().openFile(io, archive, .{}); //TODO: Handle error gracefully.
+    defer fil.close(io);
+    var arc: squashfs.Archive = try .initAdvanced(alloc, io, fil, offset, 0); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
+    defer arc.deinit(io);
     const options: squashfs.ExtractionOptions = .{
-        .threads = if (threads == 0) try std.Thread.getCpuCount() else threads,
+        .single_threaded = threads == 1,
         .verbose = verbose,
         .verbose_writer = if (verbose) &out.interface else null,
+        .ignore_xattr = ignore_xattrs,
+        .ignore_permissions = ignore_permissions,
     };
-    try arc.extract(alloc, extLoc, options); //TODO: Handle error gracefully.
+    if (force)
+        try Io.Dir.cwd().deleteTree(io, extLoc);
+    try arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully.
 }
 
-fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
-    var args = try std.process.argsWithAllocator(alloc);
-    defer args.deinit();
-    _ = args.next(); // args[0] is the application launch command.
-    while (args.next()) |arg| {
+fn handleArgs(out: *Writer, args: std.process.Args) !void {
+    var arg_iter = args.iterate();
+    _ = arg_iter.next(); // args[0] is the application launch command.
+    while (arg_iter.next()) |arg| {
         if (std.mem.eql(u8, arg, "-o")) {
-            const nxt = args.next();
+            const nxt = arg_iter.next();
             if (nxt == null or nxt.?.len == 0) {
                 try out.print("-o must be followed by a number\n", .{});
                 return errors.InvalidArguments;
@@ -71,7 +86,7 @@ fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
             };
             continue;
         } else if (std.mem.eql(u8, arg, "-d")) {
-            const nxt = args.next();
+            const nxt = arg_iter.next();
             if (nxt == null or nxt.?.len == 0) {
                 try out.print("-d must be followed by a location\n", .{});
                 return errors.InvalidArguments;
@@ -79,7 +94,7 @@ fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
             extLoc = nxt.?;
             continue;
         } else if (std.mem.eql(u8, arg, "-p")) {
-            const nxt = args.next();
+            const nxt = arg_iter.next();
             if (nxt == null or nxt.?.len == 0) {
                 try out.print("-p must be followed by a number\n", .{});
                 return errors.InvalidArguments;
@@ -92,6 +107,15 @@ fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
         } else if (std.mem.eql(u8, arg, "-v")) {
             verbose = true;
             continue;
+        } else if (std.mem.eql(u8, arg, "-dx")) {
+            ignore_xattrs = true;
+            continue;
+        } else if (std.mem.eql(u8, arg, "-dp")) {
+            ignore_permissions = true;
+            continue;
+        } else if (std.mem.eql(u8, arg, "--force")) {
+            force = true;
+            continue;
         } else if (std.mem.eql(u8, arg, "--version")) {
             try out.print("zig-unsquashfs v", .{});
             try config.version.format(out);
@@ -111,3 +135,7 @@ fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
         archive = arg;
     }
 }
+
+test {
+    std.testing.refAllDecls(squashfs.Archive);
+}

src/c.h

@@ -0,0 +1,7 @@
+#ifdef ALLOW_LZO
+#include <lzo/minilzo.h>
+#endif
+#include <zlib-ng.h>
+#include <zstd.h>
+#include <lz4.h>
+#include <lzma.h>

src/decomp.zig

@@ -1,260 +0,0 @@
-//! Implementations for decompression.
-//! TODO: change to vtable interface to allow for shared decompressors for better performance/resource usage.
-
-const std = @import("std");
-const Reader = std.Io.Reader;
-const builtin = @import("builtin");
-
-const config = if (builtin.is_test) .{
-    .use_c_libs = builtin.link_libc == true,
-    .allow_lzo = false, // Change once LZO compilation is fixed
-} else @import("config");
-
-const c = @cImport({
-    @cInclude("zlib.h");
-    @cInclude("lzma.h");
-    @cInclude("lz4.h");
-    @cInclude("zstd.h");
-    @cInclude("zstd_errors.h");
-    if (config.allow_lzo)
-        @cInclude("lzo/minilzo.h");
-});
-
-pub const CompressionType = enum(u16) {
-    gzip = 1,
-    lzma,
-    lzo,
-    xz,
-    lz4,
-    zstd,
-};
-
-pub const DecompFn = *const fn (alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize; // TODO: replace anyerror to definitive error types.
-
-pub const gzipDecompress = if (config.use_c_libs) cGzip else zigGzip;
-
-fn zigGzip(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    var rdr: Reader = .fixed(in);
-    const buf = try alloc.alloc(u8, out.len);
-    defer alloc.free(buf);
-    var decomp = std.compress.flate.Decompress.init(&rdr, .zlib, buf);
-    return decomp.reader.readSliceShort(out);
-}
-fn cGzip(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    _ = alloc;
-    var out_len: usize = out.len;
-    const res = c.uncompress(out.ptr, &out_len, in.ptr, in.len);
-    return switch (res) {
-        c.Z_OK => out_len,
-        c.Z_MEM_ERROR => error.NotEnoughMemory,
-        c.Z_BUF_ERROR => error.OutBufferTooSmall,
-        c.Z_DATA_ERROR => error.BadData,
-        else => error.UnknownResult,
-    };
-}
-
-pub const lzmaDecompress = if (config.use_c_libs) cLzma else zigLzma;
-
-fn zigLzma(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    var rdr: Reader = .fixed(in);
-    var decomp = try std.compress.lzma.decompress(alloc, rdr.adaptToOldInterface());
-    return decomp.read(out);
-}
-fn cLzma(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    _ = alloc;
-    var stream: c.lzma_stream = .{
-        .next_in = in.ptr,
-        .avail_in = in.len,
-        .next_out = out.ptr,
-        .avail_out = out.len,
-    };
-    var res = c.lzma_alone_decoder(&stream, in.len * 2);
-    switch (res) {
-        c.LZMA_OK => {},
-        c.LZMA_MEM_ERROR => return error.LzmaMemoryError,
-        c.LZMA_PROG_ERROR => return error.LzmaProgramError,
-        else => return error.UnknownResult,
-    }
-    defer c.lzma_end(&stream);
-    while (res == c.LZMA_OK)
-        res = c.lzma_code(&stream, c.LZMA_RUN);
-    return switch (res) {
-        c.LZMA_STREAM_END => stream.total_out,
-        c.LZMA_MEM_ERROR => error.LzmaMemoryError,
-        c.LZMA_MEMLIMIT_ERROR => error.LzmaMemoryLimit,
-        c.LZMA_FORMAT_ERROR => error.LzmaBadFormat,
-        c.LZMA_DATA_ERROR => error.LzmaDataCorrupt,
-        c.LZMA_BUF_ERROR => error.LzmaCannotProgress,
-        c.LZMA_PROG_ERROR => error.LzmaProgramError,
-        else => error.UnknownResult,
-    };
-}
-
-// pub const lzoDecompress = if (config.use_c_libs) cLzo else zigLzo;
-
-// fn zigLzo(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-//     _ = alloc;
-//     _ = in;
-//     _ = out;
-//     return error.LzoUnsupported;
-// }
-pub fn cLzo(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    _ = alloc;
-    var res = c.lzo_init();
-    if (res != 0) return error.LzoInitFailed;
-    var out_len: usize = out.len;
-    res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &out_len, null);
-    return switch (res) {
-        c.LZO_E_OK => out_len,
-        c.LZO_E_ERROR => error.LzoError,
-        c.LZO_E_OUT_OF_MEMORY => error.LzoOutOfMemory,
-        c.LZO_E_NOT_COMPRESSIBLE => error.LzoNotCompressible,
-        c.LZO_E_INPUT_OVERRUN => error.LzoInputOverrun,
-        c.LZO_E_OUTPUT_OVERRUN => error.LzoOutputOverrun,
-        c.LZO_E_LOOKBEHIND_OVERRUN => error.LzoLookbehindOverrun,
-        c.LZO_E_EOF_NOT_FOUND => error.LzoEofNotFound,
-        c.LZO_E_INPUT_NOT_CONSUMED => error.LzoInputNotConsumed,
-        c.LZO_E_NOT_YET_IMPLEMENTED => error.LzoNotYetImplemented,
-        c.LZO_E_INVALID_ARGUMENT => error.LzoInvalidArgument,
-        c.LZO_E_INVALID_ALIGNMENT => error.LzoInvalidAlignment,
-        c.LZO_E_OUTPUT_NOT_CONSUMED => error.LzoOutputNotConsumed,
-        c.LZO_E_INTERNAL_ERROR => error.LzoInternalError,
-        else => error.UnknownResult,
-    };
-}
-
-pub const xzDecompress = if (config.use_c_libs) cXz else zigXz;
-
-fn zigXz(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    var rdr: Reader = .fixed(in);
-    var decomp = try std.compress.xz.decompress(alloc, rdr.adaptToOldInterface());
-    return decomp.read(out);
-}
-fn cXz(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    _ = alloc;
-    var stream: c.lzma_stream = .{
-        .next_in = in.ptr,
-        .avail_in = in.len,
-        .next_out = out.ptr,
-        .avail_out = out.len,
-    };
-    var res = c.lzma_stream_decoder(&stream, in.len * 2, 0);
-    switch (res) {
-        c.LZMA_OK => {},
-        c.LZMA_MEM_ERROR => return error.LzmaMemoryError,
-        c.LZMA_PROG_ERROR => return error.LzmaProgramError,
-        else => return error.UnknownResult,
-    }
-    defer c.lzma_end(&stream);
-    while (res == c.LZMA_OK)
-        res = c.lzma_code(&stream, c.LZMA_RUN);
-    return switch (res) {
-        c.LZMA_STREAM_END => stream.total_out,
-        c.LZMA_MEM_ERROR => error.LzmaMemoryError,
-        c.LZMA_MEMLIMIT_ERROR => error.LzmaMemoryLimit,
-        c.LZMA_FORMAT_ERROR => error.LzmaBadFormat,
-        c.LZMA_DATA_ERROR => error.LzmaDataCorrupt,
-        c.LZMA_BUF_ERROR => error.LzmaCannotProgress,
-        c.LZMA_PROG_ERROR => error.LzmaProgramError,
-        else => error.UnknownResult,
-    };
-}
-
-// pub const lz4Decompress = if (config.use_c_libs) cLz4 else zigLz4;
-
-// fn zigLz4(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-//     _ = alloc;
-//     _ = in;
-//     _ = out;
-//     return error.Lz4Unsupported;
-// }
-pub fn cLz4(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    _ = alloc;
-    const res = c.LZ4_decompress_safe(in.ptr, out.ptr, @intCast(in.len), @intCast(out.len));
-    if (res > 0) return @abs(res); // TODO: Find out what error values it can return.
-    return error.Lz4DecompressFailed;
-}
-
-pub const zstdDecompress = if (config.use_c_libs) cZstd else zigZstd;
-
-pub fn zigZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    var rdr: Reader = .fixed(in);
-    const buf = try alloc.alloc(u8, 1024 * 1024);
-    defer alloc.free(buf);
-    var decomp = std.compress.zstd.Decompress.init(&rdr, buf, .{});
-    return decomp.reader.readSliceShort(out) catch |err| {
-        return decomp.err orelse err;
-    };
-}
-fn cZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    _ = alloc;
-    const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
-    if (c.ZSTD_isError(res) == 0) return res;
-    return switch (c.ZSTD_getErrorCode(res)) {
-        c.ZSTD_error_prefix_unknown => cZstdError.PrefixUnknown,
-        c.ZSTD_error_version_unsupported => cZstdError.VersionUnsupported,
-        c.ZSTD_error_frameParameter_unsupported => cZstdError.FrameParameterUnsupported,
-        c.ZSTD_error_frameParameter_windowTooLarge => cZstdError.FrameParameterWindowTooLarge,
-        c.ZSTD_error_corruption_detected => cZstdError.CorruptionDetected,
-        c.ZSTD_error_checksum_wrong => cZstdError.ChecksumWrong,
-        c.ZSTD_error_literals_headerWrong => cZstdError.LiteralsHeaderWrong,
-        c.ZSTD_error_dictionary_corrupted => cZstdError.DictionaryCorrupted,
-        c.ZSTD_error_dictionary_wrong => cZstdError.DictionaryWrong,
-        c.ZSTD_error_dictionaryCreation_failed => cZstdError.DictionaryCreationFailed,
-        c.ZSTD_error_parameter_unsupported => cZstdError.ParameterUnsupported,
-        c.ZSTD_error_parameter_combination_unsupported => cZstdError.ParameterCombinationUnsupported,
-        c.ZSTD_error_parameter_outOfBound => cZstdError.ParameterOutOfBound,
-        c.ZSTD_error_tableLog_tooLarge => cZstdError.TableLogTooLarge,
-        c.ZSTD_error_maxSymbolValue_tooLarge => cZstdError.MaxSymbolValueTooLarge,
-        c.ZSTD_error_maxSymbolValue_tooSmall => cZstdError.MaxSymbolValueTooSmall,
-        c.ZSTD_error_stabilityCondition_notRespected => cZstdError.StabilityConditionNotRespected,
-        c.ZSTD_error_stage_wrong => cZstdError.StageWrong,
-        c.ZSTD_error_init_missing => cZstdError.InitMissing,
-        c.ZSTD_error_memory_allocation => cZstdError.MemoryAllocation,
-        c.ZSTD_error_workSpace_tooSmall => cZstdError.WorkSpaceTooSmall,
-        c.ZSTD_error_dstSize_tooSmall => cZstdError.DstSizeTooSmall,
-        c.ZSTD_error_srcSize_wrong => cZstdError.SrcSizeWrong,
-        c.ZSTD_error_dstBuffer_null => cZstdError.DstBufferNull,
-        c.ZSTD_error_noForwardProgress_destFull => cZstdError.NoForwardProgressDestFull,
-        c.ZSTD_error_noForwardProgress_inputEmpty => cZstdError.NoForwardProgressInputEmpty,
-        else => cZstdError.Generic,
-    };
-}
-
-pub const cZstdError = error{
-    Generic,
-    PrefixUnknown,
-    VersionUnsupported,
-    FrameParameterUnsupported,
-    FrameParameterWindowTooLarge,
-    CorruptionDetected,
-    ChecksumWrong,
-    LiteralsHeaderWrong,
-    DictionaryCorrupted,
-    DictionaryWrong,
-    DictionaryCreationFailed,
-    ParameterUnsupported,
-    ParameterCombinationUnsupported,
-    ParameterOutOfBound,
-    TableLogTooLarge,
-    MaxSymbolValueTooLarge,
-    MaxSymbolValueTooSmall,
-    CannotProduceUncompressedBlock,
-    StabilityConditionNotRespected,
-    StageWrong,
-    InitMissing,
-    MemoryAllocation,
-    WorkSpaceTooSmall,
-    DstSizeTooSmall,
-    SrcSizeWrong,
-    DstBufferNull,
-    NoForwardProgressDestFull,
-    NoForwardProgressInputEmpty,
-    FrameIndexTooLarge,
-    SeekableIo,
-    DstBufferWrong,
-    SrcBufferWrong,
-    SequenceProducerFailed,
-    ExternalSequencesInvalid,
-    MaxCode,
-};

src/dir_entry.zig

@@ -1,60 +1,67 @@
-//! Directory entry from the directory table.
-
 const std = @import("std");
 const Reader = std.Io.Reader;
 
-const InodeType = @import("inode.zig").InodeType;
+const Inode = @import("inode.zig");
 
-const Entry = @This();
-
-const Header = extern struct { // use extern due to bad alignment with packed.
+const Header = extern struct {
     count: u32,
     block_start: u32,
     num: u32,
 };
-
-const RawEntry = packed struct {
-    offset: u16,
-    inode_offset: i16,
-    inode_type: InodeType,
+const Entry = extern struct {
+    block_offset: u16,
+    num_offset: i16,
+    inode_type: Inode.Type,
     name_size: u16,
 };
 
-block_start: u32,
-block_offset: u16,
-num: u32,
-inode_type: InodeType,
+pub const Error = error{OutOfMemory} || std.Io.Reader.Error;
+
+const DirEntry = @This();
+
+inode_type: Inode.Type,
 name: []const u8,
 
-pub fn readDir(alloc: std.mem.Allocator, rdr: *Reader, size: u32) ![]Entry {
-    var cur_red: u32 = 3; // start at 3 due to "." & ".." being counted in the dir size.
-    var hdr: Header = undefined;
-    var raw: RawEntry = undefined;
-    var out: std.ArrayList(Entry) = try .initCapacity(alloc, 100); // Start out with a decent capacity instead of needing to allocate per header.
-    errdefer out.deinit(alloc);
-    while (cur_red < size) {
-        try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
-        cur_red += @sizeOf(Header);
-        try out.ensureUnusedCapacity(alloc, hdr.count + 1);
-        for (0..hdr.count + 1) |_| {
-            try rdr.readSliceEndian(RawEntry, @ptrCast(&raw), .little);
-            const name = try alloc.alloc(u8, raw.name_size + 1);
-            errdefer alloc.free(name);
-            try rdr.readSliceEndian(u8, name, .little);
-            const val = out.addOneAssumeCapacity();
-            val.* = .{
-                .block_start = hdr.block_start,
-                .block_offset = raw.offset,
-                .num = @abs(hdr.num + raw.offset),
-                .inode_type = raw.inode_type,
-                .name = name,
-            };
-            cur_red += @sizeOf(RawEntry) + raw.name_size + 1;
-        }
-    }
-    return out.toOwnedSlice(alloc);
-}
+block_start: u32,
+block_offset: u32,
+num: u32,
 
-pub fn deinit(self: Entry, alloc: std.mem.Allocator) void {
+pub fn deinit(self: DirEntry, alloc: std.mem.Allocator) void {
     alloc.free(self.name);
 }
+
+pub fn readEntries(alloc: std.mem.Allocator, rdr: *Reader, size: u32) Error![]DirEntry {
+    var out: std.ArrayList(DirEntry) = try .initCapacity(alloc, 50);
+    errdefer out.deinit(alloc);
+
+    var tot_read: u32 = 3;
+    while (tot_read < size) {
+        var hdr: Header = undefined;
+        try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
+        tot_read += @sizeOf(Header);
+
+        try out.ensureUnusedCapacity(alloc, hdr.count + 1);
+
+        for (0..hdr.count + 1) |_| {
+            var ent: Entry = undefined;
+            try rdr.readSliceEndian(Entry, @ptrCast(&ent), .little);
+            tot_read += @sizeOf(Entry) + ent.name_size + 1;
+
+            const name = try alloc.alloc(u8, ent.name_size + 1);
+            errdefer alloc.free(name);
+
+            try rdr.readSliceEndian(u8, name, .little);
+
+            out.appendAssumeCapacity(.{
+                .inode_type = ent.inode_type,
+                .name = name,
+
+                .block_offset = ent.block_offset,
+                .block_start = hdr.block_start,
+                .num = @intCast(@as(i64, @intCast(hdr.num)) + ent.num_offset),
+            });
+        }
+    }
+
+    return out.toOwnedSlice(alloc);
+}

src/file.zig

@@ -1,202 +1,126 @@
-//! A file/directory within the squashfs archive.
+//! A wrapper around an Inode to make common activities easier.
 
 const std = @import("std");
-const File = std.fs.File;
-const WaitGroup = std.Thread.WaitGroup;
-const Mutex = std.Thread.Mutex;
+const Io = std.Io;
 
 const Archive = @import("archive.zig");
 const DirEntry = @import("dir_entry.zig");
 const ExtractionOptions = @import("options.zig");
 const Inode = @import("inode.zig");
-const BlockSize = @import("inode_data/file.zig").BlockSize;
-const DataReader = @import("util/data.zig");
+const LookupTable = @import("lookup_table.zig");
 const MetadataReader = @import("util/metadata.zig");
 
-const FileError = error{
-    NotDirectory,
-    NotRegularFile,
-    NotSymlink,
-    NotDevice,
+pub const Error = error{
     NotFound,
-    ExtractionPathExists,
 };
 
-const SfsFile = @This();
+const File = @This();
 
+alloc: std.mem.Allocator,
 archive: *Archive,
 
-inode: Inode,
 name: []const u8,
+inode: Inode,
+
+pub fn fromEntry(alloc: std.mem.Allocator, io: Io, archive: *Archive, entry: DirEntry) !File {
+    var meta: MetadataReader = .init(io, &archive.cache, archive.super.inode_start + entry.block_start);
+    defer meta.deinit();
+    try meta.interface.discardAll(entry.block_offset);
+
+    const new_name = try alloc.alloc(u8, entry.name.len);
+    errdefer alloc.free(new_name);
+    @memcpy(new_name, entry.name);
 
-/// Initialize a new File.
-/// name is copied to the File so can be safely freed afterwards.
-pub fn init(archive: *Archive, inode: Inode, name: []const u8) !SfsFile {
-    const new_name = try archive.allocator().alloc(u8, name.len);
-    @memcpy(new_name, name);
     return .{
+        .alloc = alloc,
         .archive = archive,
-        .inode = inode,
+
+        .name = new_name,
+        .inode = try .fromReader(alloc, &meta.interface, archive.super.block_size),
+    };
+}
+/// Create a File from an Inode.Ref. name should be created using the alloc given.
+pub fn fromRef(alloc: std.mem.Allocator, io: Io, archive: *Archive, name: []const u8, ref: Inode.Ref) !File {
+    return .{
+        .alloc = alloc,
+        .archive = archive,
+
+        .name = name,
+        .inode = try .fromRef(
+            alloc,
+            io,
+            &archive.cache,
+            archive.super.inode_start,
+            archive.super.block_size,
+            ref,
+        ),
+    };
+}
+pub fn copy(alloc: std.mem.Allocator, from: File) !File {
+    const new_name = try alloc.alloc(u8, from.name.len);
+    errdefer alloc.free(new_name);
+    @memcpy(new_name, from.name);
+
+    return .{
+        .alloc = alloc,
+        .archive = from.archive,
+
+        .inode = try .copy(alloc, from.inode),
         .name = new_name,
     };
 }
-pub fn fromEntry(archive: *Archive, entry: DirEntry) !SfsFile {
-    var rdr = try archive.fil.readerAt(entry.block_start + archive.super.inode_start, &[0]u8{});
-    var meta: MetadataReader = .init(archive.allocator(), &rdr.interface, archive.decomp);
-    try meta.interface.discardAll(entry.block_offset);
-    const inode: Inode = try .read(archive.allocator(), &meta.interface, archive.super.block_size);
-    errdefer inode.deinit(archive.allocator());
-    return .init(archive, inode, entry.name);
+pub fn deinit(self: File) void {
+    self.alloc.free(self.name);
+    self.inode.deinit(self.alloc);
 }
 
-pub fn deinit(self: SfsFile) void {
-    var alloc = self.archive.allocator();
-    alloc.free(self.name);
-    self.inode.deinit(alloc);
-}
+pub fn open(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
+    const path = std.mem.trim(u8, filepath, "/");
 
-fn getEntries(self: SfsFile) ![]DirEntry {
-    return self.inode.dirEntries(self.archive.allocator(), self.archive.*);
-}
+    if (path.len == 0 or std.mem.eql(u8, path, ".")) return .copy(alloc, self);
 
-pub fn ownerUid(self: SfsFile) !u16 {
-    return self.archive.id(self.inode.hdr.uid_idx);
-}
-pub fn ownerGid(self: SfsFile) !u16 {
-    return self.archive.id(self.inode.hdr.gid_idx);
-}
-pub fn permissions(self: SfsFile) u16 {
-    return self.inode.hdr.permissions;
-}
+    const first_element = std.mem.sliceTo(path, '/');
 
-pub fn isRegular(self: SfsFile) bool {
-    return switch (self.inode.hdr.inode_type) {
-        .file, .ext_file => true,
-        else => false,
-    };
-}
-/// The returned DataReader will no longer work if the File's deinit function is called
-/// or, more specifically, it's inode's deinit function is called.
-pub fn dataReader(self: SfsFile) !DataReader {
-    return self.inode.dataReader(self.archive);
-}
-
-pub fn isDir(self: SfsFile) bool {
-    return switch (self.inode.hdr.inode_type) {
-        .dir, .ext_dir => true,
-        else => false,
-    };
-}
-pub fn iterate(self: SfsFile) !Iterator {
-    if (!self.isDir()) return FileError.NotDirectory;
-    return .{
-        .entries = try self.getEntries(),
-        .archive = self.archive,
-    };
-}
-/// Open a sub-file/folder within a directory at the given path.
-/// If path is "", ".", "/", or "./", this File is returned.
-pub fn open(self: SfsFile, alloc: std.mem.Allocator, path: []const u8) !SfsFile {
-    if (!self.isDir()) return FileError.NotDirectory;
-    if (pathIsSelf(path)) return self;
-
-    // Recursively stip ending & leading path separators.
-    if (path[0] == '/') return self.open(path[1..]);
-    if (path[path.len - 1] == '/') return self.open(path[0 .. path.len - 1]);
-
-    const idx = std.mem.indexOf(u8, path, "/") orelse path.len;
-    const first_element = path[0..idx];
-    if (std.mem.eql(u8, first_element, ".")) return self.open(path[idx + 1 ..]);
-    const entries = try self.getEntries();
+    const entries = try self.inode.readDirectory(alloc, io, &self.archive.cache, self.archive.super.dir_start);
     defer {
-        for (entries) |e| {
-            e.deinit(alloc);
-        }
+        for (entries) |entry|
+            entry.deinit(alloc);
         alloc.free(entries);
     }
-    var cur_slice = entries;
-    var split = cur_slice.len / 2;
-    while (cur_slice.len > 0) {
-        split = cur_slice.len / 2;
-        const comp = std.mem.order(u8, first_element, cur_slice[split].name);
-        switch (comp) {
-            .eq => {
-                var fil: SfsFile = try .fromEntry(self.archive, cur_slice[split]);
-                if (idx == path.len) {
-                    return fil;
+
+    // Potentially I could use linear searching on small dir tables...
+    var search_slice = entries;
+    var idx = search_slice.len / 2;
+    while (search_slice.len > 0) {
+        const order = std.mem.order(u8, first_element, search_slice[idx].name);
+        switch (order) {
+            .eq => break,
+            .gt => search_slice = search_slice[idx..],
+            .lt => search_slice = search_slice[0..idx],
         }
+        idx = search_slice.len / 2;
+    }
+    if (search_slice.len == 0) return Error.NotFound;
+
+    var fil: File = try .fromEntry(alloc, io, self.archive, search_slice[idx]);
+    if (path.len == first_element.len) return fil;
     defer fil.deinit();
-                return fil.open(alloc, path[idx + 1 ..]);
-            },
-            .lt => cur_slice = cur_slice[0..split],
-            .gt => cur_slice = cur_slice[split + 1 ..],
-        }
-    }
-    return FileError.NotFound;
+
+    return fil.open(alloc, io, filepath[first_element.len..]);
 }
 
-pub fn isSymlink(self: SfsFile) bool {
-    return switch (self.inode.hdr.inode_type) {
-        .symlink, .ext_symlink => true,
-        else => false,
-    };
+pub fn extract(self: File, alloc: std.mem.Allocator, io: Io, path: []const u8, options: ExtractionOptions) !void {
+    return self.inode.extract(
+        alloc,
+        io,
+        &self.archive.cache,
+        self.archive.super.dir_start,
+        self.archive.super.inode_start,
+        self.archive.super.frag_start,
+        self.archive.super.block_size,
+        self.archive.super.id_start,
+        self.archive.super.xattr_start,
+        path,
+        options,
+    );
 }
-pub fn symlinkPath(self: SfsFile) ![]const u8 {
-    if (!self.isSymlink()) return FileError.NotSymlink;
-    return switch (self.inode.data) {
-        .symlink => |s| s.target,
-        .ext_symlink => |s| s.target,
-        else => unreachable,
-    };
-}
-
-/// Check if the File is a block or character device.
-pub fn isDevice(self: SfsFile) bool {
-    return switch (self.inode.hdr.inode_type) {
-        .block_dev, .char_dev, .ext_block_dev, .ext_char_dev => true,
-        else => false,
-    };
-}
-/// If the File is a block or character device, get's it's device number.
-pub fn devNum(self: SfsFile) !u32 {
-    if (!self.isDevice()) return FileError.NotDevice;
-    return switch (self.inode.data) {
-        .block_dev, .char_dev => |d| d.dev,
-        .ext_block_dev, .ext_char_dev => |d| d.dev,
-        else => unreachable,
-    };
-}
-
-/// Extract the given File to the path. If File is a regular file, the path must be a directory or not exist.
-/// If the gievn path is a folder, the File's contents will be extracted within.
-pub fn extract(self: *SfsFile, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void {
-    return self.inode.extractToThreaded(alloc, self.archive, path, options);
-}
-
-/// Utility function.
-pub fn pathIsSelf(path: []const u8) bool {
-    if (path.len == 0) return true;
-    if (path.len == 1 and (path[0] == '/' or path[0] == '.')) return true;
-    if (path.len == 2 and (path[0] == '.' and path[1] == '/')) return true;
-    return false;
-}
-
-pub const Iterator = struct {
-    entries: []DirEntry,
-    archive: *Archive,
-
-    idx: u32 = 0,
-
-    pub fn next(self: *Iterator) !?SfsFile {
-        if (self.idx >= self.entries.len) return null;
-        defer self.idx += 1;
-        return try SfsFile.fromEntry(self.archive, self.entries[self.idx]);
-    }
-    pub fn deinit(self: Iterator) void {
-        var alloc = self.archive.allocator();
-        for (self.entries) |e| {
-            e.deinit(alloc);
-        }
-        alloc.free(self.entries);
-    }
-};

src/frag.zig

@@ -0,0 +1,7 @@
+const BlockSize = @import("inode_data/file.zig").BlockSize;
+
+pub const FragEntry = extern struct {
+    start: u64,
+    size: BlockSize,
+    _: u32,
+};

src/inode.zig

@@ -1,28 +1,27 @@
 //! A file-system object. Represents a File or directory.
 
 const std = @import("std");
-const Reader = std.Io.Reader;
-const WaitGroup = std.Thread.WaitGroup;
-const Pool = std.Thread.Pool;
-const Mutex = std.Thread.Mutex;
+const Io = std.Io;
+const Reader = Io.Reader;
 
-const Archive = @import("archive.zig");
 const DirEntry = @import("dir_entry.zig");
 const ExtractionOptions = @import("options.zig");
-const dir = @import("inode_data/dir.zig");
-const file = @import("inode_data/file.zig");
-const misc = @import("inode_data/misc.zig");
-const DataReader = @import("util/data.zig");
-const ThreadedDataReader = @import("util/data_threaded.zig");
+const FragEntry = @import("frag.zig").FragEntry;
+const DirTypes = @import("inode_data/dir.zig");
+const FileTypes = @import("inode_data/file.zig");
+const MiscTypes = @import("inode_data/misc.zig");
+const LookupTable = @import("lookup_table.zig");
+const DataExtract = @import("util/data_extract.zig");
+const DecompCache = @import("util/decomp_cache.zig");
 const MetadataReader = @import("util/metadata.zig");
 
-pub const Ref = packed struct {
+pub const Ref = packed struct(u64) {
     block_offset: u16,
     block_start: u32,
     _: u16,
 };
 
-pub const InodeType = enum(u16) {
+pub const Type = enum(u16) {
     dir = 1,
     file,
     symlink,
@@ -39,25 +38,25 @@ pub const InodeType = enum(u16) {
     ext_socket,
 };
 
-pub const InodeData = union(InodeType) {
-    dir: dir.Dir,
-    file: file.File,
-    symlink: misc.Symlink,
-    block_dev: misc.Dev,
-    char_dev: misc.Dev,
-    fifo: misc.IPC,
-    socket: misc.IPC,
-    ext_dir: dir.ExtDir,
-    ext_file: file.ExtFile,
-    ext_symlink: misc.ExtSymlink,
-    ext_block_dev: misc.ExtDev,
-    ext_char_dev: misc.ExtDev,
-    ext_fifo: misc.ExtIPC,
-    ext_socket: misc.ExtIPC,
+pub const Data = union(Type) {
+    dir: DirTypes.Dir,
+    file: FileTypes.File,
+    symlink: MiscTypes.Symlink,
+    block_dev: MiscTypes.Dev,
+    char_dev: MiscTypes.Dev,
+    fifo: MiscTypes.IPC,
+    socket: MiscTypes.IPC,
+    ext_dir: DirTypes.ExtDir,
+    ext_file: FileTypes.ExtFile,
+    ext_symlink: MiscTypes.ExtSymlink,
+    ext_block_dev: MiscTypes.ExtDev,
+    ext_char_dev: MiscTypes.ExtDev,
+    ext_fifo: MiscTypes.ExtIPC,
+    ext_socket: MiscTypes.ExtIPC,
 };
 
 pub const Header = packed struct {
-    inode_type: InodeType,
+    inode_type: Type,
     permissions: u16,
     uid_idx: u16,
     gid_idx: u16,
@@ -65,12 +64,22 @@ pub const Header = packed struct {
     num: u32,
 };
 
+pub const Error = error{
+    NotDirectory,
+};
+
 const Inode = @This();
 
 hdr: Header,
-data: InodeData,
+data: Data,
 
-pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode {
+pub fn fromRef(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, inode_start: u64, block_size: u32, ref: Ref) !Inode {
+    var meta: MetadataReader = .init(io, cache, ref.block_start + inode_start);
+    defer meta.deinit();
+    try meta.interface.discardAll(ref.block_offset);
+    return fromReader(alloc, &meta.interface, block_size);
+}
+pub fn fromReader(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode {
     var hdr: Header = undefined;
     try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
     return .{
@@ -93,13 +102,31 @@ pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode {
         },
     };
 }
-pub fn readFromEntry(alloc: std.mem.Allocator, archive: *Archive, entry: DirEntry) !Inode {
-    var rdr = try archive.fil.readerAt(archive.super.inode_start + entry.block_start, &[0]u8{});
-    var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp);
-    try meta.interface.discardAll(entry.block_offset);
-    return read(alloc, &meta.interface, archive.super.block_size);
+pub fn copy(alloc: std.mem.Allocator, from: Inode) !Inode {
+    var new = from;
+    switch (from.data) {
+        .file => |f| {
+            new.data.file.block_sizes = try alloc.alloc(FileTypes.BlockSize, f.block_sizes.len);
+            @memcpy(new.data.file.block_sizes, f.block_sizes);
+        },
+        .ext_file => |f| {
+            new.data.ext_file.block_sizes = try alloc.alloc(FileTypes.BlockSize, f.block_sizes.len);
+            @memcpy(new.data.ext_file.block_sizes, f.block_sizes);
+        },
+        .symlink => |s| {
+            const new_target = try alloc.alloc(u8, s.target.len);
+            @memcpy(new_target, s.target);
+            new.data.symlink.target = new_target;
+        },
+        .ext_symlink => |s| {
+            const new_target = try alloc.alloc(u8, s.target.len);
+            @memcpy(new_target, s.target);
+            new.data.ext_symlink.target = new_target;
+        },
+        else => {},
+    }
+    return new;
 }
-
 pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
     switch (self.data) {
         .file => |f| alloc.free(f.block_sizes),
@@ -110,430 +137,298 @@ pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
     }
 }
 
-/// Get the data reader for a file inode.
-pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: *Archive) !DataReader {
-    return switch (self.hdr.inode_type) {
-        .file => readerFromData(alloc, archive, self.data.file),
-        .ext_file => readerFromData(alloc, archive, self.data.ext_file),
-        else => error.NotRegularFile,
+pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, io: Io, cache: *DecompCache, dir_start: u64) ![]DirEntry {
+    return switch (self.data) {
+        .dir => |d| readDirectoryFromData(alloc, io, cache, dir_start, d),
+        .ext_dir => |d| readDirectoryFromData(alloc, io, cache, dir_start, d),
+        else => Error.NotDirectory,
     };
 }
-fn readerFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !DataReader {
-    var out: DataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size);
-    if (data.frag_idx != 0xFFFFFFFF)
-        out.addFragment(try archive.frag_table.get(data.frag_idx), data.frag_block_offset);
-    return out;
-}
-/// Get a threaded data reader for a file inode.
-pub fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: *Archive) !ThreadedDataReader {
-    return switch (self.hdr.inode_type) {
-        .file => threadedReaderFromData(alloc, archive, self.data.file),
-        .ext_file => threadedReaderFromData(alloc, archive, self.data.ext_file),
-        else => error.NotRegularFile,
-    };
-}
-fn threadedReaderFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !ThreadedDataReader {
-    var out: ThreadedDataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size);
-    if (data.frag_idx != 0xFFFFFFFF)
-        out.addFragment(try archive.frag_table.get(data.frag_idx), data.frag_block_offset);
-    return out;
+fn readDirectoryFromData(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, dir_start: u64, d: anytype) ![]DirEntry {
+    var meta: MetadataReader = .init(io, cache, dir_start + d.block_start);
+    defer meta.deinit();
+    try meta.interface.discardAll(d.block_offset);
+
+    return DirEntry.readEntries(alloc, &meta.interface, d.size);
 }
 
-/// Get the directory entries for a directory inode.
-pub fn dirEntries(self: Inode, alloc: std.mem.Allocator, archive: Archive) ![]DirEntry {
-    return switch (self.hdr.inode_type) {
-        .dir => entriesFromData(alloc, archive, self.data.dir),
-        .ext_dir => entriesFromData(alloc, archive, self.data.ext_dir),
-        else => error.NotDirectory,
-    };
-}
-fn entriesFromData(alloc: std.mem.Allocator, archive: Archive, data: anytype) ![]DirEntry {
-    var rdr = try archive.fil.readerAt(archive.super.dir_start + data.block_start, &[0]u8{});
-    var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp);
-    try meta.interface.discardAll(data.block_offset);
-    return DirEntry.readDir(alloc, &meta.interface, data.size);
-}
+// Extraction
 
-/// Extract the inode to the given path. Single threaded.
-pub fn extractTo(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void {
-    if (options.threads > 1) return self.extractToThreaded(alloc, archive, path, options);
-    switch (self.hdr.inode_type) {
-        .dir, .ext_dir => {
-            // Removing any trailing separators since that's the easiest path forward.
-            if (path[path.len - 1] == '/') return self.extractTo(alloc, archive, path[0 .. path.len - 1], options);
-            std.fs.cwd().makeDir(path) catch |err| {
-                if (err != std.fs.Dir.MakeError.PathAlreadyExists) return err;
-            };
-            const entries = try self.dirEntries(alloc, archive.*);
-            defer {
-                for (entries) |entry| entry.deinit(alloc);
-                alloc.free(entries);
-            }
-            for (entries) |entry| {
-                var new_path = try alloc.alloc(u8, path.len + 1 + entry.name.len);
-                @memcpy(new_path[0..path.len], path);
-                @memcpy(new_path[path.len + 1 ..], entry.name);
-                new_path[path.len] = '/';
-                defer alloc.free(new_path);
-
-                var inode: Inode = try readFromEntry(alloc, archive, entry);
-                defer inode.deinit(alloc);
-                try inode.extractTo(alloc, archive, new_path, options);
-            }
-        },
-        .file, .ext_file => try self.extractRegFile(alloc, archive, path, options),
-        .symlink, .ext_symlink => try self.extractSymlink(path),
-        else => try self.extractDevice(archive, path, options),
-    }
-}
-
-const Parent = struct {
-    alloc: std.mem.Allocator,
-
-    path: []const u8,
-    uid: u16,
-    gid: u16,
-    perm: u16,
-    mod_time: u32,
-
-    ignore_permissions: bool,
-    ignore_xattr: bool,
-
-    wg: WaitGroup = .{},
-    mut: Mutex = .{},
-
-    fn create(alloc: std.mem.Allocator, hdr: Header, archive: *Archive, path: []const u8, options: ExtractionOptions, dir_size: usize) !*Parent {
-        const out = try alloc.create(Parent);
-        errdefer alloc.destroy(out);
-        out.* = .{
-            .alloc = alloc,
-
-            .path = path,
-            .uid = try archive.id_table.get(hdr.uid_idx),
-            .gid = try archive.id_table.get(hdr.gid_idx),
-            .perm = hdr.permissions,
-            .mod_time = hdr.mod_time,
-
-            .ignore_permissions = options.ignore_permissions,
-            .ignore_xattr = options.ignore_xattr,
-        };
-        out.wg.startMany(dir_size);
-        return out;
-    }
-
-    fn finish(p: *Parent) !void {
-        p.mut.lock();
-        {
-            defer p.mut.unlock();
-            p.wg.finish();
-            if (!p.wg.isDone()) return;
-        }
-        defer p.alloc.destroy(p);
-        var fil = try std.fs.cwd().openFile(p.path, .{});
-        defer fil.close();
-        const time = @as(i128, p.mod_time) * 1000000000;
-        try fil.updateTimes(time, time);
-        if (p.ignore_permissions) {
-            try fil.chmod(p.perm);
-            try fil.chown(p.uid, p.gid);
-        }
-    }
-};
-
-/// Extract the inode to the given path. Multi-threaded.
-/// Functions identically to extractTo on all but regular files and directories.
-///
-/// If threads <= 1, then this just calls extractTo.
-fn extractToThreaded(self: Inode, allocator: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void {
-    switch (self.hdr.inode_type) {
-        .dir, .ext_dir => {
-            // Removing any trailing separators since that's the easiest path forward.
-            if (path[path.len - 1] == '/') return self.extractToThreaded(allocator, archive, path[0 .. path.len - 1], options);
-
-            // Fixed Allocator
-            // const mem_buf = archive.allocator().alloc(u8, 2 * 1024 * 1024 * 1024);
-            // defer archive.allocator().free(mem_buf);
-            // var fixed_alloc: std.heap.FixedBufferAllocator = .init(mem_buf);
-            // const alloc = fixed_alloc.threadSafeAllocator();
-
-            // Arena Allocator
-            var arena_alloc: std.heap.ArenaAllocator = .init(allocator);
-            defer arena_alloc.deinit();
-            var thread_alloc: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena_alloc.allocator() };
-            const alloc = thread_alloc.allocator();
-
-            var wg: WaitGroup = .{};
-            // defer if(!options.ignore_permissions) perms.?.deinit(alloc); We don't need to do this due to ArenaAllocator
-            var pool: Pool = undefined;
-            try pool.init(.{ .allocator = allocator, .n_jobs = options.threads - 1 });
-            defer pool.deinit();
-            var out_err: ?anyerror = null;
-
-            wg.start();
-            self.extractThread(alloc, archive, path, options, &wg, &pool, &out_err, null);
-            pool.waitAndWork(&wg);
-            if (out_err != null) return out_err.?;
-
-            var fil = try std.fs.cwd().openFile(path, .{});
-            defer fil.close();
-            const time = @as(i128, self.hdr.mod_time) * 1000000000;
-            try fil.updateTimes(time, time);
-            if (options.ignore_permissions) {
-                try fil.chmod(self.hdr.permissions);
-                try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx));
-            }
-        },
-        .file, .ext_file => {
-            var pool: Pool = undefined;
-            try pool.init(.{ .allocator = allocator, .n_jobs = options.threads - 1 });
-            defer pool.deinit();
-
-            var arena_alloc: std.heap.ArenaAllocator = .init(allocator);
-            defer arena_alloc.deinit();
-            var thread_alloc: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena_alloc.allocator() };
-            const alloc = thread_alloc.allocator();
-
-            try self.extractRegFileThreaded(alloc, archive, path, options, &pool);
-
-            var fil = try std.fs.cwd().openFile(path, .{});
-            defer fil.close();
-            const time = @as(i128, self.hdr.mod_time) * 1000000000;
-            try fil.updateTimes(time, time);
-            if (!options.ignore_permissions) {
-                try fil.chmod(self.hdr.permissions);
-                try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx));
-            }
-        },
-        .symlink, .ext_symlink => try self.extractSymlink(path),
-        else => try self.extractDevice(archive, path, options),
-    }
-}
-
-fn extractThreadEntry(
-    entry: DirEntry,
-    alloc: std.mem.Allocator,
-    archive: *Archive,
-    path: []const u8,
-    options: ExtractionOptions,
-    wg: *WaitGroup,
-    pool: *Pool,
-    out_err: *?anyerror,
-    parent: ?*Parent,
-) void {
-    var new_path = alloc.alloc(u8, path.len + entry.name.len + 1) catch |err| {
-        wg.finish();
-        out_err.* = err;
-        return;
-    };
-    @memcpy(new_path[0..path.len], path);
-    @memcpy(new_path[path.len + 1 ..], entry.name);
-    new_path[path.len] = '/';
-    var inode = readFromEntry(alloc, archive, entry) catch |err| {
-        out_err.* = err;
-        wg.finish();
-        return;
-    };
-    inode.extractThread(alloc, archive, new_path, options, wg, pool, out_err, parent);
-}
-
-/// Extract threadedly the inode to the path.
-fn extractThread(
+pub fn extract(
     self: Inode,
     alloc: std.mem.Allocator,
-    archive: *Archive,
-    path: []const u8,
+    io: Io,
+    cache: *DecompCache,
+    dir_start: u64,
+    inode_start: u64,
+    frag_start: u64,
+    block_size: u32,
+    id_start: u64,
+    xattr_start: u64,
+    ext_loc: []const u8,
     options: ExtractionOptions,
-    wg: *WaitGroup,
-    pool: *Pool,
-    out_err: *?anyerror,
-    parent: ?*Parent,
-) void {
-    if (options.verbose)
-        options.verbose_writer.?.print("Extracting inode #{} to {s}\n", .{ self.hdr.num, path }) catch {};
-    defer {
-        if (parent != null) parent.?.finish() catch |err| {
-            if (options.verbose)
-                options.verbose_writer.?.print("Error setting folder permission to {s}: {}\n", .{ path, err }) catch {};
-            out_err.* = err;
+) !void {
+    const path = std.mem.trimEnd(u8, ext_loc, "/");
+
+    var sel_val: std.atomic.Value(usize) = .init(1);
+
+    var sel_buf: [5]ExtractUnion = undefined;
+    var sel: Io.Select(ExtractUnion) = .init(io, &sel_buf);
+    defer sel.cancelDiscard();
+
+    var meta_loop = io.async(metadataLoop, .{ alloc, io, cache, id_start, xattr_start, &sel, &sel_val, options });
+    defer _ = meta_loop.cancel(io) catch {};
+
+    sel.async(.ret, extractReal, .{ self, alloc, io, cache, dir_start, inode_start, frag_start, block_size, &sel, &sel_val, path, true });
+
+    try meta_loop.await(io);
+}
+fn extractReal(
+    self: Inode,
+    alloc: std.mem.Allocator,
+    io: Io,
+    cache: *DecompCache,
+    dir_start: u64,
+    inode_start: u64,
+    frag_start: u64,
+    block_size: u32,
+    sel: *Io.Select(ExtractUnion),
+    sel_val: *std.atomic.Value(usize),
+    path: []const u8,
+    origin: bool,
+) ExtractionError!ExtractReturn {
+    errdefer if (!origin) {
+        self.deinit(alloc);
+        alloc.free(path);
     };
-        wg.finish();
-    }
-    if (out_err.* != null) return;
     switch (self.hdr.inode_type) {
         .dir, .ext_dir => {
-            _ = std.fs.cwd().makePathStatus(path) catch |err| {
-                if (options.verbose)
-                    options.verbose_writer.?.print("Error creating {s}: {}\n", .{ path, err }) catch {};
-                out_err.* = err;
-                return;
-            };
+            try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777));
 
-            const entries = self.dirEntries(alloc, archive.*) catch |err| {
-                if (options.verbose)
-                    options.verbose_writer.?.print("Error getting directory entries for inode #{} (extracting to {s}): {}\n", .{ self.hdr.num, path, err }) catch {};
-                out_err.* = err;
-                return;
+            const entries = self.readDirectory(alloc, io, cache, dir_start) catch |err| switch (err) {
+                error.NotDirectory => unreachable,
+                else => |e| return e,
             };
-            const p = Parent.create(alloc, self.hdr, archive, path, options, entries.len) catch |err| {
-                out_err.* = err;
-                return;
-            };
-            wg.startMany(entries.len);
-            // defer files.deinit(alloc); We don't need to do this due to ArenaAllocator
+            defer {
+                for (entries) |entry|
+                    entry.deinit(alloc);
+                alloc.free(entries);
+            }
+
+            if (entries.len != 0) {
+                _ = sel_val.fetchAdd(entries.len, .acq_rel);
+
                 for (entries) |entry| {
-                if (entry.inode_type == .dir) {
-                    extractThreadEntry(entry, alloc, archive, path, options, wg, pool, out_err, p);
-                    continue;
+                    var meta: MetadataReader = .init(io, cache, inode_start + entry.block_start);
+                    defer meta.deinit();
+                    try meta.interface.discardAll(entry.block_offset);
+
+                    var new_inode: Inode = try .fromReader(alloc, &meta.interface, block_size);
+                    errdefer new_inode.deinit(alloc);
+
+                    const new_path = try std.mem.concat(alloc, u8, &.{ path, "/", entry.name });
+                    errdefer alloc.free(new_path);
+                    sel.async(
+                        .ret,
+                        extractReal,
+                        .{ new_inode, alloc, io, cache, dir_start, inode_start, frag_start, block_size, sel, sel_val, new_path, false },
+                    );
                 }
-                pool.spawn(
-                    extractThreadEntry,
-                    .{
-                        entry,
-                        alloc,
-                        archive,
-                        path,
-                        options,
-                        wg,
-                        pool,
-                        out_err,
-                        p,
-                    },
-                ) catch |err| {
-                    wg.finish();
-                    if (options.verbose)
-                        options.verbose_writer.?.print("Error starting extraction thread: {}\n", .{err}) catch {};
-                    out_err.* = err;
-                    continue;
-                };
             }
         },
         .file, .ext_file => {
-            self.extractRegFileThreaded(alloc, archive, path, options, pool) catch |err| {
-                if (options.verbose)
-                    options.verbose_writer.?.print("Error extracting file inode #{} to {s}: {}\n", .{ self.hdr.num, path, err }) catch {};
-                out_err.* = err;
-            };
+            std.debug.print("{s} {}\n", .{ path, self.data });
+            // var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{});
+            // defer atomic.deinit(io);
+
+            // var data: DataExtract = undefined;
+            // var frag_offset: ?u64 = null;
+            // switch (self.data) {
+            //     .file => |f| {
+            //         data = .init(cache.decomp, cache.map, block_size, f.block_start, f.size, f.block_sizes);
+            //         if (f.frag_idx != 0xFFFFFFFF) {
+            //             const entry: FragEntry = try LookupTable.lookup(FragEntry, io, cache, frag_start, f.frag_idx);
+            //             if (entry.size.uncompressed) {
+            //                 data.addFrag(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset);
+            //             } else {
+            //                 frag_offset = entry.start;
+            //                 const block = try cache.checkoutBlock(io, entry.start, entry.size.size, block_size);
+            //                 data.addFrag(block, f.frag_offset);
+            //             }
+            //         }
+            //     },
+            //     .ext_file => |f| {
+            //         data = .init(cache.decomp, cache.map, block_size, f.block_start, f.size, f.block_sizes);
+            //         if (f.frag_idx != 0xFFFFFFFF) {
+            //             const entry: FragEntry = try LookupTable.lookup(FragEntry, io, cache, frag_start, f.frag_idx);
+            //             if (entry.size.uncompressed) {
+            //                 data.addFrag(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset);
+            //             } else {
+            //                 frag_offset = entry.start;
+            //                 const block = try cache.checkoutBlock(io, entry.start, entry.size.size, block_size);
+            //                 data.addFrag(block, f.frag_offset);
+            //             }
+            //         }
+            //     },
+            //     else => unreachable,
+            // }
+            // defer if (frag_offset != null) cache.checkinBlock(io, frag_offset.?) catch {};
+            // try data.asyncExtract(alloc, io, atomic.file);
+
+            // try atomic.link(io);
         },
         .symlink, .ext_symlink => {
-            self.extractSymlink(path) catch |err| {
-                if (options.verbose)
-                    options.verbose_writer.?.print("Error extracting symlink inode #{} to {s}: {}\n", .{ self.hdr.num, path, err }) catch {};
-                out_err.* = err;
-            };
-        },
-        else => {
-            self.extractDevice(archive, path, options) catch |err| {
-                if (options.verbose)
-                    options.verbose_writer.?.print("Error extracting device/IPC inode #{} to {s}: {}\n", .{ self.hdr.num, path, err }) catch {};
-                out_err.* = err;
-            };
-        },
-    }
-}
-/// Creates and writes the inode file contents to the given path.
-/// Optionally set owner & permissions.
-///
-/// Assumes the inode is a file or ext_file type.
-fn extractRegFile(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void {
-    var fil = try std.fs.cwd().createFile(path, .{ .exclusive = true });
-    defer fil.close();
-    var wrt = fil.writer(&[0]u8{});
-    var dat_rdr = try self.dataReader(alloc, archive);
-    defer dat_rdr.deinit();
-    _ = try dat_rdr.interface.streamRemaining(&wrt.interface);
-    try wrt.interface.flush();
-
-    const time = @as(i128, self.hdr.mod_time) * 1000000000;
-    try fil.updateTimes(time, time);
-    if (!options.ignore_permissions) {
-        try fil.chmod(self.hdr.permissions);
-        try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx));
-    }
-}
-/// Extract the inode file contents to the given path threadedly.
-/// pool is used to spawn threads.
-///
-/// Assumes the inode is a file or ext_file type.
-fn extractRegFileThreaded(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions, pool: *Pool) !void {
-    var fil = try std.fs.cwd().createFile(path, .{});
-    defer fil.close();
-    var data = try self.threadedDataReader(alloc, archive);
-    try data.extractThreaded(fil, pool);
-
-    const time = @as(i128, self.hdr.mod_time) * 1000000000;
-    try fil.updateTimes(time, time);
-    if (!options.ignore_permissions) {
-        try fil.chmod(self.hdr.permissions);
-        try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx));
-    }
-}
-/// Creates the symlink described by the inode.
-///
-/// Assumes the inode is a symlink or ext_symlink type.
-fn extractSymlink(self: Inode, path: []const u8) !void {
             const target = switch (self.data) {
                 .symlink => |s| s.target,
                 .ext_symlink => |s| s.target,
                 else => unreachable,
             };
-    try std.fs.cwd().symLink(target, path, .{});
-}
-/// Creates the device described by the inode.
-///
-/// Optionally set owner & permissions.
-/// Assumes the inode is a char_dev, block_dev, fifo, socket, or their extended counterparts.
-fn extractDevice(self: Inode, archive: *Archive, path: []const u8, options: ExtractionOptions) !void {
-    var mode: u32 = undefined;
+            try Io.Dir.cwd().symLink(io, target, path, .{});
+        },
+        else => {
             var dev: u32 = 0;
+            var mode: u32 = undefined;
+
+            const DT = std.os.linux.DT;
+
             switch (self.data) {
-        .char_dev => |d| {
-            mode = std.posix.S.IFCHR;
-            dev = d.dev;
-        },
-        .ext_char_dev => |d| {
-            mode = std.posix.S.IFCHR;
-            dev = d.dev;
-        },
                 .block_dev => |d| {
-            mode = std.posix.S.IFBLK;
+                    mode = DT.BLK;
                     dev = d.dev;
                 },
                 .ext_block_dev => |d| {
-            mode = std.posix.S.IFBLK;
+                    mode = DT.BLK;
                     dev = d.dev;
                 },
-        .fifo, .ext_fifo => mode = std.posix.S.IFIFO,
-        .socket, .ext_socket => mode = std.posix.S.IFSOCK,
+                .char_dev => |d| {
+                    mode = DT.CHR;
+                    dev = d.dev;
+                },
+                .ext_char_dev => |d| {
+                    mode = DT.CHR;
+                    dev = d.dev;
+                },
+                .fifo, .ext_fifo => mode = DT.FIFO,
+                .socket, .ext_socket => mode = DT.SOCK,
                 else => unreachable,
             }
-    const res: std.os.linux.E = @enumFromInt(std.os.linux.mknod(@ptrCast(path), mode, dev));
-    switch (res) {
-        .SUCCESS => {},
-        .ACCES => return std.fs.Dir.MakeError.AccessDenied,
-        .DQUOT => return std.fs.Dir.MakeError.DiskQuota,
-        .EXIST => return std.fs.Dir.MakeError.PathAlreadyExists,
-        .FAULT, .NOENT => return std.fs.Dir.MakeError.BadPathName,
-        .LOOP => return std.fs.Dir.MakeError.SymLinkLoop,
-        .NAMETOOLONG => return std.fs.Dir.MakeError.NameTooLong,
-        .NOMEM => return std.fs.Dir.MakeError.SystemResources,
-        .NOSPC => return std.fs.Dir.MakeError.NoSpaceLeft,
-        .NOTDIR => return std.fs.Dir.MakeError.NotDir,
-        .PERM => return std.fs.Dir.MakeError.PermissionDenied,
-        .ROFS => return std.fs.Dir.MakeError.ReadOnlyFileSystem,
-        else => return blk: {
-            std.debug.print("unhandled mknod result: {}\n", .{res});
-            break :blk std.fs.Dir.MakeError.Unexpected;
+
+            const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &.{path}, 0);
+            defer alloc.free(sentinel_path);
+            const res = std.os.linux.mknod(sentinel_path, mode, dev);
+            if (res != 0)
+                return ExtractionError.Mknod;
         },
     }
-    var fil = try std.fs.cwd().openFile(path, .{});
-    defer fil.close();
-    const time = @as(i128, self.hdr.mod_time) * 1000000000;
-    try fil.updateTimes(time, time);
-    if (!options.ignore_permissions) {
-        try fil.chmod(self.hdr.permissions);
-        try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx));
+
+    return .{
+        .path = path,
+        .inode = self,
+        .origin = origin,
+    };
+}
+
+const ExtractUnion = union { ret: ExtractionError!ExtractReturn };
+pub const ExtractionError = error{ SetXattr, Mknod, Canceled } || DirEntry.Error || Io.Dir.CreateFileAtomicError || DataExtract.Error || Io.File.Atomic.LinkError ||
+    Io.Dir.SymLinkError;
+
+const ExtractReturn = struct {
+    path: []const u8,
+    inode: Inode,
+    origin: bool,
+
+    fn deinit(self: ExtractReturn, alloc: std.mem.Allocator) void {
+        if (self.origin) return;
+        alloc.free(self.path);
+        self.inode.deinit(alloc);
     }
-    if (!options.ignore_xattr) {
-        // TODO
+    fn setMetadata(self: ExtractReturn, alloc: std.mem.Allocator, io: Io, cache: *DecompCache, id_start: u64, xattr_start: u64, options: ExtractionOptions) !void {
+        if (options.ignore_permissions and options.ignore_xattr) return;
+
+        var fil = try Io.Dir.cwd().openFile(io, self.path, .{});
+        defer fil.close(io);
+
+        if (!options.ignore_permissions) {
+            try fil.setTimestamps(io, .{ .modify_timestamp = .{
+                .new = .{ .nanoseconds = @as(i96, @intCast(self.inode.hdr.mod_time)) * std.time.ns_per_s },
+            } });
+            try fil.setPermissions(io, @enumFromInt(self.inode.hdr.permissions));
+            try fil.setOwner(
+                io,
+                try LookupTable.lookup(u16, io, cache, id_start, self.inode.hdr.uid_idx),
+                try LookupTable.lookup(u16, io, cache, id_start, self.inode.hdr.gid_idx),
+            );
+        }
+        if (options.ignore_xattr) return;
+        const xattr_idx: u32 = switch (self.inode.data) {
+            .ext_dir => |d| d.xattr_idx,
+            .ext_file => |f| f.xattr_idx,
+            .ext_symlink => |s| s.xattr_idx,
+            .ext_block_dev, .ext_char_dev => |d| d.xattr_idx,
+            .ext_fifo, .ext_socket => |i| i.xattr_idx,
+            else => return,
+        };
+        if (xattr_idx == 0xFFFFFFFF) return;
+        const xattrs = try LookupTable.xattrLookup(alloc, io, cache, xattr_start, xattr_idx);
+        defer {
+            for (xattrs) |kv|
+                kv.deinit(alloc);
+            alloc.free(xattrs);
+        }
+
+        for (xattrs) |kv| {
+            const res = std.os.linux.fsetxattr(fil.handle, kv.key.ptr, kv.value.ptr, kv.value.len, 0);
+            if (res != 0)
+                return ExtractionError.SetXattr;
+        }
+    }
+};
+
+fn metadataLoop(
+    alloc: std.mem.Allocator,
+    io: Io,
+    cache: *DecompCache,
+    id_start: u64,
+    xattr_start: u64,
+    sel: *Io.Select(ExtractUnion),
+    sel_val: *std.atomic.Value(usize),
+    options: ExtractionOptions,
+) !void {
+    errdefer {
+        while (sel.group.token.load(.unordered) != null) {
+            const ret = sel.queue.getOne(io) catch break;
+
+            const res = ret.ret catch continue;
+            res.deinit(alloc);
+        }
+    }
+    var dir_queue: std.PriorityDequeue(ExtractReturn, void, dirReturnQueueOrder) = .empty;
+    defer {
+        while (dir_queue.popMax()) |ret|
+            ret.deinit(alloc);
+        dir_queue.deinit(alloc);
+    }
+
+    while (sel_val.load(.unordered) > 0) {
+        defer _ = sel_val.fetchSub(1, .acq_rel);
+        const ret = try sel.queue.getOne(io);
+
+        const res = try ret.ret;
+
+        if (res.inode.hdr.inode_type == .dir or res.inode.hdr.inode_type == .ext_dir) {
+            try dir_queue.push(alloc, res);
+        } else {
+            defer res.deinit(alloc);
+            try res.setMetadata(alloc, io, cache, id_start, xattr_start, options);
+        }
+    }
+
+    while (dir_queue.popMax()) |res| {
+        defer res.deinit(alloc);
+        try res.setMetadata(alloc, io, cache, id_start, xattr_start, options);
     }
 }
+
+fn dirReturnQueueOrder(_: void, a: ExtractReturn, b: ExtractReturn) std.math.Order {
+    return std.math.order(std.mem.count(u8, a.path, "/"), std.mem.count(u8, b.path, "/"));
+}

src/inode_data/dir.zig

@@ -1,6 +1,6 @@
 const Reader = @import("std").Io.Reader;
 
-pub const Dir = packed struct {
+pub const Dir = extern struct {
     block_start: u32,
     hard_links: u32,
     size: u16,
@@ -14,19 +14,19 @@ pub const Dir = packed struct {
     }
 };
 
-pub const ExtDir = packed struct {
+pub const ExtDir = extern struct {
     hard_links: u32,
     size: u32,
     block_start: u32,
     parent_num: u32,
     idx_count: u16,
     block_offset: u16,
-    xattr_id: u32,
+    xattr_idx: u32,
     // index: []DirIndex
 
     pub fn read(rdr: *Reader) !ExtDir {
         var d: ExtDir = undefined;
-        try rdr.readSliceEndian(Dir, @ptrCast(&d), .little);
+        try rdr.readSliceEndian(ExtDir, @ptrCast(&d), .little);
         return d;
     }
 };

src/inode_data/file.zig

@@ -1,7 +1,7 @@
 const std = @import("std");
 const Reader = std.Io.Reader;
 
-pub const BlockSize = packed struct {
+pub const BlockSize = packed struct(u32) {
     size: u24,
     uncompressed: bool,
     _: u7,
@@ -10,25 +10,31 @@ pub const BlockSize = packed struct {
 pub const File = struct {
     block_start: u32, // bytes 0-3
     frag_idx: u32, // bytes 4-7
-    frag_block_offset: u32, // bytes 8-11
+    frag_offset: u32, // bytes 8-11
     size: u32, // bytes 12-15
     block_sizes: []BlockSize,
 
     pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !File {
-        var start: [16]u8 = undefined;
-        try rdr.readSliceAll(&start);
-        const frag_idx: u32 = std.mem.readInt(u32, start[4..8], .little);
-        const size: u32 = std.mem.readInt(u32, start[12..16], .little);
-        var num_blocks: u32 = size / block_size;
-        if (size % block_size != 0 and frag_idx == 0xFFFFFFFF) num_blocks += 1;
+        const raw_values = extern struct {
+            block_start: u32, // bytes 0-3
+            frag_idx: u32, // bytes 4-7
+            frag_offset: u32, // bytes 8-11
+            size: u32, // bytes 12-15
+        };
+        var values: raw_values = undefined;
+        try rdr.readSliceEndian(@TypeOf(values), @ptrCast(&values), .little);
+
+        var num_blocks: u32 = values.size / block_size;
+        if (values.size % block_size != 0 and values.frag_idx == 0xFFFFFFFF) num_blocks += 1;
         const sizes = try alloc.alloc(BlockSize, num_blocks);
         errdefer alloc.free(sizes);
         try rdr.readSliceEndian(BlockSize, sizes, .little);
+
         return .{
-            .block_start = std.mem.readInt(u32, start[0..4], .little),
-            .frag_idx = frag_idx,
-            .frag_block_offset = std.mem.readInt(u32, start[8..12], .little),
-            .size = size,
+            .block_start = values.block_start,
+            .frag_idx = values.frag_idx,
+            .frag_offset = values.frag_offset,
+            .size = values.size,
             .block_sizes = sizes,
         };
     }
@@ -44,28 +50,37 @@ pub const ExtFile = struct {
     sparse: u64, // bytes 16-23
     hard_links: u32, // bytes 24-27
     frag_idx: u32, // bytes 28-31
-    frag_block_offset: u32, // bytes 32-35
+    frag_offset: u32, // bytes 32-35
     xattr_idx: u32, // bytes 36-39
     block_sizes: []BlockSize,
 
     pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !ExtFile {
-        var start: [40]u8 = undefined;
-        try rdr.readSliceAll(&start);
-        const frag_idx: u32 = std.mem.readInt(u32, start[28..32], .little);
-        const size: u64 = std.mem.readInt(u64, start[8..16], .little);
-        var num_blocks: u32 = @truncate(size / block_size);
-        if (size % block_size != 0 and frag_idx == 0xFFFFFFFF) num_blocks += 1;
+        const raw_values = extern struct {
+            block_start: u64, // bytes 0-7
+            size: u64, // bytes 8-15
+            sparse: u64, // bytes 16-23
+            hard_links: u32, // bytes 24-27
+            frag_idx: u32, // bytes 28-31
+            frag_offset: u32, // bytes 32-35
+            xattr_idx: u32, // bytes 36-39
+        };
+        var values: raw_values = undefined;
+        try rdr.readSliceEndian(@TypeOf(values), @ptrCast(&values), .little);
+
+        var num_blocks: u32 = @truncate(values.size / block_size);
+        if (values.size % block_size != 0 and values.frag_idx == 0xFFFFFFFF) num_blocks += 1;
         const sizes = try alloc.alloc(BlockSize, num_blocks);
         errdefer alloc.free(sizes);
         try rdr.readSliceEndian(BlockSize, sizes, .little);
+
         return .{
-            .block_start = std.mem.readInt(u64, start[0..8], .little),
-            .size = size,
-            .sparse = std.mem.readInt(u64, start[16..24], .little),
-            .hard_links = std.mem.readInt(u32, start[24..28], .little),
-            .frag_idx = frag_idx,
-            .frag_block_offset = std.mem.readInt(u32, start[32..36], .little),
-            .xattr_idx = std.mem.readInt(u32, start[36..40], .little),
+            .block_start = values.block_start,
+            .size = values.size,
+            .sparse = values.sparse,
+            .hard_links = values.hard_links,
+            .frag_idx = values.frag_idx,
+            .frag_offset = values.frag_offset,
+            .xattr_idx = values.xattr_idx,
             .block_sizes = sizes,
         };
     }

src/inode_data/misc.zig

@@ -50,7 +50,7 @@ pub const ExtSymlink = struct {
 };
 
 /// A block or character device.
-pub const Dev = packed struct {
+pub const Dev = extern struct {
     hard_links: u32,
     dev: u32,
 
@@ -62,7 +62,7 @@ pub const Dev = packed struct {
 };
 
 /// An extended block or character device.
-pub const ExtDev = packed struct {
+pub const ExtDev = extern struct {
     hard_links: u32,
     dev: u32,
     xattr_idx: u32,
@@ -75,7 +75,7 @@ pub const ExtDev = packed struct {
 };
 
 /// A socket or FIFO file.
-pub const IPC = packed struct {
+pub const IPC = extern struct {
     hard_links: u32,
 
     pub fn read(rdr: *Reader) !IPC {
@@ -86,7 +86,7 @@ pub const IPC = packed struct {
 };
 
 /// An extended socket or FIFO file.
-pub const ExtIPC = packed struct {
+pub const ExtIPC = extern struct {
     hard_links: u32,
     xattr_idx: u32,
 

src/lookup_table.zig

@@ -0,0 +1,124 @@
+const std = @import("std");
+const Io = std.Io;
+
+const Inode = @import("inode.zig");
+const DecompCache = @import("util/decomp_cache.zig");
+const MetadataReader = @import("util/metadata.zig");
+
+pub fn lookup(comptime T: anytype, io: Io, cache: *DecompCache, table_start: u64, idx: u32) !T {
+    const PER_BLOCK = 8192 / @sizeOf(T);
+
+    const block_idx = idx / PER_BLOCK;
+    const block_offset = idx % PER_BLOCK;
+
+    if (table_start + (block_idx * 8) > cache.map.memory.len) return error.ReadFailed;
+    const offset: u64 = std.mem.readInt(u64, cache.map.memory[table_start + (block_idx * 8) ..][0..8], .little);
+
+    var meta: MetadataReader = .init(io, cache, offset);
+    defer meta.deinit();
+    try meta.interface.discardAll(block_offset * @sizeOf(T));
+
+    var new: T = undefined;
+    try meta.interface.readSliceEndian(T, @ptrCast(&new), .little);
+    return new;
+}
+
+pub const XattrKV = struct {
+    key: [:0]u8,
+    value: []u8,
+
+    pub fn deinit(self: XattrKV, alloc: std.mem.Allocator) void {
+        alloc.free(self.key);
+        alloc.free(self.value);
+    }
+};
+
+const LookupValue = extern struct {
+    ref: Inode.Ref,
+    count: u32,
+    size: u32,
+};
+const KeyEntry = extern struct {
+    prefix: packed struct(u16) {
+        prefix: enum(u8) {
+            user,
+            trusted,
+            security,
+        },
+        out_of_line: bool,
+        _: u7,
+    },
+    name_size: u16,
+};
+
+pub fn xattrLookup(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, xattr_start: u64, idx: u32) ![]XattrKV {
+    const table_start = std.mem.readInt(u64, cache.map.memory[xattr_start..][0..8], .little);
+
+    const val: LookupValue = try lookup(
+        LookupValue,
+        io,
+        cache,
+        xattr_start + 16,
+        idx,
+    );
+
+    const out = try alloc.alloc(XattrKV, val.count);
+    errdefer alloc.free(out);
+
+    var meta: MetadataReader = .init(io, cache, table_start + val.ref.block_start);
+    defer meta.deinit();
+    try meta.interface.discardAll(val.ref.block_offset);
+
+    for (out) |*kv| {
+        var key_entry: KeyEntry = undefined;
+        try meta.interface.readSliceEndian(KeyEntry, @ptrCast(&key_entry), .little);
+
+        const prefix_len: u16 = switch (key_entry.prefix.prefix) {
+            .user => 5,
+            .trusted => 8,
+            .security => 9,
+        };
+        var key_len = key_entry.name_size;
+        key_len += prefix_len;
+
+        kv.key = try alloc.allocSentinel(u8, key_len, 0);
+        errdefer alloc.free(kv.key);
+
+        try meta.interface.readSliceEndian(u8, kv.key[prefix_len..], .little);
+        switch (key_entry.prefix.prefix) {
+            .user => @memcpy(kv.key[0..prefix_len], "user."),
+            .trusted => @memcpy(kv.key[0..prefix_len], "trusted."),
+            .security => @memcpy(kv.key[0..prefix_len], "security."),
+        }
+
+        if (key_entry.prefix.out_of_line) {
+            try meta.interface.discardAll(8);
+
+            var ool_ref: Inode.Ref = undefined;
+            try meta.interface.readSliceEndian(Inode.Ref, @ptrCast(&ool_ref), .little);
+
+            var ool_meta: MetadataReader = .init(io, cache, table_start + ool_ref.block_start);
+            defer ool_meta.deinit();
+            try ool_meta.interface.discardAll(ool_ref.block_offset);
+
+            kv.value = try readValue(alloc, &ool_meta.interface);
+            errdefer alloc.free(kv.value);
+        } else {
+            kv.value = try readValue(alloc, &meta.interface);
+        }
+    }
+
+    return out;
+}
+
+fn readValue(alloc: std.mem.Allocator, rdr: *Io.Reader) ![]u8 {
+    var val_size: u32 = undefined;
+    try rdr.readSliceEndian(u32, @ptrCast(&val_size), .little);
+
+    const val = try alloc.alloc(u8, val_size);
+    errdefer alloc.free(val);
+
+    try rdr.readSliceEndian(u8, val, .little);
+
+    return val;
+}

src/options.zig

@@ -5,25 +5,22 @@ const Writer = std.Io.Writer;
 
 const ExtractionOptions = @This();
 
-/// The number of threads used for extraction. 0 implies single threaded.
-threads: usize = 1,
-/// Don't set the file's owner & permissions after extraction
+/// Force single-threaded extraction. Io.Threaded.global_single_threaded also works.
+single_threaded: bool = false,
+/// Don't set the file's owner, permissions, & modify time after extraction.
 ignore_permissions: bool = false,
-/// Don't set xattr values. Currently xattrs are never set anyway.
+/// Don't set xattr values.
 ignore_xattr: bool = false,
-/// Replace symlinks with their target.
+/// Replace symlinks with their target. Currently doesn't do anything.
 dereference_symlinks: bool = false,
 /// Verbose logging. If true, verbose_writer must be set
 verbose: bool = false,
 /// Where to print verbose log.
 verbose_writer: ?*Writer = null,
 
-pub const SingleThreadedDefault: ExtractionOptions = .{};
-pub fn Default() !ExtractionOptions {
-    return .{
-        .threads = try std.Thread.getCpuCount(),
-    };
-}
+pub const defaultSingleThreaded: ExtractionOptions = .{ .single_threaded = true };
+pub const default: ExtractionOptions = .{};
+
 pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions {
     return .{
         .verbose = true,

src/root.zig

@@ -1,2 +1,7 @@
 pub const Archive = @import("archive.zig");
 pub const ExtractionOptions = @import("options.zig");
+const Test = @import("test.zig");
+
+test {
+    @import("std").testing.refAllDecls(Test);
+}

src/super.zig

@@ -1,8 +1,8 @@
 const std = @import("std");
 const math = std.math;
 
-const CompressionType = @import("decomp.zig").CompressionType;
 const InodeRef = @import("inode.zig").Ref;
+const CompressionType = @import("util/decompress.zig").CompressionType;
 
 const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little);
 
@@ -14,7 +14,7 @@ const SuperblockError = error{
 };
 
 /// A squashfs Superblock
-pub const Superblock = packed struct {
+pub const Superblock = extern struct {
     magic: u32,
     inode_count: u32,
     mod_time: u32,
@@ -22,7 +22,7 @@ pub const Superblock = packed struct {
     frag_count: u32,
     compression: CompressionType,
     block_log: u16,
-    flags: packed struct {
+    flags: packed struct(u16) {
         inode_uncompressed: bool,
         data_uncompressed: bool,
         check: bool,

src/table.zig

@@ -1,77 +0,0 @@
-const std = @import("std");
-const Mutex = std.Thread.Mutex;
-
-const DecompFn = @import("decomp.zig").DecompFn;
-const MetadataReader = @import("util/metadata.zig");
-const OffsetFile = @import("util/offset_file.zig");
-
-const TableError = error{
-    InvalidIndex,
-};
-
-/// A two-layer metadata table.
-pub fn Table(T: anytype) type {
-    return struct {
-        const Self = @This();
-
-        const VALS_PER_BLOCK = 8192 / @sizeOf(T);
-
-        alloc: std.mem.Allocator,
-        fil: OffsetFile,
-        decomp: DecompFn,
-        tab_start: u64,
-
-        tab: std.AutoHashMap(u32, []T),
-        values: u32,
-
-        mut: Mutex = .{},
-
-        pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: DecompFn, tab_start: u64, values: u32) !Self {
-            return .{
-                .alloc = alloc,
-                .fil = fil,
-                .decomp = decomp,
-                .tab_start = tab_start,
-
-                .tab = .init(alloc),
-                .values = values,
-            };
-        }
-
-        pub fn deinit(self: *Self) void {
-            var iter = self.tab.valueIterator();
-            while (iter.next()) |s| {
-                self.alloc.free(s.*);
-            }
-            self.tab.deinit();
-        }
-
-        pub fn get(self: *Self, idx: u32) !T {
-            if (idx >= self.values) return TableError.InvalidIndex;
-            const block_num = idx / VALS_PER_BLOCK;
-            const idx_offset = idx - (block_num * VALS_PER_BLOCK);
-            if (self.tab.contains(block_num)) {
-                const block = self.tab.get(block_num).?;
-                return block[idx_offset];
-            }
-            self.mut.lock();
-            defer self.mut.unlock();
-            // Double check in case of race condition..
-            if (self.tab.contains(block_num)) {
-                const block = self.tab.get(block_num).?;
-                return block[idx_offset];
-            }
-            const is_last = (self.values - 1) / VALS_PER_BLOCK == block_num;
-            const slice_size = if (is_last) self.values - (block_num * VALS_PER_BLOCK) else VALS_PER_BLOCK;
-            const slice = try self.alloc.alloc(T, slice_size);
-            var rdr = try self.fil.readerAt(self.tab_start + (8 * block_num), &[0]u8{});
-            var offset: u64 = 0;
-            try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
-            rdr = try self.fil.readerAt(offset, &[0]u8{});
-            var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
-            try meta.interface.readSliceEndian(T, @ptrCast(slice), .little);
-            try self.tab.put(block_num, slice);
-            return slice[idx_offset];
-        }
-    };
-}

src/test.zig

@@ -1,4 +1,5 @@
 const std = @import("std");
+const Io = std.Io;
 const stuff = @import("builtin");
 
 const Archive = @import("archive.zig");
@@ -7,40 +8,64 @@ const Superblock = @import("super.zig").Superblock;
 const TestArchive = "testing/LinuxPATest.sfs";
 
 test "Basics" {
-    var fil = try std.fs.cwd().openFile(TestArchive, .{});
-    defer fil.close();
-    var sfs: Archive = try .init(std.testing.allocator, fil);
-    defer sfs.deinit();
-    if (sfs.super != LinuxPATestCorrectSuperblock) {
-        std.debug.print("Superblock wrong\nShould be: {}\n\nis: {}\n", .{ LinuxPATestCorrectSuperblock, sfs.super });
-        return error.BadSuperblock;
-    }
+    const io = std.testing.io;
+    const alloc = std.testing.allocator;
+
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    var sfs: Archive = try .init(alloc, io, fil);
+    defer sfs.deinit(io);
+    try std.testing.expectEqualDeep(sfs.super, LinuxPATestCorrectSuperblock);
 }
 
 const TestFile = "Start.exe";
 const TestFileExtractLocation = "testing/Start.exe";
 
 test "ExtractSingleFile" {
-    std.fs.cwd().deleteFile(TestFileExtractLocation) catch {};
-    var fil = try std.fs.cwd().openFile(TestArchive, .{});
-    defer fil.close();
-    var sfs: Archive = try .init(std.testing.allocator, fil);
-    defer sfs.deinit();
-    var test_fil = try sfs.open(TestFile);
+    const io = std.testing.io;
+    const alloc = std.testing.allocator;
+
+    Io.Dir.cwd().deleteFile(io, TestFileExtractLocation) catch {};
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    var sfs: Archive = try .init(alloc, io, fil);
+    defer sfs.deinit(io);
+    var test_fil = try sfs.open(alloc, io, TestFile);
     defer test_fil.deinit();
-    try test_fil.extract(TestFileExtractLocation, .Default);
+    try test_fil.extract(alloc, io, TestFileExtractLocation, .default);
+    //TODO: validate extracted file.
+}
+
+const TestDir = "Documents";
+const TestDirExtractLocation = "testing/Documents";
+
+test "ExtractSmallDir" {
+    const io = std.testing.io;
+    const alloc = std.testing.allocator;
+
+    Io.Dir.cwd().deleteTree(io, TestDirExtractLocation) catch {};
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    var sfs: Archive = try .init(alloc, io, fil);
+    defer sfs.deinit(io);
+    var test_fil = try sfs.open(alloc, io, TestDir);
+    defer test_fil.deinit();
+    try test_fil.extract(alloc, io, TestDirExtractLocation, .default);
     //TODO: validate extracted file.
 }
 
 const TestFullExtractLocation = "testing/TestExtract";
 
 test "ExtractCompleteArchive" {
-    std.fs.cwd().deleteTree(TestFullExtractLocation) catch {};
-    var fil = try std.fs.cwd().openFile(TestArchive, .{});
-    defer fil.close();
-    var sfs: Archive = try .init(std.testing.allocator, fil);
-    defer sfs.deinit();
-    try sfs.extract(TestFullExtractLocation, .Default);
+    const io = std.testing.io;
+    const alloc = std.testing.allocator;
+
+    Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    var sfs: Archive = try .init(alloc, io, fil);
+    defer sfs.deinit(io);
+    try sfs.extract(alloc, io, TestFullExtractLocation, .default);
 }
 
 const LinuxPATestCorrectSuperblock: Superblock = .{

src/util/c_decomp.zig

@@ -0,0 +1,61 @@
+const std = @import("std");
+
+const c = @import("c");
+
+const Error = @import("decompress.zig").Error;
+
+pub fn zlibDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var strem: c.zng_stream = .{
+        .next_in = in.ptr,
+        .avail_in = @truncate(in.len),
+        .next_out = out.ptr,
+        .avail_out = @truncate(out.len),
+    };
+    var res = c.zng_inflateInit(&strem);
+    if (res != c.Z_OK) return Error.ReadFailed;
+    defer _ = c.zng_inflateEnd(&strem);
+
+    res = c.zng_inflate(&strem, c.Z_FULL_FLUSH);
+    if (res != c.Z_OK) return Error.ReadFailed;
+
+    return strem.total_out;
+}
+pub fn lzmaDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var strem: c.lzma_stream = .{
+        .next_in = in.ptr,
+        .avail_in = in.len,
+        .next_out = out.ptr,
+        .avail_out = out.len,
+    };
+    var res = c.lzma_auto_decoder(&strem, out.len * 2, 0);
+    if (res != c.LZMA_OK) return Error.ReadFailed;
+    defer c.lzma_end(&strem);
+
+    while (res == c.LZMA_OK)
+        res = c.lzma_code(&strem, c.LZMA_RUN);
+    if (res != c.LZMA_FINISH) return Error.ReadFailed;
+
+    return strem.total_out;
+}
+pub fn lzoDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var out_len = out.len;
+    const res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &out_len, null);
+    if (res != c.LZO_E_OK) return Error.ReadFailed;
+    return out_len;
+}
+pub fn lz4Decompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    const res = c.LZ4_decompress_safe(
+        in.ptr,
+        out.ptr,
+        @bitCast(@as(u32, @truncate(in.len))),
+        @bitCast(@as(u32, @truncate(out.len))),
+    );
+    if (res < 0) return Error.ReadFailed;
+    return @abs(res);
+}
+pub fn zstdDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
+    if (c.ZSTD_isError(res) != 0)
+        return Error.ReadFailed;
+    return res;
+}

src/util/data.zig

@@ -1,174 +0,0 @@
-//! A reader for a regular file.
-
-const std = @import("std");
-const Reader = std.Io.Reader;
-const Writer = std.Io.Writer;
-const Limit = std.Io.Limit;
-
-const Archive = @import("../archive.zig");
-const FragEntry = Archive.FragEntry;
-const DecompFn = @import("../decomp.zig").DecompFn;
-const BlockSize = @import("../inode_data/file.zig").BlockSize;
-const OffsetFile = @import("offset_file.zig");
-
-const DataReader = @This();
-
-alloc: std.mem.Allocator,
-fil: OffsetFile,
-decomp: DecompFn,
-block_size: u32,
-
-blocks: []BlockSize,
-
-frag: ?FragEntry = null, // TODO: do something better?
-frag_offset: u32 = 0,
-size: u64,
-
-interface: Reader,
-
-cur_offset: u64,
-block_idx: u32 = 0,
-
-pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64) DataReader {
-    return .{
-        .alloc = alloc,
-        .fil = archive.fil,
-        .decomp = archive.decomp,
-        .block_size = archive.super.block_size,
-        .blocks = blocks,
-        .size = size,
-        .cur_offset = start,
-        .interface = .{
-            .end = 0,
-            .seek = 0,
-            .buffer = &[0]u8{},
-            .vtable = &.{
-                .stream = stream,
-                .discard = discard,
-                .readVec = readVec,
-            },
-        },
-    };
-}
-pub fn deinit(self: *DataReader) void {
-    self.alloc.free(self.interface.buffer);
-    self.interface.end = 0;
-    self.interface.seek = 0;
-}
-
-pub fn addFragment(self: *DataReader, entry: FragEntry, frag_offset: u32) void {
-    self.frag = entry;
-    self.frag_offset = frag_offset;
-}
-
-fn numBlocks(self: DataReader) usize {
-    var res = self.blocks.len;
-    if (self.frag != null) res += 1;
-    return res;
-}
-
-fn advance(self: *DataReader) !void {
-    if (self.block_idx > self.blocks.len or (self.block_idx == self.blocks.len and self.frag == null)) {
-        if (self.interface.buffer.len > 0) {
-            self.alloc.free(self.interface.buffer);
-            self.interface.buffer = &[0]u8{};
-            self.interface.end = 0;
-            self.interface.seek = 0;
-        }
-        return Reader.Error.EndOfStream;
-    }
-    defer self.block_idx += 1;
-    const cur_block_size = if (self.block_idx == self.numBlocks() - 1) self.size % self.block_size else self.block_size;
-    try self.resizeBuffer(cur_block_size);
-    self.interface.seek = 0;
-    self.interface.end = cur_block_size;
-    if (self.block_idx == self.blocks.len) { // fragment
-        var rdr = try self.fil.readerAt(self.frag.?.start, &[0]u8{});
-        if (self.frag.?.size.uncompressed) {
-            try rdr.interface.discardAll(self.frag_offset);
-            try rdr.interface.readSliceAll(self.interface.buffer);
-            return;
-        }
-        const tmp_buf = try self.alloc.alloc(u8, self.frag.?.size.size);
-        defer self.alloc.free(tmp_buf);
-        try rdr.interface.readSliceAll(tmp_buf);
-        const needed_block = try self.alloc.alloc(u8, self.block_size);
-        defer self.alloc.free(needed_block);
-        _ = try self.decomp(self.alloc, tmp_buf, needed_block);
-        @memcpy(self.interface.buffer, needed_block[self.frag_offset .. self.frag_offset + cur_block_size]);
-        return;
-    }
-    const block = self.blocks[self.block_idx];
-    if (block.size == 0) {
-        @memset(self.interface.buffer, 0);
-        return;
-    }
-    var rdr = try self.fil.readerAt(self.cur_offset, &[0]u8{});
-    self.cur_offset += block.size;
-    if (block.uncompressed) {
-        try rdr.interface.readSliceAll(self.interface.buffer);
-        return;
-    }
-    const tmp_buf = try self.alloc.alloc(u8, block.size);
-    defer self.alloc.free(tmp_buf);
-    try rdr.interface.readSliceAll(tmp_buf);
-    _ = try self.decomp(self.alloc, tmp_buf, self.interface.buffer);
-}
-/// Does not guarentee that data currently in the buffer is retained.
-fn resizeBuffer(self: *DataReader, size: usize) !void {
-    if (self.interface.buffer.len == size) return;
-    if (!self.alloc.resize(self.interface.buffer, size)) {
-        self.alloc.free(self.interface.buffer);
-        self.interface.buffer = self.alloc.alloc(u8, size) catch |err| {
-            self.interface.buffer = &[0]u8{};
-            return err;
-        };
-    } else {
-        self.interface.buffer.len = size;
-    }
-}
-
-fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) Reader.StreamError!usize {
-    var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr));
-    if (rdr.seek >= rdr.end) self.advance() catch |err| {
-        if (err == error.EndOfStream) return error.EndOfStream;
-        std.log.err("Error advancing data reader: {}\n", .{err});
-        return Reader.Error.ReadFailed;
-    };
-    if (limit == .nothing) return 0;
-    const to_read = @min(rdr.end - rdr.seek, @intFromEnum(limit));
-    const res = try wrt.write(rdr.buffer[rdr.seek .. rdr.seek + to_read]);
-    rdr.seek += res;
-    return res;
-}
-
-fn discard(rdr: *Reader, limit: Limit) Reader.Error!usize {
-    var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr));
-    if (rdr.seek >= rdr.end) self.advance() catch |err| {
-        if (err == error.EndOfStream) return error.EndOfStream;
-        std.log.err("Error advancing data reader: {}\n", .{err});
-        return Reader.Error.ReadFailed;
-    };
-    if (limit == .nothing) return 0;
-    const to_adv = @min(rdr.end - rdr.seek, @intFromEnum(limit));
-    rdr.seek += to_adv;
-    return to_adv;
-}
-
-fn readVec(rdr: *Reader, vec: [][]u8) Reader.Error!usize {
-    var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr));
-    if (rdr.seek >= rdr.end) self.advance() catch |err| {
-        if (err == error.EndOfStream) return error.EndOfStream;
-        std.log.err("Error advancing data reader: {}\n", .{err});
-        return Reader.Error.ReadFailed;
-    };
-    var cur_red: usize = 0;
-    for (vec) |s| {
-        const to_copy: usize = @min(rdr.end - rdr.seek, s.len);
-        @memcpy(s[0..to_copy], rdr.buffer[rdr.seek .. rdr.seek + to_copy]);
-        rdr.seek += to_copy;
-        cur_red += to_copy;
-        if (rdr.end == rdr.seek) break;
-    }
-    return cur_red;
-}

src/util/data_extract.zig

@@ -0,0 +1,85 @@
+const std = @import("std");
+const Io = std.Io;
+
+const BlockSize = @import("../inode_data/file.zig").BlockSize;
+const Decompress = @import("decompress.zig");
+
+const DataExtract = @This();
+
+decomp: Decompress.Fn,
+map: Io.File.MemoryMap,
+
+block_size: u32,
+block_start: u64,
+size: u64,
+blocks: []BlockSize,
+
+frag_data: ?[]u8 = null,
+frag_offset: u32 = undefined,
+
+pub fn init(decomp: Decompress.Fn, map: Io.File.MemoryMap, block_size: u32, block_start: u64, size: u64, blocks: []BlockSize) DataExtract {
+    return .{
+        .decomp = decomp,
+        .map = map,
+
+        .block_size = block_size,
+        .block_start = block_start,
+        .size = size,
+        .blocks = blocks,
+    };
+}
+pub fn addFrag(self: *DataExtract, frag_block: []u8, frag_offset: u32) void {
+    self.frag_data = frag_block;
+    self.frag_offset = frag_offset;
+}
+
+pub const Error = error{} || Io.File.MemoryMap.CreateError || Io.File.WritePositionalError || Decompress.Error || Io.File.MemoryMap.SetLengthError;
+
+pub fn asyncExtract(self: DataExtract, alloc: std.mem.Allocator, io: Io, fil: Io.File) Error!void {
+    if (self.size == 0) return;
+
+    try fil.writePositionalAll(io, &.{0}, self.size - 1);
+    var map = try fil.createMemoryMap(io, .{ .len = self.size, .protection = .{ .write = true }, .undefined_contents = true });
+    defer map.destroy(io);
+
+    var group: Io.Group = .init;
+    defer group.cancel(io);
+
+    var ret_err: ?Error = null;
+
+    var offset: u64 = self.block_start;
+    for (0..self.blocks.len) |i| {
+        group.async(io, blockThread, .{ self, alloc, map, offset, i, &ret_err });
+        offset += self.blocks[i].size;
+    }
+    if (self.frag_data != null)
+        group.async(io, fragThread, .{ self, map });
+
+    try group.await(io);
+    if (ret_err != null) return ret_err.?;
+    return map.write(io);
+}
+fn blockThread(self: DataExtract, alloc: std.mem.Allocator, map: Io.File.MemoryMap, read_offset: u64, idx: usize, ret_err: *?Error) error{Canceled}!void {
+    const block = self.blocks[idx];
+    const write_offset = idx * self.block_size;
+
+    const size = if (self.frag_data == null and idx == self.blocks.len - 1)
+        self.size % self.block_size
+    else
+        self.block_size;
+
+    if (block.size == 0) {
+        @memset(map.memory[write_offset..][0..size], 0);
+        return;
+    } else if (block.uncompressed) {
+        @memcpy(map.memory[write_offset..][0..size], self.map.memory[read_offset..][0..block.size]);
+    }
+    _ = self.decomp(alloc, self.map.memory[read_offset..][0..block.size], map.memory[write_offset..][0..size]) catch |err| {
+        ret_err.* = err;
+        return error.Canceled;
+    };
+}
+fn fragThread(self: DataExtract, map: Io.File.MemoryMap) error{Canceled}!void {
+    const size = self.size % self.block_size;
+    @memcpy(map.memory[self.blocks.len * self.block_size ..][0..size], self.frag_data.?[self.frag_offset..][0..size]);
+}

src/util/data_threaded.zig

@@ -1,183 +0,0 @@
-//! Similiar to DataReader, but set-up for threaded writing to files.
-
-const std = @import("std");
-const Reader = std.Io.Reader;
-const Writer = std.Io.Writer;
-const Limit = std.Io.Limit;
-const WaitGroup = std.Thread.WaitGroup;
-const Pool = std.Thread.Pool;
-
-const Archive = @import("../archive.zig");
-const FragEntry = Archive.FragEntry;
-const DecompFn = @import("../decomp.zig").DecompFn;
-const BlockSize = @import("../inode_data/file.zig").BlockSize;
-const OffsetFile = @import("offset_file.zig");
-
-const ThreadedDataReader = @This();
-
-alloc: std.mem.Allocator,
-fil: OffsetFile,
-decomp: DecompFn,
-block_size: u32,
-
-blocks: []BlockSize,
-
-frag: ?FragEntry = null, // TODO: do something better?
-frag_offset: u32 = 0,
-size: u64,
-
-start_offset: u64,
-
-pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64) ThreadedDataReader {
-    return .{
-        .alloc = alloc,
-        .fil = archive.fil,
-        .decomp = archive.decomp,
-        .block_size = archive.super.block_size,
-        .blocks = blocks,
-        .size = size,
-        .start_offset = start,
-    };
-}
-
-pub fn addFragment(self: *ThreadedDataReader, entry: FragEntry, frag_offset: u32) void {
-    self.frag = entry;
-    self.frag_offset = frag_offset;
-}
-
-fn numBlocks(self: ThreadedDataReader) usize {
-    var res = self.blocks.len;
-    if (self.frag != null) res += 1;
-    return res;
-}
-
-/// Extract the data to the file threadedly, using pool to spawn threads.
-/// If multiple errors occur, thread spawning errors will have, then the last decompression error that occurs;
-///
-/// The function must be called from an unused DataReader. The DataReader is still usable afterwards.
-/// If only extractThreaded is used, there is no need to call deinit() afterwards.
-///
-/// The file will always be written to starting at 0.
-pub fn extractThreaded(self: ThreadedDataReader, file: std.fs.File, pool: *Pool) !void {
-    var wg: WaitGroup = .{};
-    wg.startMany(self.numBlocks());
-    var out_err: ?anyerror = null;
-
-    var cur_write_offset: u64 = 0;
-    var cur_read_offset: u64 = self.start_offset;
-    for (0..self.blocks.len) |i| {
-        const cur_block_size = if (i == self.numBlocks() - 1) self.size % self.block_size else self.block_size;
-        try pool.spawn(workThreadBlocks, .{ self, file, cur_write_offset, cur_read_offset, self.blocks[i], cur_block_size, &wg, &out_err });
-        cur_write_offset += cur_block_size;
-        cur_read_offset += self.blocks[i].size;
-    }
-    if (self.frag != null) {
-        try pool.spawn(workThreadFragment, .{ self, file, cur_write_offset, &wg, &out_err });
-    }
-    pool.waitAndWork(&wg);
-    if (out_err != null) return out_err.?;
-}
-
-fn workThreadBlocks(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, read_offset: u64, block: BlockSize, cur_block_size: u64, wg: *WaitGroup, out_err: *?anyerror) void {
-    defer wg.finish();
-    var wrt = fil.writer(&[0]u8{});
-    wrt.seekTo(write_offset) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    defer wrt.interface.flush() catch |err| {
-        out_err.* = err;
-    };
-    if (block.size == 0) {
-        wrt.interface.splatByteAll(0, cur_block_size) catch |err| {
-            out_err.* = err;
-            return;
-        };
-        return;
-    }
-    var rdr = self.fil.readerAt(read_offset, &[0]u8{}) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    if (block.uncompressed) {
-        rdr.interface.streamExact(&wrt.interface, block.size) catch |err| {
-            out_err.* = err;
-            return;
-        };
-        return;
-    }
-    // TODO: shared buffers
-    const read_buf = self.alloc.alloc(u8, block.size) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    defer self.alloc.free(read_buf);
-    rdr.interface.readSliceAll(read_buf) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    // TODO: shared buffers
-    const res_buf = self.alloc.alloc(u8, cur_block_size) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    defer self.alloc.free(res_buf);
-    _ = self.decomp(self.alloc, read_buf, res_buf) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    wrt.interface.writeAll(res_buf) catch |err| {
-        out_err.* = err;
-        return;
-    };
-}
-fn workThreadFragment(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, wg: *WaitGroup, out_err: *?anyerror) void {
-    defer wg.finish();
-
-    var wrt = fil.writer(&[0]u8{});
-    wrt.seekTo(write_offset) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    defer wrt.interface.flush() catch |err| {
-        out_err.* = err;
-    };
-
-    var rdr = self.fil.readerAt(self.frag.?.start, &[0]u8{}) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    if (self.frag.?.size.uncompressed) {
-        rdr.interface.discardAll(self.frag_offset) catch |err| {
-            out_err.* = err;
-            return;
-        };
-        rdr.interface.streamExact(&wrt.interface, self.size % self.block_size) catch |err| {
-            out_err.* = err;
-            return;
-        };
-        return;
-    }
-    const tmp_buf = self.alloc.alloc(u8, self.frag.?.size.size) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    defer self.alloc.free(tmp_buf);
-    rdr.interface.readSliceAll(tmp_buf) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    const needed_block = self.alloc.alloc(u8, self.block_size) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    defer self.alloc.free(needed_block);
-    _ = self.decomp(self.alloc, tmp_buf, needed_block) catch |err| {
-        out_err.* = err;
-        return;
-    };
-    wrt.interface.writeAll(needed_block[self.frag_offset .. self.frag_offset + (self.size % self.block_size)]) catch |err| {
-        out_err.* = err;
-        return;
-    };
-}

src/util/decomp_cache.zig

@@ -0,0 +1,112 @@
+const std = @import("std");
+const Io = std.Io;
+const ArrayHashMap = std.array_hash_map.Auto;
+const Atomic = std.atomic.Value;
+
+const Decompress = @import("decompress.zig");
+const Fn = Decompress.Fn;
+const DecompressType = Decompress.CompressionType;
+
+const DecompCache = @This();
+
+const Cache = struct {
+    cache: []u8,
+    usage: Atomic(u32),
+};
+
+arena: std.heap.ArenaAllocator,
+decomp: Fn,
+
+map: Io.File.MemoryMap,
+
+cache: ArrayHashMap(u64, Cache),
+mut: Io.RwLock = .init,
+cond: Io.Condition = .init,
+
+max_size: u64,
+cur_size: u64 = 0,
+
+pub fn init(alloc: std.mem.Allocator, map: Io.File.MemoryMap, decomp_type: DecompressType, max_size: u64) !DecompCache {
+    return .{
+        .arena = .init(alloc),
+        .decomp = try Decompress.getDecompressFn(decomp_type),
+
+        .map = map,
+
+        .cache = .empty,
+
+        .max_size = max_size,
+    };
+}
+pub fn deinit(self: *DecompCache, io: Io) void {
+    self.mut.lockUncancelable(io);
+    self.cache.deinit(self.arena.child_allocator);
+    self.arena.deinit();
+    self.map.destroy(io);
+}
+
+fn makeRoom(self: *DecompCache, io: Io, size: u32) !void {
+    if (size + self.cur_size < self.max_size) return;
+    var iter = self.cache.iterator();
+    while (iter.next()) |ent| {
+        const val = ent.value_ptr;
+        if (val.usage.load(.unordered) == 0) {
+            self.cur_size -= val.cache.len;
+            _ = self.cache.orderedRemove(ent.key_ptr.*);
+        }
+        if (size + self.cur_size < self.max_size) return;
+    }
+    try self.cond.wait(io, &self.mut.mutex);
+    return self.makeRoom(io, size);
+}
+
+pub fn checkinBlock(self: *DecompCache, io: Io, offset: u64) !void {
+    self.mut.lockSharedUncancelable(io);
+    defer self.mut.unlockShared(io);
+
+    const get = self.cache.getPtr(offset);
+    if (get == null) return error.NotACachedBlock;
+    const res = get.?.usage.fetchSub(1, .acq_rel);
+    if (res == 0) self.cond.broadcast(io);
+}
+pub fn checkoutBlock(self: *DecompCache, io: Io, offset: u64, data_size: u32, max_result_size: u32) ![]u8 {
+    {
+        try self.mut.lockShared(io);
+        defer self.mut.unlockShared(io);
+
+        const get = self.cache.getPtr(offset);
+        if (get != null) {
+            _ = get.?.usage.fetchAdd(1, .acq_rel);
+            return get.?.cache;
+        }
+    }
+    try self.mut.lock(io);
+    defer self.mut.unlock(io);
+
+    try self.makeRoom(io, max_result_size);
+
+    var alloc = self.arena.allocator();
+    const buf_alloc = self.arena.child_allocator;
+
+    var out = try alloc.alloc(u8, max_result_size);
+    errdefer alloc.free(out);
+
+    const out_size = try self.decomp(buf_alloc, self.map.memory[offset..][0..data_size], out);
+    if (out_size != max_result_size) {
+        if (alloc.resize(out, out_size)) {
+            out.len = out_size;
+        } else {
+            const new_out = try alloc.alloc(u8, out_size);
+            @memcpy(new_out, out[0..out_size]);
+            alloc.free(out);
+            out = new_out;
+        }
+    }
+
+    try self.cache.put(buf_alloc, offset, .{
+        .cache = out,
+        .usage = .init(1),
+    });
+
+    return out;
+}

src/util/decompress.zig

@@ -0,0 +1,38 @@
+const std = @import("std");
+const Io = std.Io;
+
+const config = @import("config");
+
+const c_decomp = @import("c_decomp.zig");
+const zig_decomp = @import("zig_decomp.zig");
+
+pub const Error = Io.Reader.Error || std.mem.Allocator.Error;
+
+pub const Fn = *const fn (alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize;
+
+pub const CompressionType = enum(u16) {
+    gzip = 1,
+    lzma,
+    lzo,
+    xz,
+    lz4,
+    zstd,
+};
+
+pub fn getDecompressFn(t: CompressionType) !Fn {
+    return if (config.use_zig_decomp) switch (t) {
+        .lzo => error.LzoUnsupported,
+        .lz4 => error.Lz4Unsupported,
+        .gzip => zig_decomp.zlibDecompress,
+        .lzma => zig_decomp.lzmaDecompress,
+        .xz => zig_decomp.xzDecompress,
+        .zstd => zig_decomp.zstdDecompress,
+    } else switch (t) {
+        .gzip => c_decomp.zlibDecompress,
+        .lzma => c_decomp.lzmaDecompress,
+        .lzo => if (config.allow_lzo) c_decomp.lzoDecompress else error.LzoUnsupported,
+        .xz => c_decomp.lzmaDecompress,
+        .lz4 => c_decomp.lz4Decompress,
+        .zstd => c_decomp.zstdDecompress,
+    };
+}

src/util/metadata.zig

@@ -1,33 +1,30 @@
+//! A cache for decompressed blocks. Used for Metadata & fragments.
+
 const std = @import("std");
-const Reader = std.Io.Reader;
-const Writer = std.Io.Writer;
-const Limit = std.Io.Limit;
-const StreamError = std.Io.Reader.StreamError;
+const Io = std.Io;
+const Reader = Io.Reader;
+const Writer = Io.Writer;
+const Limit = Io.Limit;
 
-const DecompFn = @import("../decomp.zig").DecompFn;
+const DecompCache = @import("decomp_cache.zig");
 
-const BlockHeader = packed struct {
+const MetadataReader = @This();
+
+const BlockHeader = packed struct(u16) {
     size: u15,
     uncompressed: bool,
 };
 
-const This = @This();
+io: Io,
 
-alloc: std.mem.Allocator,
-rdr: *Reader,
-decomp: DecompFn,
+cur_offset: u64 = 0,
+next_offset: u64,
 
-buf: [8192]u8 = undefined,
+cache: *DecompCache,
 
-interface: Reader,
-err: ?anyerror = null,
+buf_uncompress: bool = false,
 
-pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: DecompFn) This {
-    return .{
-        .alloc = alloc,
-        .rdr = rdr,
-        .decomp = decomp,
-        .interface = .{
+interface: Reader = .{
     .buffer = &[0]u8{},
     .end = 0,
     .seek = 0,
@@ -36,62 +33,78 @@ pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: DecompFn) This {
         .discard = discard,
         .readVec = readVec,
     },
-        },
+},
+
+pub fn init(io: Io, cache: *DecompCache, offset: u64) MetadataReader {
+    return .{
+        .io = io,
+
+        .next_offset = offset,
+
+        .cache = cache,
     };
 }
+pub fn deinit(self: *MetadataReader) void {
+    if (self.cur_offset != 0 and !self.buf_uncompress)
+        self.cache.checkinBlock(self.io, self.cur_offset) catch {};
+}
 
-fn advance(self: *This) !void {
-    self.interface.seek = 0;
-    var hdr: BlockHeader = undefined;
-    try self.rdr.readSliceEndian(BlockHeader, @ptrCast(&hdr), .little);
+fn advance(self: *MetadataReader) !void {
+    if (self.interface.buffer.len > 0 and !self.buf_uncompress)
+        self.cache.checkinBlock(self.io, self.cur_offset) catch |err| {
+            std.debug.print("UH OH! {}\n", .{err});
+            return error.ReadFailed;
+        };
+    const hdr: BlockHeader = @bitCast(std.mem.readInt(u16, self.cache.map.memory[self.next_offset..][0..2], .little));
+    self.cur_offset = self.next_offset + 2;
+    self.next_offset = self.cur_offset + hdr.size;
+
+    self.buf_uncompress = hdr.uncompressed;
     if (hdr.uncompressed) {
-        try self.rdr.readSliceEndian(u8, self.buf[0..hdr.size], .little);
+        self.interface.buffer = self.cache.map.memory[self.cur_offset..][0..hdr.size];
         self.interface.end = hdr.size;
-        self.interface.buffer = self.buf[0..hdr.size];
+        self.interface.seek = 0;
         return;
     }
-    var tmp_buf: [8192]u8 = undefined;
-    try self.rdr.readSliceAll(tmp_buf[0..hdr.size]);
-    self.interface.end = try self.decomp(self.alloc, tmp_buf[0..hdr.size], &self.buf);
-    self.interface.buffer = self.buf[0..self.interface.end];
+    self.interface.buffer = try self.cache.checkoutBlock(self.io, self.cur_offset, hdr.size, 8192);
+    self.interface.end = self.interface.buffer.len;
+    self.interface.seek = 0;
 }
 
-fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) StreamError!usize {
-    const self: *This = @fieldParentPtr("interface", rdr);
-    if (rdr.end == rdr.seek) self.advance() catch |err| {
-        self.err = err;
-        return StreamError.ReadFailed;
-    };
-    if (@intFromEnum(limit) == 0) return 0;
-    const to_write = @min(rdr.end - rdr.seek, @intFromEnum(limit));
-    const wrote = try wrt.write(self.buf[rdr.seek .. rdr.seek + to_write]);
-    self.interface.seek += wrote;
+fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {
+    if (r.seek == r.end) {
+        var self: *MetadataReader = @fieldParentPtr("interface", r);
+        self.advance() catch return Reader.Error.ReadFailed;
+    }
+    if (limit == .nothing) return 0;
+    const to_write = @min(r.end - r.seek, @intFromEnum(limit));
+    const wrote = try w.write(r.buffer[r.seek..][0..to_write]);
+    r.seek += wrote;
     return wrote;
 }
-fn discard(rdr: *Reader, limit: Limit) error{ EndOfStream, ReadFailed }!usize {
-    const self: *This = @fieldParentPtr("interface", rdr);
-    if (rdr.end == rdr.seek) self.advance() catch |err| {
-        self.err = err;
-        return StreamError.ReadFailed;
-    };
-    if (@intFromEnum(limit) == 0) return 0;
-    const to_skip = @min(rdr.end - rdr.seek, @intFromEnum(limit));
-    rdr.seek += to_skip;
+fn discard(r: *Reader, limit: Limit) Reader.Error!usize {
+    if (r.seek == r.end) {
+        var self: *MetadataReader = @fieldParentPtr("interface", r);
+        self.advance() catch return Reader.Error.ReadFailed;
+    }
+    if (limit == .nothing) return 0;
+    const to_skip = @min(r.end - r.seek, @intFromEnum(limit));
+    r.seek += to_skip;
     return to_skip;
 }
-fn readVec(rdr: *Reader, vec: [][]u8) error{ EndOfStream, ReadFailed }!usize {
-    const self: *This = @fieldParentPtr("interface", rdr);
-    if (rdr.end == rdr.seek) self.advance() catch |err| {
-        self.err = err;
-        return StreamError.ReadFailed;
-    };
-    var cur_red: usize = 0;
-    for (vec) |s| {
-        const to_copy: usize = @min(rdr.end - rdr.seek, s.len);
-        @memcpy(s[0..to_copy], self.buf[rdr.seek .. rdr.seek + to_copy]);
-        rdr.seek += to_copy;
-        cur_red += to_copy;
-        if (rdr.end == rdr.seek) break;
+fn readVec(r: *Reader, vec: [][]u8) Reader.Error!usize {
+    if (r.seek == r.end) {
+        var self: *MetadataReader = @fieldParentPtr("interface", r);
+        self.advance() catch return Reader.Error.ReadFailed;
     }
-    return cur_red;
+    if (vec.len == 0) return 0;
+    var total_copied: usize = 0;
+    for (vec) |v| {
+        const to_cpy = @min(r.end - r.seek, v.len);
+        @memcpy(v[0..to_cpy], r.buffer[r.seek..][0..to_cpy]);
+        r.seek += to_cpy;
+        total_copied += to_cpy;
+        if (r.seek == r.end) break;
+    }
+    return total_copied;
 }

src/util/offset_file.zig

@@ -1,20 +0,0 @@
-//! A File where it's meaningful (to us) content starts at a given offset.
-
-const std = @import("std");
-const File = std.fs.File;
-const Reader = std.fs.File.Reader;
-
-const OffsetFile = @This();
-
-fil: File,
-offset: u64,
-
-pub fn init(fil: File, init_offset: u64) OffsetFile {
-    return .{ .fil = fil, .offset = init_offset };
-}
-
-pub fn readerAt(self: OffsetFile, offset: u64, buffer: []u8) !Reader {
-    var rdr = self.fil.reader(buffer);
-    try rdr.seekTo(self.offset + offset);
-    return rdr;
-}

src/util/zig_decomp.zig

@@ -0,0 +1,40 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+const flate = std.compress.flate;
+const zstd = std.compress.zstd;
+const xz = std.compress.xz;
+const lzma = std.compress.lzma;
+
+const Error = @import("decompress.zig").Error;
+
+pub fn zlibDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var buf: [flate.max_window_len]u8 = undefined;
+
+    var rdr: Reader = .fixed(in);
+    var decomp: flate.Decompress = .init(&rdr, .zlib, &buf);
+
+    return decomp.reader.readSliceShort(out);
+}
+pub fn zstdDecompress(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    const buf = try alloc.alloc(u8, in.len + zstd.block_size_max);
+    defer alloc.free(buf);
+
+    var rdr: Reader = .fixed(in);
+    var decomp: zstd.Decompress = .init(&rdr, buf, .{ .window_len = in.len });
+
+    return decomp.reader.readSliceShort(out);
+}
+pub fn lzmaDecompress(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var rdr: Reader = .fixed(in);
+    var decomp: lzma.Decompress = .initOptions(&rdr, alloc, &[0]u8{}, .{}, 2 * out.len);
+    defer decomp.deinit();
+
+    return decomp.reader.readSliceShort(out);
+}
+pub fn xzDecompress(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var rdr: Reader = .fixed(in);
+    var decomp: xz.Decompress = .init(&rdr, alloc, &[0]u8{});
+    defer decomp.deinit();
+
+    return decomp.reader.readSliceShort(out);
+}