Fixed using the wrong decompression function for gzip

.github/workflows/release-build.yml

@@ -13,13 +13,13 @@ jobs:
         uses: actions/checkout@v6
       - uses: mlugg/setup-zig@v2
       - name: Install deps
-        run: sudo apt update && sudo apt install -y liblzma-dev liblzo2-dev
+        run: sudo apt update && sudo apt install -y zlib1g-dev libzstd-dev liblzma-dev liblz4-dev liblzo2-dev
       - name: Build normal version
-        run: zig build --release=fast -Duse_zig_decomp=true -Dversion=${{ github.ref_name }}
+        run: zig build --release=fast -Dversion=${{ github.ref_name }}
-      - name: Move zig build out
+      - name: Move normal build out
         run: mv zig-out/bin/unsquashfs ./unsquashfs-x86_64-zig-libs
       - name: Rebuild with C libraries
-        run: zig build --release=fast -Dversion="${{ github.ref_name }}"
+        run: zig build --release=fast -Duse_c_libs=true -Dversion="${{ github.ref_name }}"
       - name: Move C build out
         run: mv zig-out/bin/unsquashfs ./unsquashfs-x86_64-c-libs
       - name: Release

.gitignore

@@ -2,4 +2,3 @@ testing/
 
 .zig-cache/
 zig-out/
-zig-pkg/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "extern/zstd"]
+	path = extern/zstd
+	url = https://github.com/facebook/zstd

.zed/debug.json

@@ -11,7 +11,7 @@
 
 		"build": {
 			"command": "zig",
-			"args": ["build", "-Ddebug=true"],
+			"args": ["build", "-Duse_c_libs=true", "-Ddebug=true"],
 		},
 
 		"program": "zig-out/bin/unsquashfs",

README.md

@@ -10,13 +10,9 @@ Overall works, but currently is missing some features ([see below](#capabilities
 
 ## Build options
 
-> `-Duse_zig_decomp=true`
+> `-Duse_c_libs=true`
 
-Instead of using C libraries for decompression, use Zig's standard library for decompression. If using this option LZO and LZ4 decomrpession types are unsupported and decompression times will be significantly longer.
+Instead of using Zig's standard library for decompression, use the system's C libraries. Has the benefit of being much faster and enabling LZO and LZ4 decompression.
-
-> `-Ddynamic=true`
-
-Dynamicly link C libraries (if they're used) instead of statically linking them.
 
 > `-Dallow_lzo=true`
 
@@ -39,22 +35,22 @@ Most features are present except for the following:
 
 ## Performance
 
-This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive which is fairly small and uses zstd compression with `-Doptimize=ReleaseFast`.
+This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive (which is fairly small and uses zstd compression) and with `--release=fast`.
 
 Currently, my only performance checks are checking execution time, nothing deeper.
 
-* Currently, using my test archive, performance aproximately matches `unsquashfs` when multi-threaded, but significantly slower when single-threaded.
+* Under ideal circumstances, my library is ~70% slower (.12s vs .20s).
-* Using Zig decompression libraries *significantly* increases decompression time.
+* Using Zig decompression libraries *significantly* increases decompression time by ~600%. Under ideal circumstances.
 * Performance improvements/regressions will be common. I'm still learning Zig.
 
 Example Times:
 
-* *unsquashfs, multi-threaded*: .11s
+* *unsquashfs, multi-threaded*: .12s
 * *unsquashfs, single-threaded*: .13s
-* *C-libs, multi-threaded*: .10s
+* *C-libs, single-threaded*: .45s
-* *C-libs, single-threaded*: ..28s
+* *C-libs, multi-threaded*: .20s
-* *Zig-libs, single-threaded*: .74s
+* *Zig-libs, single-threaded*: 5.78s
-* *Zig-libs, multi-threaded*: 2.70s
+* *Zig-libs, multi-threaded*: 1.08s
 
 ## Build considerations
 

benchmark.sh

@@ -1,17 +0,0 @@
-#! /usr/bin/env bash
-
-ARCHIVE="testing/LinuxPATest.sfs"
-
-REF_EXT_LOC="testing/LinuxPAReference"
-PROG_EXT_LOC="testing/LinuxPABinTest"
-
-echo "Testing Multi-threaded Performance"
-echo ""
-
-hyperfine --warmup 5 --prepare "rm -rf $REF_EXT_LOC && rm -rf $PROG_EXT_LOC" "unsquashfs -d $REF_EXT_LOC $ARCHIVE" "zig-out/bin/unsquashfs -d $PROG_EXT_LOC $ARCHIVE"
-
-echo ""
-echo "Testing Single-threaded Performance"
-echo ""
-
-hyperfine --warmup 5 --prepare "rm -rf $REF_EXT_LOC && rm -rf $PROG_EXT_LOC" "unsquashfs -p 1 -d $REF_EXT_LOC $ARCHIVE" "zig-out/bin/unsquashfs -p 1 -d $PROG_EXT_LOC $ARCHIVE"

build.zig

@@ -1,10 +1,13 @@
 const std = @import("std");
+const Compile = std.Build.Step.Compile;
+const ResolvedTarget = std.Build.ResolvedTarget;
+const OptimizeMode = std.builtin.OptimizeMode;
+const Module = std.Build.Module;
 
 pub fn build(b: *std.Build) !void {
-    const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
+    const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use Zig standard library for decompression instead of C libraries.") orelse false;
     const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
-    const dynamic = b.option(bool, "dynamic", "Dynamicly link C decompression libraries") orelse false;
+    const debug = b.option(bool, "debug", "Enable options to make debugging easier.");
-    var debug = b.option(bool, "debug", "Enable options to make debugging easier.");
     const version_string_option = b.option([]const u8, "version", "Version of the library/binary");
 
     const zig_squashfs_options = b.addOptions();
@@ -12,43 +15,30 @@ pub fn build(b: *std.Build) !void {
     zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo);
 
     const target = b.standardTargetOptions(.{});
-    var optimize = b.standardOptimizeOption(.{});
+    const optimize = b.standardOptimizeOption(.{});
-
+    const mod = b.addModule("zig_squashfs", .{
-    if (debug == true)
+        .root_source_file = b.path("src/root.zig"),
-        optimize = .Debug;
+        .target = target,
-    if (optimize == .Debug)
+        .optimize = if (debug == true) .Debug else optimize,
-        debug = true;
+        .link_libc = !use_zig_decomp,
-
+        .valgrind = debug,
-    const lib = b.addLibrary(.{
+        .error_tracing = debug,
-        .name = "squashfs",
+        .strip = if (debug == true) false else null,
-        .root_module = b.createModule(.{
-            .optimize = if (debug == true) .Debug else optimize,
-            .target = target,
-            .valgrind = debug,
-            .root_source_file = b.path("src/root.zig"),
-            .imports = &.{
-                .{ .name = "options", .module = zig_squashfs_options.createModule() },
-            },
-        }),
-        .use_llvm = debug,
     });
-
+    mod.addOptions("config", zig_squashfs_options);
-    const deps = try dependencies(b, optimize, target, use_zig_decomp, allow_lzo, dynamic);
-    defer b.allocator.free(deps);
-    for (deps) |d|
-        lib.root_module.linkLibrary(d);
 
     if (!use_zig_decomp) {
-        const c = b.addTranslateC(.{
+        var zlib = b.dependency("zlib_ng", .{});
-            .optimize = optimize,
+        mod.linkLibrary(zlib.artifact("zng"));
-            .target = target,
-            .root_source_file = b.path("src/c.h"),
-        });
-        if (allow_lzo) c.defineCMacro("ALLOW_LZO", null);
-        lib.root_module.addImport("c", c.createModule());
 
-        if (dynamic)
+        mod.linkSystemLibrary("lzma", .{ .preferred_link_mode = .static });
-            dynamicLinkLibraries(c, allow_lzo);
+        if (allow_lzo == true)
+            mod.linkSystemLibrary("minilzo", .{ .preferred_link_mode = .static });
+        mod.linkSystemLibrary("lz4", .{ .preferred_link_mode = .static });
+
+        const zstd_lib = buildZstdLibrary(b, target, optimize, debug);
+        mod.linkLibrary(zstd_lib);
+        mod.addIncludePath(b.path("extern/zstd/lib/"));
     }
 
     var version = version_string_option orelse "0.0.0-testing";
@@ -59,106 +49,115 @@ pub fn build(b: *std.Build) !void {
         "version",
         try std.SemanticVersion.parse(version),
     );
+
+    var exe_mod = b.createModule(.{
+        .root_source_file = b.path("src/bin/unsquashfs.zig"),
+        .target = target,
+        .optimize = if (debug == true) .Debug else optimize,
+        .link_libc = !use_zig_decomp,
+        .imports = &.{
+            .{ .name = "zig_squashfs", .module = mod },
+        },
+        .valgrind = debug,
+        .error_tracing = debug,
+        .strip = if (debug == true) false else null,
+    });
+    exe_mod.addOptions("config", unsquashfs_options);
     const exe = b.addExecutable(.{
         .name = "unsquashfs",
-        .root_module = b.createModule(.{
+        .root_module = exe_mod,
-            .optimize = if (debug == true) .Debug else optimize,
+        .use_llvm = debug,
-            .target = target,
+    });
-            .valgrind = debug,
+
-            .root_source_file = b.path("src/bin/unsquashfs.zig"),
+    const lib = b.addLibrary(.{
-            .imports = &.{
+        .name = "squashfs",
-                .{ .name = "zig_squashfs", .module = lib.root_module },
+        .root_module = mod,
-            },
-        }),
         .use_llvm = debug,
     });
-    exe.root_module.addOptions("config", unsquashfs_options);
 
     b.installArtifact(lib);
     b.installArtifact(exe);
 
     const mod_tests = b.addTest(.{
-        .root_module = b.createModule(.{
+        .root_module = mod,
-            .optimize = optimize,
-            .target = target,
-            .root_source_file = b.path("src/root.zig"),
-            .imports = &.{
-                .{ .name = "options", .module = zig_squashfs_options.createModule() },
-            },
-            .valgrind = debug,
-        }),
-        .use_llvm = debug,
     });
-
-    for (deps) |d|
-        mod_tests.root_module.linkLibrary(d);
-
-    if (!use_zig_decomp) {
-        const c = b.addTranslateC(.{
-            .optimize = optimize,
-            .target = target,
-            .root_source_file = b.path("src/c.h"),
-        });
-        mod_tests.root_module.addImport("c", c.createModule());
-        if (allow_lzo) c.defineCMacro("ALLOW_LZO", null);
-
-        if (dynamic)
-            dynamicLinkLibraries(c, allow_lzo);
-    }
-
     const run_mod_tests = b.addRunArtifact(mod_tests);
+    const exe_tests = b.addTest(.{
+        .root_module = exe.root_module,
+    });
+    const run_exe_tests = b.addRunArtifact(exe_tests);
     const test_step = b.step("test", "Run tests");
     test_step.dependOn(&run_mod_tests.step);
+    test_step.dependOn(&run_exe_tests.step);
 
     // zls build check steps
     const lib_check = b.addLibrary(.{
         .name = "squashfs",
-        .root_module = exe.root_module,
+        .root_module = mod,
     });
     const exe_check = b.addExecutable(.{
         .name = "unsquashfs",
-        .root_module = lib.root_module,
+        .root_module = exe_mod,
     });
     const check = b.step("check", "Check if unsquashfs compiles");
     check.dependOn(&lib_check.step);
     check.dependOn(&exe_check.step);
 }
 
-pub fn dynamicLinkLibraries(mod: *std.Build.Step.TranslateC, allow_lzo: bool) void {
+fn buildZstdLibrary(b: *std.Build, target: ResolvedTarget, optimize: OptimizeMode, debug: ?bool) *Compile {
-    mod.linkSystemLibrary("zstd", .{});
+    var zstd_lib = b.addLibrary(.{
-    mod.linkSystemLibrary("zlib-ng", .{});
+        .name = "zstd",
-    mod.linkSystemLibrary("lzma", .{});
+        .linkage = .static,
-    mod.linkSystemLibrary("lz4", .{});
+        .root_module = b.createModule(.{
-    if (allow_lzo)
+            .target = target,
-        mod.linkSystemLibrary("minilzo", .{});
+            .optimize = if (debug == true) .Debug else optimize,
-}
+            .link_libc = true,
-fn dependencies(
+        }),
-    b: *std.Build,
+        .use_llvm = debug,
-    optimize: std.builtin.OptimizeMode,
+    });
-    target: std.Build.ResolvedTarget,
+    zstd_lib.root_module.addCSourceFiles(.{
-    use_zig_decomp: bool,
+        .root = b.path("extern/zstd/lib/"),
-    allow_lzo: bool,
+        .files = &.{
-    dynamic: bool,
+            "common/debug.c",
-) ![]*std.Build.Step.Compile {
+            "common/entropy_common.c",
-    if (use_zig_decomp or dynamic) return &.{};
+            "common/error_private.c",
-
+            "common/fse_decompress.c",
-    var list: std.ArrayList(*std.Build.Step.Compile) = .empty;
+            "common/pool.c",
-
+            "common/threading.c",
-    const zstd = b.dependency("zstd", .{ .optimize = optimize, .target = target });
+            "common/xxhash.c",
-    try list.append(b.allocator, zstd.artifact("zstd"));
+            "common/zstd_common.c",
-
+            "compress/fse_compress.c",
-    const zng = b.dependency("zlib_ng", .{ .optimize = optimize, .target = target });
+            "compress/hist.c",
-    try list.append(b.allocator, zng.artifact("zng"));
+            "compress/huf_compress.c",
-
+            "compress/zstd_compress.c",
-    const xz = b.dependency("xz", .{ .optimize = optimize, .target = target });
+            "compress/zstd_compress_literals.c",
-    try list.append(b.allocator, xz.artifact("lzma"));
+            "compress/zstd_compress_sequences.c",
-
+            "compress/zstd_compress_superblock.c",
-    const lz4 = b.dependency("lz4", .{ .optimize = optimize, .target = target });
+            "compress/zstd_double_fast.c",
-    try list.append(b.allocator, lz4.artifact("lz4"));
+            "compress/zstd_fast.c",
-
+            "compress/zstd_lazy.c",
-    if (allow_lzo) {
+            "compress/zstd_ldm.c",
-        const minilzo = b.dependency("minilzo", .{ .optimize = optimize, .target = target });
+            "compress/zstdmt_compress.c",
-        try list.append(b.allocator, minilzo.artifact("minilzo"));
+            "compress/zstd_opt.c",
-    }
+            "compress/zstd_preSplit.c",
-    return list.toOwnedSlice(b.allocator);
+            "decompress/huf_decompress.c",
+            "decompress/zstd_ddict.c",
+            "decompress/zstd_decompress_block.c",
+            "decompress/zstd_decompress.c",
+            "dictBuilder/cover.c",
+            "dictBuilder/divsufsort.c",
+            "dictBuilder/fastcover.c",
+            "dictBuilder/zdict.c",
+        },
+    });
+    zstd_lib.root_module.addCSourceFiles(.{
+        .root = b.path("extern/zstd/lib/decompress"),
+        .files = &.{"huf_decompress_amd64.S"},
+    });
+    zstd_lib.installHeadersDirectory(b.path("extern/zstd/lib/"), &.{}, .{});
+    zstd_lib.installHeadersDirectory(b.path("extern/zstd/lib/common/"), &.{}, .{});
+    zstd_lib.installHeadersDirectory(b.path("extern/zstd/lib/compress/"), &.{}, .{});
+    zstd_lib.installHeadersDirectory(b.path("extern/zstd/lib/dictBuilder/"), &.{}, .{});
+    zstd_lib.installHeadersDirectory(b.path("extern/zstd/lib/"), &.{}, .{});
+    return zstd_lib;
 }

build.zig.zon

@@ -5,24 +5,9 @@
     .minimum_zig_version = "0.15.2",
     .dependencies = .{
         .zlib_ng = .{
-            .url = "git+https://github.com/CalebQ42/zig-zlib-ng#5f2f02dfb28acca2517dacbbd09e9b987f57b133",
+            // .url = "https://github.com/CalebQ42/zig-zlib-ng/archive/refs/tags/2.3.3.tar.gz",
-            .hash = "zlib_ng-2.3.3-pre1-2HYS4ClFAABW8KlHMyBHtlNKE3V7kCS8wqfxawG7xeaa",
+            // .hash = "zlib_ng-2.3.3-2HYS4Bw_AADjgv7tkrqjjz2fVz5kRTvha8wN9LEcjYNp",
-        },
+            .path = "../zig-zlib-ng",
-        .zstd = .{
-            .url = "git+https://github.com/allyourcodebase/zstd.git?ref=1.5.7-1#e1a501be57f42c541e8a5597e4b59a074dfd09a3",
-            .hash = "zstd-1.5.7-1-KEItkAMwAAD6OKY3m0OOmXG7aL-aLUfrDqbP5J5oYapU",
-        },
-        .lz4 = .{
-            .url = "git+https://github.com/allyourcodebase/lz4.git?ref=1.10.0-6#41f52ab227caf9d48cf88c89a4d2946caa12b102",
-            .hash = "lz4-1.10.0-6-ewyzw-4NAAAWDpY4xpiqr4LQhZQAC0x_rGnW2iPh6jk2",
-        },
-        .minilzo = .{
-            .url = "git+https://github.com/CalebQ42/zig-minilzo.git#7cbae997b91a44d74b7cd6c073584dc9562a6c90",
-            .hash = "minilzo-2.10.0-Ij7BO8wLAADeWI4Pe4jp8XTDsDaquZR14oZ7_9yKKDWP",
-        },
-        .xz = .{
-            .url = "git+https://github.com/akunaakwei/zig-xz.git#e2d389262c8291907e3e4c6fb119819141c16c0f",
-            .hash = "xz-5.8.2-6v47_JYeAABSL-jonprpL5-E_YaaGc4B5xrbe93WsJ3G",
         },
     },
     .paths = .{

run_tests.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+zig test \
+	-lc \
+	-lz \
+	-llzma \
+	-lminilzo \
+	-llz4 \
+	-lzstd \
+		src/test.zig

src/archive.zig

@@ -1,288 +1,97 @@
+//! A squashfs archive read from a file.
+//! Can be used to directly access File's contents or extract to the filesystem.
+
 const std = @import("std");
-const Io = std.Io;
+const File = std.fs.File;
+const builtin = @import("builtin");
 
 const Decomp = @import("decomp.zig");
 const ExtractionOptions = @import("options.zig");
-const File = @import("file.zig");
 const Inode = @import("inode.zig");
-const LookupTable = @import("lookup_table.zig");
+const InodeRef = Inode.Ref;
-const Decompressor = @import("util/decompressor.zig");
+const BlockSize = @import("inode_data/file.zig").BlockSize;
+const SfsFile = @import("file.zig");
+const Superblock = @import("super.zig").Superblock;
+const Tables = @import("tables.zig");
 const MetadataReader = @import("util/metadata.zig");
-const Utils = @import("util/misc.zig");
 const OffsetFile = @import("util/offset_file.zig");
+const XattrTable = @import("xattr.zig");
+
+const config = if (builtin.is_test) .{
+    .use_zig_decomp = builtin.link_libc != true,
+    .allow_lzo = false,
+} else @import("config");
 
 const Archive = @This();
 
-file: OffsetFile,
+alloc: std.mem.Allocator,
+
+fil: OffsetFile,
+decomp: Decomp.DecompFn,
+
 super: Superblock,
 
-stateless_decomp: Decompressor,
+tables: ?Tables = null,
 
-pub fn init(io: Io, file: std.Io.File, offset: u64) !Archive {
+/// Default settings using std.Thread.getCpuCount() threads and the minimum of 4gb or half of system memory for memory usage.
-    var rdr = file.reader(io, &[0]u8{});
+pub fn init(alloc: std.mem.Allocator, fil: File, offset: u64) !Archive {
-    try rdr.seekTo(offset);
     var super: Superblock = undefined;
-    try rdr.interface.readSliceEndian(Superblock, @ptrCast(&super), .little);
+    const red = try fil.pread(@ptrCast(&super), offset);
+    std.debug.assert(red == @sizeOf(Superblock));
     try super.validate();
-
+    const off_fil: OffsetFile = .init(fil, offset);
+    const decomp: Decomp.DecompFn = switch (super.compression) {
+        .gzip => Decomp.gzipDecompress,
+        .lzma => Decomp.lzmaDecompress,
+        .xz => Decomp.xzDecompress,
+        .zstd => Decomp.zstdDecompress,
+        .lz4 => if (!config.use_zig_decomp) Decomp.cLz4 else return error.Lz4Unsupported,
+        .lzo => if (!config.use_zig_decomp and config.allow_lzo) Decomp.lzoDecompress else return error.LzoUnsupported,
+    };
     return .{
-        .file = try .init(io, file, super.size, offset),
+        .alloc = alloc,
+        .fil = off_fil,
+        .decomp = decomp,
         .super = super,
-
-        .stateless_decomp = try Decomp.StatelessDecomp(super.compression),
     };
 }
-pub fn deinit(self: *Archive, io: Io) void {
+pub fn deinit(self: *Archive) void {
-    self.file.deinit(io);
+    if (self.tables != null)
+        self.tables.?.deinit();
 }
 
-/// The root folder of the Archive. Used to open other Files.
+pub fn inode(self: *Archive, alloc: std.mem.Allocator, num: u32) !Inode {
-pub fn root(self: *Archive, alloc: std.mem.Allocator) !File {
+    if (self.tables == null)
-    const root_inode = try Utils.inodeFromRef(
+        self.tables = try .init(alloc, self);
-        alloc,
+    const ref = try self.export_table.get(num - 1);
-        self.file,
+    var rdr = try self.fil.readerAt(ref.block_start + self.super.inode_start, &[0]u8{});
-        &self.stateless_decomp,
+    var meta: MetadataReader = .init(alloc, &rdr.interface, &self.decomp);
-        self.super.inode_start,
+    try meta.interface.discardAll(ref.block_offset);
-        self.super.block_size,
+    return try .read(alloc, &meta.interface, self.super.block_size);
-        self.super.root_ref,
-    );
-    return .init(alloc, self.*, root_inode, "");
-}
-/// Opens a File within the archive.
-pub fn open(self: *Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
-    var root_file = try self.root(alloc);
-    const path = std.mem.trim(u8, filepath, "/");
-    if (Utils.pathIsSelf(path))
-        return root_file;
-    defer root_file.deinit();
-    return root_file.open(alloc, io, filepath);
 }
 
-/// Returns the inode with the given inode number.
+pub fn root(self: *Archive, alloc: std.mem.Allocator) !SfsFile {
-/// Requires that the archive is exportable (has an export lookup table).
+    if (self.tables == null)
-pub fn inode(self: *Archive, alloc: std.mem.Allocator, io: Io, num: u32) !Inode {
+        self.tables = try .init(alloc, self);
-    if (!self.super.flags.exportable)
+    var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{});
-        return error.NotExportable;
+    var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
-    const ref = try LookupTable.lookupValue(
+    try meta.interface.discardAll(self.super.root_ref.block_offset);
-        Inode.Ref,
+    const in: Inode = try .read(alloc, &meta.interface, self.super.block_size);
-        alloc,
+    return .init(self, in, "");
-        io,
-        &self.stateless_decomp,
-        self.file,
-        self.super.export_start,
-        num + 1,
-    );
-    return Utils.inodeFromRef(
-        alloc,
-        io,
-        self.file,
-        &self.stateless_decomp,
-        self.super.inode_start,
-        self.super.block_size,
-        ref,
-    );
-}
-/// Returns a value at the given index from the Archive's id (uid/gid) table.
-pub fn idTable(self: *Archive, alloc: std.mem.Allocator, io: Io, idx: u32) !u16 {
-    return LookupTable.lookupValue(
-        u16,
-        alloc,
-        io,
-        &self.stateless_decomp,
-        self.file,
-        self.super.id_start,
-        idx,
-    );
 }
 
-/// Extract the entire archive contents to the given directory.
+pub fn open(self: *Archive, alloc: std.mem.Allocator, path: []const u8) !SfsFile {
-pub fn extract(self: *Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void {
+    if (self.tables == null)
-    const root_inode = try Utils.inodeFromRef(
+        self.tables = try .init(alloc, self);
-        alloc,
+    var root_fil = try self.root(alloc);
-        self.file,
+    defer if (!SfsFile.pathIsSelf(path)) root_fil.deinit();
-        &self.stateless_decomp,
+    return root_fil.open(path);
-        self.super.inode_start,
-        self.super.block_size,
-        self.super.root_ref,
-    );
-    return root_inode.extract(alloc, io, self.file, self.super, extract_dir, options);
 }
 
-// Superblock
+pub fn extract(self: Archive, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void {
-
+    var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{});
-const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little);
+    var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
-
+    try meta.interface.discardAll(self.super.root_ref.block_offset);
-const SuperblockError = error{
+    const in: Inode = try .read(self.alloc, &meta.interface, self.super.block_size);
-    InvalidMagic,
+    try in.extractTo(alloc, self, path, options);
-    InvalidBlockLog,
-    InvalidVersion,
-    InvalidCheck,
-};
-
-/// A squashfs Superblock
-pub const Superblock = extern struct {
-    magic: u32,
-    inode_count: u32,
-    mod_time: u32,
-    block_size: u32,
-    frag_count: u32,
-    compression: Decomp.Enum,
-    block_log: u16,
-    flags: packed struct(u16) {
-        inode_uncompressed: bool,
-        data_uncompressed: bool,
-        check: bool,
-        frag_uncompressed: bool,
-        fragment_never: bool,
-        fragment_always: bool,
-        duplicates: bool,
-        exportable: bool,
-        xattr_uncompressed: bool,
-        xattr_never: bool,
-        compression_options: bool,
-        ids_uncompressed: bool,
-        _: u4,
-    },
-    id_count: u16,
-    ver_maj: u16,
-    ver_min: u16,
-    root_ref: Inode.Ref,
-    size: u64,
-    id_start: u64,
-    xattr_start: u64,
-    inode_start: u64,
-    dir_start: u64,
-    frag_start: u64,
-    export_start: u64,
-
-    /// Validate the Superblock. If an error is returned, it's likely the archive is corrupted or not a squashfs archive.
-    pub fn validate(self: Superblock) !void {
-        if (self.magic != SQUASHFS_MAGIC)
-            return SuperblockError.InvalidMagic;
-        if (self.flags.check)
-            return SuperblockError.InvalidCheck;
-        if (self.ver_maj != 4 or self.ver_min != 0)
-            return SuperblockError.InvalidVersion;
-        if (std.math.log2(self.block_size) != self.block_log)
-            return SuperblockError.InvalidBlockLog;
-    }
-};
-
-// Tests
-
-const TestArchive = "testing/LinuxPATest.sfs";
-
-test "Basics" {
-    std.debug.print("Starting test: Basics...\n", .{});
-
-    const alloc = std.testing.allocator;
-    const io = std.testing.io;
-
-    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer fil.close(io);
-    var sfs: Archive = try .init(io, fil, 0);
-    defer sfs.deinit(io);
-    try std.testing.expectEqualDeep(sfs.super, LinuxPATestCorrectSuperblock);
-    const root_file = try sfs.root(alloc);
-    defer root_file.deinit();
 }
-
-const TestFile = "Start.exe";
-const TestFileExtractLocation = "testing/Start.exe";
-
-test "ExtractSingleFile" {
-    std.debug.print("Starting test: ExtractSingleFile...\n", .{});
-
-    const alloc = std.testing.allocator;
-    const io = std.testing.io;
-
-    Io.Dir.cwd().deleteFile(io, TestFileExtractLocation) catch {};
-    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer fil.close(io);
-    var sfs: Archive = try .init(io, fil, 0);
-    defer sfs.deinit(io);
-    var test_fil = try sfs.open(alloc, io, TestFile);
-    defer test_fil.deinit();
-    try test_fil.extract(alloc, io, TestFileExtractLocation, .default);
-    //TODO: validate extracted file.
-}
-
-const TestFullExtractLocation = "testing/TestExtract";
-
-test "ExtractCompleteArchive" {
-    std.debug.print("Starting test: ExtractCompleteArchive...\n", .{});
-
-    const alloc = std.testing.allocator;
-    const io = std.testing.io;
-
-    Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
-    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer fil.close(io);
-    var sfs: Archive = try .init(io, fil, 0);
-    defer sfs.deinit(io);
-    try sfs.extract(alloc, io, TestFullExtractLocation, .default);
-}
-
-test "ExtractCompleteArchiveSingleThreaded" {
-    std.debug.print("Starting test: ExtractCompleteArchive...\n", .{});
-
-    const alloc = std.testing.allocator;
-    const io = std.testing.io;
-
-    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer fil.close(io);
-    {
-        std.debug.print("First testing using Threaded.global_single_threaded...\n", .{});
-        Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
-        var sfs: Archive = try .init(Io.Threaded.global_single_threaded.io(), fil, 0);
-        defer sfs.deinit(Io.Threaded.global_single_threaded.io());
-        try sfs.extract(alloc, Io.Threaded.global_single_threaded.io(), TestFullExtractLocation, .default);
-    }
-    {
-        std.debug.print("Next testing using ExtractionOptions.single_threaded...\n", .{});
-        Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
-        var sfs: Archive = try .init(io, fil, 0);
-        defer sfs.deinit(io);
-        try sfs.extract(alloc, io, TestFullExtractLocation, .default_single_threaded);
-    }
-}
-
-const LinuxPATestCorrectSuperblock: Superblock = .{
-    .magic = std.mem.readInt(u32, "hsqs", .little),
-    .inode_count = 2974,
-    .mod_time = 1632696724,
-    .block_size = 131072,
-    .frag_count = 264,
-    .compression = .zstd,
-    .block_log = 17,
-    .flags = .{
-        .inode_uncompressed = false,
-        .data_uncompressed = false,
-        .check = false,
-        .frag_uncompressed = false,
-        .fragment_never = false,
-        .fragment_always = false,
-        .duplicates = true,
-        .exportable = true,
-        .xattr_uncompressed = false,
-        .xattr_never = false,
-        .compression_options = false,
-        .ids_uncompressed = false,
-        ._ = 0,
-    },
-    .id_count = 1,
-    .ver_maj = 4,
-    .ver_min = 0,
-    .root_ref = .{
-        .block_offset = 1363,
-        .block_start = 29237,
-        ._ = 0,
-    },
-    .size = 106841744,
-    .id_start = 106841632,
-    .xattr_start = 106841720,
-    .inode_start = 106778274,
-    .dir_start = 106807998,
-    .frag_start = 106837747,
-    .export_start = 106841602,
-};

src/bin/unsquashfs.zig

@@ -1,6 +1,5 @@
 const std = @import("std");
-const Io = std.Io;
+const Writer = std.Io.Writer;
-const Writer = Io.Writer;
 const builtin = @import("builtin");
 
 const config = @import("config");
@@ -18,7 +17,7 @@ const help_mgs =
     \\  -dx             Don't set xattr values
     \\  -dp             Don't set permissions (includes setting uid & gid owner)
     \\
-    \\  -p <threads>    Specify how many threads to use. If not present or zero, the system's logical cores count is used.
+    \\  -p <threads>    Specify how many threads to use. If no present or zero, the system's logical cores count is used.
     \\  -v              Verbose
     \\
     \\  --force         Force extraction. If the destination already exists, it will be deleted.
@@ -39,58 +38,40 @@ var ignore_xattrs: bool = false;
 var ignore_permissions: bool = false;
 var force: bool = false;
 
-pub fn main(init: std.process.Init) !void {
+pub fn main() !void {
-    const alloc = init.gpa;
+    const alloc = std.heap.smp_allocator;
-    // const io = init.io;
+    var stdout = std.fs.File.stdout();
-    var evented: Io.Evented = undefined;
+    var out = stdout.writer(&[0]u8{});
-    try evented.init(alloc, .{});
-    const io = evented.io();
-
-    var stdout = std.Io.File.stdout();
-    defer stdout.close(io);
-    var out = stdout.writer(io, &[0]u8{});
     defer out.interface.flush() catch {};
-
+    try handleArgs(alloc, &out.interface);
-    try handleArgs(init.minimal.args, &out.interface);
     if (archive.len == 0) {
         try out.interface.print("You must provide a squashfs archive\n", .{});
         try out.interface.print(help_mgs, .{});
         return;
     }
-
+    var fil: std.fs.File = try std.fs.cwd().openFile(archive, .{}); //TODO: Handle error gracefully.
-    var fil = try Io.Dir.cwd().openFile(io, archive, .{}); //TODO: Handle error gracefully.
+    defer fil.close();
-    defer fil.close(io);
+    var arc: squashfs.Archive = try .init(alloc, fil, offset); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
-
+    defer arc.deinit();
-    var arc: squashfs.Archive = try .init(io, fil, offset); //TODO: Handle error gracefully.
     const options: squashfs.ExtractionOptions = .{
-        .single_threaded = threads == 1,
+        .threads = if (threads == 0) try std.Thread.getCpuCount() else threads,
         .verbose = verbose,
         .verbose_writer = if (verbose) &out.interface else null,
         .ignore_xattr = ignore_xattrs,
         .ignore_permissions = ignore_permissions,
     };
-
     if (force)
-        try Io.Dir.cwd().deleteTree(io, extLoc);
+        try std.fs.cwd().deleteTree(extLoc);
-    if (threads != 0) {
+    try arc.extract(alloc, extLoc, options); //TODO: Handle error gracefully.
-        var limited_io = Io.Threaded.init(alloc, .{
-            .async_limit = .limited(threads - 1),
-            .concurrent_limit = .limited(threads - 1),
-            .argv0 = .init(init.minimal.args),
-            .environ = init.minimal.environ,
-        });
-        return arc.extract(alloc, limited_io.io(), extLoc, options); //TODO: Handle error gracefully.
-    }
-    return arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully.
 }
 
-fn handleArgs(args: std.process.Args, out: *Writer) !void {
+fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
-    var arg_iter = args.iterate();
+    var args = try std.process.argsWithAllocator(alloc);
-    defer arg_iter.deinit();
+    defer args.deinit();
-    _ = arg_iter.next(); // args[0] is the application launch command.
+    _ = args.next(); // args[0] is the application launch command.
-    while (arg_iter.next()) |arg| {
+    while (args.next()) |arg| {
         if (std.mem.eql(u8, arg, "-o")) {
-            const nxt = arg_iter.next();
+            const nxt = args.next();
             if (nxt == null or nxt.?.len == 0) {
                 try out.print("-o must be followed by a number\n", .{});
                 return errors.InvalidArguments;
@@ -101,7 +82,7 @@ fn handleArgs(args: std.process.Args, out: *Writer) !void {
             };
             continue;
         } else if (std.mem.eql(u8, arg, "-d")) {
-            const nxt = arg_iter.next();
+            const nxt = args.next();
             if (nxt == null or nxt.?.len == 0) {
                 try out.print("-d must be followed by a location\n", .{});
                 return errors.InvalidArguments;
@@ -109,7 +90,7 @@ fn handleArgs(args: std.process.Args, out: *Writer) !void {
             extLoc = nxt.?;
             continue;
         } else if (std.mem.eql(u8, arg, "-p")) {
-            const nxt = arg_iter.next();
+            const nxt = args.next();
             if (nxt == null or nxt.?.len == 0) {
                 try out.print("-p must be followed by a number\n", .{});
                 return errors.InvalidArguments;

src/c.h

@@ -1,7 +0,0 @@
-#include <zstd.h>
-#include <zlib-ng.h>
-#include <lzma.h>
-#ifdef ALLOW_LZO
-#include <lzo/minilzo.h>
-#endif
-#include <lz4.h>

src/decomp.zig

@@ -1,18 +1,26 @@
+//! Implementations for decompression.
+//! TODO: change to vtable interface to allow for shared decompressors for better performance/resource usage.
+
 const std = @import("std");
-const Io = std.Io;
+const Reader = std.Io.Reader;
+const builtin = @import("builtin");
 
-const options = @import("options");
+const config = if (builtin.is_test) .{
+    .use_zig_decomp = builtin.link_libc != true,
+    .allow_lzo = false, // Change once LZO compilation is fixed
+} else @import("config");
 
-const Decompressor = @import("util/decompressor.zig");
+const c = @cImport({
+    @cInclude("zlib-ng.h");
+    @cInclude("lzma.h");
+    @cInclude("lz4.h");
+    @cInclude("zstd.h");
+    @cInclude("zstd_errors.h");
+    if (config.allow_lzo)
+        @cInclude("lzo/minilzo.h");
+});
 
-const zlib = if (options.use_zig_decomp) @import("decomp/zig_zlib.zig") else @import("decomp/c_zlib.zig");
+pub const CompressionType = enum(u16) {
-const lzma = if (options.use_zig_decomp) @import("decomp/zig_lzma.zig") else @import("decomp/c_lzma.zig");
-const lzo = if (options.use_zig_decomp or !options.allow_lzo) void else @import("decomp/c_lzo.zig");
-const xz = if (options.use_zig_decomp) @import("decomp/zig_xz.zig") else @import("decomp/c_xz.zig");
-const lz4 = if (options.use_zig_decomp) void else @import("decomp/c_lz4.zig");
-const zstd = if (options.use_zig_decomp) @import("decomp/zig_zstd.zig") else @import("decomp/c_zstd.zig");
-
-pub const Enum = enum(u16) {
     gzip = 1,
     lzma,
     lzo,
@@ -21,67 +29,233 @@ pub const Enum = enum(u16) {
     zstd,
 };
 
-pub fn StatelessDecomp(val: Enum) !Decompressor {
+pub const DecompFn = *const fn (alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize; // TODO: replace anyerror to definitive error types.
-    return switch (val) {
+
-        .gzip => zlib.stateless_decompressor,
+pub const gzipDecompress = if (!config.use_zig_decomp) cGzip else zigGzip;
-        .lzma => lzma.stateless_decompressor,
+
-        .lzo => if (options.use_zig_decomp or !options.allow_lzo)
+fn zigGzip(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-            error.LzoUnsupported
+    var rdr: Reader = .fixed(in);
-        else
+    const buf = try alloc.alloc(u8, out.len);
-            lzo.stateless_decompressor,
+    defer alloc.free(buf);
-        .xz => xz.stateless_decompressor,
+    var decomp = std.compress.flate.Decompress.init(&rdr, .zlib, buf);
-        .lz4 => if (options.use_zig_decomp)
+    return decomp.reader.readSliceShort(out);
-            error.Lz4Unsupported
+}
-        else
+fn cGzip(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-            lz4.stateless_decompressor,
+    _ = alloc;
-        .zstd => zstd.stateless_decompressor,
+    var out_len: usize = out.len;
+    const res = c.zng_uncompress(out.ptr, &out_len, in.ptr, in.len);
+    return switch (res) {
+        c.Z_OK => out_len,
+        c.Z_MEM_ERROR => error.NotEnoughMemory,
+        c.Z_BUF_ERROR => error.OutBufferTooSmall,
+        c.Z_DATA_ERROR => error.BadData,
+        else => error.UnknownResult,
     };
 }
 
-pub const Decomp = union(enum) {
+pub const lzmaDecompress = if (!config.use_zig_decomp) cLzma else zigLzma;
-    gzip: zlib,
-    lzma: lzma,
-    lzo: lzo,
-    xz: xz,
-    lz4: lz4,
-    zstd: zstd,
 
-    pub fn init(val: Enum, alloc: std.mem.Allocator, io: Io, block_size: u32) !Decomp {
+fn zigLzma(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-        return switch (val) {
+    var rdr: Reader = .fixed(in);
-            .gzip => .{ .gzip = if (options.use_zig_decomp) try zlib.init(alloc, io, block_size) else try zlib.init(alloc, io) },
+    var decomp = try std.compress.lzma.decompress(alloc, rdr.adaptToOldInterface());
-            .lzma => .{ .lzma = if (options.use_zig_decomp) try lzma.init(alloc, io, block_size) else .{} },
+    return decomp.read(out);
-            .lzo => if (options.use_zig_decomp or !options.allow_lzo) error.LzoUnsupported else .{ .lzo = .{} },
+}
-            .xz => .{ .xz = if (options.use_zig_decomp) try xz.init(alloc, io, block_size) else .{} },
+fn cLzma(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-            .lz4 => if (options.use_zig_decomp) error.Lz4Unsupported else .{ .lz4 = .{} },
+    _ = alloc;
-            .zstd => .{ .zstd = if (options.use_zig_decomp) try zstd.init(alloc, io, block_size) else try zstd.init(alloc, io) },
+    var stream: c.lzma_stream = .{
-        };
+        .next_in = in.ptr,
-    }
+        .avail_in = in.len,
-    pub fn deinit(self: *Decomp, alloc: std.mem.Allocator) void {
+        .next_out = out.ptr,
-        if (options.use_zig_decomp) {
+        .avail_out = out.len,
-            switch (self.*) {
+    };
-                .gzip => self.gzip.deinit(),
+    var res = c.lzma_alone_decoder(&stream, in.len * 2);
-                .lzma => self.lzma.deinit(),
+    switch (res) {
-                .xz => self.xz.deinit(),
+        c.LZMA_OK => {},
-                .zstd => self.zstd.deinit(),
+        c.LZMA_MEM_ERROR => return error.LzmaMemoryError,
-                else => {},
+        c.LZMA_PROG_ERROR => return error.LzmaProgramError,
-            }
+        else => return error.UnknownResult,
-        } else {
-            switch (self.*) {
-                .gzip => self.gzip.deinit(alloc),
-                .zstd => self.zstd.deinit(alloc),
-                else => {},
-            }
-        }
     }
+    defer c.lzma_end(&stream);
+    while (res == c.LZMA_OK)
+        res = c.lzma_code(&stream, c.LZMA_RUN);
+    return switch (res) {
+        c.LZMA_STREAM_END => stream.total_out,
+        c.LZMA_MEM_ERROR => error.LzmaMemoryError,
+        c.LZMA_MEMLIMIT_ERROR => error.LzmaMemoryLimit,
+        c.LZMA_FORMAT_ERROR => error.LzmaBadFormat,
+        c.LZMA_DATA_ERROR => error.LzmaDataCorrupt,
+        c.LZMA_BUF_ERROR => error.LzmaCannotProgress,
+        c.LZMA_PROG_ERROR => error.LzmaProgramError,
+        else => error.UnknownResult,
+    };
+}
 
-    pub fn decompressor(self: *Decomp) *Decompressor {
+// pub const lzoDecompress = if (!config.use_zig_decomp) cLzo else zigLzo;
-        return switch (self.*) {
+
-            .gzip => &self.gzip.interface,
+// fn zigLzo(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-            .lzma => &self.lzma.interface,
+//     _ = alloc;
-            .lzo => if (options.use_zig_decomp or !options.allow_lzo) unreachable else &self.lzo.interface,
+//     _ = in;
-            .xz => &self.xz.interface,
+//     _ = out;
-            .lz4 => if (options.use_zig_decomp) unreachable else &self.lz4.interface,
+//     return error.LzoUnsupported;
-            .zstd => &self.zstd.interface,
+// }
-        };
+pub fn cLzo(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
+    _ = alloc;
+    var res = c.lzo_init();
+    if (res != 0) return error.LzoInitFailed;
+    var out_len: usize = out.len;
+    res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &out_len, null);
+    return switch (res) {
+        c.LZO_E_OK => out_len,
+        c.LZO_E_ERROR => error.LzoError,
+        c.LZO_E_OUT_OF_MEMORY => error.LzoOutOfMemory,
+        c.LZO_E_NOT_COMPRESSIBLE => error.LzoNotCompressible,
+        c.LZO_E_INPUT_OVERRUN => error.LzoInputOverrun,
+        c.LZO_E_OUTPUT_OVERRUN => error.LzoOutputOverrun,
+        c.LZO_E_LOOKBEHIND_OVERRUN => error.LzoLookbehindOverrun,
+        c.LZO_E_EOF_NOT_FOUND => error.LzoEofNotFound,
+        c.LZO_E_INPUT_NOT_CONSUMED => error.LzoInputNotConsumed,
+        c.LZO_E_NOT_YET_IMPLEMENTED => error.LzoNotYetImplemented,
+        c.LZO_E_INVALID_ARGUMENT => error.LzoInvalidArgument,
+        c.LZO_E_INVALID_ALIGNMENT => error.LzoInvalidAlignment,
+        c.LZO_E_OUTPUT_NOT_CONSUMED => error.LzoOutputNotConsumed,
+        c.LZO_E_INTERNAL_ERROR => error.LzoInternalError,
+        else => error.UnknownResult,
+    };
+}
+
+pub const xzDecompress = if (!config.use_zig_decomp) cXz else zigXz;
+
+fn zigXz(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
+    var rdr: Reader = .fixed(in);
+    var decomp = try std.compress.xz.decompress(alloc, rdr.adaptToOldInterface());
+    return decomp.read(out);
+}
+fn cXz(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
+    _ = alloc;
+    var stream: c.lzma_stream = .{
+        .next_in = in.ptr,
+        .avail_in = in.len,
+        .next_out = out.ptr,
+        .avail_out = out.len,
+        // .allocator = _, TODO: create a custom allocator based on alloc,
+    };
+    var res = c.lzma_stream_decoder(&stream, in.len * 2, 0);
+    switch (res) {
+        c.LZMA_OK => {},
+        c.LZMA_MEM_ERROR => return error.LzmaMemoryError,
+        c.LZMA_PROG_ERROR => return error.LzmaProgramError,
+        else => return error.UnknownResult,
     }
+    defer c.lzma_end(&stream);
+    while (res == c.LZMA_OK)
+        res = c.lzma_code(&stream, c.LZMA_RUN);
+    return switch (res) {
+        c.LZMA_STREAM_END => stream.total_out,
+        c.LZMA_MEM_ERROR => error.LzmaMemoryError,
+        c.LZMA_MEMLIMIT_ERROR => error.LzmaMemoryLimit,
+        c.LZMA_FORMAT_ERROR => error.LzmaBadFormat,
+        c.LZMA_DATA_ERROR => error.LzmaDataCorrupt,
+        c.LZMA_BUF_ERROR => error.LzmaCannotProgress,
+        c.LZMA_PROG_ERROR => error.LzmaProgramError,
+        else => error.UnknownResult,
+    };
+}
+
+// pub const lz4Decompress = if (!config.use_zig_decomp) cLz4 else zigLz4;
+
+// fn zigLz4(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
+//     _ = alloc;
+//     _ = in;
+//     _ = out;
+//     return error.Lz4Unsupported;
+// }
+pub fn cLz4(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
+    _ = alloc;
+    const res = c.LZ4_decompress_safe(in.ptr, out.ptr, @intCast(in.len), @intCast(out.len));
+    if (res > 0) return @abs(res); // TODO: Find out what error values it can return.
+    return error.Lz4DecompressFailed;
+}
+
+pub const zstdDecompress = if (!config.use_zig_decomp) cZstd else zigZstd;
+
+pub fn zigZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
+    var rdr: Reader = .fixed(in);
+    const buf = try alloc.alloc(u8, 1024 * 1024);
+    defer alloc.free(buf);
+    var decomp = std.compress.zstd.Decompress.init(&rdr, buf, .{});
+    return decomp.reader.readSliceShort(out) catch |err| {
+        return decomp.err orelse err;
+    };
+}
+fn cZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
+    _ = alloc;
+    const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
+    if (c.ZSTD_isError(res) == 0) return res;
+    return switch (c.ZSTD_getErrorCode(res)) {
+        c.ZSTD_error_prefix_unknown => cZstdError.PrefixUnknown,
+        c.ZSTD_error_version_unsupported => cZstdError.VersionUnsupported,
+        c.ZSTD_error_frameParameter_unsupported => cZstdError.FrameParameterUnsupported,
+        c.ZSTD_error_frameParameter_windowTooLarge => cZstdError.FrameParameterWindowTooLarge,
+        c.ZSTD_error_corruption_detected => cZstdError.CorruptionDetected,
+        c.ZSTD_error_checksum_wrong => cZstdError.ChecksumWrong,
+        c.ZSTD_error_literals_headerWrong => cZstdError.LiteralsHeaderWrong,
+        c.ZSTD_error_dictionary_corrupted => cZstdError.DictionaryCorrupted,
+        c.ZSTD_error_dictionary_wrong => cZstdError.DictionaryWrong,
+        c.ZSTD_error_dictionaryCreation_failed => cZstdError.DictionaryCreationFailed,
+        c.ZSTD_error_parameter_unsupported => cZstdError.ParameterUnsupported,
+        c.ZSTD_error_parameter_combination_unsupported => cZstdError.ParameterCombinationUnsupported,
+        c.ZSTD_error_parameter_outOfBound => cZstdError.ParameterOutOfBound,
+        c.ZSTD_error_tableLog_tooLarge => cZstdError.TableLogTooLarge,
+        c.ZSTD_error_maxSymbolValue_tooLarge => cZstdError.MaxSymbolValueTooLarge,
+        c.ZSTD_error_maxSymbolValue_tooSmall => cZstdError.MaxSymbolValueTooSmall,
+        c.ZSTD_error_stabilityCondition_notRespected => cZstdError.StabilityConditionNotRespected,
+        c.ZSTD_error_stage_wrong => cZstdError.StageWrong,
+        c.ZSTD_error_init_missing => cZstdError.InitMissing,
+        c.ZSTD_error_memory_allocation => cZstdError.MemoryAllocation,
+        c.ZSTD_error_workSpace_tooSmall => cZstdError.WorkSpaceTooSmall,
+        c.ZSTD_error_dstSize_tooSmall => cZstdError.DstSizeTooSmall,
+        c.ZSTD_error_srcSize_wrong => cZstdError.SrcSizeWrong,
+        c.ZSTD_error_dstBuffer_null => cZstdError.DstBufferNull,
+        c.ZSTD_error_noForwardProgress_destFull => cZstdError.NoForwardProgressDestFull,
+        c.ZSTD_error_noForwardProgress_inputEmpty => cZstdError.NoForwardProgressInputEmpty,
+        else => cZstdError.Generic,
+    };
+}
+
+pub const cZstdError = error{
+    Generic,
+    PrefixUnknown,
+    VersionUnsupported,
+    FrameParameterUnsupported,
+    FrameParameterWindowTooLarge,
+    CorruptionDetected,
+    ChecksumWrong,
+    LiteralsHeaderWrong,
+    DictionaryCorrupted,
+    DictionaryWrong,
+    DictionaryCreationFailed,
+    ParameterUnsupported,
+    ParameterCombinationUnsupported,
+    ParameterOutOfBound,
+    TableLogTooLarge,
+    MaxSymbolValueTooLarge,
+    MaxSymbolValueTooSmall,
+    CannotProduceUncompressedBlock,
+    StabilityConditionNotRespected,
+    StageWrong,
+    InitMissing,
+    MemoryAllocation,
+    WorkSpaceTooSmall,
+    DstSizeTooSmall,
+    SrcSizeWrong,
+    DstBufferNull,
+    NoForwardProgressDestFull,
+    NoForwardProgressInputEmpty,
+    FrameIndexTooLarge,
+    SeekableIo,
+    DstBufferWrong,
+    SrcBufferWrong,
+    SequenceProducerFailed,
+    ExternalSequencesInvalid,
+    MaxCode,
 };

src/decomp/c_lz4.zig

@@ -1,17 +0,0 @@
-const std = @import("std");
-
-const c = @import("c");
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-interface: Decompressor = .{ .decomp_fn = statelessDecomp },
-
-fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    const out_len: c_int = @bitCast(@as(u32, @truncate(out.len)));
-    const res = c.LZ4_decompress_fast(in.ptr, out.ptr, out_len);
-    if (res < 0) return Error.ReadFailed;
-    return @abs(res);
-}

src/decomp/c_lzma.zig

@@ -1,46 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Reader = std.Io.Reader;
-const zstd = std.compress.zstd;
-const Node = std.SinglyLinkedList.Node;
-
-const c = @import("c");
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Queue = std.Io.Queue(c.lzma_stream);
-
-const Self = @This();
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-interface: Decompressor = .{ .decomp_fn = statelessDecomp },
-
-fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var stream: c.lzma_stream = .{
-        .next_in = in.ptr,
-        .avail_in = in.len,
-        .next_out = out.ptr,
-        .avail_out = out.len,
-    };
-
-    var res = c.lzma_alone_decoder(&stream, stream.avail_out * 2);
-    if (res != c.LZMA_OK) return Error.ReadFailed;
-    while (res == c.LZMA_OK)
-        res = c.lzma_code(&stream, c.LZMA_RUN);
-    if (res != c.LZMA_FINISH) return Error.ReadFailed;
-    return stream.total_out;
-}
-
-// lzma_allocator
-
-// fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque {
-//     var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
-//     return alloc.rawAlloc(size, .@"1", 0);
-// }
-// fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
-//     if (mem_ptr == null) return;
-//     var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
-//     alloc.free(@as([*]u8, @ptrCast(mem_ptr.?)));
-// }

src/decomp/c_lzo.zig

@@ -1,26 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Reader = std.Io.Reader;
-const zstd = std.compress.zstd;
-const Node = std.SinglyLinkedList.Node;
-
-const c = @import("c");
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Queue = std.Io.Queue(c.lzma_stream);
-
-const Self = @This();
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-interface: Decompressor = .{ .decomp_fn = statelessDecomp },
-
-fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    _ = c.lzo_init();
-    var out_len = out.len;
-    const res = c.lzo1x_decompress_safe(in.ptr, in.len, out.ptr, &out_len, null);
-    if (res != c.LZO_E_OK) return Error.ReadFailed;
-    return out_len;
-}

src/decomp/c_xz.zig

@@ -1,47 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Reader = std.Io.Reader;
-const zstd = std.compress.zstd;
-const Node = std.SinglyLinkedList.Node;
-
-const c = @import("c");
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Queue = std.Io.Queue(c.lzma_stream);
-
-const Self = @This();
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-interface: Decompressor = .{ .decomp_fn = statelessDecomp },
-
-fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var stream: c.lzma_stream = .{
-        .next_in = in.ptr,
-        .avail_in = in.len,
-        .next_out = out.ptr,
-        .avail_out = out.len,
-    };
-
-    var res = c.lzma_alone_decoder(&stream, stream.avail_out * 2);
-    if (res != c.LZMA_OK) return Error.ReadFailed;
-    while (res == c.LZMA_OK)
-        res = c.lzma_code(&stream, c.LZMA_RUN);
-    if (res != c.LZMA_FINISH) return Error.ReadFailed;
-    return stream.total_out;
-}
-
-// lzma_allocator
-
-// fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque {
-//     var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
-//     const mem = alloc.alloc(u8, size) catch return null;
-//     return mem.ptr;
-// }
-// fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
-//     if (mem_ptr == null) return;
-//     var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
-//     alloc.free(@as([*]u8, @ptrCast(mem_ptr.?)));
-// }

src/decomp/c_zlib.zig

@@ -1,80 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Reader = std.Io.Reader;
-const zstd = std.compress.zstd;
-const Node = std.SinglyLinkedList.Node;
-
-const c = @import("c");
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Queue = std.Io.Queue(c.zng_stream);
-
-const Self = @This();
-
-interface: Decompressor = .{ .decomp_fn = decomp },
-
-io: Io,
-
-ctx: []c.zng_stream,
-ctx_queue: Queue,
-
-pub fn init(alloc: std.mem.Allocator, io: Io) !Self {
-    const buf = try alloc.alloc(c.zng_stream, 20); // TODO: Choose a better number instead of a random one.
-    var queue: Queue = .init(buf);
-    for (0..20) |_|
-        try queue.putOne(io, .{});
-
-    return .{
-        .io = io,
-
-        .ctx = buf,
-        .ctx_queue = queue,
-    };
-}
-pub fn deinit(self: *Self, alloc: std.mem.Allocator) void {
-    self.ctx_queue.close(self.io);
-    alloc.free(self.ctx);
-}
-
-fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    if (d == null) {
-        return statelessDecomp(d, alloc, in, out);
-    }
-    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
-
-    var stream = self.ctx_queue.getOne(self.io) catch return Error.ReadFailed;
-    defer self.ctx_queue.putOne(self.io, stream) catch {};
-
-    stream.next_in = in.ptr;
-    stream.avail_in = @truncate(in.len);
-    stream.next_out = out.ptr;
-    stream.avail_out = @truncate(out.len);
-
-    try zlibDecomp(&stream);
-
-    return stream.total_out;
-}
-
-inline fn zlibDecomp(stream: *c.zng_stream) !void {
-    _ = c.zng_inflateReset(stream);
-
-    const res = c.zng_inflate(stream, c.Z_FULL_FLUSH);
-    if (res != c.Z_OK) return Error.ReadFailed;
-}
-
-// Stateless
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var stream: c.zng_stream = .{
-        .next_in = in.ptr,
-        .avail_in = @truncate(in.len),
-        .next_out = out.ptr,
-        .avail_out = @truncate(out.len),
-    };
-    try zlibDecomp(&stream);
-    return stream.total_out;
-}

src/decomp/c_zstd.zig

@@ -1,69 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Reader = std.Io.Reader;
-const zstd = std.compress.zstd;
-const Node = std.SinglyLinkedList.Node;
-
-const c = @import("c");
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Queue = std.Io.Queue(?*c.ZSTD_DCtx);
-
-const Self = @This();
-
-interface: Decompressor = .{ .decomp_fn = decomp },
-
-io: Io,
-
-ctx: []?*c.ZSTD_DCtx,
-ctx_queue: Queue,
-
-pub fn init(alloc: std.mem.Allocator, io: Io) !Self {
-    const buf = try alloc.alloc(?*c.ZSTD_DCtx, 20); // TODO: Choose a better number instead of a random one.
-    var queue: Queue = .init(buf);
-    for (0..20) |_|
-        try queue.putOne(io, c.ZSTD_createDCtx());
-
-    return .{
-        .io = io,
-
-        .ctx = buf,
-        .ctx_queue = queue,
-    };
-}
-pub fn deinit(self: *Self, alloc: std.mem.Allocator) void {
-    self.ctx_queue.close(self.io);
-    for (self.ctx) |ctx|
-        _ = c.ZSTD_freeDCtx(ctx);
-    alloc.free(self.ctx);
-}
-
-fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    if (d == null) {
-        return statelessDecomp(d, alloc, in, out);
-    }
-    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
-
-    const ctx = self.ctx_queue.getOne(self.io) catch return Error.ReadFailed;
-    defer self.ctx_queue.putOne(self.io, ctx) catch {};
-
-    _ = c.ZSTD_DCtx_reset(ctx, c.ZSTD_reset_session_only);
-
-    const res = c.ZSTD_decompressDCtx(ctx, out.ptr, out.len, in.ptr, in.len);
-    if (c.ZSTD_isError(res) != 0)
-        return Error.ReadFailed;
-    return res;
-}
-
-// Stateless
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
-    if (c.ZSTD_isError(res) != 0)
-        return Error.ReadFailed;
-    return res;
-}

src/decomp/zig_lzma.zig

@@ -1,81 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Reader = std.Io.Reader;
-const lzma = std.compress.lzma;
-const Node = std.SinglyLinkedList.Node;
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Queue = Io.Queue([]u8);
-
-const Self = @This();
-
-const Buffer = struct {
-    node: Node,
-    buf: []u8,
-};
-
-interface: Decompressor = .{ .decomp_fn = decomp },
-
-alloc: std.mem.Allocator,
-io: Io,
-
-block_size: u32,
-buf: [][]u8,
-buf_queue: Queue,
-
-pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self {
-    const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one.
-    var queue: Queue = .init(buf);
-    for (0..20) |_|
-        try queue.putOne(io, try alloc.alloc(u8, block_size));
-
-    return .{
-        .alloc = alloc,
-        .io = io,
-
-        .block_size = block_size,
-        .buf = buf,
-        .buf_queue = queue,
-    };
-}
-pub fn deinit(self: *Self) void {
-    self.buf_queue.close(self.io);
-    for (self.buf) |buf|
-        self.alloc.free(buf);
-    self.alloc.free(self.buf);
-}
-
-fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    if (d == null) {
-        return statelessDecomp(d, alloc, in, out);
-    }
-    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
-
-    var buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed;
-    defer self.buf_queue.putOne(self.io, buf) catch {};
-
-    return lzmaDecomp(self.alloc, &buf, in, out) catch return Error.ReadFailed;
-}
-
-inline fn lzmaDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize {
-    var rdr: Reader = .fixed(in);
-    var d = try lzma.Decompress.initOptions(&rdr, alloc, buffer.*, .{}, in.len * 2);
-    defer {
-        buffer.* = d.takeBuffer();
-        d.deinit();
-    }
-
-    return d.reader.readSliceShort(out);
-}
-
-// Stateless
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var buf = try alloc.alloc(u8, in.len);
-    defer alloc.free(buf);
-    return lzmaDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
-}

src/decomp/zig_xz.zig

@@ -1,81 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Reader = std.Io.Reader;
-const xz = std.compress.xz;
-const Node = std.SinglyLinkedList.Node;
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Queue = Io.Queue([]u8);
-
-const Self = @This();
-
-const Buffer = struct {
-    node: Node,
-    buf: []u8,
-};
-
-interface: Decompressor = .{ .decomp_fn = decomp },
-
-alloc: std.mem.Allocator,
-io: Io,
-
-block_size: u32,
-buf: [][]u8,
-buf_queue: Queue,
-
-pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self {
-    const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one.
-    var queue: Queue = .init(buf);
-    for (0..20) |_|
-        try queue.putOne(io, try alloc.alloc(u8, block_size));
-
-    return .{
-        .alloc = alloc,
-        .io = io,
-
-        .block_size = block_size,
-        .buf = buf,
-        .buf_queue = queue,
-    };
-}
-pub fn deinit(self: *Self) void {
-    self.buf_queue.close(self.io);
-    for (self.buf) |buf|
-        self.alloc.free(buf);
-    self.alloc.free(self.buf);
-}
-
-fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    if (d == null) {
-        return statelessDecomp(d, alloc, in, out);
-    }
-    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
-
-    var buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed;
-    defer self.buf_queue.putOne(self.io, buf) catch {};
-
-    return xzDecomp(self.alloc, &buf, in, out) catch return Error.ReadFailed;
-}
-
-inline fn xzDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize {
-    var rdr: Reader = .fixed(in);
-    var d = try xz.Decompress.init(&rdr, alloc, buffer.*);
-    defer {
-        buffer.* = d.takeBuffer();
-        d.deinit();
-    }
-
-    return d.reader.readSliceShort(out);
-}
-
-// Stateless
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var buf = try alloc.alloc(u8, in.len);
-    defer alloc.free(buf);
-    return xzDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
-}

src/decomp/zig_zlib.zig

@@ -1,77 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const flate = std.compress.flate;
-const Node = std.SinglyLinkedList.Node;
-const Reader = Io.Reader;
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Queue = Io.Queue([]u8);
-
-const Self = @This();
-
-const Buffer = struct {
-    node: Node,
-    buf: []u8,
-};
-
-interface: Decompressor = .{ .decomp_fn = decomp },
-
-alloc: std.mem.Allocator,
-io: Io,
-
-block_size: u32,
-buf: [][]u8,
-buf_queue: Queue,
-
-pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self {
-    const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one.
-    var queue: Queue = .init(buf);
-    for (0..20) |_|
-        try queue.putOne(io, try alloc.alloc(u8, block_size));
-
-    return .{
-        .alloc = alloc,
-        .io = io,
-
-        .block_size = block_size,
-        .buf = buf,
-        .buf_queue = queue,
-    };
-}
-pub fn deinit(self: *Self) void {
-    self.buf_queue.close(self.io);
-    for (self.buf) |buf|
-        self.alloc.free(buf);
-    self.alloc.free(self.buf);
-}
-
-fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    if (d == null) {
-        return statelessDecomp(d, alloc, in, out);
-    }
-    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
-
-    const buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed;
-    defer self.buf_queue.putOne(self.io, buf) catch {};
-
-    return zlibDecomp(buf, in, out);
-}
-
-inline fn zlibDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
-    var rdr: Reader = .fixed(in);
-    var d = flate.Decompress.init(&rdr, .zlib, buffer);
-
-    return d.reader.readSliceShort(out);
-}
-
-// Stateless
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    const buf = try alloc.alloc(u8, out.len);
-    defer alloc.free(buf);
-    return zlibDecomp(buf, in, out);
-}

src/decomp/zig_zstd.zig

@@ -1,73 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Reader = std.Io.Reader;
-const zstd = std.compress.zstd;
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Queue = std.Io.Queue([]u8);
-
-const Self = @This();
-
-interface: Decompressor = .{ .decomp_fn = decomp },
-
-alloc: std.mem.Allocator,
-io: Io,
-
-block_size: u32,
-buf: [][]u8,
-buf_queue: Queue,
-
-pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self {
-    const buf = try alloc.alloc([]u8, 5); // TODO: Choose a better number instead of a random one.
-    var queue: Queue = .init(buf);
-    for (buf) |_|
-        try queue.putOne(io, try alloc.alloc(u8, block_size + zstd.block_size_max));
-
-    return .{
-        .alloc = alloc,
-        .io = io,
-
-        .block_size = block_size,
-        .buf = buf,
-        .buf_queue = queue,
-    };
-}
-pub fn deinit(self: *Self) void {
-    self.buf_queue.close(self.io);
-    for (self.buf) |buf|
-        self.alloc.free(buf);
-    self.alloc.free(self.buf);
-}
-
-fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    if (d == null) {
-        const buf = try alloc.alloc(u8, in.len * 2);
-        defer alloc.free(buf);
-        return zstdDecomp(buf, in, out);
-    }
-    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
-
-    const buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed;
-    defer self.buf_queue.putOne(self.io, buf) catch {};
-
-    return zstdDecomp(buf, in, out);
-}
-
-inline fn zstdDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
-    var rdr: Reader = .fixed(in);
-    var d = zstd.Decompress.init(&rdr, buffer, .{ .window_len = @truncate(out.len) });
-
-    return d.reader.readSliceShort(out);
-}
-
-// Stateless
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    const buf = try alloc.alloc(u8, out.len + zstd.block_size_max);
-    defer alloc.free(buf);
-    return zstdDecomp(buf, in, out);
-}

src/dir_entry.zig

@@ -0,0 +1,60 @@
+//! Directory entry from the directory table.
+
+const std = @import("std");
+const Reader = std.Io.Reader;
+
+const InodeType = @import("inode.zig").InodeType;
+
+const Entry = @This();
+
+const Header = extern struct { // use extern due to bad alignment with packed.
+    count: u32,
+    block_start: u32,
+    num: u32,
+};
+
+const RawEntry = packed struct {
+    offset: u16,
+    inode_offset: i16,
+    inode_type: InodeType,
+    name_size: u16,
+};
+
+block_start: u32,
+block_offset: u16,
+num: u32,
+inode_type: InodeType,
+name: []const u8,
+
+pub fn readDir(alloc: std.mem.Allocator, rdr: *Reader, size: u32) ![]Entry {
+    var cur_red: u32 = 3; // start at 3 due to "." & ".." being counted in the dir size.
+    var hdr: Header = undefined;
+    var raw: RawEntry = undefined;
+    var out: std.ArrayList(Entry) = try .initCapacity(alloc, 100); // Start out with a decent capacity instead of needing to allocate per header.
+    errdefer out.deinit(alloc);
+    while (cur_red < size) {
+        try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
+        cur_red += @sizeOf(Header);
+        try out.ensureUnusedCapacity(alloc, hdr.count + 1);
+        for (0..hdr.count + 1) |_| {
+            try rdr.readSliceEndian(RawEntry, @ptrCast(&raw), .little);
+            const name = try alloc.alloc(u8, raw.name_size + 1);
+            errdefer alloc.free(name);
+            try rdr.readSliceEndian(u8, name, .little);
+            const val = out.addOneAssumeCapacity();
+            val.* = .{
+                .block_start = hdr.block_start,
+                .block_offset = raw.offset,
+                .num = @abs(hdr.num + raw.offset),
+                .inode_type = raw.inode_type,
+                .name = name,
+            };
+            cur_red += @sizeOf(RawEntry) + raw.name_size + 1;
+        }
+    }
+    return out.toOwnedSlice(alloc);
+}
+
+pub fn deinit(self: Entry, alloc: std.mem.Allocator) void {
+    alloc.free(self.name);
+}

src/directory.zig

@@ -1,69 +0,0 @@
-const std = @import("std");
-const Reader = std.Io.Reader;
-
-const Inode = @import("inode.zig");
-
-pub const Error = error{OutOfMemory} || Reader.Error;
-
-const DirEntry = @This();
-
-block_start: u32,
-block_offset: u16,
-type: Inode.Type,
-name: []const u8,
-
-pub fn deinit(self: DirEntry, alloc: std.mem.Allocator) void {
-    alloc.free(self.name);
-}
-
-pub fn readDirectory(alloc: std.mem.Allocator, rdr: *Reader, size: u32) Error![]DirEntry {
-    var hdr: Header = undefined;
-    var raw: RawEntry = undefined;
-    var out: std.ArrayList(DirEntry) = try .initCapacity(alloc, 30);
-    errdefer {
-        for (out.items) |ent|
-            alloc.free(ent.name);
-        out.deinit(alloc);
-    }
-
-    var tot_red: u32 = 3;
-    while (tot_red < size) {
-        try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
-        try out.ensureUnusedCapacity(alloc, hdr.count + 1);
-
-        tot_red += @sizeOf(Header);
-
-        for (0..hdr.count + 1) |_| {
-            try rdr.readSliceEndian(RawEntry, @ptrCast(&raw), .little);
-
-            const new_name = try alloc.alloc(u8, raw.name_size + 1);
-            try rdr.readSliceEndian(u8, new_name, .little);
-
-            const new = out.addOneAssumeCapacity();
-            new.* = .{
-                .block_start = hdr.block_start,
-                .block_offset = raw.block_offset,
-                .type = raw.type,
-                .name = new_name,
-            };
-
-            tot_red += @sizeOf(RawEntry) + raw.name_size + 1;
-        }
-    }
-    return out.toOwnedSlice(alloc);
-}
-
-// Types
-
-const Header = extern struct {
-    count: u32,
-    block_start: u32,
-    num: u32,
-};
-
-const RawEntry = extern struct {
-    block_offset: u16,
-    num_offset: i16,
-    type: Inode.Type,
-    name_size: u16,
-};

src/file.zig

@@ -1,93 +1,202 @@
-//! An easier to use wrapper around an inode.
+//! A file/directory within the squashfs archive.
 
 const std = @import("std");
-const Io = std.Io;
+const File = std.fs.File;
+const WaitGroup = std.Thread.WaitGroup;
+const Mutex = std.Thread.Mutex;
 
 const Archive = @import("archive.zig");
-const DirEntry = @import("directory.zig");
+const DirEntry = @import("dir_entry.zig");
 const ExtractionOptions = @import("options.zig");
 const Inode = @import("inode.zig");
-const DataExtractor = @import("util/data_extractor.zig");
+const BlockSize = @import("inode_data/file.zig").BlockSize;
-const Decompressor = @import("util/decompressor.zig");
+const DataReader = @import("util/data.zig");
 const MetadataReader = @import("util/metadata.zig");
 
-const File = @This();
+const FileError = error{
+    NotDirectory,
+    NotRegularFile,
+    NotSymlink,
+    NotDevice,
+    NotFound,
+    ExtractionPathExists,
+};
 
-alloc: std.mem.Allocator,
+const SfsFile = @This();
 
-archive: Archive,
+archive: *Archive,
 
 inode: Inode,
 name: []const u8,
 
-/// Creates a new File from an inode. Takes ownership of the Inode and creates a copy of the given name.
+/// Initialize a new File.
-/// Requires the given allocator was used to create the Inode.
+/// name is copied to the File so can be safely freed afterwards.
-pub fn init(alloc: std.mem.Allocator, archive: Archive, in: Inode, name: []const u8) !File {
+pub fn init(archive: *Archive, inode: Inode, name: []const u8) !SfsFile {
-    const new_name = try alloc.alloc(u8, name.len);
+    const new_name = try archive.allocator().alloc(u8, name.len);
     @memcpy(new_name, name);
     return .{
-        .alloc = alloc,
-
         .archive = archive,
-
+        .inode = inode,
-        .inode = in,
         .name = new_name,
     };
 }
-pub fn fromDirEntry(alloc: std.mem.Allocator, archive: *Archive, ent: DirEntry) !File {
+pub fn fromEntry(archive: *Archive, entry: DirEntry) !SfsFile {
-    var rdr = archive.file.readerAt(archive.super.inode_start + ent.block_start);
+    var rdr = try archive.fil.readerAt(entry.block_start + archive.super.inode_start, &[0]u8{});
-    var meta: MetadataReader = .init(alloc, &rdr, &archive.stateless_decomp);
+    var meta: MetadataReader = .init(archive.allocator(), &rdr.interface, archive.decomp);
-    try meta.interface.discardAll(ent.block_offset);
+    try meta.interface.discardAll(entry.block_offset);
-
+    const inode: Inode = try .read(archive.allocator(), &meta.interface, archive.super.block_size);
-    var in: Inode = try .read(alloc, &meta.interface, archive.super.block_size);
+    errdefer inode.deinit(archive.allocator());
-    errdefer in.deinit(alloc);
+    return .init(archive, inode, entry.name);
-    return .init(alloc, archive.*, in, ent.name);
-}
-pub fn deinit(self: File) void {
-    self.alloc.free(self.name);
-    self.inode.deinit(self.alloc);
 }
 
-pub fn open(self: *File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
+pub fn deinit(self: SfsFile) void {
-    const entries = try self.inode.readDirectory(
+    var alloc = self.archive.allocator();
-        alloc,
+    alloc.free(self.name);
-        self.archive.file,
+    self.inode.deinit(alloc);
-        &self.archive.stateless_decomp,
+}
-        self.archive.super.dir_start,
+
-    );
+fn getEntries(self: SfsFile) ![]DirEntry {
+    return self.inode.dirEntries(self.archive.allocator(), self.archive.*);
+}
+
+pub fn ownerUid(self: SfsFile) !u16 {
+    return self.archive.id(self.inode.hdr.uid_idx);
+}
+pub fn ownerGid(self: SfsFile) !u16 {
+    return self.archive.id(self.inode.hdr.gid_idx);
+}
+pub fn permissions(self: SfsFile) u16 {
+    return self.inode.hdr.permissions;
+}
+
+pub fn isRegular(self: SfsFile) bool {
+    return switch (self.inode.hdr.inode_type) {
+        .file, .ext_file => true,
+        else => false,
+    };
+}
+/// The returned DataReader will no longer work if the File's deinit function is called
+/// or, more specifically, it's inode's deinit function is called.
+pub fn dataReader(self: SfsFile) !DataReader {
+    return self.inode.dataReader(self.archive);
+}
+
+pub fn isDir(self: SfsFile) bool {
+    return switch (self.inode.hdr.inode_type) {
+        .dir, .ext_dir => true,
+        else => false,
+    };
+}
+pub fn iterate(self: SfsFile) !Iterator {
+    if (!self.isDir()) return FileError.NotDirectory;
+    return .{
+        .entries = try self.getEntries(),
+        .archive = self.archive,
+    };
+}
+/// Open a sub-file/folder within a directory at the given path.
+/// If path is "", ".", "/", or "./", this File is returned.
+pub fn open(self: SfsFile, alloc: std.mem.Allocator, path: []const u8) !SfsFile {
+    if (!self.isDir()) return FileError.NotDirectory;
+    if (pathIsSelf(path)) return self;
+
+    // Recursively stip ending & leading path separators.
+    if (path[0] == '/') return self.open(path[1..]);
+    if (path[path.len - 1] == '/') return self.open(path[0 .. path.len - 1]);
+
+    const idx = std.mem.indexOf(u8, path, "/") orelse path.len;
+    const first_element = path[0..idx];
+    if (std.mem.eql(u8, first_element, ".")) return self.open(path[idx + 1 ..]);
+    const entries = try self.getEntries();
     defer {
-        for (entries) |ent|
+        for (entries) |e| {
-            alloc.free(ent.name);
+            e.deinit(alloc);
+        }
         alloc.free(entries);
     }
-    const path = std.mem.trim(u8, filepath, "/");
+    var cur_slice = entries;
-    const first_element: []const u8 = std.mem.sliceTo(path, '/');
+    var split = cur_slice.len / 2;
-
+    while (cur_slice.len > 0) {
-    var search_slice = entries;
+        split = cur_slice.len / 2;
-    var idx: usize = undefined;
+        const comp = std.mem.order(u8, first_element, cur_slice[split].name);
-    while (search_slice.len > 0) {
+        switch (comp) {
-        idx = search_slice.len / 2;
+            .eq => {
-        const middle = search_slice[idx];
+                var fil: SfsFile = try .fromEntry(self.archive, cur_slice[split]);
-        switch (std.mem.order(u8, first_element, middle.name)) {
+                if (idx == path.len) {
-            .eq => break,
+                    return fil;
-            .lt => search_slice = search_slice[0..idx],
+                }
-            .gt => search_slice = search_slice[idx + 1 ..],
+                defer fil.deinit();
+                return fil.open(alloc, path[idx + 1 ..]);
+            },
+            .lt => cur_slice = cur_slice[0..split],
+            .gt => cur_slice = cur_slice[split + 1 ..],
         }
-    } else return Error.FileNotFound;
+    }
-
+    return FileError.NotFound;
-    var first_elem_file = try fromDirEntry(alloc, &self.archive, search_slice[idx]);
-    if (first_element.len == path.len)
-        return first_elem_file;
-    defer first_elem_file.deinit();
-    return first_elem_file.open(alloc, io, path[first_element.len + 1 ..]);
 }
 
-pub fn extract(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8, options: ExtractionOptions) !void {
+pub fn isSymlink(self: SfsFile) bool {
-    return self.inode.extract(alloc, io, self.archive.file, self.archive.super, filepath, options);
+    return switch (self.inode.hdr.inode_type) {
+        .symlink, .ext_symlink => true,
+        else => false,
+    };
+}
+pub fn symlinkPath(self: SfsFile) ![]const u8 {
+    if (!self.isSymlink()) return FileError.NotSymlink;
+    return switch (self.inode.data) {
+        .symlink => |s| s.target,
+        .ext_symlink => |s| s.target,
+        else => unreachable,
+    };
 }
 
-// Types
+/// Check if the File is a block or character device.
+pub fn isDevice(self: SfsFile) bool {
+    return switch (self.inode.hdr.inode_type) {
+        .block_dev, .char_dev, .ext_block_dev, .ext_char_dev => true,
+        else => false,
+    };
+}
+/// If the File is a block or character device, get's it's device number.
+pub fn devNum(self: SfsFile) !u32 {
+    if (!self.isDevice()) return FileError.NotDevice;
+    return switch (self.inode.data) {
+        .block_dev, .char_dev => |d| d.dev,
+        .ext_block_dev, .ext_char_dev => |d| d.dev,
+        else => unreachable,
+    };
+}
 
-pub const Error = error{
+/// Extract the given File to the path. If File is a regular file, the path must be a directory or not exist.
-    FileNotFound,
+/// If the gievn path is a folder, the File's contents will be extracted within.
-} || Inode.Error;
+pub fn extract(self: *SfsFile, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void {
+    return self.inode.extractToThreaded(alloc, self.archive, path, options);
+}
+
+/// Utility function.
+pub fn pathIsSelf(path: []const u8) bool {
+    if (path.len == 0) return true;
+    if (path.len == 1 and (path[0] == '/' or path[0] == '.')) return true;
+    if (path.len == 2 and (path[0] == '.' and path[1] == '/')) return true;
+    return false;
+}
+
+pub const Iterator = struct {
+    entries: []DirEntry,
+    archive: *Archive,
+
+    idx: u32 = 0,
+
+    pub fn next(self: *Iterator) !?SfsFile {
+        if (self.idx >= self.entries.len) return null;
+        defer self.idx += 1;
+        return try SfsFile.fromEntry(self.archive, self.entries[self.idx]);
+    }
+    pub fn deinit(self: Iterator) void {
+        var alloc = self.archive.allocator();
+        for (self.entries) |e| {
+            e.deinit(alloc);
+        }
+        alloc.free(self.entries);
+    }
+};

src/frag.zig

@@ -1,85 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const BlockSize = @import("inode_data/file.zig").BlockSize;
-const LookupTable = @import("lookup_table.zig");
-const Decompressor = @import("util/decompressor.zig");
-const MetadataReader = @import("util/metadata.zig");
-const OffsetFile = @import("util/offset_file.zig");
-
-const FragManager = @This();
-
-pub const FragEntry = extern struct {
-    start: u64,
-    size: BlockSize,
-    _: u32,
-};
-
-alloc: std.mem.Allocator,
-fil: OffsetFile,
-decomp: *Decompressor,
-block_size: u32,
-
-entries: []FragEntry,
-
-frag_cache: std.array_hash_map.Auto(u32, []u8),
-cache_mut: std.Io.RwLock = .init,
-
-pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, frag_start: u64, frag_num: u32, block_size: u32) !FragManager {
-    const first_offset: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[frag_start .. frag_start + 8]), .little);
-
-    var rdr = fil.readerAt(first_offset);
-    var meta: MetadataReader = .init(alloc, &rdr, decomp);
-
-    const entries = try alloc.alloc(FragEntry, frag_num);
-    errdefer alloc.free(entries);
-
-    try meta.interface.readSliceEndian(FragEntry, entries, .little);
-
-    return .{
-        .alloc = alloc,
-        .fil = fil,
-        .decomp = decomp,
-        .block_size = block_size,
-
-        .entries = entries,
-
-        .frag_cache = .empty,
-    };
-}
-pub fn deinit(self: *FragManager, io: Io) void {
-    self.cache_mut.lockUncancelable(io);
-    self.alloc.free(self.entries);
-    for (self.frag_cache.values()) |v|
-        self.alloc.free(v);
-    self.frag_cache.deinit(self.alloc);
-}
-
-pub fn get(self: *FragManager, io: Io, idx: u32) ![]u8 {
-    {
-        try self.cache_mut.lockShared(io);
-        defer self.cache_mut.unlockShared(io);
-        if (self.frag_cache.contains(idx))
-            return self.frag_cache.get(idx).?;
-    }
-
-    try self.cache_mut.lock(io);
-    defer self.cache_mut.unlock(io);
-
-    if (self.frag_cache.contains(idx))
-        return self.frag_cache.get(idx).?;
-
-    const entry = self.entries[idx];
-
-    const out = try self.alloc.alloc(u8, if (entry.size.uncompressed) entry.size.size else self.block_size);
-
-    if (entry.size.uncompressed) {
-        @memcpy(out, self.fil.map.memory[entry.start .. entry.start + entry.size.size]);
-    } else {
-        @branchHint(.likely);
-        _ = try self.decomp.Decompress(self.alloc, self.fil.map.memory[entry.start .. entry.start + entry.size.size], out);
-    }
-
-    try self.frag_cache.put(self.alloc, idx, out);
-    return out;
-}

src/inode.zig

@@ -2,30 +2,77 @@
 
 const std = @import("std");
 const Reader = std.Io.Reader;
-const Io = std.Io;
+const WaitGroup = std.Thread.WaitGroup;
+const Pool = std.Thread.Pool;
+const Mutex = std.Thread.Mutex;
 
 const Archive = @import("archive.zig");
-const Decomp = @import("decomp.zig").Decomp;
+const DirEntry = @import("dir_entry.zig");
-const DirEntry = @import("directory.zig");
 const ExtractionOptions = @import("options.zig");
-const FragEntry = @import("frag.zig").FragEntry;
-const FragManager = @import("frag.zig");
 const dir = @import("inode_data/dir.zig");
 const file = @import("inode_data/file.zig");
 const misc = @import("inode_data/misc.zig");
-const LookupTable = @import("lookup_table.zig");
+const Tables = @import("tables.zig");
-const CachedTable = LookupTable.CachedTable;
+const DataReader = @import("util/data.zig");
-const DataExtractor = @import("util/data_extractor.zig");
+const ThreadedDataReader = @import("util/data_threaded.zig");
-const DataReader = @import("util/data_reader.zig");
+const InodeExtract = @import("util/extract.zig");
-const Decompressor = @import("util/decompressor.zig");
+const InodeFinish = @import("util/inode_finish.zig");
+const FinishUnion = InodeFinish.FinishUnion;
 const MetadataReader = @import("util/metadata.zig");
-const OffsetFile = @import("util/offset_file.zig");
+
-const XattrTable = @import("xattr_table.zig");
+pub const Ref = packed struct {
+    block_offset: u16,
+    block_start: u32,
+    _: u16,
+};
+
+pub const InodeType = enum(u16) {
+    dir = 1,
+    file,
+    symlink,
+    block_dev,
+    char_dev,
+    fifo,
+    socket,
+    ext_dir,
+    ext_file,
+    ext_symlink,
+    ext_block_dev,
+    ext_char_dev,
+    ext_fifo,
+    ext_socket,
+};
+
+pub const InodeData = union(InodeType) {
+    dir: dir.Dir,
+    file: file.File,
+    symlink: misc.Symlink,
+    block_dev: misc.Dev,
+    char_dev: misc.Dev,
+    fifo: misc.IPC,
+    socket: misc.IPC,
+    ext_dir: dir.ExtDir,
+    ext_file: file.ExtFile,
+    ext_symlink: misc.ExtSymlink,
+    ext_block_dev: misc.ExtDev,
+    ext_char_dev: misc.ExtDev,
+    ext_fifo: misc.ExtIPC,
+    ext_socket: misc.ExtIPC,
+};
+
+pub const Header = packed struct {
+    inode_type: InodeType,
+    permissions: u16,
+    uid_idx: u16,
+    gid_idx: u16,
+    mod_time: u32,
+    num: u32,
+};
 
 const Inode = @This();
 
 hdr: Header,
-data: Data,
+data: InodeData,
 
 pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode {
     var hdr: Header = undefined;
@@ -50,584 +97,93 @@ pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode {
         },
     };
 }
+pub fn readFromEntry(alloc: std.mem.Allocator, archive: Archive, entry: DirEntry) !Inode {
+    var rdr = try archive.fil.readerAt(archive.super.inode_start + entry.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp);
+    try meta.interface.discardAll(entry.block_offset);
+    return read(alloc, &meta.interface, archive.super.block_size);
+}
+
 pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
     switch (self.data) {
-        .file => |d| d.deinit(alloc),
+        .file => |f| alloc.free(f.block_sizes),
-        .symlink => |d| d.deinit(alloc),
+        .ext_file => |f| alloc.free(f.block_sizes),
-        .ext_file => |d| d.deinit(alloc),
+        .symlink => |s| alloc.free(s.target),
-        .ext_symlink => |d| d.deinit(alloc),
+        .ext_symlink => |s| alloc.free(s.target),
         else => {},
     }
 }
 
-// Utility Functions
+/// Get the data reader for a file inode.
+pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive, tables: *Tables) !DataReader {
+    return switch (self.hdr.inode_type) {
+        .file => readerFromData(alloc, archive, tables, self.data.file),
+        .ext_file => readerFromData(alloc, archive, tables, self.data.ext_file),
+        else => error.NotRegularFile,
+    };
+}
+fn readerFromData(alloc: std.mem.Allocator, archive: Archive, tables: *Tables, data: anytype) !DataReader {
+    var out: DataReader = .init(alloc, archive, data.block_sizes, data.block_start, data.size);
+    if (data.frag_idx != 0xFFFFFFFF)
+        out.addFragment(try tables.frag_table.get(data.frag_idx), data.frag_block_offset);
+    return out;
+}
 
-/// Read the directory entries
+/// Get the directory entries for a directory inode.
-pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, dir_offset: u64) ![]DirEntry {
+pub fn dirEntries(self: Inode, alloc: std.mem.Allocator, archive: Archive) ![]DirEntry {
+    return switch (self.hdr.inode_type) {
+        .dir => entriesFromData(alloc, archive, self.data.dir),
+        .ext_dir => entriesFromData(alloc, archive, self.data.ext_dir),
+        else => error.NotDirectory,
+    };
+}
+fn entriesFromData(alloc: std.mem.Allocator, archive: Archive, data: anytype) ![]DirEntry {
+    var rdr = try archive.fil.readerAt(archive.super.dir_start + data.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp);
+    try meta.interface.discardAll(data.block_offset);
+    return DirEntry.readDir(alloc, &meta.interface, data.size);
+}
+
+/// Returns the xattr index for the given inode. If the inode isn't an extended variant or doesn't have any, the u32 max is returned (0xFFFFFFFF).
+pub fn xattrIdx(self: Inode) u32 {
     return switch (self.data) {
-        .dir => |d| readDirFromData(alloc, fil, decomp, dir_offset, d),
+        .ext_dir => |d| d.xattr_id,
-        .ext_dir => |d| readDirFromData(alloc, fil, decomp, dir_offset, d),
+        .ext_file => |f| f.xattr_idx,
-        else => Error.NotDirectory,
+        .ext_symlink => |s| s.xattr_idx,
+        .ext_block_dev, .ext_char_dev => |d| d.xattr_idx,
+        .ext_fifo, .ext_socket => |i| i.xattr_idx,
+        else => 0xFFFFFFFF,
     };
 }
-fn readDirFromData(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, dir_offset: u64, d: anytype) ![]DirEntry {
-    var rdr = fil.readerAt(dir_offset + d.block_start);
-    var meta: MetadataReader = .init(alloc, &rdr, decomp);
-    try meta.interface.discardAll(d.block_offset);
-
-    return DirEntry.readDirectory(alloc, &meta.interface, d.size);
-}
-/// Get a reader for a regular file's data.
-pub fn dataReader(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8) !DataReader {
-    return switch (self.data) {
-        .file => |f| getReaderFromData(alloc, io, fil, decomp, block_size, frag_block, f),
-        .ext_file => |f| getReaderFromData(alloc, io, fil, decomp, block_size, frag_block, f),
-        else => Error.NotRegularFile,
-    };
-}
-fn getReaderFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8, d: anytype) !DataReader {
-    const ext: DataReader = .init(alloc, io, fil, decomp, block_size, d.size, d.block_start, d.blocks);
-    if (d.frag_block_offset == 0xFFFFFFFF) {
-        if (frag_block == null) return error.FragBlockNotProvided;
-        ext.addFrag(d.frag_block_offset, frag_block.?);
-    }
-    return ext;
-}
-/// Get an extractor for a regular file's data.
-pub fn dataExtractor(self: Inode, fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8) !DataExtractor {
-    return switch (self.data) {
-        .file => |f| getExtractorFromData(fil, decomp, block_size, frag_block, f),
-        .ext_file => |f| getExtractorFromData(fil, decomp, block_size, frag_block, f),
-        else => Error.NotRegularFile,
-    };
-}
-fn getExtractorFromData(fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8, d: anytype) !DataExtractor {
-    const ext: DataExtractor = .init(fil, decomp, block_size, d.size, d.block_start, d.blocks);
-    if (d.frag_block_offset == 0xFFFFFFFF) {
-        if (frag_block == null) return error.FragBlockNotProvided;
-        ext.addFrag(d.frag_block_offset, frag_block.?);
-    }
-    return ext;
-}
-/// Get a symlink's target path
-pub fn symlinkTarget(self: Inode) ![]const u8 {
-    return switch (self.data) {
-        .symlink => |s| s.target,
-        .ext_symlink => |s| s.target,
-        else => Error.NotSymlink,
-    };
-}
-/// Get inode's gid
-pub fn gid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, id_table_start: u64) !u16 {
-    return LookupTable.lookupValue(u16, alloc, io, decomp, fil, id_table_start, self.hdr.gid_idx);
-}
-/// Get inode's uid
-pub fn uid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, id_table_start: u64) !u16 {
-    return LookupTable.lookupValue(u16, alloc, io, decomp, fil, id_table_start, self.hdr.uid_idx);
-}
-/// Get the inode's xattr values as an index into the Archive's xattr table.
-/// Returns error.NoXattr if the inode doesn't have extended attributes.
-pub fn xattrIndex(self: Inode) !u32 {
-    const idx = switch (self.data) {
-        .ext_dir => |e| e.xattr_idx,
-        .ext_file => |e| e.xattr_idx,
-        .ext_symlink => |e| e.xattr_idx,
-        .ext_block_dev => |e| e.xattr_idx,
-        .ext_char_dev => |e| e.xattr_idx,
-        .ext_fifo => |e| e.xattr_idx,
-        .ext_socket => |e| e.xattr_idx,
-        else => return error.NoXattr,
-    };
-    if (idx == 0xFFFFFFFF) return error.NoXattr;
-    return idx;
-}
-// Get an inode's xattr values. If the inode does not have xattr values (including if the inode is not an extended type), an empty slice is returned.
-pub fn xattrValues(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, xattr_table_start: u64) ![]XattrTable.XattrOwned {
-    const idx = self.xattrIndex() catch &[0]XattrTable.XattrOwned{};
-    return XattrTable.statelessLookup(alloc, io, decomp, fil, xattr_table_start, idx);
-}
-
-// Types
-
-pub const Error = error{
-    NotDirectory,
-    NotRegularFile,
-    NotSymlink,
-    NotExtended,
-};
-
-pub const Ref = packed struct(u64) {
-    block_offset: u16,
-    block_start: u32,
-    _: u16 = 0,
-};
-
-pub const Type = enum(u16) {
-    dir = 1,
-    file,
-    symlink,
-    block_dev,
-    char_dev,
-    fifo,
-    socket,
-    ext_dir,
-    ext_file,
-    ext_symlink,
-    ext_block_dev,
-    ext_char_dev,
-    ext_fifo,
-    ext_socket,
-};
-
-pub const Data = union(Type) {
-    dir: dir.Dir,
-    file: file.File,
-    symlink: misc.Symlink,
-    block_dev: misc.Dev,
-    char_dev: misc.Dev,
-    fifo: misc.IPC,
-    socket: misc.IPC,
-    ext_dir: dir.ExtDir,
-    ext_file: file.ExtFile,
-    ext_symlink: misc.ExtSymlink,
-    ext_block_dev: misc.ExtDev,
-    ext_char_dev: misc.ExtDev,
-    ext_fifo: misc.ExtIPC,
-    ext_socket: misc.ExtIPC,
-};
-
-pub const Header = extern struct {
-    inode_type: Type,
-    permissions: u16,
-    uid_idx: u16,
-    gid_idx: u16,
-    mod_time: u32,
-    num: u32,
-};
-
-// Extract
-
-const ExtractError = error{ MknodFailed, CannotSetXattr } || DataExtractor.Error || DirEntry.Error ||
-    Decompressor.Error || Io.File.Atomic.InitError || Io.File.Atomic.LinkError || Io.Dir.SymLinkError;
-const PathRet = struct {
-    path: []const u8,
-    inode: Inode,
-    origin: bool,
-
-    fn deinit(self: PathRet, alloc: std.mem.Allocator) void {
-        if (self.origin) return;
-        alloc.free(self.path);
-        self.inode.deinit(alloc);
-    }
-    fn setMetadata(self: PathRet, alloc: std.mem.Allocator, io: Io, id_table: *CachedTable(u16), xattr_table: ?*XattrTable, options: ExtractionOptions) !void {
-        var fil = try Io.Dir.cwd().openFile(io, self.path, .{});
-        defer fil.close(io);
-
-        const inode = self.inode;
-
-        if (!options.ignore_permissions) {
-            try fil.setPermissions(io, @enumFromInt(inode.hdr.permissions));
-            try fil.setOwner(io, try id_table.get(io, inode.hdr.uid_idx), try id_table.get(io, inode.hdr.gid_idx));
-        }
-        if (xattr_table != null) {
-            const idx = inode.xattrIndex() catch return;
-
-            const xattrs = try xattr_table.?.get(alloc, io, idx);
-            defer {
-                for (xattrs) |x|
-                    x.deinit(alloc);
-                alloc.free(xattrs);
-            }
-
-            const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{self.path}, 0);
-            defer alloc.free(sentinel_path);
-            for (xattrs) |x| {
-                const xattr_ret = std.os.linux.fsetxattr(fil.handle, x.key, x.value.ptr, x.value.len, 0);
-                if (xattr_ret != 0)
-                    return ExtractError.CannotSetXattr;
-            }
-        }
-    }
-};
-fn DirCompare(_: void, a: PathRet, b: PathRet) std.math.Order {
-    return std.math.order(std.mem.count(u8, a.path, "/"), std.mem.count(u8, b.path, "/"));
-}
-const ExtractReturnUnion = union(enum) {
-    path_ret: ExtractError!PathRet,
-};
-const Tables = struct {
-    id: LookupTable.CachedTable(u16),
-    frag: LookupTable.CachedTable(FragEntry),
-    xattr: XattrTable,
-};
-
-/// Extracts the given inode to the given path. If the inode not a directory, the given path must not exist.
-/// If the inode is a directory the path must not exist or be a directory.
-pub fn extract(
-    self: Inode,
-    alloc: std.mem.Allocator,
-    io: Io,
-    fil: OffsetFile,
-    super: Archive.Superblock,
-    filepath: []const u8,
-    options: ExtractionOptions,
-) !void {
-    const path = std.mem.trimEnd(u8, filepath, "/");
-
-    var decomp_base: Decompressor = try @import("decomp.zig").StatelessDecomp(super.compression); // TODO: Replace with actual Decomp value to share states & caches for efficiency.
-    const decomp = &decomp_base;
-
-    var frag_mgr: FragManager = try .init(alloc, fil, decomp, super.frag_start, super.frag_count, super.block_size);
-    defer frag_mgr.deinit(io);
-
-    if (options.single_threaded)
-        return self.extractSinglethreaded(alloc, io, fil, super, path, options, decomp, &frag_mgr);
-
-    var sel_buf: [10]ExtractReturnUnion = undefined;
-    var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf);
-    defer sel.cancelDiscard();
-
-    var loop = io.async(finishLoop, .{ alloc, io, fil, decomp, super, options, &sel });
-
-    sel.async(.path_ret, extractRealAsync, .{ self, alloc, io, fil, super, decomp, &sel, &frag_mgr, path, true });
-
-    try loop.await(io);
-}
-fn extractRealAsync(
-    self: Inode,
-    alloc: std.mem.Allocator,
-    io: Io,
-    fil: OffsetFile,
-    super: Archive.Superblock,
-    decomp: *Decompressor,
-    sel: *Io.Select(ExtractReturnUnion),
-    frag_mgr: *FragManager,
-    path: []const u8,
-    origin: bool,
-) ExtractError!PathRet {
-    errdefer {
-        if (!origin) {
-            self.deinit(alloc);
-            alloc.free(path);
-        }
-    }
-    switch (self.hdr.inode_type) {
-        .dir, .ext_dir => {
-            try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777));
-
-            const entries = self.readDirectory(alloc, fil, decomp, super.dir_start) catch |err| switch (err) {
-                Error.NotDirectory, Error.NotExtended, Error.NotRegularFile, Error.NotSymlink => unreachable,
-                else => |e| return e,
-            };
-            defer {
-                for (entries) |e|
-                    e.deinit(alloc);
-                alloc.free(entries);
-            }
-
-            for (entries) |e| {
-                const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", e.name });
-                errdefer alloc.free(new_path);
-
-                var rdr = fil.readerAt(super.inode_start + e.block_start);
-                var meta: MetadataReader = .init(alloc, &rdr, decomp);
-                try meta.interface.discardAll(e.block_offset);
-
-                const new_inode = try read(alloc, &meta.interface, super.block_size);
-                errdefer new_inode.deinit(alloc);
-
-                sel.async(.path_ret, extractRealAsync, .{ new_inode, alloc, io, fil, super, decomp, sel, frag_mgr, new_path, false });
-            }
-        },
-        .file, .ext_file => {
-            var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{ .make_path = true });
-            defer atomic.deinit(io);
-
-            var ext: DataExtractor = switch (self.data) {
-                .file => |f| blk: {
-                    var ext: DataExtractor = .init(fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
-                    if (f.frag_idx != 0xFFFFFFFF)
-                        ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
-                    break :blk ext;
-                },
-                .ext_file => |f| blk: {
-                    var ext: DataExtractor = .init(fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
-                    if (f.frag_idx != 0xFFFFFFFF)
-                        ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
-                    break :blk ext;
-                },
-                else => unreachable,
-            };
-
-            try ext.extractAsync(alloc, io, atomic.file);
-
-            try atomic.link(io);
-        },
-        .symlink, .ext_symlink => try Io.Dir.cwd().symLink(io, self.symlinkTarget() catch unreachable, path, .{}),
-        else => {
-            var mode: u32 = undefined;
-            var dev: u32 = 0;
-
-            const DT = std.posix.DT;
-
-            switch (self.data) {
-                .char_dev => |d| {
-                    dev = d.dev;
-                    mode = DT.CHR;
-                },
-                .ext_char_dev => |d| {
-                    dev = d.dev;
-                    mode = DT.CHR;
-                },
-                .block_dev => |d| {
-                    dev = d.dev;
-                    mode = DT.BLK;
-                },
-                .ext_block_dev => |d| {
-                    dev = d.dev;
-                    mode = DT.BLK;
-                },
-                .fifo, .ext_fifo => mode = DT.FIFO,
-                .socket, .ext_socket => mode = DT.SOCK,
-                else => unreachable,
-            }
-
-            const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
-            const res = std.os.linux.mknod(sentinel_path, mode, dev);
-            alloc.free(sentinel_path);
-            if (res != 0)
-                return ExtractError.MknodFailed;
-        },
-    }
-    return .{
-        .path = path,
-        .inode = self,
-        .origin = origin,
-    };
-}
-fn finishLoop(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, super: Archive.Superblock, options: ExtractionOptions, sel: *Io.Select(ExtractReturnUnion)) !void {
-    var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count);
-    defer id_table.deinit(io);
-
-    var xattr_table: ?XattrTable = if (super.flags.xattr_never or options.ignore_xattr or !@hasField(std.os, "linux"))
-        null
-    else
-        try .init(alloc, fil, decomp, super.xattr_start);
-    defer if (xattr_table != null) xattr_table.?.deinit(io);
-
-    var dir_queue: std.PriorityDequeue(PathRet, void, DirCompare) = .empty;
-    defer dir_queue.deinit(alloc);
-
-    while (true) {
-        if (sel.group.token.load(.unordered) == null) break;
-
-        const ret = try sel.await();
-        const path_ret = try ret.path_ret;
-
-        if (options.ignore_permissions and xattr_table == null) {
-            path_ret.deinit(alloc);
-            continue;
-        }
-
-        if (path_ret.inode.hdr.inode_type == .dir or path_ret.inode.hdr.inode_type == .ext_dir) {
-            try dir_queue.push(alloc, path_ret);
-            continue;
-        }
-        defer path_ret.deinit(alloc);
-
-        try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options);
-    }
-
-    while (sel.cancel()) |ret| {
-        const path_ret = try ret.path_ret;
-
-        if (options.ignore_permissions and xattr_table == null) {
-            path_ret.deinit(alloc);
-            continue;
-        }
-
-        if (path_ret.inode.hdr.inode_type == .dir or path_ret.inode.hdr.inode_type == .ext_dir) {
-            try dir_queue.push(alloc, path_ret);
-            continue;
-        }
-
-        defer path_ret.deinit(alloc);
-        try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options);
-    }
-
-    var iter = dir_queue.iterator();
-    while (iter.next()) |path_ret| {
-        defer path_ret.deinit(alloc);
-        try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options);
-    }
-}
-/// Extracts the given inode to the given path. If the inode not a directory, the given path must not exist.
-/// If the inode is a directory the path must not exist or be a directory.
-fn extractSinglethreaded(
-    self: Inode,
-    alloc: std.mem.Allocator,
-    io: Io,
-    fil: OffsetFile,
-    super: Archive.Superblock,
-    path: []const u8,
-    options: ExtractionOptions,
-    decomp: *Decompressor,
-    frag: *FragManager,
-) !void {
-    var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count);
-    defer id_table.deinit(io);
-
-    var xattr_table: ?XattrTable = if (super.flags.xattr_never or options.ignore_xattr or !@hasField(std.os, "linux"))
-        null
-    else
-        try .init(alloc, fil, decomp, super.xattr_start);
-    defer if (xattr_table != null) xattr_table.?.deinit(io);
-
-    return self.extractReal(
-        alloc,
-        io,
-        fil,
-        super,
-        decomp,
-        frag,
-        &id_table,
-        if (xattr_table == null) null else &xattr_table.?,
-        path,
-        options,
-    );
-}
-fn extractReal(
-    self: Inode,
-    alloc: std.mem.Allocator,
-    io: Io,
-    fil: OffsetFile,
-    super: Archive.Superblock,
-    decomp: *Decompressor,
-    frag_mgr: *FragManager,
-    id_table: *CachedTable(u16),
-    xattr_table: ?*XattrTable,
-    path: []const u8,
-    options: ExtractionOptions,
-) !void {
-    switch (self.hdr.inode_type) {
-        .dir, .ext_dir => {
-            try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777));
-
-            const entries = self.readDirectory(alloc, fil, decomp, super.dir_start) catch |err| switch (err) {
-                Error.NotDirectory, Error.NotExtended, Error.NotRegularFile, Error.NotSymlink => unreachable,
-                else => |e| return e,
-            };
-            defer {
-                for (entries) |e|
-                    e.deinit(alloc);
-                alloc.free(entries);
-            }
-
-            for (entries) |e| {
-                const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", e.name });
-                defer alloc.free(new_path);
-
-                var rdr = fil.readerAt(super.inode_start + e.block_start);
-                var meta: MetadataReader = .init(alloc, &rdr, decomp);
-                try meta.interface.discardAll(e.block_offset);
-
-                const new_inode = try read(alloc, &meta.interface, super.block_size);
-                defer new_inode.deinit(alloc);
-
-                try new_inode.extractReal(alloc, io, fil, super, decomp, frag_mgr, id_table, xattr_table, new_path, options);
-            }
-        },
-        .file, .ext_file => {
-            var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{ .make_path = true });
-            defer atomic.deinit(io);
-
-            var rdr: DataReader = switch (self.data) {
-                .file => |f| blk: {
-                    var ext: DataReader = try .init(alloc, io, fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
-                    if (f.frag_idx != 0xFFFFFFFF)
-                        ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
-                    break :blk ext;
-                },
-                .ext_file => |f| blk: {
-                    var ext: DataReader = try .init(alloc, io, fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
-                    if (f.frag_idx != 0xFFFFFFFF)
-                        ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
-                    break :blk ext;
-                },
-                else => unreachable,
-            };
-            defer rdr.deinit();
-
-            var buf: [512 * 1024]u8 = undefined;
-            var wrt = atomic.file.writer(io, &buf);
-
-            _ = try rdr.interface.streamRemaining(&wrt.interface);
-
-            try wrt.flush();
-
-            try atomic.link(io);
-        },
-        .symlink, .ext_symlink => try Io.Dir.cwd().symLink(io, self.symlinkTarget() catch unreachable, path, .{}),
-        else => {
-            var mode: u32 = undefined;
-            var dev: u32 = 0;
-
-            const DT = std.posix.DT;
-
-            switch (self.data) {
-                .char_dev => |d| {
-                    dev = d.dev;
-                    mode = DT.CHR;
-                },
-                .ext_char_dev => |d| {
-                    dev = d.dev;
-                    mode = DT.CHR;
-                },
-                .block_dev => |d| {
-                    dev = d.dev;
-                    mode = DT.BLK;
-                },
-                .ext_block_dev => |d| {
-                    dev = d.dev;
-                    mode = DT.BLK;
-                },
-                .fifo, .ext_fifo => mode = DT.FIFO,
-                .socket, .ext_socket => mode = DT.SOCK,
-                else => unreachable,
-            }
-
-            const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
-            const res = std.os.linux.mknod(sentinel_path, mode, dev);
-            alloc.free(sentinel_path);
-            if (res != 0)
-                return ExtractError.MknodFailed;
-        },
-    }
-    if (options.ignore_permissions and options.ignore_xattr) return;
-
-    var f = try Io.Dir.cwd().openFile(io, path, .{});
-    defer f.close(io);
 
+/// Applies the Inode's metadata to the given File.
+/// Mod time is always set, but permissions and xattrs are set based on the given ExtractionOptions.
+pub fn setMetadata(self: Inode, alloc: std.mem.Allocator, tables: *Tables, fil: std.fs.File, options: ExtractionOptions) !void {
+    const time = @as(i128, self.hdr.mod_time) * 1000000000;
+    try fil.updateTimes(time, time);
     if (!options.ignore_permissions) {
-        try f.setPermissions(io, @enumFromInt(self.hdr.permissions));
+        try fil.chmod(self.hdr.permissions);
-        try f.setOwner(io, try id_table.get(io, self.hdr.uid_idx), try id_table.get(io, self.hdr.gid_idx));
+        try fil.chown(try tables.id_table.get(self.hdr.uid_idx), try tables.id_table.get(self.hdr.gid_idx));
     }
-    if (xattr_table != null) {
+    if (!options.ignore_xattr) {
-        const idx = self.xattrIndex() catch return;
+        const idx = self.xattrIdx();
-
+        if (idx == 0xFFFFFFFF) return;
-        const xattrs = try xattr_table.?.get(alloc, io, idx);
+        const xattrs = try tables.xattr_table.get(alloc, idx);
-        defer {
+        defer alloc.free(xattrs);
-            for (xattrs) |x|
+        for (xattrs) |kv| {
-                x.deinit(alloc);
+            const res = std.os.linux.fsetxattr(fil.handle, kv.key, kv.value.ptr, kv.value.len, 0);
-            alloc.free(xattrs);
+            alloc.free(kv.key);
-        }
+            alloc.free(kv.value);
-
+            if (res != 0) {
-        const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
+                if (options.verbose)
-        defer alloc.free(sentinel_path);
+                    options.verbose_writer.?.print("fsetxattr has result of: {}\n", .{res}) catch {};
-        for (xattrs) |x| {
+                return error.SetXattr;
-            const xattr_ret = std.os.linux.fsetxattr(f.handle, x.key, x.value.ptr, x.value.len, 0);
+            }
-            if (xattr_ret != 0)
-                return ExtractError.CannotSetXattr;
         }
     }
 }
+
+/// Extract the inode to the given path.
+pub fn extractTo(self: Inode, alloc: std.mem.Allocator, archive: Archive, path: []const u8, options: ExtractionOptions) !void {
+    return InodeExtract.extractTo(alloc, self, archive, path, options);
+}

src/inode_data/dir.zig

@@ -1,6 +1,6 @@
 const Reader = @import("std").Io.Reader;
 
-pub const Dir = extern struct {
+pub const Dir = packed struct {
     block_start: u32,
     hard_links: u32,
     size: u16,
@@ -14,19 +14,19 @@ pub const Dir = extern struct {
     }
 };
 
-pub const ExtDir = extern struct {
+pub const ExtDir = packed struct {
     hard_links: u32,
     size: u32,
     block_start: u32,
     parent_num: u32,
     idx_count: u16,
     block_offset: u16,
-    xattr_idx: u32,
+    xattr_id: u32,
     // index: []DirIndex
 
     pub fn read(rdr: *Reader) !ExtDir {
         var d: ExtDir = undefined;
-        try rdr.readSliceEndian(ExtDir, @ptrCast(&d), .little);
+        try rdr.readSliceEndian(Dir, @ptrCast(&d), .little);
         return d;
     }
 };

src/inode_data/file.zig

@@ -1,43 +1,34 @@
 const std = @import("std");
 const Reader = std.Io.Reader;
 
-pub const BlockSize = packed struct(u32) {
+pub const BlockSize = packed struct {
     size: u24,
     uncompressed: bool,
     _: u7,
 };
 
-const FileRawRead = extern struct {
-    block_start: u32,
-    frag_idx: u32,
-    frag_block_offset: u32,
-    size: u32,
-};
-
 pub const File = struct {
-    block_start: u32,
+    block_start: u32, // bytes 0-3
-    frag_idx: u32,
+    frag_idx: u32, // bytes 4-7
-    frag_block_offset: u32,
+    frag_block_offset: u32, // bytes 8-11
-    size: u32,
+    size: u32, // bytes 12-15
     block_sizes: []BlockSize,
 
     pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !File {
-        var raw: FileRawRead = undefined;
+        var start: [16]u8 = undefined;
-        try rdr.readSliceEndian(FileRawRead, @ptrCast(&raw), .little);
+        try rdr.readSliceAll(&start);
-
+        const frag_idx: u32 = std.mem.readInt(u32, start[4..8], .little);
-        var num_blocks: u32 = raw.size / block_size;
+        const size: u32 = std.mem.readInt(u32, start[12..16], .little);
-        if (raw.size % block_size != 0 and raw.frag_idx == 0xFFFFFFFF)
+        var num_blocks: u32 = size / block_size;
-            num_blocks += 1;
+        if (size % block_size != 0 and frag_idx == 0xFFFFFFFF) num_blocks += 1;
-
         const sizes = try alloc.alloc(BlockSize, num_blocks);
         errdefer alloc.free(sizes);
         try rdr.readSliceEndian(BlockSize, sizes, .little);
-
         return .{
-            .block_start = raw.block_start,
+            .block_start = std.mem.readInt(u32, start[0..4], .little),
-            .frag_idx = raw.frag_idx,
+            .frag_idx = frag_idx,
-            .frag_block_offset = raw.frag_block_offset,
+            .frag_block_offset = std.mem.readInt(u32, start[8..12], .little),
-            .size = raw.size,
+            .size = size,
             .block_sizes = sizes,
         };
     }
@@ -47,46 +38,34 @@ pub const File = struct {
     }
 };
 
-const ExtFileRawRead = extern struct {
-    block_start: u64,
-    size: u64,
-    sparse: u64,
-    hard_links: u32,
-    frag_idx: u32,
-    frag_block_offset: u32,
-    xattr_idx: u32,
-};
-
 pub const ExtFile = struct {
-    block_start: u64,
+    block_start: u64, // bytes 0-7
-    size: u64,
+    size: u64, // bytes 8-15
-    sparse: u64,
+    sparse: u64, // bytes 16-23
-    hard_links: u32,
+    hard_links: u32, // bytes 24-27
-    frag_idx: u32,
+    frag_idx: u32, // bytes 28-31
-    frag_block_offset: u32,
+    frag_block_offset: u32, // bytes 32-35
-    xattr_idx: u32,
+    xattr_idx: u32, // bytes 36-39
     block_sizes: []BlockSize,
 
     pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !ExtFile {
-        var raw: ExtFileRawRead = undefined;
+        var start: [40]u8 = undefined;
-        try rdr.readSliceEndian(ExtFileRawRead, @ptrCast(&raw), .little);
+        try rdr.readSliceAll(&start);
-
+        const frag_idx: u32 = std.mem.readInt(u32, start[28..32], .little);
-        var num_blocks: u32 = @truncate(raw.size / block_size);
+        const size: u64 = std.mem.readInt(u64, start[8..16], .little);
-        if (raw.size % block_size != 0 and raw.frag_idx == 0xFFFFFFFF)
+        var num_blocks: u32 = @truncate(size / block_size);
-            num_blocks += 1;
+        if (size % block_size != 0 and frag_idx == 0xFFFFFFFF) num_blocks += 1;
-
         const sizes = try alloc.alloc(BlockSize, num_blocks);
         errdefer alloc.free(sizes);
         try rdr.readSliceEndian(BlockSize, sizes, .little);
-
         return .{
-            .block_start = raw.block_start,
+            .block_start = std.mem.readInt(u64, start[0..8], .little),
-            .size = raw.size,
+            .size = size,
-            .sparse = raw.sparse,
+            .sparse = std.mem.readInt(u64, start[16..24], .little),
-            .hard_links = raw.hard_links,
+            .hard_links = std.mem.readInt(u32, start[24..28], .little),
-            .frag_idx = raw.frag_idx,
+            .frag_idx = frag_idx,
-            .frag_block_offset = raw.frag_block_offset,
+            .frag_block_offset = std.mem.readInt(u32, start[32..36], .little),
-            .xattr_idx = raw.xattr_idx,
+            .xattr_idx = std.mem.readInt(u32, start[36..40], .little),
             .block_sizes = sizes,
         };
     }

src/inode_data/misc.zig

@@ -50,7 +50,7 @@ pub const ExtSymlink = struct {
 };
 
 /// A block or character device.
-pub const Dev = extern struct {
+pub const Dev = packed struct {
     hard_links: u32,
     dev: u32,
 
@@ -62,7 +62,7 @@ pub const Dev = extern struct {
 };
 
 /// An extended block or character device.
-pub const ExtDev = extern struct {
+pub const ExtDev = packed struct {
     hard_links: u32,
     dev: u32,
     xattr_idx: u32,
@@ -75,7 +75,7 @@ pub const ExtDev = extern struct {
 };
 
 /// A socket or FIFO file.
-pub const IPC = extern struct {
+pub const IPC = packed struct {
     hard_links: u32,
 
     pub fn read(rdr: *Reader) !IPC {
@@ -86,7 +86,7 @@ pub const IPC = extern struct {
 };
 
 /// An extended socket or FIFO file.
-pub const ExtIPC = extern struct {
+pub const ExtIPC = packed struct {
     hard_links: u32,
     xattr_idx: u32,
 

src/lookup_table.zig

@@ -1,125 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const Decompressor = @import("util/decompressor.zig");
-const MetadataReader = @import("util/metadata.zig");
-const OffsetFile = @import("util/offset_file.zig");
-
-pub fn lookupValue(comptime T: anytype, alloc: std.mem.Allocator, decomp: *Decompressor, file: OffsetFile, table_start: u64, idx: u32) !T {
-    const T_PER_BLOCK: u16 = 8192 / @sizeOf(T);
-
-    const block = idx / T_PER_BLOCK;
-    const block_offset = idx % T_PER_BLOCK;
-
-    const offset_pos = table_start + (8 * block);
-    const offset: u64 = std.mem.readInt(u64, @ptrCast(file.map.memory[offset_pos .. offset_pos + 8]), .little);
-
-    var rdr = file.readerAt(offset);
-    var meta: MetadataReader = .init(alloc, &rdr, decomp);
-
-    try meta.interface.discardAll(@sizeOf(T) * block_offset);
-    var out: T = undefined;
-    try meta.interface.readSliceEndian(T, @ptrCast(&out), .little);
-    return out;
-}
-
-pub const Error = Io.Cancelable || Io.File.Reader.SeekError || Io.Reader.ReadAllocError;
-
-pub fn CachedTable(comptime T: anytype) type {
-    return struct {
-        const T_PER_BLOCK: u16 = 8192 / @sizeOf(T);
-
-        const Table = @This();
-
-        alloc: std.mem.Allocator,
-        fil: OffsetFile,
-        decomp: *Decompressor,
-
-        table_start: u64,
-        total_num: u32,
-
-        table: std.AutoHashMap(u32, []T),
-
-        mut: Io.RwLock = .init,
-
-        pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, offset: u64, total_num: u32) Table {
-            return .{
-                .alloc = alloc,
-                .fil = fil,
-                .decomp = decomp,
-
-                .table_start = offset,
-                .total_num = total_num,
-
-                .table = .init(alloc),
-            };
-        }
-        pub fn deinit(self: *Table, io: Io) void {
-            self.mut.lockUncancelable(io);
-            var iter = self.table.valueIterator();
-            while (iter.next()) |val|
-                self.alloc.free(val.*);
-            self.table.deinit();
-        }
-
-        pub fn fill(self: *Table, io: Io) Error!void {
-            try self.mut.lock(io);
-            defer self.mut.unlock(io);
-
-            var num_blocks = self.total_num / T_PER_BLOCK;
-            if (self.total_num % T_PER_BLOCK > 0)
-                num_blocks += 1;
-
-            for (0..num_blocks) |block| {
-                const offset_pos = self.table_start + (8 * block);
-                const offset: u64 = std.mem.readInt(u64, @ptrCast(self.fil.map.memory[offset_pos .. offset_pos + 8]), .little);
-
-                const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK)
-                    @truncate(self.total_num % T_PER_BLOCK)
-                else
-                    T_PER_BLOCK;
-
-                var rdr = self.fil.readerAt(offset);
-                var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
-
-                const slice = try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little);
-                try self.table.put(@truncate(block), slice);
-            }
-        }
-
-        pub fn get(self: *Table, io: Io, idx: u32) Error!T {
-            const block = idx / T_PER_BLOCK;
-            const block_offset = idx % T_PER_BLOCK;
-
-            {
-                try self.mut.lockShared(io);
-                defer self.mut.unlockShared(io);
-
-                if (self.table.contains(block))
-                    return self.table.get(block).?[block_offset];
-            }
-
-            try self.mut.lock(io);
-            defer self.mut.unlock(io);
-
-            if (self.table.contains(block))
-                return self.table.get(block).?[block_offset];
-
-            const offset_pos = self.table_start + (8 * block);
-            const offset: u64 = std.mem.readInt(u64, @ptrCast(self.fil.map.memory[offset_pos .. offset_pos + 8]), .little);
-
-            const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK)
-                @truncate(self.total_num % T_PER_BLOCK)
-            else
-                T_PER_BLOCK;
-
-            var rdr = self.fil.readerAt(offset);
-            var meta: MetadataReader = .init(self.alloc, &rdr, self.decomp);
-
-            const slice = try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little);
-            try self.table.put(@truncate(block), slice);
-
-            return slice[block_offset];
-        }
-    };
-}

src/options.zig

@@ -5,10 +5,8 @@ const Writer = std.Io.Writer;
 
 const ExtractionOptions = @This();
 
-/// Extract single-threaded only.
+/// The number of threads used for extraction. 0 implies single threaded.
-/// Though not necessary if using Threaded.single_threaded,
+threads: usize = 1,
-/// setting single_threaded is more efficient.
-single_threaded: bool = false,
 /// Don't set the file's owner & permissions after extraction
 ignore_permissions: bool = false,
 /// Don't set xattr values. Currently xattrs are never set anyway.
@@ -20,12 +18,16 @@ verbose: bool = false,
 /// Where to print verbose log.
 verbose_writer: ?*Writer = null,
 
-pub const default: ExtractionOptions = .{};
+pub const SingleThreadedDefault: ExtractionOptions = .{};
-pub const default_single_threaded: ExtractionOptions = .{ .single_threaded = true };
+pub fn Default() !ExtractionOptions {
-
+    return .{
+        .threads = try std.Thread.getCpuCount(),
+    };
+}
 pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions {
     return .{
         .verbose = true,
         .verbose_writer = wrt,
+        .threads = try std.Thread.getCpuCount(),
     };
 }

src/root.zig

@@ -1,6 +1,2 @@
 pub const Archive = @import("archive.zig");
 pub const ExtractionOptions = @import("options.zig");
-
-test {
-    @import("std").testing.refAllDecls(@This());
-}

src/super.zig

@@ -0,0 +1,63 @@
+const std = @import("std");
+const math = std.math;
+
+const CompressionType = @import("decomp.zig").CompressionType;
+const InodeRef = @import("inode.zig").Ref;
+
+const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little);
+
+const SuperblockError = error{
+    InvalidMagic,
+    InvalidBlockLog,
+    InvalidVersion,
+    InvalidCheck,
+};
+
+/// A squashfs Superblock
+pub const Superblock = packed struct {
+    magic: u32,
+    inode_count: u32,
+    mod_time: u32,
+    block_size: u32,
+    frag_count: u32,
+    compression: CompressionType,
+    block_log: u16,
+    flags: packed struct {
+        inode_uncompressed: bool,
+        data_uncompressed: bool,
+        check: bool,
+        frag_uncompressed: bool,
+        fragment_never: bool,
+        fragment_always: bool,
+        duplicates: bool,
+        exportable: bool,
+        xattr_uncompressed: bool,
+        xattr_never: bool,
+        compression_options: bool,
+        ids_uncompressed: bool,
+        _: u4,
+    },
+    id_count: u16,
+    ver_maj: u16,
+    ver_min: u16,
+    root_ref: InodeRef,
+    size: u64,
+    id_start: u64,
+    xattr_start: u64,
+    inode_start: u64,
+    dir_start: u64,
+    frag_start: u64,
+    export_start: u64,
+
+    /// Validate the Superblock. If an error is returned, it's likely the archive is corrupted or not a squashfs archive.
+    pub fn validate(self: Superblock) !void {
+        if (self.magic != SQUASHFS_MAGIC)
+            return SuperblockError.InvalidMagic;
+        if (self.flags.check)
+            return SuperblockError.InvalidCheck;
+        if (self.ver_maj != 4 or self.ver_min != 0)
+            return SuperblockError.InvalidVersion;
+        if (math.log2(self.block_size) != self.block_log)
+            return SuperblockError.InvalidBlockLog;
+    }
+};

src/tables.zig

@@ -0,0 +1,108 @@
+const std = @import("std");
+const Mutex = std.Thread.Mutex;
+
+const Archive = @import("archive.zig");
+const DecompFn = @import("decomp.zig").DecompFn;
+const BlockSize = @import("inode_data/file.zig").BlockSize;
+const InodeRef = @import("inode.zig").Ref;
+const Superblock = @import("super.zig").Superblock;
+const MetadataReader = @import("util/metadata.zig");
+const OffsetFile = @import("util/offset_file.zig");
+const XattrTable = @import("xattr.zig");
+
+/// Information about a fragment section. Multiple fragments are contained in the block described by a single FragEntry.
+/// The offset into the block and fragment size is stored in the file's inode.
+pub const FragEntry = packed struct {
+    start: u64,
+    size: BlockSize,
+    _: u32,
+};
+
+const Tables = @This();
+
+frag_table: Table(FragEntry),
+id_table: Table(u16),
+export_table: Table(InodeRef),
+xattr_table: XattrTable,
+
+pub fn init(alloc: std.mem.Allocator, archive: Archive) !Tables {
+    return .{
+        .frag_table = try .init(alloc, archive.fil, archive.decomp, archive.super.frag_start, archive.super.frag_count),
+        .id_table = try .init(alloc, archive.fil, archive.decomp, archive.super.id_start, archive.super.id_count),
+        .export_table = try .init(alloc, archive.fil, archive.decomp, archive.super.export_start, archive.super.inode_count),
+        .xattr_table = try .init(alloc, archive.fil, archive.decomp, archive.super.xattr_start),
+    };
+}
+pub fn deinit(self: *Tables) void {
+    self.frag_table.deinit();
+    self.id_table.deinit();
+    self.export_table.deinit();
+    self.xattr_table.deinit();
+}
+
+/// A two-layer metadata table.
+pub fn Table(T: anytype) type {
+    return struct {
+        const Self = @This();
+
+        const VALS_PER_BLOCK = 8192 / @sizeOf(T);
+
+        alloc: std.mem.Allocator,
+        fil: OffsetFile,
+        decomp: DecompFn,
+        tab_start: u64,
+
+        tab: std.AutoHashMap(u32, []T),
+        values: u32,
+
+        mut: Mutex = .{},
+
+        pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: DecompFn, tab_start: u64, values: u32) !Self {
+            return .{
+                .alloc = alloc,
+                .fil = fil,
+                .decomp = decomp,
+                .tab_start = tab_start,
+
+                .tab = .init(alloc),
+                .values = values,
+            };
+        }
+
+        pub fn deinit(self: *Self) void {
+            var iter = self.tab.valueIterator();
+            while (iter.next()) |s| {
+                self.alloc.free(s.*);
+            }
+            self.tab.deinit();
+        }
+
+        pub fn get(self: *Self, idx: u32) !T {
+            if (idx >= self.values) return error.InvalidIndex;
+            const block_num = idx / VALS_PER_BLOCK;
+            const idx_offset = idx - (block_num * VALS_PER_BLOCK);
+            if (self.tab.contains(block_num)) {
+                const block = self.tab.get(block_num).?;
+                return block[idx_offset];
+            }
+            self.mut.lock();
+            defer self.mut.unlock();
+            // Double check in case of race condition..
+            if (self.tab.contains(block_num)) {
+                const block = self.tab.get(block_num).?;
+                return block[idx_offset];
+            }
+            const is_last = (self.values - 1) / VALS_PER_BLOCK == block_num;
+            const slice_size = if (is_last) self.values - (block_num * VALS_PER_BLOCK) else VALS_PER_BLOCK;
+            const slice = try self.alloc.alloc(T, slice_size);
+            var rdr = try self.fil.readerAt(self.tab_start + (8 * block_num), &[0]u8{});
+            var offset: u64 = 0;
+            try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
+            rdr = try self.fil.readerAt(offset, &[0]u8{});
+            var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
+            try meta.interface.readSliceEndian(T, @ptrCast(slice), .little);
+            try self.tab.put(block_num, slice);
+            return slice[idx_offset];
+        }
+    };
+}

src/test.zig

@@ -0,0 +1,84 @@
+const std = @import("std");
+const stuff = @import("builtin");
+
+const Archive = @import("archive.zig");
+const Superblock = @import("super.zig").Superblock;
+
+const TestArchive = "testing/LinuxPATest.sfs";
+
+test "Basics" {
+    var fil = try std.fs.cwd().openFile(TestArchive, .{});
+    defer fil.close();
+    var sfs: Archive = try .init(std.testing.allocator, fil);
+    defer sfs.deinit();
+    if (sfs.super != LinuxPATestCorrectSuperblock) {
+        std.debug.print("Superblock wrong\nShould be: {}\n\nis: {}\n", .{ LinuxPATestCorrectSuperblock, sfs.super });
+        return error.BadSuperblock;
+    }
+}
+
+const TestFile = "Start.exe";
+const TestFileExtractLocation = "testing/Start.exe";
+
+test "ExtractSingleFile" {
+    std.fs.cwd().deleteFile(TestFileExtractLocation) catch {};
+    var fil = try std.fs.cwd().openFile(TestArchive, .{});
+    defer fil.close();
+    var sfs: Archive = try .init(std.testing.allocator, fil);
+    defer sfs.deinit();
+    var test_fil = try sfs.open(TestFile);
+    defer test_fil.deinit();
+    try test_fil.extract(TestFileExtractLocation, .Default);
+    //TODO: validate extracted file.
+}
+
+const TestFullExtractLocation = "testing/TestExtract";
+
+test "ExtractCompleteArchive" {
+    std.fs.cwd().deleteTree(TestFullExtractLocation) catch {};
+    var fil = try std.fs.cwd().openFile(TestArchive, .{});
+    defer fil.close();
+    var sfs: Archive = try .init(std.testing.allocator, fil);
+    defer sfs.deinit();
+    try sfs.extract(TestFullExtractLocation, .Default);
+}
+
+const LinuxPATestCorrectSuperblock: Superblock = .{
+    .magic = std.mem.readInt(u32, "hsqs", .little),
+    .inode_count = 2974,
+    .mod_time = 1632696724,
+    .block_size = 131072,
+    .frag_count = 264,
+    .compression = .zstd,
+    .block_log = 17,
+    .flags = .{
+        .inode_uncompressed = false,
+        .data_uncompressed = false,
+        .check = false,
+        .frag_uncompressed = false,
+        .fragment_never = false,
+        .fragment_always = false,
+        .duplicates = true,
+        .exportable = true,
+        .xattr_uncompressed = false,
+        .xattr_never = false,
+        .compression_options = false,
+        .ids_uncompressed = false,
+        ._ = 0,
+    },
+    .id_count = 1,
+    .ver_maj = 4,
+    .ver_min = 0,
+    .root_ref = .{
+        .block_offset = 1363,
+        .block_start = 29237,
+        ._ = 0,
+    },
+    .size = 106841744,
+    .id_start = 106841632,
+    .xattr_start = 106841720,
+    .inode_start = 106778274,
+    .dir_start = 106807998,
+    .frag_start = 106837747,
+    .export_start = 106841602,
+};

src/util/data.zig

@@ -0,0 +1,174 @@
+//! A reader for a regular file.
+
+const std = @import("std");
+const Reader = std.Io.Reader;
+const Writer = std.Io.Writer;
+const Limit = std.Io.Limit;
+
+const Archive = @import("../archive.zig");
+const DecompFn = @import("../decomp.zig").DecompFn;
+const BlockSize = @import("../inode_data/file.zig").BlockSize;
+const FragEntry = @import("../tables.zig").FragEntry;
+const OffsetFile = @import("offset_file.zig");
+
+const DataReader = @This();
+
+alloc: std.mem.Allocator,
+fil: OffsetFile,
+decomp: DecompFn,
+block_size: u32,
+
+blocks: []BlockSize,
+
+frag: ?FragEntry = null, // TODO: do something better?
+frag_offset: u32 = 0,
+size: u64,
+
+interface: Reader,
+
+cur_offset: u64,
+block_idx: u32 = 0,
+
+pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64) DataReader {
+    return .{
+        .alloc = alloc,
+        .fil = archive.fil,
+        .decomp = archive.decomp,
+        .block_size = archive.super.block_size,
+        .blocks = blocks,
+        .size = size,
+        .cur_offset = start,
+        .interface = .{
+            .end = 0,
+            .seek = 0,
+            .buffer = &[0]u8{},
+            .vtable = &.{
+                .stream = stream,
+                .discard = discard,
+                .readVec = readVec,
+            },
+        },
+    };
+}
+pub fn deinit(self: *DataReader) void {
+    self.alloc.free(self.interface.buffer);
+    self.interface.end = 0;
+    self.interface.seek = 0;
+}
+
+pub fn addFragment(self: *DataReader, entry: FragEntry, frag_offset: u32) void {
+    self.frag = entry;
+    self.frag_offset = frag_offset;
+}
+
+fn numBlocks(self: DataReader) usize {
+    var res = self.blocks.len;
+    if (self.frag != null) res += 1;
+    return res;
+}
+
+fn advance(self: *DataReader) !void {
+    if (self.block_idx > self.blocks.len or (self.block_idx == self.blocks.len and self.frag == null)) {
+        if (self.interface.buffer.len > 0) {
+            self.alloc.free(self.interface.buffer);
+            self.interface.buffer = &[0]u8{};
+            self.interface.end = 0;
+            self.interface.seek = 0;
+        }
+        return Reader.Error.EndOfStream;
+    }
+    defer self.block_idx += 1;
+    const cur_block_size = if (self.block_idx == self.numBlocks() - 1) self.size % self.block_size else self.block_size;
+    try self.resizeBuffer(cur_block_size);
+    self.interface.seek = 0;
+    self.interface.end = cur_block_size;
+    if (self.block_idx == self.blocks.len) { // fragment
+        var rdr = try self.fil.readerAt(self.frag.?.start, &[0]u8{});
+        if (self.frag.?.size.uncompressed) {
+            try rdr.interface.discardAll(self.frag_offset);
+            try rdr.interface.readSliceAll(self.interface.buffer);
+            return;
+        }
+        const tmp_buf = try self.alloc.alloc(u8, self.frag.?.size.size);
+        defer self.alloc.free(tmp_buf);
+        try rdr.interface.readSliceAll(tmp_buf);
+        const needed_block = try self.alloc.alloc(u8, self.block_size);
+        defer self.alloc.free(needed_block);
+        _ = try self.decomp(self.alloc, tmp_buf, needed_block);
+        @memcpy(self.interface.buffer, needed_block[self.frag_offset .. self.frag_offset + cur_block_size]);
+        return;
+    }
+    const block = self.blocks[self.block_idx];
+    if (block.size == 0) {
+        @memset(self.interface.buffer, 0);
+        return;
+    }
+    var rdr = try self.fil.readerAt(self.cur_offset, &[0]u8{});
+    self.cur_offset += block.size;
+    if (block.uncompressed) {
+        try rdr.interface.readSliceAll(self.interface.buffer);
+        return;
+    }
+    const tmp_buf = try self.alloc.alloc(u8, block.size);
+    defer self.alloc.free(tmp_buf);
+    try rdr.interface.readSliceAll(tmp_buf);
+    _ = try self.decomp(self.alloc, tmp_buf, self.interface.buffer);
+}
+/// Does not guarentee that data currently in the buffer is retained.
+fn resizeBuffer(self: *DataReader, size: usize) !void {
+    if (self.interface.buffer.len == size) return;
+    if (!self.alloc.resize(self.interface.buffer, size)) {
+        self.alloc.free(self.interface.buffer);
+        self.interface.buffer = self.alloc.alloc(u8, size) catch |err| {
+            self.interface.buffer = &[0]u8{};
+            return err;
+        };
+    } else {
+        self.interface.buffer.len = size;
+    }
+}
+
+fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) Reader.StreamError!usize {
+    var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr));
+    if (rdr.seek >= rdr.end) self.advance() catch |err| {
+        if (err == error.EndOfStream) return error.EndOfStream;
+        std.log.err("Error advancing data reader: {}\n", .{err});
+        return Reader.Error.ReadFailed;
+    };
+    if (limit == .nothing) return 0;
+    const to_read = @min(rdr.end - rdr.seek, @intFromEnum(limit));
+    const res = try wrt.write(rdr.buffer[rdr.seek .. rdr.seek + to_read]);
+    rdr.seek += res;
+    return res;
+}
+
+fn discard(rdr: *Reader, limit: Limit) Reader.Error!usize {
+    var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr));
+    if (rdr.seek >= rdr.end) self.advance() catch |err| {
+        if (err == error.EndOfStream) return error.EndOfStream;
+        std.log.err("Error advancing data reader: {}\n", .{err});
+        return Reader.Error.ReadFailed;
+    };
+    if (limit == .nothing) return 0;
+    const to_adv = @min(rdr.end - rdr.seek, @intFromEnum(limit));
+    rdr.seek += to_adv;
+    return to_adv;
+}
+
+fn readVec(rdr: *Reader, vec: [][]u8) Reader.Error!usize {
+    var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr));
+    if (rdr.seek >= rdr.end) self.advance() catch |err| {
+        if (err == error.EndOfStream) return error.EndOfStream;
+        std.log.err("Error advancing data reader: {}\n", .{err});
+        return Reader.Error.ReadFailed;
+    };
+    var cur_red: usize = 0;
+    for (vec) |s| {
+        const to_copy: usize = @min(rdr.end - rdr.seek, s.len);
+        @memcpy(s[0..to_copy], rdr.buffer[rdr.seek .. rdr.seek + to_copy]);
+        rdr.seek += to_copy;
+        cur_red += to_copy;
+        if (rdr.end == rdr.seek) break;
+    }
+    return cur_red;
+}

src/util/data_extractor.zig

@@ -1,162 +0,0 @@
-//! The DataExtractor is meant to extract a regular file's data to a given file asyncronously.
-
-const std = @import("std");
-const Io = std.Io;
-
-const FragEntry = @import("../frag.zig").FragEntry;
-const BlockSize = @import("../inode_data/file.zig").BlockSize;
-const Decompressor = @import("decompressor.zig");
-const OffsetFile = @import("offset_file.zig");
-
-// const SharedCache = @import("shared_cache.zig");
-
-pub const Error = Decompressor.Error || Io.File.MemoryMap.CreateError || Io.File.WritePositionalError;
-
-const DataExtractor = @This();
-
-fil: OffsetFile,
-decomp: *Decompressor,
-block_size: u32,
-
-file_size: u64,
-start: u64,
-blocks: []BlockSize,
-
-frag_offset: u32 = 0,
-frag_block: ?[]u8 = null,
-
-err: ?Error = null,
-
-pub fn init(fil: OffsetFile, decomp: *Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) DataExtractor {
-    return .{
-        .fil = fil,
-        .decomp = decomp,
-        .block_size = block_size,
-
-        .file_size = file_size,
-        .start = data_start,
-        .blocks = blocks,
-    };
-}
-pub fn addFrag(self: *DataExtractor, frag_offset: u32, block: []u8) void {
-    self.frag_offset = frag_offset;
-    self.frag_block = block;
-}
-
-fn numBlocks(self: DataExtractor) usize {
-    var num = self.blocks.len;
-    if (self.frag_block != null) num += 1;
-    return num;
-}
-
-/// Starts extracting the data using the given group to spawn async tasks.
-pub fn extractConcurrent(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.File) (Error || Io.ConcurrentError)!void {
-    var group: Io.Group = .init;
-    defer group.cancel(io);
-    var err: ?Error = null;
-
-    var read_offset: u64 = self.start;
-    for (0..self.blocks.len) |idx| {
-        try group.concurrent(io, blockThread, .{ self, alloc, io, fil, read_offset, idx, &err });
-        read_offset += self.blocks[idx].size;
-    }
-    if (self.frag_block != null)
-        try group.concurrent(io, fragThread, .{ self, io, fil, &err });
-
-    group.await(io) catch |cancel| return err orelse cancel;
-}
-
-/// Starts extracting the data using the given group to spawn async tasks.
-pub fn extractAsync(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.File) Error!void {
-    var group: Io.Group = .init;
-    defer group.cancel(io);
-    var err: ?Error = null;
-
-    var read_offset: u64 = self.start;
-    for (0..self.blocks.len) |idx| {
-        group.async(io, blockThread, .{ self, alloc, io, fil, read_offset, idx, &err });
-        read_offset += self.blocks[idx].size;
-    }
-    if (self.frag_block != null)
-        group.async(io, fragThread, .{ self, io, fil, &err });
-
-    group.await(io) catch |cancel| return err orelse cancel;
-}
-
-fn blockThread(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.File, read_offset: u64, idx: usize, ret_err: *?Error) Io.Cancelable!void {
-    const block = self.blocks[idx];
-
-    const cur_block_size = if (idx == self.numBlocks() - 1)
-        self.file_size % self.block_size
-    else
-        self.block_size;
-
-    const write_offset = self.block_size * idx;
-
-    var wrt = fil.writer(io, &[0]u8{});
-    wrt.seekTo(write_offset) catch |err| {
-        ret_err.* = err;
-        if (err == error.Canceled) io.recancel();
-        return Io.Cancelable.Canceled;
-    };
-
-    if (block.size == 0) {
-        wrt.interface.splatByteAll(0, cur_block_size) catch |err| {
-            ret_err.* = err;
-            if (err == error.Canceled) io.recancel();
-            return Io.Cancelable.Canceled;
-        };
-    } else {
-        if (block.uncompressed) {
-            wrt.interface.writeAll(self.fil.map.memory[read_offset..][0..cur_block_size]) catch |err| {
-                ret_err.* = err;
-                if (err == error.Canceled) io.recancel();
-                return Io.Cancelable.Canceled;
-            };
-        } else {
-            @branchHint(.likely);
-
-            var tmp: [1024 * 1024]u8 = undefined;
-
-            _ = self.decomp.Decompress(alloc, self.fil.map.memory[read_offset..][0..block.size], tmp[0..cur_block_size]) catch |err| {
-                ret_err.* = err;
-                return Io.Cancelable.Canceled;
-            };
-
-            wrt.interface.writeAll(tmp[0..cur_block_size]) catch |err| {
-                ret_err.* = err;
-                if (err == error.Canceled) io.recancel();
-                return Io.Cancelable.Canceled;
-            };
-        }
-    }
-    wrt.flush() catch |err| {
-        ret_err.* = err;
-        if (err == error.Canceled) io.recancel();
-        return Io.Cancelable.Canceled;
-    };
-}
-fn fragThread(self: DataExtractor, io: Io, fil: Io.File, ret_err: *?Error) Io.Cancelable!void {
-    const cur_block_size = self.file_size % self.block_size;
-
-    const write_offset = self.blocks.len * self.block_size;
-
-    var wrt = fil.writer(io, &[0]u8{});
-    wrt.seekTo(write_offset) catch |err| {
-        ret_err.* = err;
-        if (err == error.Canceled) io.recancel();
-        return Io.Cancelable.Canceled;
-    };
-
-    wrt.interface.writeAll(self.frag_block.?[self.frag_offset..][0..cur_block_size]) catch |err| {
-        ret_err.* = err;
-        if (err == error.Canceled) io.recancel();
-        return Io.Cancelable.Canceled;
-    };
-
-    wrt.flush() catch |err| {
-        ret_err.* = err;
-        if (err == error.Canceled) io.recancel();
-        return Io.Cancelable.Canceled;
-    };
-}

src/util/data_reader.zig

@@ -1,189 +0,0 @@
-//! DataReader reads a regular file's data linearly from start to finish using Io.Reader interface.
-
-const std = @import("std");
-const Io = std.Io;
-const Reader = Io.Reader;
-const Writer = Io.Writer;
-const Limit = Io.Limit;
-
-const BlockSize = @import("../inode_data/file.zig").BlockSize;
-const Decompressor = @import("decompressor.zig");
-const OffsetFile = @import("offset_file.zig");
-
-// const SharedCache = @import("shared_cache.zig");
-
-const DataReader = @This();
-
-alloc: std.mem.Allocator,
-
-fil: OffsetFile,
-io: Io,
-decomp: *Decompressor,
-block_size: u32,
-
-file_size: u64,
-cur_offset: u64,
-blocks: []BlockSize,
-
-frag_offset: u32 = 0,
-frag_block: ?[]u8 = null,
-
-block_idx: usize = 0,
-sparse_block: bool = false,
-
-interface: Io.Reader,
-
-pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) !DataReader {
-    return .{
-        .alloc = alloc,
-
-        .fil = fil,
-        .io = io,
-        .decomp = decomp,
-        .block_size = block_size,
-
-        .file_size = file_size,
-        .cur_offset = data_start,
-        .blocks = blocks,
-
-        .interface = .{
-            .buffer = try alloc.alloc(u8, block_size),
-            .seek = 0,
-            .end = 0,
-            .vtable = &.{
-                .stream = stream,
-                .discard = discard,
-                .readVec = readVec,
-            },
-        },
-    };
-}
-pub fn deinit(self: *DataReader) void {
-    self.alloc.free(self.interface.buffer);
-}
-pub fn addFrag(self: *DataReader, frag_offset: u32, block: []u8) void {
-    self.frag_offset = frag_offset;
-    self.frag_block = block;
-}
-
-fn numBlocks(self: DataReader) usize {
-    var num = self.blocks.len;
-    if (self.frag_block != null) num += 1;
-    return num;
-}
-fn advanceBuffer(self: *DataReader) !void {
-    if (self.block_idx >= self.numBlocks())
-        return Reader.Error.EndOfStream;
-
-    errdefer self.interface.end = 0;
-    defer self.block_idx += 1;
-
-    self.interface.end = if (self.block_idx == self.numBlocks() - 1)
-        self.file_size % self.block_size
-    else
-        self.block_size;
-
-    // Fragment
-    if (self.block_idx == self.blocks.len) {
-        @memcpy(self.interface.buffer[0..self.interface.end], self.frag_block.?[self.frag_offset .. self.frag_offset + self.interface.end]);
-        self.interface.seek = 0;
-        return;
-    }
-
-    // Normal Block
-    const block = self.blocks[self.block_idx];
-    if (block.size == 0) {
-        self.interface.seek = 0;
-        self.sparse_block = true;
-        return;
-    } else {
-        self.sparse_block = false;
-    }
-    if (block.uncompressed) {
-        @memcpy(self.interface.buffer[0..self.interface.end], self.fil.map.memory[self.cur_offset .. self.cur_offset + self.interface.end]);
-        self.cur_offset += self.interface.end;
-    } else {
-        @branchHint(.likely);
-
-        _ = try self.decomp.Decompress(self.alloc, self.fil.map.memory[self.cur_offset .. self.cur_offset + block.size], self.interface.buffer[0..self.interface.end]);
-        self.cur_offset += block.size;
-    }
-    self.interface.seek = 0;
-}
-
-fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) Reader.StreamError!usize {
-    var data: *DataReader = @fieldParentPtr("interface", rdr);
-    if (rdr.seek == rdr.end)
-        data.advanceBuffer() catch |err| return switch (err) {
-            error.ReadFailed => error.ReadFailed,
-            error.EndOfStream => error.EndOfStream,
-            else => error.ReadFailed,
-        };
-
-    switch (limit) {
-        .nothing => return 0,
-        .unlimited => {
-            const wrote = if (data.sparse_block)
-                try wrt.splatByte(0, rdr.end - rdr.seek)
-            else
-                try wrt.write(rdr.buffer[rdr.seek..rdr.end]);
-            rdr.seek += wrote;
-            return wrote;
-        },
-        else => {
-            const to_read = @min(rdr.end - rdr.seek, @intFromEnum(limit));
-            const wrote = if (data.sparse_block)
-                try wrt.splatByte(0, to_read)
-            else
-                try wrt.write(rdr.buffer[rdr.seek .. rdr.seek + to_read]);
-            rdr.seek += wrote;
-            return wrote;
-        },
-    }
-}
-fn discard(rdr: *Reader, limit: Limit) Reader.Error!usize {
-    var data: *DataReader = @fieldParentPtr("interface", rdr);
-    if (rdr.seek == rdr.end)
-        data.advanceBuffer() catch |err| return switch (err) {
-            error.ReadFailed => error.ReadFailed,
-            error.EndOfStream => error.EndOfStream,
-            else => error.ReadFailed,
-        };
-
-    switch (limit) {
-        .nothing => return 0,
-        .unlimited => {
-            const adv = rdr.end - rdr.seek;
-            rdr.seek = rdr.end;
-            return adv;
-        },
-        else => {
-            const adv = @min(rdr.end - rdr.seek, @intFromEnum(limit));
-            rdr.seek += adv;
-            return adv;
-        },
-    }
-}
-fn readVec(rdr: *Reader, vec: [][]u8) Reader.Error!usize {
-    var data: *DataReader = @fieldParentPtr("interface", rdr);
-    if (rdr.seek == rdr.end)
-        data.advanceBuffer() catch |err| return switch (err) {
-            error.ReadFailed => error.ReadFailed,
-            error.EndOfStream => error.EndOfStream,
-            else => error.ReadFailed,
-        };
-
-    var wrote: usize = 0;
-    for (vec) |buf| {
-        if (rdr.seek == rdr.end) break;
-
-        const to_copy = @min(rdr.end - rdr.seek, buf.len);
-        if (data.sparse_block)
-            @memset(buf[0..to_copy], 0)
-        else
-            @memcpy(buf[0..to_copy], rdr.buffer[rdr.seek .. rdr.seek + to_copy]);
-        rdr.seek += to_copy;
-        wrote += to_copy;
-    }
-    return wrote;
-}

src/util/data_threaded.zig

@@ -0,0 +1,207 @@
+//! Similiar to DataReader, but set-up for threaded writing to files.
+
+const std = @import("std");
+const Reader = std.Io.Reader;
+const Writer = std.Io.Writer;
+const Limit = std.Io.Limit;
+const WaitGroup = std.Thread.WaitGroup;
+const Pool = std.Thread.Pool;
+
+const Archive = @import("../archive.zig");
+const DecompFn = @import("../decomp.zig").DecompFn;
+const BlockSize = @import("../inode_data/file.zig").BlockSize;
+const FragEntry = @import("../tables.zig").FragEntry;
+const InodeFinish = @import("inode_finish.zig");
+const OffsetFile = @import("offset_file.zig");
+
+const ThreadedDataReader = @This();
+
+alloc: std.mem.Allocator,
+fil: OffsetFile,
+decomp: DecompFn,
+block_size: u32,
+
+blocks: []BlockSize,
+
+frag: ?FragEntry = null, // TODO: do something better?
+frag_offset: u32 = 0,
+size: u64,
+num_blocks: usize,
+
+start_offset: u64,
+
+pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64) ThreadedDataReader {
+    return .{
+        .alloc = alloc,
+        .fil = archive.fil,
+        .decomp = archive.decomp,
+        .block_size = archive.super.block_size,
+
+        .blocks = blocks,
+
+        .size = size,
+        .num_blocks = blocks.len,
+
+        .start_offset = start,
+    };
+}
+
+pub fn addFragment(self: *ThreadedDataReader, entry: FragEntry, frag_offset: u32) void {
+    self.frag = entry;
+    self.frag_offset = frag_offset;
+    self.num_blocks = self.blocks.len + 1;
+}
+
+/// Extract the data to the file threadedly, using pool to spawn threads.
+/// If errors occur, they are set to finish.out_err.
+pub fn extractThreaded(self: ThreadedDataReader, file: std.fs.File, pool: *Pool, finish: *InodeFinish) void {
+    var cur_write_offset: u64 = 0;
+    var cur_read_offset: u64 = self.start_offset;
+    for (0..self.blocks.len) |i| {
+        const cur_block_size = if (i == self.num_blocks - 1) self.size % self.block_size else self.block_size;
+        pool.spawn(workThreadBlocks, .{ self, file, cur_write_offset, cur_read_offset, self.blocks[i], cur_block_size, finish }) catch |res_err| {
+            finish.logError("Can't spawn pool task: {}", .{res_err});
+            finish.out_err.* = res_err;
+            finish.finish();
+        };
+        cur_write_offset += cur_block_size;
+        cur_read_offset += self.blocks[i].size;
+    }
+    if (self.frag != null)
+        pool.spawn(workThreadFragment, .{ self, file, cur_write_offset, finish }) catch |res_err| {
+            finish.logError("Can't spawn pool task: {}", .{res_err});
+            finish.out_err.* = res_err;
+            finish.finish();
+        };
+}
+
+fn workThreadBlocks(
+    self: ThreadedDataReader,
+    fil: std.fs.File,
+    write_offset: u64,
+    read_offset: u64,
+    block: BlockSize,
+    cur_block_size: u64,
+    finish: *InodeFinish,
+) void {
+    defer finish.finish();
+    var wrt = fil.writer(&[0]u8{});
+    wrt.seekTo(write_offset) catch |err| {
+        finish.logError("Error seeking file writer: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    defer wrt.interface.flush() catch |err| {
+        finish.logError("Error flushing file writer: {}", .{err});
+        finish.out_err.* = err;
+    };
+    if (block.size == 0) {
+        wrt.interface.splatByteAll(0, cur_block_size) catch |err| {
+            finish.logError("Error writing zeroes: {}", .{err});
+            finish.out_err.* = err;
+            return;
+        };
+        return;
+    }
+    var rdr = self.fil.readerAt(read_offset, &[0]u8{}) catch |err| {
+        finish.logError("Error creating file reader: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    if (block.uncompressed) {
+        rdr.interface.streamExact(&wrt.interface, block.size) catch |err| {
+            finish.logError("Error streaming data: {}", .{err});
+            finish.out_err.* = err;
+            return;
+        };
+        return;
+    }
+    // TODO: shared buffers
+    const read_buf = self.alloc.alloc(u8, block.size) catch |err| {
+        finish.logError("Error creating reader buffer: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    defer self.alloc.free(read_buf);
+    rdr.interface.readSliceAll(read_buf) catch |err| {
+        finish.logError("Error reading data into reader buffer: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    // TODO: shared buffers
+    const res_buf = self.alloc.alloc(u8, cur_block_size) catch |err| {
+        finish.logError("Error creating result buffer: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    defer self.alloc.free(res_buf);
+    _ = self.decomp(self.alloc, read_buf, res_buf) catch |err| {
+        finish.logError("Error decompressing data block: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    wrt.interface.writeAll(res_buf) catch |err| {
+        finish.logError("Error writing to file: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+}
+fn workThreadFragment(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, finish: *InodeFinish) void {
+    defer finish.finish();
+
+    var wrt = fil.writer(&[0]u8{});
+    wrt.seekTo(write_offset) catch |err| {
+        finish.logError("Error seeking file writer for file fragment: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    defer wrt.interface.flush() catch |err| {
+        finish.out_err.* = err;
+    };
+
+    var rdr = self.fil.readerAt(self.frag.?.start, &[0]u8{}) catch |err| {
+        finish.logError("Error creating file reader for file fragment: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    if (self.frag.?.size.uncompressed) {
+        rdr.interface.discardAll(self.frag_offset) catch |err| {
+            finish.logError("Error discarding useless fragment data: {}", .{err});
+            finish.out_err.* = err;
+            return;
+        };
+        rdr.interface.streamExact(&wrt.interface, self.size % self.block_size) catch |err| {
+            finish.logError("Error streaming fragment data: {}", .{err});
+            finish.out_err.* = err;
+            return;
+        };
+        return;
+    }
+    const tmp_buf = self.alloc.alloc(u8, self.frag.?.size.size) catch |err| {
+        finish.logError("Error creating a temporary buffer for a file fragment: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    defer self.alloc.free(tmp_buf);
+    rdr.interface.readSliceAll(tmp_buf) catch |err| {
+        finish.logError("Error reading data into fragment buffer: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    const needed_block = self.alloc.alloc(u8, self.block_size) catch |err| {
+        finish.logError("Error allocating fragment decompression results: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    defer self.alloc.free(needed_block);
+    _ = self.decomp(self.alloc, tmp_buf, needed_block) catch |err| {
+        finish.logError("Error decompressing fragment: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+    wrt.interface.writeAll(needed_block[self.frag_offset .. self.frag_offset + (self.size % self.block_size)]) catch |err| {
+        finish.logError("Error writing fragment: {}", .{err});
+        finish.out_err.* = err;
+        return;
+    };
+}

src/util/decompressor.zig

@@ -1,15 +0,0 @@
-//! A decompression interface
-
-const std = @import("std");
-
-const Decompressor = @This();
-
-pub const Error = std.Io.Reader.StreamError || std.mem.Allocator.Error;
-
-/// The actual decompression function.
-/// If the given decompressor is null, then the decompression should be done "stateless" without lasting allocations.
-decomp_fn: *const fn (?*Decompressor, std.mem.Allocator, in: []u8, out: []u8) Error!usize,
-
-pub fn Decompress(self: *Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    return self.decomp_fn(self, alloc, in, out);
-}

src/util/extract.zig

@@ -0,0 +1,353 @@
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const Pool = std.Thread.Pool;
+const WaitGroup = std.Thread.WaitGroup;
+
+const Archive = @import("../archive.zig");
+const DirEntry = @import("../dir_entry.zig");
+const Inode = @import("../inode.zig");
+const ExtractionOptions = @import("../options.zig");
+const Tables = @import("../tables.zig");
+const InodeFinish = @import("inode_finish.zig");
+const FinishUnion = InodeFinish.FinishUnion;
+const ThreadedDataReader = @import("data_threaded.zig");
+
+// 1 MB
+const STACK_ALLOC_SIZE = 1024 * 1024;
+
+pub fn extractTo(
+    allocator: Allocator,
+    inode: Inode,
+    archive: Archive,
+    path: []const u8,
+    options: ExtractionOptions,
+) !void {
+    if (path[path.len - 1] == '/')
+        return extractTo(allocator, inode, archive, path[0 .. path.len - 2], options);
+
+    var stack_alloc = std.heap.stackFallback(STACK_ALLOC_SIZE, allocator);
+    var arena: std.heap.ArenaAllocator = .init(stack_alloc.get());
+    defer arena.deinit();
+    if (options.threads <= 1) {
+        const alloc = arena.allocator();
+        var tables: Tables = try .init(alloc, archive);
+        return extractSingleThread(arena.allocator(), inode, archive, &tables, path, options);
+    }
+
+    var thread_alloc = std.heap.ThreadSafeAllocator{ .child_allocator = arena.allocator() };
+    const alloc = thread_alloc.allocator();
+    var tables: Tables = try .init(alloc, archive);
+
+    var pool_alloc = std.heap.stackFallback(10 * 1024, alloc);
+    var pool: Pool = undefined;
+    try pool.init(.{ .allocator = pool_alloc.get(), .n_jobs = options.threads - 1 });
+
+    var wg: WaitGroup = .{};
+    var err: ?anyerror = null;
+    wg.start();
+    try pool.spawn(extractMultiThread, .{
+        alloc,
+        inode,
+        archive,
+        &tables,
+        path,
+        options,
+        &pool,
+        FinishUnion{ .wg = &wg },
+        &err,
+    });
+    pool.waitAndWork(&wg);
+    if (err != null) return err.?;
+}
+
+fn extractSingleThread(
+    alloc: Allocator,
+    inode: Inode,
+    archive: Archive,
+    tables: *Tables,
+    path: []const u8,
+    options: ExtractionOptions,
+) !void {
+    switch (inode.hdr.inode_type) {
+        .dir, .ext_dir => {
+            _ = std.fs.cwd().makeDir(path) catch |err| switch (err) {
+                std.fs.Dir.MakeError.PathAlreadyExists => {},
+                else => return err,
+            };
+
+            // Currently we are ignoring any deinit or free calls since we know we are under an ArenaAllocator.
+            // Possibly in the future, do some simple math to see if it would be safe to ONLY deinit via Arena,
+            // otherwise be more conscientious about freeing memory.
+            // For now, this is good enough.
+
+            const entries = try inode.dirEntries(alloc, archive);
+            for (entries) |ent| {
+                const sub_inode: Inode = try .readFromEntry(alloc, archive, ent);
+                const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", ent.name });
+                try extractSingleThread(alloc, sub_inode, archive, tables, new_path, options);
+            }
+
+            const fil = try std.fs.cwd().openFile(path, .{});
+            defer fil.close();
+            try inode.setMetadata(alloc, tables, fil, options);
+        },
+        .file, .ext_file => {
+            var fil = try std.fs.cwd().createFile(path, .{ .exclusive = true });
+            defer fil.close();
+            var wrt = fil.writer(&[0]u8{});
+            var dat_rdr = try inode.dataReader(alloc, archive, tables);
+            defer dat_rdr.deinit();
+            _ = try dat_rdr.interface.streamRemaining(&wrt.interface);
+            try wrt.interface.flush();
+
+            try inode.setMetadata(alloc, tables, fil, options);
+        },
+        .symlink, .ext_symlink => return extractSymlink(inode, path),
+        else => return extractDeviceAndIPC(inode, alloc, tables, path, options),
+    }
+}
+
+fn extractMultiThread(
+    alloc: Allocator,
+    inode: Inode,
+    archive: Archive,
+    tables: *Tables,
+    path: []const u8,
+    options: ExtractionOptions,
+    pool: *Pool,
+    fin: FinishUnion,
+    err: *?anyerror,
+) void {
+    if (err.* != null) {
+        fin.finish();
+        return;
+    }
+    switch (inode.hdr.inode_type) {
+        .dir, .ext_dir => {
+            _ = std.fs.cwd().makeDir(path) catch |res_err| switch (res_err) {
+                std.fs.Dir.MakeError.PathAlreadyExists => {},
+                else => {
+                    err.* = res_err;
+                    fin.finish();
+                    return;
+                },
+            };
+
+            // Currently we are ignoring any deinit or free calls since we know we are under an ArenaAllocator.
+            // Possibly in the future, do some simple math to see if it would be safe to ONLY deinit via Arena,
+            // otherwise be more conscientious about freeing memory.
+            // For now, this is good enough.
+
+            const entries = inode.dirEntries(alloc, archive) catch |res_err| {
+                err.* = res_err;
+                fin.finish();
+                return;
+            };
+
+            if (entries.len == 0) {
+                fin.finish();
+                return;
+            }
+
+            var dir_fin = InodeFinish.create(
+                alloc,
+                inode,
+                path,
+                tables,
+                options,
+                fin,
+                err,
+                null,
+                entries.len,
+            ) catch |res_err| {
+                err.* = res_err;
+                fin.finish();
+                return;
+            };
+
+            for (entries) |ent| {
+                if (ent.inode_type == .dir) {
+                    extractEntry(
+                        alloc,
+                        ent,
+                        archive,
+                        tables,
+                        path,
+                        options,
+                        pool,
+                        .{ .fin = dir_fin },
+                        err,
+                    );
+                    continue;
+                }
+
+                pool.spawn(
+                    extractEntry,
+                    .{ alloc, ent, archive, tables, path, options, pool, FinishUnion{ .fin = dir_fin }, err },
+                ) catch |res_err| {
+                    err.* = res_err;
+                    dir_fin.finish();
+                    return;
+                };
+            }
+        },
+        .file, .ext_file => {
+            const fil = std.fs.cwd().createFile(path, .{ .exclusive = true }) catch |res_err| {
+                if (options.verbose)
+                    options.verbose_writer.?.print("Can't create file at {s}: {}\n", .{ path, res_err }) catch {};
+                err.* = res_err;
+                fin.finish();
+                return;
+            };
+
+            var data_rdr = threadedDataReader(inode, alloc, archive, tables) catch |res_err| {
+                if (options.verbose)
+                    options.verbose_writer.?.print("Can't create data reader for inode #{} (extracting to {s}): {}\n", .{ inode.hdr.num, path, res_err }) catch {};
+                err.* = res_err;
+                fin.finish();
+                return;
+            };
+            if (data_rdr == null) {
+                inode.setMetadata(alloc, tables, fil, options) catch |res_err| {
+                    if (options.verbose)
+                        options.verbose_writer.?.print("Can't set metadata to {s}: {}\n", .{ path, res_err }) catch {};
+                    err.* = res_err;
+                };
+                fin.finish();
+                return;
+            }
+            const file_fin = InodeFinish.create(
+                alloc,
+                inode,
+                path,
+                tables,
+                options,
+                fin,
+                err,
+                fil,
+                data_rdr.?.num_blocks,
+            ) catch |res_err| {
+                if (options.verbose)
+                    options.verbose_writer.?.print("Can't create callback for inode #{} (extracting to {s}): {}\n", .{ inode.hdr.num, path, res_err }) catch {};
+                err.* = res_err;
+                fin.finish();
+                return;
+            };
+
+            data_rdr.?.extractThreaded(fil, pool, file_fin);
+        },
+        .symlink, .ext_symlink => {
+            extractSymlink(inode, path) catch |res_err| {
+                err.* = res_err;
+            };
+            fin.finish();
+        },
+        else => {
+            extractDeviceAndIPC(inode, alloc, tables, path, options) catch |res_err| {
+                err.* = res_err;
+            };
+            fin.finish();
+        },
+    }
+}
+
+fn extractEntry(
+    alloc: Allocator,
+    ent: DirEntry,
+    archive: Archive,
+    tables: *Tables,
+    path: []const u8,
+    options: ExtractionOptions,
+    pool: *Pool,
+    fin: FinishUnion,
+    err: *?anyerror,
+) void {
+    const new_path = std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", ent.name }) catch |res_err| {
+        err.* = res_err;
+        fin.finish();
+        return;
+    };
+
+    const inode = Inode.readFromEntry(alloc, archive, ent) catch |res_err| {
+        err.* = res_err;
+        fin.finish();
+        return;
+    };
+    extractMultiThread(alloc, inode, archive, tables, new_path, options, pool, fin, err);
+}
+
+/// Get a threaded data reader for a file inode.
+fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive, tables: *Tables) !?ThreadedDataReader {
+    return switch (self.hdr.inode_type) {
+        .file => threadedReaderFromData(alloc, archive, tables, self.data.file),
+        .ext_file => threadedReaderFromData(alloc, archive, tables, self.data.ext_file),
+        else => error.NotRegularFile,
+    };
+}
+fn threadedReaderFromData(alloc: std.mem.Allocator, archive: Archive, tables: *Tables, data: anytype) !?ThreadedDataReader {
+    if (data.block_sizes.len == 0 and data.frag_idx == 0xFFFFFFFF) return null;
+    var out: ThreadedDataReader = .init(alloc, archive, data.block_sizes, data.block_start, data.size);
+    if (data.frag_idx != 0xFFFFFFFF)
+        out.addFragment(try tables.frag_table.get(data.frag_idx), data.frag_block_offset);
+    return out;
+}
+
+/// Creates the symlink described by the inode.
+/// Sets metadata.
+fn extractSymlink(self: Inode, path: []const u8) !void {
+    const target = switch (self.data) {
+        .symlink => |s| s.target,
+        .ext_symlink => |s| s.target,
+        else => unreachable,
+    };
+    try std.fs.cwd().symLink(target, path, .{});
+}
+/// Creates the device described by the inode.
+/// Sets metadata.
+fn extractDeviceAndIPC(self: Inode, alloc: std.mem.Allocator, tables: *Tables, path: []const u8, options: ExtractionOptions) !void {
+    var mode: u32 = undefined;
+    var dev: u32 = 0;
+    switch (self.data) {
+        .char_dev => |d| {
+            mode = std.posix.S.IFCHR;
+            dev = d.dev;
+        },
+        .ext_char_dev => |d| {
+            mode = std.posix.S.IFCHR;
+            dev = d.dev;
+        },
+        .block_dev => |d| {
+            mode = std.posix.S.IFBLK;
+            dev = d.dev;
+        },
+        .ext_block_dev => |d| {
+            mode = std.posix.S.IFBLK;
+            dev = d.dev;
+        },
+        .fifo, .ext_fifo => mode = std.posix.S.IFIFO,
+        .socket, .ext_socket => mode = std.posix.S.IFSOCK,
+        else => unreachable,
+    }
+    const res: std.os.linux.E = @enumFromInt(std.os.linux.mknod(@ptrCast(path), mode, dev));
+    switch (res) {
+        .SUCCESS => {},
+        .ACCES => return std.fs.Dir.MakeError.AccessDenied,
+        .DQUOT => return std.fs.Dir.MakeError.DiskQuota,
+        .EXIST => return std.fs.Dir.MakeError.PathAlreadyExists,
+        .FAULT, .NOENT => return std.fs.Dir.MakeError.BadPathName,
+        .LOOP => return std.fs.Dir.MakeError.SymLinkLoop,
+        .NAMETOOLONG => return std.fs.Dir.MakeError.NameTooLong,
+        .NOMEM => return std.fs.Dir.MakeError.SystemResources,
+        .NOSPC => return std.fs.Dir.MakeError.NoSpaceLeft,
+        .NOTDIR => return std.fs.Dir.MakeError.NotDir,
+        .PERM => return std.fs.Dir.MakeError.PermissionDenied,
+        .ROFS => return std.fs.Dir.MakeError.ReadOnlyFileSystem,
+        else => return blk: {
+            std.debug.print("unhandled mknod result: {}\n", .{res});
+            break :blk std.fs.Dir.MakeError.Unexpected;
+        },
+    }
+    var fil = try std.fs.cwd().openFile(path, .{});
+    defer fil.close();
+    try self.setMetadata(alloc, tables, fil, options);
+}

src/util/inode_finish.zig

@@ -0,0 +1,101 @@
+const std = @import("std");
+const WaitGroup = std.Thread.WaitGroup;
+const Mutex = std.Thread.Mutex;
+
+const Archive = @import("../archive.zig");
+const Inode = @import("../inode.zig");
+const ExtractionOptions = @import("../options.zig");
+const Tables = @import("../tables.zig");
+
+const InodeFinish = @This();
+
+const FinishEnum = enum {
+    wg,
+    fin,
+};
+pub const FinishUnion = union(FinishEnum) {
+    wg: *WaitGroup,
+    fin: *InodeFinish,
+
+    pub fn finish(self: FinishUnion) void {
+        switch (self) {
+            .wg => |wg| wg.finish(),
+            .fin => |fin| fin.finish(),
+        }
+    }
+};
+
+alloc: std.mem.Allocator,
+
+inode: Inode,
+path: []const u8,
+tables: *Tables,
+options: ExtractionOptions,
+parent_finish: FinishUnion,
+fil: ?std.fs.File,
+out_err: *?anyerror,
+
+wg: WaitGroup = .{},
+mut: Mutex = .{},
+
+pub fn create(
+    alloc: std.mem.Allocator,
+    inode: Inode,
+    path: []const u8,
+    tables: *Tables,
+    options: ExtractionOptions,
+    parent_finish: FinishUnion,
+    out_err: *?anyerror,
+    fil: ?std.fs.File,
+    work_size: usize,
+) !*InodeFinish {
+    if (work_size == 0)
+        return error.InvalidWorkSize;
+    const out = try alloc.create(InodeFinish);
+    errdefer alloc.destroy(out);
+    out.* = .{
+        .alloc = alloc,
+
+        .inode = inode,
+        .path = path,
+        .tables = tables,
+        .options = options,
+        .parent_finish = parent_finish,
+        .out_err = out_err,
+        .fil = fil,
+    };
+    out.wg.startMany(work_size);
+    return out;
+}
+
+pub fn logError(self: *InodeFinish, comptime fmt: []const u8, args: anytype) void {
+    if (self.options.verbose)
+        self.options.verbose_writer.?.print(fmt, args) catch {};
+}
+
+pub fn finish(self: *InodeFinish) void {
+    self.mut.lock();
+    {
+        defer self.mut.unlock();
+        self.wg.finish();
+        if (!self.wg.isDone()) return;
+    }
+    defer {
+        self.parent_finish.finish();
+        self.alloc.destroy(self);
+    }
+    if (self.fil == null)
+        self.fil = std.fs.cwd().openFile(self.path, .{}) catch |err| {
+            if (self.options.verbose)
+                self.options.verbose_writer.?.print("Error opening {s} to set metadata: {}\n", .{ self.path, err }) catch {};
+            self.out_err.* = err;
+            return;
+        };
+    defer self.fil.?.close();
+    self.inode.setMetadata(self.alloc, self.tables, self.fil.?, self.options) catch |err| {
+        if (self.options.verbose)
+            self.options.verbose_writer.?.print("Error setting metadata to {s}: {}\n", .{ self.path, err }) catch {};
+        self.out_err.* = err;
+        return;
+    };
+}

src/util/meta_cache.zig

@@ -0,0 +1,41 @@
+const std = @import("std");
+
+const OffsetFile = @import("offset_file.zig");
+
+const MetadataCache = @This();
+
+alloc: std.mem.Allocator,
+
+buf: []u8,
+fixed_alloc: std.heap.FixedBufferAllocator,
+
+cache: std.AutoArrayHashMap(u64, [8192]u8),
+
+mut: std.Thread.Mutex = .{},
+cache_mut: std.AutoArrayHashMap(u64, std.Thread.Mutex),
+
+fil: OffsetFile,
+
+pub fn init(alloc: std.mem.Allocator, cache_size: u64) !MetadataCache {}
+pub fn deinit(self: *MetadataCache) void {
+    self.mut.lock();
+    defer self.mut.unlock();
+    self.cache.deinit();
+    self.cache_mut.deinit();
+    self.alloc.free(self.buf);
+}
+
+pub fn getChunk(self: *MetadataCache, offset: u64) ![8192]u8 {
+    var res = self.cache.get(offset);
+    if (res != null) return res.?;
+    var offset_mut = blk: {
+        self.mut.lock();
+        defer self.mut.unlock();
+        const mut = try self.cache_mut.getOrPut(offset);
+        if (!mut.found_existing)
+            mut.value_ptr.* = .{};
+        break :blk mut.value_ptr;
+    };
+    offset_mut.lock();
+    defer offset_mut.unlock();
+}

src/util/metadata.zig

@@ -4,9 +4,9 @@ const Writer = std.Io.Writer;
 const Limit = std.Io.Limit;
 const StreamError = std.Io.Reader.StreamError;
 
-const Decompressor = @import("decompressor.zig");
+const DecompFn = @import("../decomp.zig").DecompFn;
 
-const BlockHeader = packed struct(u16) {
+const BlockHeader = packed struct {
     size: u15,
     uncompressed: bool,
 };
@@ -15,30 +15,28 @@ const This = @This();
 
 alloc: std.mem.Allocator,
 rdr: *Reader,
-decomp: *Decompressor,
+decomp: DecompFn,
 
-cur_block_start: u32 = 0,
-next_start_start: u32 = 0,
 buf: [8192]u8 = undefined,
 
+interface: Reader,
 err: ?anyerror = null,
-interface: Reader = .{
-    .buffer = &[0]u8{},
-    .end = 0,
-    .seek = 0,
-    .vtable = &.{
-        .stream = stream,
-        .discard = discard,
-        .readVec = readVec,
-        // TODO: Potentially add rebase so that we can guarentee that self.block_start & interface.seek is correct.
-    },
-},
 
-pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: *Decompressor) This {
+pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: DecompFn) This {
     return .{
         .alloc = alloc,
         .rdr = rdr,
         .decomp = decomp,
+        .interface = .{
+            .buffer = &[0]u8{},
+            .end = 0,
+            .seek = 0,
+            .vtable = &.{
+                .stream = stream,
+                .discard = discard,
+                .readVec = readVec,
+            },
+        },
     };
 }
 
@@ -46,20 +44,16 @@ fn advance(self: *This) !void {
     self.interface.seek = 0;
     var hdr: BlockHeader = undefined;
     try self.rdr.readSliceEndian(BlockHeader, @ptrCast(&hdr), .little);
-    self.cur_block_start = self.next_start_start;
-    self.next_start_start += hdr.size;
     if (hdr.uncompressed) {
         try self.rdr.readSliceEndian(u8, self.buf[0..hdr.size], .little);
         self.interface.end = hdr.size;
         self.interface.buffer = self.buf[0..hdr.size];
         return;
-    } else {
-        @branchHint(.likely);
-        var tmp_buf: [8192]u8 = undefined;
-        try self.rdr.readSliceAll(tmp_buf[0..hdr.size]);
-        self.interface.end = try self.decomp.Decompress(self.alloc, tmp_buf[0..hdr.size], &self.buf);
-        self.interface.buffer = self.buf[0..self.interface.end];
     }
+    var tmp_buf: [8192]u8 = undefined;
+    try self.rdr.readSliceAll(tmp_buf[0..hdr.size]);
+    self.interface.end = try self.decomp(self.alloc, tmp_buf[0..hdr.size], &self.buf);
+    self.interface.buffer = self.buf[0..self.interface.end];
 }
 
 fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) StreamError!usize {
@@ -74,22 +68,22 @@ fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) StreamError!usize {
     self.interface.seek += wrote;
     return wrote;
 }
-fn discard(rdr: *Reader, limit: Limit) Reader.Error!usize {
+fn discard(rdr: *Reader, limit: Limit) error{ EndOfStream, ReadFailed }!usize {
     const self: *This = @fieldParentPtr("interface", rdr);
     if (rdr.end == rdr.seek) self.advance() catch |err| {
         self.err = err;
-        return error.ReadFailed;
+        return StreamError.ReadFailed;
     };
     if (@intFromEnum(limit) == 0) return 0;
     const to_skip = @min(rdr.end - rdr.seek, @intFromEnum(limit));
     rdr.seek += to_skip;
     return to_skip;
 }
-fn readVec(rdr: *Reader, vec: [][]u8) Reader.Error!usize {
+fn readVec(rdr: *Reader, vec: [][]u8) error{ EndOfStream, ReadFailed }!usize {
     const self: *This = @fieldParentPtr("interface", rdr);
     if (rdr.end == rdr.seek) self.advance() catch |err| {
         self.err = err;
-        return error.ReadFailed;
+        return StreamError.ReadFailed;
     };
     var cur_red: usize = 0;
     for (vec) |s| {

src/util/misc.zig

@@ -1,25 +0,0 @@
-//! Miscellaneous utility functions.
-
-const std = @import("std");
-const Io = std.Io;
-
-const Inode = @import("../inode.zig");
-const Decompressor = @import("decompressor.zig");
-const MetadataReader = @import("metadata.zig");
-const OffsetFile = @import("offset_file.zig");
-
-/// check is the path is referencing itself ("" or ".").
-/// separators must be trimmed before calling this function for it to work properly.
-pub fn pathIsSelf(path: []const u8) bool {
-    if (path.len == 0) return true;
-    if (path.len > 1) return false;
-    return path[0] == '.';
-}
-/// Creates an Inode from an Inode.Ref.
-pub fn inodeFromRef(alloc: std.mem.Allocator, file: OffsetFile, decomp: *Decompressor, inode_start: u64, block_size: u32, ref: Inode.Ref) !Inode {
-    var rdr = file.readerAt(inode_start + ref.block_start);
-    var meta: MetadataReader = .init(alloc, &rdr, decomp);
-    try meta.interface.discardAll(ref.block_offset);
-
-    return .read(alloc, &meta.interface, block_size);
-}

src/util/offset_file.zig

@@ -1,27 +1,20 @@
 //! A File where it's meaningful (to us) content starts at a given offset.
 
 const std = @import("std");
-const Io = std.Io;
+const File = std.fs.File;
-const File = Io.File;
+const Reader = std.fs.File.Reader;
-const Reader = Io.Reader;
 
 const OffsetFile = @This();
 
-map: Io.File.MemoryMap,
+fil: File,
+offset: u64,
 
-pub fn init(io: Io, fil: File, archive_size: u64, init_offset: u64) !OffsetFile {
+pub fn init(fil: File, init_offset: u64) OffsetFile {
-    return .{
+    return .{ .fil = fil, .offset = init_offset };
-        .map = try fil.createMemoryMap(io, .{
-            .protection = .{ .read = true, .write = false, .execute = false },
-            .len = archive_size,
-            .offset = init_offset,
-        }),
-    };
-}
-pub fn deinit(self: *OffsetFile, io: Io) void {
-    self.map.destroy(io);
 }
 
-pub fn readerAt(self: OffsetFile, offset: u64) Reader {
+pub fn readerAt(self: OffsetFile, offset: u64, buffer: []u8) !Reader {
-    return .fixed(self.map.memory[offset..]);
+    var rdr = self.fil.reader(buffer);
+    try rdr.seekTo(self.offset + offset);
+    return rdr;
 }

src/xattr.zig

@@ -0,0 +1,117 @@
+const std = @import("std");
+
+const DecompFn = @import("decomp.zig").DecompFn;
+const Table = @import("tables.zig").Table;
+const MetadataReader = @import("util/metadata.zig");
+const OffsetFile = @import("util/offset_file.zig");
+
+const Ref = packed struct {
+    block_offset: u16,
+    block_start: u32,
+    _: u16,
+};
+const Entry = packed struct {
+    ref: Ref,
+    count: u32,
+    size: u32,
+};
+const KeyPrefix = enum(u8) {
+    user,
+    trusted,
+    security,
+};
+const KeyRaw = packed struct {
+    type: packed struct {
+        prefix: KeyPrefix,
+        out_of_line: bool,
+        _: u7,
+    },
+    name_size: u16,
+};
+
+pub const KeyValue = struct {
+    key: [:0]u8,
+    value: []u8,
+};
+
+const XattrTable = @This();
+
+alloc: std.mem.Allocator,
+fil: OffsetFile,
+decomp: DecompFn,
+
+count: u32,
+start: u64,
+
+table: Table(Entry),
+
+pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: DecompFn, table_start: u64) !XattrTable {
+    var info = packed struct {
+        start: u64 = undefined,
+        count: u32 = undefined,
+        _: u32 = undefined,
+    }{};
+    var rdr = try fil.readerAt(table_start, &[0]u8{});
+    try rdr.interface.readSliceEndian(@TypeOf(info), @ptrCast(&info), .little);
+    return .{
+        .alloc = alloc,
+        .fil = fil,
+        .decomp = decomp,
+        .count = info.count,
+        .start = info.start,
+        .table = try .init(alloc, fil, decomp, table_start + 16, info.count),
+    };
+}
+pub fn deinit(self: *XattrTable) void {
+    self.table.deinit();
+}
+
+pub fn get(self: *XattrTable, alloc: std.mem.Allocator, idx: u32) ![]KeyValue {
+    const entry: Entry = try self.table.get(idx);
+    const out = try alloc.alloc(KeyValue, entry.count);
+
+    for (out) |*kv| {
+        var rdr = try self.fil.readerAt(self.start + entry.ref.block_start, &[0]u8{});
+        var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
+        try meta.interface.discardAll(entry.ref.block_offset);
+
+        var key_raw: KeyRaw = undefined;
+        try meta.interface.readSliceEndian(KeyRaw, @ptrCast(&key_raw), .little);
+
+        switch (key_raw.type.prefix) {
+            .user => {
+                kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 5 + 1));
+                @memcpy(kv.key[0..5], "user.");
+                try meta.interface.readSliceAll(kv.key[5 .. kv.key.len - 1]);
+                kv.key[kv.key.len - 1] = 0;
+            },
+            .security => {
+                kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 9 + 1));
+                @memcpy(kv.key[0..9], "security.");
+                try meta.interface.readSliceAll(kv.key[9 .. kv.key.len - 1]);
+                kv.key[kv.key.len - 1] = 0;
+            },
+            .trusted => {
+                kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 8 + 1));
+                @memcpy(kv.key[0..8], "trusted.");
+                try meta.interface.readSliceAll(kv.key[8 .. kv.key.len - 1]);
+                kv.key[kv.key.len - 1] = 0;
+            },
+        }
+        if (key_raw.type.out_of_line) {
+            try meta.interface.discardAll(4);
+            var ref: Ref = undefined;
+            try meta.interface.readSliceEndian(Ref, @ptrCast(&ref), .little);
+
+            rdr = try self.fil.readerAt(self.start + ref.block_start, &[0]u8{});
+            meta = .init(alloc, &rdr.interface, self.decomp);
+            try meta.interface.discardAll(ref.block_offset);
+        }
+        var value_size: u32 = undefined;
+        try meta.interface.readSliceEndian(u32, @ptrCast(&value_size), .little);
+        kv.value = try alloc.alloc(u8, value_size);
+        try meta.interface.readSliceAll(kv.value);
+    }
+
+    return out;
+}

src/xattr_table.zig

@@ -1,292 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const InodeRef = @import("inode.zig").Ref;
-const LookupTable = @import("lookup_table.zig");
-const Decompressor = @import("util/decompressor.zig");
-const MetadataReader = @import("util/metadata.zig");
-const OffsetFile = @import("util/offset_file.zig");
-
-const XattrCachedTable = @This();
-
-alloc: std.mem.Allocator,
-
-fil: OffsetFile,
-decomp: *Decompressor,
-
-kv_start: u64,
-
-table: LookupTable.CachedTable(TableValue),
-value_cache: std.AutoHashMap(InodeRef, []const u8),
-value_mut: Io.RwLock = .init,
-
-pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, xattr_start: u64) !XattrCachedTable {
-    const start: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[xattr_start .. xattr_start + 8]), .little);
-    const num: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[xattr_start + 8 .. xattr_start + 16]), .little);
-
-    return .{
-        .alloc = alloc,
-
-        .fil = fil,
-        .decomp = decomp,
-
-        .kv_start = start,
-
-        .table = .init(alloc, fil, decomp, xattr_start + 16, num),
-        .value_cache = .init(alloc),
-    };
-}
-pub fn deinit(self: *XattrCachedTable, io: Io) void {
-    self.value_mut.lockUncancelable(io);
-    self.table.deinit(io);
-    self.value_cache.deinit();
-}
-
-pub fn get(self: *XattrCachedTable, alloc: std.mem.Allocator, io: Io, idx: u32) ![]XattrSemiOwned {
-    const lookup = try self.table.get(io, idx);
-
-    var rdr = self.fil.readerAt(self.kv_start + lookup.ref.block_start);
-    var meta: MetadataReader = .init(alloc, &rdr, self.decomp);
-    try meta.interface.discardAll(lookup.ref.block_offset);
-
-    const out = try alloc.alloc(XattrSemiOwned, lookup.count);
-    errdefer alloc.free(out);
-
-    for (0..lookup.count) |i| {
-        var key_entry: KeyEntry = undefined;
-        try meta.interface.readSliceEndian(KeyEntry, @ptrCast(&key_entry), .little);
-
-        const key: [:0]u8 = switch (key_entry.type.namespace) {
-            .user => blk: {
-                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 5);
-                errdefer alloc.free(tmp);
-                try meta.interface.readSliceEndian(u8, tmp[5 .. tmp.len - 1], .little);
-                @memcpy(tmp[0..5], "user.");
-                tmp[tmp.len - 1] = 0;
-                break :blk @ptrCast(tmp);
-            },
-            .trusted => blk: {
-                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 8);
-                errdefer alloc.free(tmp);
-                try meta.interface.readSliceEndian(u8, tmp[8 .. tmp.len - 1], .little);
-                @memcpy(tmp[0..8], "trusted.");
-                tmp[tmp.len - 1] = 0;
-                break :blk @ptrCast(tmp);
-            },
-            .security => blk: {
-                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 9);
-                errdefer alloc.free(tmp);
-                try meta.interface.readSliceEndian(u8, tmp[9 .. tmp.len - 1], .little);
-                @memcpy(tmp[0..9], "security.");
-                tmp[tmp.len - 1] = 0;
-                break :blk @ptrCast(tmp);
-            },
-        };
-        errdefer alloc.free(key);
-
-        if (key_entry.type.out_of_line) {
-            var value: ValueOutOfLineEntry = undefined;
-            try meta.interface.readSliceEndian(ValueOutOfLineEntry, @ptrCast(&value), .little);
-
-            out[i] = .{
-                .key = key,
-                .value = try self.valueAt(io, value.ref),
-            };
-            continue;
-        }
-        const val_ref: InodeRef = .{ .block_start = meta.cur_block_start, .block_offset = @truncate(meta.interface.seek) };
-
-        {
-            try self.value_mut.lockShared(io);
-            defer self.value_mut.unlockShared(io);
-            if (self.value_cache.contains(val_ref)) {
-                out[i] = .{
-                    .key = key,
-                    .value = try self.valueAt(io, val_ref),
-                };
-                continue;
-            }
-        }
-        try self.value_mut.lock(io);
-        defer self.value_mut.unlock(io);
-
-        if (self.value_cache.contains(val_ref)) {
-            out[i] = .{
-                .key = key,
-                .value = try self.valueAt(io, val_ref),
-            };
-            continue;
-        }
-
-        var val_size: u32 = undefined;
-        try meta.interface.readSliceEndian(u32, @ptrCast(&val_size), .little);
-
-        const val = try self.alloc.alloc(u8, val_size);
-        errdefer alloc.free(val);
-        try meta.interface.readSliceEndian(u8, val, .little);
-
-        try self.value_cache.put(val_ref, val);
-        out[i] = .{
-            .key = key,
-            .value = val,
-        };
-    }
-    return out;
-}
-
-fn valueAt(self: *XattrCachedTable, io: Io, ref: InodeRef) ![]const u8 {
-    try self.value_mut.lock(io);
-    defer self.value_mut.unlock(io);
-
-    if (self.value_cache.contains(ref)) return self.value_cache.get(ref).?;
-
-    var rdr = self.fil.readerAt(self.kv_start + ref.block_start);
-    var meta: MetadataReader = .init(self.alloc, &rdr, self.decomp);
-    try meta.interface.discardAll(ref.block_offset);
-
-    var val_size: u32 = undefined;
-    try meta.interface.readSliceEndian(u32, @ptrCast(&val_size), .little);
-
-    const val = try self.alloc.alloc(u8, val_size);
-    errdefer self.alloc.free(val);
-    try meta.interface.readSliceEndian(u8, val, .little);
-
-    try self.value_cache.put(ref, val);
-    return val;
-}
-
-// Types
-
-/// An Xattr return value where the reciever only owns the key value.
-pub const XattrSemiOwned = struct {
-    key: [:0]const u8,
-    value: []const u8,
-
-    pub fn deinit(self: XattrSemiOwned, alloc: std.mem.Allocator) void {
-        alloc.free(self.key);
-    }
-};
-/// An Xattr return value where the reciever owns both the key & value.
-pub const XattrOwned = struct {
-    key: [:0]const u8,
-    value: []const u8,
-
-    pub fn deinit(self: XattrSemiOwned, alloc: std.mem.Allocator) void {
-        alloc.free(self.key);
-        alloc.free(self.value);
-    }
-};
-
-const TableValue = extern struct {
-    ref: InodeRef,
-    count: u32,
-    size: u32,
-};
-
-const KeyEntry = extern struct {
-    type: XattrPrefix,
-    name_size: u16,
-};
-const ValueOutOfLineEntry = extern struct {
-    _: u32,
-    ref: InodeRef,
-};
-
-const XattrPrefix = packed struct(u16) {
-    namespace: enum(u8) {
-        user,
-        trusted,
-        security,
-
-        fn prefixSize(self: @This()) u16 {
-            return switch (self) {
-                .user => 5,
-                .trusted => 8,
-                .security => 9,
-            };
-        }
-    },
-    out_of_line: bool,
-    _: u7,
-};
-
-// Stateless
-
-pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *Decompressor, fil: OffsetFile, table_start: u64, idx: u16) ![]XattrOwned {
-    const kv_start: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[table_start .. table_start + 8]), .little);
-
-    const lookup = try LookupTable.lookupValue(TableValue, alloc, io, decomp, fil, table_start + 16, idx);
-
-    var rdr = fil.readerAt(kv_start + lookup.ref.block_start);
-    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
-    try meta.interface.discardAll(lookup.ref.block_offset);
-
-    const out = try alloc.alloc(XattrOwned, lookup.count);
-    errdefer alloc.free(out);
-
-    for (0..lookup.count) |i| {
-        const key_entry: KeyEntry = undefined;
-        try meta.interface.readSliceEndian(KeyEntry, @ptrCast(&key_entry), .little);
-
-        const key = switch (key_entry.type.namespace) {
-            .user => blk: {
-                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 5);
-                errdefer alloc.free(tmp);
-                try meta.interface.readSliceEndian(u8, tmp[5 .. tmp.len - 1], .little);
-                @memset(tmp[0..5], "user.");
-                break :blk tmp;
-            },
-            .trusted => blk: {
-                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 8);
-                errdefer alloc.free(tmp);
-                try meta.interface.readSliceEndian(u8, tmp[8 .. tmp.len - 1], .little);
-                @memset(tmp[0..8], "trusted.");
-                break :blk tmp;
-            },
-            .security => blk: {
-                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 9);
-                errdefer alloc.free(tmp);
-                try meta.interface.readSliceEndian(u8, tmp[9 .. tmp.len - 1], .little);
-                @memset(tmp[0..9], "security.");
-                break :blk tmp;
-            },
-        };
-        key[key.len - 1] = 0;
-        errdefer alloc.free(key);
-
-        if (key_entry.type.out_of_line) {
-            const value: ValueOutOfLineEntry = undefined;
-            try meta.interface.readSliceEndian(ValueOutOfLineEntry, @ptrCast(&value), .little);
-
-            var ool_rdr = fil.readerAt(kv_start + value.ref.block_start);
-            var ool_meta: MetadataReader = .init(alloc, &ool_rdr.interface, decomp);
-            try ool_meta.interface.discardAll(value.ref.block_offset);
-
-            var val_size: u32 = undefined;
-            try ool_meta.interface.readSliceEndian(val_size, @ptrCast(&val_size), .little);
-
-            const val = try alloc.alloc(u8, val_size);
-            errdefer alloc.free(val);
-            try ool_meta.interface.readSliceEndian(u8, val, .little);
-
-            out[i] = .{
-                .key = key,
-                .value = val,
-            };
-            continue;
-        }
-
-        var val_size: u32 = undefined;
-        try meta.interface.readSliceEndian(val_size, @ptrCast(&val_size), .little);
-
-        const val = try alloc.alloc(u8, val_size);
-        errdefer alloc.free(val);
-        try meta.interface.readSliceEndian(u8, val, .little);
-
-        out[i] = .{
-            .key = key,
-            .value = val,
-        };
-    }
-    return out;
-}