Fixed missed merge text (oops)

Changed all *const Decompressor to *Decompressor
Changed all decompressors to only stateless (my queues are borked)

README.md

@@ -10,9 +10,13 @@ Overall works, but currently is missing some features ([see below](#capabilities
 
 ## Build options
 
-> `-Duse_c_libs=true`
+> `-Duse_zig_decomp=true`
 
-Instead of using Zig's standard library for decompression, use the system's C libraries. Has the benefit of being much faster and enabling LZO and LZ4 decompression.
+Instead of using C libraries for decompression, use Zig's standard library for decompression. If using this option LZO and LZ4 decomrpession types are unsupported and decompression times will be significantly longer.
+
+> `-Ddynamic=true`
+
+Dynamicly link C libraries (if they're used) instead of statically linking them.
 
 > `-Dallow_lzo=true`
 
@@ -35,22 +39,22 @@ Most features are present except for the following:
 
 ## Performance
 
-This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive (which is fairly small and uses zstd compression) and with `--release=fast`.
+This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive which is fairly small and uses zstd compression with `-Doptimize=ReleaseFast`.
 
 Currently, my only performance checks are checking execution time, nothing deeper.
 
-* Under ideal circumstances, my library is ~70% slower (.12s vs .20s).
-* Using Zig decompression libraries *significantly* increases decompression time by ~600%. Under ideal circumstances.
+* Currently, using my test archive, performance aproximately matches `unsquashfs` when multi-threaded, but significantly slower when single-threaded.
+* Using Zig decompression libraries *significantly* increases decompression time.
 * Performance improvements/regressions will be common. I'm still learning Zig.
 
 Example Times:
 
-* *unsquashfs, multi-threaded*: .12s
+* *unsquashfs, multi-threaded*: .11s
 * *unsquashfs, single-threaded*: .13s
-* *C-libs, single-threaded*: .45s
-* *C-libs, multi-threaded*: .20s
-* *Zig-libs, single-threaded*: 5.78s
-* *Zig-libs, multi-threaded*: 1.08s
+* *C-libs, multi-threaded*: .10s
+* *C-libs, single-threaded*: ..28s
+* *Zig-libs, single-threaded*: .74s
+* *Zig-libs, multi-threaded*: 2.70s
 
 ## Build considerations
 

benchmark.sh

@@ -0,0 +1,17 @@
+#! /usr/bin/env bash
+
+ARCHIVE="testing/LinuxPATest.sfs"
+
+REF_EXT_LOC="testing/LinuxPAReference"
+PROG_EXT_LOC="testing/LinuxPABinTest"
+
+echo "Testing Multi-threaded Performance"
+echo ""
+
+hyperfine --warmup 5 --prepare "rm -rf $REF_EXT_LOC && rm -rf $PROG_EXT_LOC" "unsquashfs -d $REF_EXT_LOC $ARCHIVE" "zig-out/bin/unsquashfs -d $PROG_EXT_LOC $ARCHIVE"
+
+echo ""
+echo "Testing Single-threaded Performance"
+echo ""
+
+hyperfine --warmup 5 --prepare "rm -rf $REF_EXT_LOC && rm -rf $PROG_EXT_LOC" "unsquashfs -p 1 -d $REF_EXT_LOC $ARCHIVE" "zig-out/bin/unsquashfs -p 1 -d $PROG_EXT_LOC $ARCHIVE"

build.zig

@@ -1,17 +1,23 @@
 const std = @import("std");
 
 pub fn build(b: *std.Build) !void {
-    // const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
-    // const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
-    const debug = b.option(bool, "debug", "Enable options to make debugging easier.");
+    const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
+    const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
+    const dynamic = b.option(bool, "dynamic", "Dynamicly link C decompression libraries") orelse false;
+    var debug = b.option(bool, "debug", "Enable options to make debugging easier.");
     const version_string_option = b.option([]const u8, "version", "Version of the library/binary");
 
-    // const zig_squashfs_options = b.addOptions();
-    // zig_squashfs_options.addOption(bool, "use_zig_decomp", use_zig_decomp);
-    // zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo);
+    const zig_squashfs_options = b.addOptions();
+    zig_squashfs_options.addOption(bool, "use_zig_decomp", use_zig_decomp);
+    zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo);
 
     const target = b.standardTargetOptions(.{});
-    const optimize = b.standardOptimizeOption(.{});
+    var optimize = b.standardOptimizeOption(.{});
+
+    if (debug == true)
+        optimize = .Debug;
+    if (optimize == .Debug)
+        debug = true;
 
     const lib = b.addLibrary(.{
         .name = "squashfs",
@@ -20,10 +26,31 @@ pub fn build(b: *std.Build) !void {
             .target = target,
             .valgrind = debug,
             .root_source_file = b.path("src/root.zig"),
+            .imports = &.{
+                .{ .name = "options", .module = zig_squashfs_options.createModule() },
+            },
         }),
         .use_llvm = debug,
     });
 
+    const deps = try dependencies(b, optimize, target, use_zig_decomp, allow_lzo, dynamic);
+    defer b.allocator.free(deps);
+    for (deps) |d|
+        lib.root_module.linkLibrary(d);
+
+    if (!use_zig_decomp) {
+        const c = b.addTranslateC(.{
+            .optimize = optimize,
+            .target = target,
+            .root_source_file = b.path("src/c.h"),
+        });
+        if (allow_lzo) c.defineCMacro("ALLOW_LZO", null);
+        lib.root_module.addImport("c", c.createModule());
+
+        if (dynamic)
+            dynamicLinkLibraries(c, allow_lzo);
+    }
+
     var version = version_string_option orelse "0.0.0-testing";
     if (version[0] == 'v') version = version[1..];
     const unsquashfs_options = b.addOptions();
@@ -54,9 +81,31 @@ pub fn build(b: *std.Build) !void {
         .root_module = b.createModule(.{
             .optimize = optimize,
             .target = target,
-            .root_source_file = b.path("src/test.zig"),
+            .root_source_file = b.path("src/root.zig"),
+            .imports = &.{
+                .{ .name = "options", .module = zig_squashfs_options.createModule() },
+            },
+            .valgrind = debug,
         }),
+        .use_llvm = debug,
     });
+
+    for (deps) |d|
+        mod_tests.root_module.linkLibrary(d);
+
+    if (!use_zig_decomp) {
+        const c = b.addTranslateC(.{
+            .optimize = optimize,
+            .target = target,
+            .root_source_file = b.path("src/c.h"),
+        });
+        mod_tests.root_module.addImport("c", c.createModule());
+        if (allow_lzo) c.defineCMacro("ALLOW_LZO", null);
+
+        if (dynamic)
+            dynamicLinkLibraries(c, allow_lzo);
+    }
+
     const run_mod_tests = b.addRunArtifact(mod_tests);
     const test_step = b.step("test", "Run tests");
     test_step.dependOn(&run_mod_tests.step);
@@ -74,3 +123,42 @@ pub fn build(b: *std.Build) !void {
     check.dependOn(&lib_check.step);
     check.dependOn(&exe_check.step);
 }
+
+pub fn dynamicLinkLibraries(mod: *std.Build.Step.TranslateC, allow_lzo: bool) void {
+    mod.linkSystemLibrary("zstd", .{});
+    mod.linkSystemLibrary("zlib-ng", .{});
+    mod.linkSystemLibrary("lzma", .{});
+    mod.linkSystemLibrary("lz4", .{});
+    if (allow_lzo)
+        mod.linkSystemLibrary("minilzo", .{});
+}
+fn dependencies(
+    b: *std.Build,
+    optimize: std.builtin.OptimizeMode,
+    target: std.Build.ResolvedTarget,
+    use_zig_decomp: bool,
+    allow_lzo: bool,
+    dynamic: bool,
+) ![]*std.Build.Step.Compile {
+    if (use_zig_decomp or dynamic) return &.{};
+
+    var list: std.ArrayList(*std.Build.Step.Compile) = .empty;
+
+    const zstd = b.dependency("zstd", .{ .optimize = optimize, .target = target });
+    try list.append(b.allocator, zstd.artifact("zstd"));
+
+    const zng = b.dependency("zlib_ng", .{ .optimize = optimize, .target = target });
+    try list.append(b.allocator, zng.artifact("zng"));
+
+    const xz = b.dependency("xz", .{ .optimize = optimize, .target = target });
+    try list.append(b.allocator, xz.artifact("lzma"));
+
+    const lz4 = b.dependency("lz4", .{ .optimize = optimize, .target = target });
+    try list.append(b.allocator, lz4.artifact("lz4"));
+
+    if (allow_lzo) {
+        const minilzo = b.dependency("minilzo", .{ .optimize = optimize, .target = target });
+        try list.append(b.allocator, minilzo.artifact("minilzo"));
+    }
+    return list.toOwnedSlice(b.allocator);
+}

build.zig.zon

@@ -20,6 +20,10 @@
             .url = "git+https://github.com/CalebQ42/zig-minilzo.git#7cbae997b91a44d74b7cd6c073584dc9562a6c90",
             .hash = "minilzo-2.10.0-Ij7BO8wLAADeWI4Pe4jp8XTDsDaquZR14oZ7_9yKKDWP",
         },
+        .xz = .{
+            .url = "git+https://github.com/akunaakwei/zig-xz.git#e2d389262c8291907e3e4c6fb119819141c16c0f",
+            .hash = "xz-5.8.2-6v47_JYeAABSL-jonprpL5-E_YaaGc4B5xrbe93WsJ3G",
+        },
     },
     .paths = .{
         "build.zig",

run_tests.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-
-zig test \
-	-lc \
-	-lz \
-	-llzma \
-	-lminilzo \
-	-llz4 \
-	-lzstd \
-		src/test.zig

src/archive.zig

@@ -1,6 +1,7 @@
 const std = @import("std");
 const Io = std.Io;
 
+const Decomp = @import("decomp.zig");
 const ExtractionOptions = @import("options.zig");
 const File = @import("file.zig");
 const Inode = @import("inode.zig");
@@ -22,60 +23,44 @@ pub fn init(io: Io, file: std.Io.File, offset: u64) !Archive {
     try rdr.seekTo(offset);
     var super: Superblock = undefined;
     try rdr.interface.readSliceEndian(Superblock, @ptrCast(&super), .little);
+    try super.validate();
+
     return .{
-        .file = .init(file, offset),
+        .file = try .init(io, file, super.size, offset),
         .super = super,
 
-        .stateless_decomp = switch (super.compression) {
-            .gzip => @import("decomp/zlib.zig").stateless_decompressor,
-            .lzma => @import("decomp/lzma.zig").stateless_decompressor,
-            .lzo => return error.LzoUnsupported,
-            .xz => @import("decomp/xz.zig").stateless_decompressor,
-            .lz4 => return error.Lz4Unsupported,
-            .zstd => @import("decomp/zstd.zig").stateless_decompressor,
-        },
+        .stateless_decomp = try Decomp.StatelessDecomp(super.compression),
     };
 }
+pub fn deinit(self: *Archive, io: Io) void {
+    self.file.deinit(io);
+}
 
 /// The root folder of the Archive. Used to open other Files.
-pub fn root(self: Archive, alloc: std.mem.Allocator, io: Io) !File {
+pub fn root(self: *Archive, alloc: std.mem.Allocator) !File {
     const root_inode = try Utils.inodeFromRef(
         alloc,
-        io,
         self.file,
         &self.stateless_decomp,
         self.super.inode_start,
         self.super.block_size,
         self.super.root_ref,
     );
-    return .init(alloc, self, root_inode, "");
+    return .init(alloc, self.*, root_inode, "");
 }
 /// Opens a File within the archive.
-pub fn open(self: Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
-    const root_file = try self.root(alloc, io);
+pub fn open(self: *Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
+    var root_file = try self.root(alloc);
     const path = std.mem.trim(u8, filepath, "/");
     if (Utils.pathIsSelf(path))
         return root_file;
     defer root_file.deinit();
     return root_file.open(alloc, io, filepath);
 }
-/// Extract the entire archive contents to the given directory.
-pub fn extract(self: Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void {
-    const root_inode = try Utils.inodeFromRef(
-        alloc,
-        io,
-        self.file,
-        &self.stateless_decomp,
-        self.super.inode_start,
-        self.super.block_size,
-        self.super.root_ref,
-    );
-    return root_inode.extract(alloc, io, self.file, self.super, extract_dir, options);
-}
 
 /// Returns the inode with the given inode number.
 /// Requires that the archive is exportable (has an export lookup table).
-pub fn inode(self: Archive, alloc: std.mem.Allocator, io: Io, num: u32) !Inode {
+pub fn inode(self: *Archive, alloc: std.mem.Allocator, io: Io, num: u32) !Inode {
     if (!self.super.flags.exportable)
         return error.NotExportable;
     const ref = try LookupTable.lookupValue(
@@ -98,7 +83,7 @@ pub fn inode(self: Archive, alloc: std.mem.Allocator, io: Io, num: u32) !Inode {
     );
 }
 /// Returns a value at the given index from the Archive's id (uid/gid) table.
-pub fn idTable(self: Archive, alloc: std.mem.Allocator, io: Io, idx: u32) !u16 {
+pub fn idTable(self: *Archive, alloc: std.mem.Allocator, io: Io, idx: u32) !u16 {
     return LookupTable.lookupValue(
         u16,
         alloc,
@@ -110,6 +95,19 @@ pub fn idTable(self: Archive, alloc: std.mem.Allocator, io: Io, idx: u32) !u16 {
     );
 }
 
+/// Extract the entire archive contents to the given directory.
+pub fn extract(self: *Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void {
+    const root_inode = try Utils.inodeFromRef(
+        alloc,
+        self.file,
+        &self.stateless_decomp,
+        self.super.inode_start,
+        self.super.block_size,
+        self.super.root_ref,
+    );
+    return root_inode.extract(alloc, io, self.file, self.super, extract_dir, options);
+}
+
 // Superblock
 
 const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little);
@@ -128,14 +126,7 @@ pub const Superblock = extern struct {
     mod_time: u32,
     block_size: u32,
     frag_count: u32,
-    compression: enum(u16) {
-        gzip = 1,
-        lzma,
-        lzo,
-        xz,
-        lz4,
-        zstd,
-    },
+    compression: Decomp.Enum,
     block_log: u16,
     flags: packed struct(u16) {
         inode_uncompressed: bool,
@@ -176,3 +167,122 @@ pub const Superblock = extern struct {
             return SuperblockError.InvalidBlockLog;
     }
 };
+
+// Tests
+
+const TestArchive = "testing/LinuxPATest.sfs";
+
+test "Basics" {
+    std.debug.print("Starting test: Basics...\n", .{});
+
+    const alloc = std.testing.allocator;
+    const io = std.testing.io;
+
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    var sfs: Archive = try .init(io, fil, 0);
+    defer sfs.deinit(io);
+    try std.testing.expectEqualDeep(sfs.super, LinuxPATestCorrectSuperblock);
+    const root_file = try sfs.root(alloc);
+    defer root_file.deinit();
+}
+
+const TestFile = "Start.exe";
+const TestFileExtractLocation = "testing/Start.exe";
+
+test "ExtractSingleFile" {
+    std.debug.print("Starting test: ExtractSingleFile...\n", .{});
+
+    const alloc = std.testing.allocator;
+    const io = std.testing.io;
+
+    Io.Dir.cwd().deleteFile(io, TestFileExtractLocation) catch {};
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    var sfs: Archive = try .init(io, fil, 0);
+    defer sfs.deinit(io);
+    var test_fil = try sfs.open(alloc, io, TestFile);
+    defer test_fil.deinit();
+    try test_fil.extract(alloc, io, TestFileExtractLocation, .default);
+    //TODO: validate extracted file.
+}
+
+const TestFullExtractLocation = "testing/TestExtract";
+
+test "ExtractCompleteArchive" {
+    std.debug.print("Starting test: ExtractCompleteArchive...\n", .{});
+
+    const alloc = std.testing.allocator;
+    const io = std.testing.io;
+
+    Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    var sfs: Archive = try .init(io, fil, 0);
+    defer sfs.deinit(io);
+    try sfs.extract(alloc, io, TestFullExtractLocation, .default);
+}
+
+test "ExtractCompleteArchiveSingleThreaded" {
+    std.debug.print("Starting test: ExtractCompleteArchive...\n", .{});
+
+    const alloc = std.testing.allocator;
+    const io = std.testing.io;
+
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    {
+        std.debug.print("First testing using Threaded.global_single_threaded...\n", .{});
+        Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
+        var sfs: Archive = try .init(Io.Threaded.global_single_threaded.io(), fil, 0);
+        defer sfs.deinit(Io.Threaded.global_single_threaded.io());
+        try sfs.extract(alloc, Io.Threaded.global_single_threaded.io(), TestFullExtractLocation, .default);
+    }
+    {
+        std.debug.print("Next testing using ExtractionOptions.single_threaded...\n", .{});
+        Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
+        var sfs: Archive = try .init(io, fil, 0);
+        defer sfs.deinit(io);
+        try sfs.extract(alloc, io, TestFullExtractLocation, .default_single_threaded);
+    }
+}
+
+const LinuxPATestCorrectSuperblock: Superblock = .{
+    .magic = std.mem.readInt(u32, "hsqs", .little),
+    .inode_count = 2974,
+    .mod_time = 1632696724,
+    .block_size = 131072,
+    .frag_count = 264,
+    .compression = .zstd,
+    .block_log = 17,
+    .flags = .{
+        .inode_uncompressed = false,
+        .data_uncompressed = false,
+        .check = false,
+        .frag_uncompressed = false,
+        .fragment_never = false,
+        .fragment_always = false,
+        .duplicates = true,
+        .exportable = true,
+        .xattr_uncompressed = false,
+        .xattr_never = false,
+        .compression_options = false,
+        .ids_uncompressed = false,
+        ._ = 0,
+    },
+    .id_count = 1,
+    .ver_maj = 4,
+    .ver_min = 0,
+    .root_ref = .{
+        .block_offset = 1363,
+        .block_start = 29237,
+        ._ = 0,
+    },
+    .size = 106841744,
+    .id_start = 106841632,
+    .xattr_start = 106841720,
+    .inode_start = 106778274,
+    .dir_start = 106807998,
+    .frag_start = 106837747,
+    .export_start = 106841602,
+};

src/bin/unsquashfs.zig

@@ -41,7 +41,10 @@ var force: bool = false;
 
 pub fn main(init: std.process.Init) !void {
     const alloc = init.gpa;
-    const io = init.io;
+    // const io = init.io;
+    var evented: Io.Evented = undefined;
+    try evented.init(alloc, .{});
+    const io = evented.io();
 
     var stdout = std.Io.File.stdout();
     defer stdout.close(io);
@@ -60,7 +63,7 @@ pub fn main(init: std.process.Init) !void {
 
     var arc: squashfs.Archive = try .init(io, fil, offset); //TODO: Handle error gracefully.
     const options: squashfs.ExtractionOptions = .{
-        .threads = if (threads == 0) try std.Thread.getCpuCount() else threads,
+        .single_threaded = threads == 1,
         .verbose = verbose,
         .verbose_writer = if (verbose) &out.interface else null,
         .ignore_xattr = ignore_xattrs,
@@ -69,7 +72,16 @@ pub fn main(init: std.process.Init) !void {
 
     if (force)
         try Io.Dir.cwd().deleteTree(io, extLoc);
-    try arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully.
+    if (threads != 0) {
+        var limited_io = Io.Threaded.init(alloc, .{
+            .async_limit = .limited(threads - 1),
+            .concurrent_limit = .limited(threads - 1),
+            .argv0 = .init(init.minimal.args),
+            .environ = init.minimal.environ,
+        });
+        return arc.extract(alloc, limited_io.io(), extLoc, options); //TODO: Handle error gracefully.
+    }
+    return arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully.
 }
 
 fn handleArgs(args: std.process.Args, out: *Writer) !void {

src/c.h

@@ -0,0 +1,7 @@
+#include <zstd.h>
+#include <zlib-ng.h>
+#include <lzma.h>
+#ifdef ALLOW_LZO
+#include <lzo/minilzo.h>
+#endif
+#include <lz4.h>

src/decomp.zig

@@ -1,22 +1,76 @@
 const std = @import("std");
+const Io = std.Io;
+
+const options = @import("options");
 
 const Decompressor = @import("util/decompressor.zig");
 
-pub const Decomp = union(enum) {
-    gzip: @import("decomp/zlib.zig"),
-    lzma: @import("decomp/lzma.zig"),
-    lzo: void,
-    xz: @import("decomp/xz.zig"),
-    lz4: void,
-    zstd: @import("decomp/zstd.zig"),
+const zlib = if (options.use_zig_decomp) @import("decomp/zig_zlib.zig") else @import("decomp/c_zlib.zig");
+const lzma = if (options.use_zig_decomp) @import("decomp/zig_lzma.zig") else @import("decomp/c_lzma.zig");
+const lzo = if (options.use_zig_decomp or !options.allow_lzo) void else @import("decomp/c_lzo.zig");
+const xz = if (options.use_zig_decomp) @import("decomp/zig_xz.zig") else @import("decomp/c_xz.zig");
+const lz4 = if (options.use_zig_decomp) void else @import("decomp/c_lz4.zig");
+const zstd = if (options.use_zig_decomp) @import("decomp/zig_zstd.zig") else @import("decomp/c_zstd.zig");
 
-    pub fn deinit(self: *Decomp) void {
+pub const Enum = enum(u16) {
+    gzip = 1,
+    lzma,
+    lzo,
+    xz,
+    lz4,
+    zstd,
+};
+
+pub fn StatelessDecomp(val: Enum) !Decompressor {
+    return switch (val) {
+        .gzip => zlib.stateless_decompressor,
+        .lzma => lzma.stateless_decompressor,
+        .lzo => if (options.use_zig_decomp or !options.allow_lzo)
+            error.LzoUnsupported
+        else
+            lzo.stateless_decompressor,
+        .xz => xz.stateless_decompressor,
+        .lz4 => if (options.use_zig_decomp)
+            error.Lz4Unsupported
+        else
+            lz4.stateless_decompressor,
+        .zstd => zstd.stateless_decompressor,
+    };
+}
+
+pub const Decomp = union(enum) {
+    gzip: zlib,
+    lzma: lzma,
+    lzo: lzo,
+    xz: xz,
+    lz4: lz4,
+    zstd: zstd,
+
+    pub fn init(val: Enum, alloc: std.mem.Allocator, io: Io, block_size: u32) !Decomp {
+        return switch (val) {
+            .gzip => .{ .gzip = if (options.use_zig_decomp) try zlib.init(alloc, io, block_size) else try zlib.init(alloc, io) },
+            .lzma => .{ .lzma = if (options.use_zig_decomp) try lzma.init(alloc, io, block_size) else .{} },
+            .lzo => if (options.use_zig_decomp or !options.allow_lzo) error.LzoUnsupported else .{ .lzo = .{} },
+            .xz => .{ .xz = if (options.use_zig_decomp) try xz.init(alloc, io, block_size) else .{} },
+            .lz4 => if (options.use_zig_decomp) error.Lz4Unsupported else .{ .lz4 = .{} },
+            .zstd => .{ .zstd = if (options.use_zig_decomp) try zstd.init(alloc, io, block_size) else try zstd.init(alloc, io) },
+        };
+    }
+    pub fn deinit(self: *Decomp, alloc: std.mem.Allocator) void {
+        if (options.use_zig_decomp) {
             switch (self.*) {
                 .gzip => self.gzip.deinit(),
                 .lzma => self.lzma.deinit(),
                 .xz => self.xz.deinit(),
                 .zstd => self.zstd.deinit(),
-            else => unreachable,
+                else => {},
+            }
+        } else {
+            switch (self.*) {
+                .gzip => self.gzip.deinit(alloc),
+                .zstd => self.zstd.deinit(alloc),
+                else => {},
+            }
         }
     }
 
@@ -24,9 +78,10 @@ pub const Decomp = union(enum) {
         return switch (self.*) {
             .gzip => &self.gzip.interface,
             .lzma => &self.lzma.interface,
+            .lzo => if (options.use_zig_decomp or !options.allow_lzo) unreachable else &self.lzo.interface,
             .xz => &self.xz.interface,
+            .lz4 => if (options.use_zig_decomp) unreachable else &self.lz4.interface,
             .zstd => &self.zstd.interface,
-            else => unreachable,
         };
     }
 };

src/decomp/c_lz4.zig

@@ -0,0 +1,17 @@
+const std = @import("std");
+
+const c = @import("c");
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+interface: Decompressor = .{ .decomp_fn = statelessDecomp },
+
+fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    const out_len: c_int = @bitCast(@as(u32, @truncate(out.len)));
+    const res = c.LZ4_decompress_fast(in.ptr, out.ptr, out_len);
+    if (res < 0) return Error.ReadFailed;
+    return @abs(res);
+}

src/decomp/c_lzma.zig

@@ -0,0 +1,46 @@
+const std = @import("std");
+const Io = std.Io;
+const Reader = std.Io.Reader;
+const zstd = std.compress.zstd;
+const Node = std.SinglyLinkedList.Node;
+
+const c = @import("c");
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Queue = std.Io.Queue(c.lzma_stream);
+
+const Self = @This();
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+interface: Decompressor = .{ .decomp_fn = statelessDecomp },
+
+fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var stream: c.lzma_stream = .{
+        .next_in = in.ptr,
+        .avail_in = in.len,
+        .next_out = out.ptr,
+        .avail_out = out.len,
+    };
+
+    var res = c.lzma_alone_decoder(&stream, stream.avail_out * 2);
+    if (res != c.LZMA_OK) return Error.ReadFailed;
+    while (res == c.LZMA_OK)
+        res = c.lzma_code(&stream, c.LZMA_RUN);
+    if (res != c.LZMA_FINISH) return Error.ReadFailed;
+    return stream.total_out;
+}
+
+// lzma_allocator
+
+// fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque {
+//     var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
+//     return alloc.rawAlloc(size, .@"1", 0);
+// }
+// fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
+//     if (mem_ptr == null) return;
+//     var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
+//     alloc.free(@as([*]u8, @ptrCast(mem_ptr.?)));
+// }

src/decomp/c_lzo.zig

@@ -0,0 +1,26 @@
+const std = @import("std");
+const Io = std.Io;
+const Reader = std.Io.Reader;
+const zstd = std.compress.zstd;
+const Node = std.SinglyLinkedList.Node;
+
+const c = @import("c");
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Queue = std.Io.Queue(c.lzma_stream);
+
+const Self = @This();
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+interface: Decompressor = .{ .decomp_fn = statelessDecomp },
+
+fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    _ = c.lzo_init();
+    var out_len = out.len;
+    const res = c.lzo1x_decompress_safe(in.ptr, in.len, out.ptr, &out_len, null);
+    if (res != c.LZO_E_OK) return Error.ReadFailed;
+    return out_len;
+}

src/decomp/c_xz.zig

@@ -0,0 +1,47 @@
+const std = @import("std");
+const Io = std.Io;
+const Reader = std.Io.Reader;
+const zstd = std.compress.zstd;
+const Node = std.SinglyLinkedList.Node;
+
+const c = @import("c");
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Queue = std.Io.Queue(c.lzma_stream);
+
+const Self = @This();
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+interface: Decompressor = .{ .decomp_fn = statelessDecomp },
+
+fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var stream: c.lzma_stream = .{
+        .next_in = in.ptr,
+        .avail_in = in.len,
+        .next_out = out.ptr,
+        .avail_out = out.len,
+    };
+
+    var res = c.lzma_alone_decoder(&stream, stream.avail_out * 2);
+    if (res != c.LZMA_OK) return Error.ReadFailed;
+    while (res == c.LZMA_OK)
+        res = c.lzma_code(&stream, c.LZMA_RUN);
+    if (res != c.LZMA_FINISH) return Error.ReadFailed;
+    return stream.total_out;
+}
+
+// lzma_allocator
+
+// fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque {
+//     var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
+//     const mem = alloc.alloc(u8, size) catch return null;
+//     return mem.ptr;
+// }
+// fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
+//     if (mem_ptr == null) return;
+//     var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
+//     alloc.free(@as([*]u8, @ptrCast(mem_ptr.?)));
+// }

src/decomp/c_zlib.zig

@@ -0,0 +1,80 @@
+const std = @import("std");
+const Io = std.Io;
+const Reader = std.Io.Reader;
+const zstd = std.compress.zstd;
+const Node = std.SinglyLinkedList.Node;
+
+const c = @import("c");
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Queue = std.Io.Queue(c.zng_stream);
+
+const Self = @This();
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+io: Io,
+
+ctx: []c.zng_stream,
+ctx_queue: Queue,
+
+pub fn init(alloc: std.mem.Allocator, io: Io) !Self {
+    const buf = try alloc.alloc(c.zng_stream, 20); // TODO: Choose a better number instead of a random one.
+    var queue: Queue = .init(buf);
+    for (0..20) |_|
+        try queue.putOne(io, .{});
+
+    return .{
+        .io = io,
+
+        .ctx = buf,
+        .ctx_queue = queue,
+    };
+}
+pub fn deinit(self: *Self, alloc: std.mem.Allocator) void {
+    self.ctx_queue.close(self.io);
+    alloc.free(self.ctx);
+}
+
+fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        return statelessDecomp(d, alloc, in, out);
+    }
+    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
+
+    var stream = self.ctx_queue.getOne(self.io) catch return Error.ReadFailed;
+    defer self.ctx_queue.putOne(self.io, stream) catch {};
+
+    stream.next_in = in.ptr;
+    stream.avail_in = @truncate(in.len);
+    stream.next_out = out.ptr;
+    stream.avail_out = @truncate(out.len);
+
+    try zlibDecomp(&stream);
+
+    return stream.total_out;
+}
+
+inline fn zlibDecomp(stream: *c.zng_stream) !void {
+    _ = c.zng_inflateReset(stream);
+
+    const res = c.zng_inflate(stream, c.Z_FULL_FLUSH);
+    if (res != c.Z_OK) return Error.ReadFailed;
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var stream: c.zng_stream = .{
+        .next_in = in.ptr,
+        .avail_in = @truncate(in.len),
+        .next_out = out.ptr,
+        .avail_out = @truncate(out.len),
+    };
+    try zlibDecomp(&stream);
+    return stream.total_out;
+}

src/decomp/c_zstd.zig

@@ -0,0 +1,69 @@
+const std = @import("std");
+const Io = std.Io;
+const Reader = std.Io.Reader;
+const zstd = std.compress.zstd;
+const Node = std.SinglyLinkedList.Node;
+
+const c = @import("c");
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Queue = std.Io.Queue(?*c.ZSTD_DCtx);
+
+const Self = @This();
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+io: Io,
+
+ctx: []?*c.ZSTD_DCtx,
+ctx_queue: Queue,
+
+pub fn init(alloc: std.mem.Allocator, io: Io) !Self {
+    const buf = try alloc.alloc(?*c.ZSTD_DCtx, 20); // TODO: Choose a better number instead of a random one.
+    var queue: Queue = .init(buf);
+    for (0..20) |_|
+        try queue.putOne(io, c.ZSTD_createDCtx());
+
+    return .{
+        .io = io,
+
+        .ctx = buf,
+        .ctx_queue = queue,
+    };
+}
+pub fn deinit(self: *Self, alloc: std.mem.Allocator) void {
+    self.ctx_queue.close(self.io);
+    for (self.ctx) |ctx|
+        _ = c.ZSTD_freeDCtx(ctx);
+    alloc.free(self.ctx);
+}
+
+fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        return statelessDecomp(d, alloc, in, out);
+    }
+    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
+
+    const ctx = self.ctx_queue.getOne(self.io) catch return Error.ReadFailed;
+    defer self.ctx_queue.putOne(self.io, ctx) catch {};
+
+    _ = c.ZSTD_DCtx_reset(ctx, c.ZSTD_reset_session_only);
+
+    const res = c.ZSTD_decompressDCtx(ctx, out.ptr, out.len, in.ptr, in.len);
+    if (c.ZSTD_isError(res) != 0)
+        return Error.ReadFailed;
+    return res;
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
+    if (c.ZSTD_isError(res) != 0)
+        return Error.ReadFailed;
+    return res;
+}

src/decomp/lzma.zig

@@ -1,86 +0,0 @@
-const std = @import("std");
-const Reader = std.Io.Reader;
-const lzma = std.compress.lzma;
-const Node = std.SinglyLinkedList.Node;
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Self = @This();
-
-const Buffer = struct {
-    node: Node,
-    buf: []u8,
-};
-
-interface: Decompressor = .{ .decomp_fn = decomp },
-
-alloc: std.mem.Allocator,
-
-block_size: u32,
-buffers: std.ArrayList(Buffer),
-buffer_queue: std.SinglyLinkedList = .{},
-
-pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self {
-    return .{
-        .alloc = alloc,
-
-        .block_size = block_size,
-        .buffers = try .initCapacity(alloc, 5),
-    };
-}
-pub fn deinit(self: *Self) void {
-    for (self.buffers.items) |buf|
-        self.alloc.free(buf.buf);
-    self.buffers.deinit(self.alloc);
-}
-
-fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    if (d == null) {
-        var buf = try alloc.alloc(u8, in.len * 2);
-        defer alloc.free(buf);
-        return lzmaDecomp(alloc, &buf, in, out) catch |err| return switch (err) {
-            error.OutOfMemory => Error.OutOfMemory,
-            else => Error.ReadFailed,
-        };
-    }
-    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
-    const buf_node = self.buffer_queue.popFirst();
-    var buf: *Buffer = undefined;
-    if (buf_node == null) {
-        const new_buf = try self.buffers.addOne(self.alloc);
-        new_buf.* = .{ .node = .{}, .buf = try self.alloc.alloc(u8, self.block_size) };
-        buf = new_buf;
-    } else {
-        buf = @fieldParentPtr("node", buf_node.?);
-    }
-    defer self.buffer_queue.prepend(&buf.node);
-    return lzmaDecomp(self.alloc, &buf.buf, in, out) catch |err| {
-        // self.err = err;
-        return switch (err) {
-            error.OutOfMemory => Error.OutOfMemory,
-            else => Error.ReadFailed,
-        };
-    };
-}
-
-inline fn lzmaDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize {
-    var rdr: Reader = .fixed(in);
-    var d = try lzma.Decompress.initOptions(&rdr, alloc, buffer.*, .{ .allow_incomplete = true }, 3 * 1024 * 1024);
-    defer {
-        buffer.* = d.takeBuffer();
-        d.deinit();
-    }
-
-    return d.reader.readSliceShort(out);
-}
-
-// Stateless
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var buf = try alloc.alloc(u8, in.len);
-    defer alloc.free(buf);
-    return lzmaDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
-}

src/decomp/xz.zig

@@ -1,80 +0,0 @@
-const std = @import("std");
-const Reader = std.Io.Reader;
-const xz = std.compress.xz;
-const Node = std.SinglyLinkedList.Node;
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Self = @This();
-
-const Buffer = struct {
-    node: Node,
-    buf: []u8,
-};
-
-interface: Decompressor = .{ .decomp_fn = decomp },
-
-alloc: std.mem.Allocator,
-
-block_size: u32,
-buffers: std.ArrayList(Buffer),
-buffer_queue: std.SinglyLinkedList = .{},
-
-pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self {
-    return .{
-        .alloc = alloc,
-
-        .block_size = block_size,
-        .buffers = try .initCapacity(alloc, 5),
-    };
-}
-pub fn deinit(self: *Self) void {
-    for (self.buffers.items) |buf|
-        self.alloc.free(buf.buf);
-    self.buffers.deinit(self.alloc);
-}
-
-fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    if (d == null) {
-        var buf = try alloc.alloc(u8, in.len * 2);
-        defer alloc.free(buf);
-        return xzDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
-    }
-    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
-    const buf_node = self.buffer_queue.popFirst();
-    var buf: *Buffer = undefined;
-    if (buf_node == null) {
-        const new_buf = try self.buffers.addOne(self.alloc);
-        new_buf.* = .{ .node = .{}, .buf = try self.alloc.alloc(u8, self.block_size) };
-        buf = new_buf;
-    } else {
-        buf = @fieldParentPtr("node", buf_node.?);
-    }
-    defer self.buffer_queue.prepend(&buf.node);
-    return xzDecomp(self.alloc, &buf.buf, in, out) catch {
-        // self.err = err;
-        return Error.ReadFailed;
-    };
-}
-
-inline fn xzDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize {
-    var rdr: Reader = .fixed(in);
-    var d = try xz.Decompress.init(&rdr, alloc, buffer.*);
-    defer {
-        buffer.* = d.takeBuffer();
-        d.deinit();
-    }
-
-    return d.reader.readSliceShort(out);
-}
-
-// Stateless
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var buf = try alloc.alloc(u8, in.len);
-    defer alloc.free(buf);
-    return xzDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
-}

src/decomp/zig_lzma.zig

@@ -0,0 +1,81 @@
+const std = @import("std");
+const Io = std.Io;
+const Reader = std.Io.Reader;
+const lzma = std.compress.lzma;
+const Node = std.SinglyLinkedList.Node;
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Queue = Io.Queue([]u8);
+
+const Self = @This();
+
+const Buffer = struct {
+    node: Node,
+    buf: []u8,
+};
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+io: Io,
+
+block_size: u32,
+buf: [][]u8,
+buf_queue: Queue,
+
+pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self {
+    const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one.
+    var queue: Queue = .init(buf);
+    for (0..20) |_|
+        try queue.putOne(io, try alloc.alloc(u8, block_size));
+
+    return .{
+        .alloc = alloc,
+        .io = io,
+
+        .block_size = block_size,
+        .buf = buf,
+        .buf_queue = queue,
+    };
+}
+pub fn deinit(self: *Self) void {
+    self.buf_queue.close(self.io);
+    for (self.buf) |buf|
+        self.alloc.free(buf);
+    self.alloc.free(self.buf);
+}
+
+fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        return statelessDecomp(d, alloc, in, out);
+    }
+    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
+
+    var buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed;
+    defer self.buf_queue.putOne(self.io, buf) catch {};
+
+    return lzmaDecomp(self.alloc, &buf, in, out) catch return Error.ReadFailed;
+}
+
+inline fn lzmaDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = try lzma.Decompress.initOptions(&rdr, alloc, buffer.*, .{}, in.len * 2);
+    defer {
+        buffer.* = d.takeBuffer();
+        d.deinit();
+    }
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var buf = try alloc.alloc(u8, in.len);
+    defer alloc.free(buf);
+    return lzmaDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
+}

src/decomp/zig_xz.zig

@@ -0,0 +1,81 @@
+const std = @import("std");
+const Io = std.Io;
+const Reader = std.Io.Reader;
+const xz = std.compress.xz;
+const Node = std.SinglyLinkedList.Node;
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Queue = Io.Queue([]u8);
+
+const Self = @This();
+
+const Buffer = struct {
+    node: Node,
+    buf: []u8,
+};
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+io: Io,
+
+block_size: u32,
+buf: [][]u8,
+buf_queue: Queue,
+
+pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self {
+    const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one.
+    var queue: Queue = .init(buf);
+    for (0..20) |_|
+        try queue.putOne(io, try alloc.alloc(u8, block_size));
+
+    return .{
+        .alloc = alloc,
+        .io = io,
+
+        .block_size = block_size,
+        .buf = buf,
+        .buf_queue = queue,
+    };
+}
+pub fn deinit(self: *Self) void {
+    self.buf_queue.close(self.io);
+    for (self.buf) |buf|
+        self.alloc.free(buf);
+    self.alloc.free(self.buf);
+}
+
+fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        return statelessDecomp(d, alloc, in, out);
+    }
+    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
+
+    var buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed;
+    defer self.buf_queue.putOne(self.io, buf) catch {};
+
+    return xzDecomp(self.alloc, &buf, in, out) catch return Error.ReadFailed;
+}
+
+inline fn xzDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = try xz.Decompress.init(&rdr, alloc, buffer.*);
+    defer {
+        buffer.* = d.takeBuffer();
+        d.deinit();
+    }
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var buf = try alloc.alloc(u8, in.len);
+    defer alloc.free(buf);
+    return xzDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
+}

src/decomp/zig_zlib.zig

@@ -0,0 +1,77 @@
+const std = @import("std");
+const Io = std.Io;
+const flate = std.compress.flate;
+const Node = std.SinglyLinkedList.Node;
+const Reader = Io.Reader;
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Queue = Io.Queue([]u8);
+
+const Self = @This();
+
+const Buffer = struct {
+    node: Node,
+    buf: []u8,
+};
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+io: Io,
+
+block_size: u32,
+buf: [][]u8,
+buf_queue: Queue,
+
+pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self {
+    const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one.
+    var queue: Queue = .init(buf);
+    for (0..20) |_|
+        try queue.putOne(io, try alloc.alloc(u8, block_size));
+
+    return .{
+        .alloc = alloc,
+        .io = io,
+
+        .block_size = block_size,
+        .buf = buf,
+        .buf_queue = queue,
+    };
+}
+pub fn deinit(self: *Self) void {
+    self.buf_queue.close(self.io);
+    for (self.buf) |buf|
+        self.alloc.free(buf);
+    self.alloc.free(self.buf);
+}
+
+fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        return statelessDecomp(d, alloc, in, out);
+    }
+    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
+
+    const buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed;
+    defer self.buf_queue.putOne(self.io, buf) catch {};
+
+    return zlibDecomp(buf, in, out);
+}
+
+inline fn zlibDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = flate.Decompress.init(&rdr, .zlib, buffer);
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    const buf = try alloc.alloc(u8, out.len);
+    defer alloc.free(buf);
+    return zlibDecomp(buf, in, out);
+}

src/decomp/zig_zstd.zig

@@ -2,7 +2,6 @@ const std = @import("std");
 const Io = std.Io;
 const Reader = std.Io.Reader;
 const zstd = std.compress.zstd;
-const Node = std.SinglyLinkedList.Node;
 
 const Decompressor = @import("../util/decompressor.zig");
 const Error = Decompressor.Error;
@@ -21,9 +20,9 @@ buf: [][]u8,
 buf_queue: Queue,
 
 pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self {
-    const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one.
+    const buf = try alloc.alloc([]u8, 5); // TODO: Choose a better number instead of a random one.
     var queue: Queue = .init(buf);
-    for (0..20) |_|
+    for (buf) |_|
         try queue.putOne(io, try alloc.alloc(u8, block_size + zstd.block_size_max));
 
     return .{
@@ -42,7 +41,7 @@ pub fn deinit(self: *Self) void {
     self.alloc.free(self.buf);
 }
 
-fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
     if (d == null) {
         const buf = try alloc.alloc(u8, in.len * 2);
         defer alloc.free(buf);
@@ -67,7 +66,7 @@ inline fn zstdDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
 
 pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
 
-fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
     const buf = try alloc.alloc(u8, out.len + zstd.block_size_max);
     defer alloc.free(buf);
     return zstdDecomp(buf, in, out);

src/decomp/zlib.zig

@@ -1,73 +0,0 @@
-const std = @import("std");
-const Reader = std.Io.Reader;
-const flate = std.compress.flate;
-const Node = std.SinglyLinkedList.Node;
-
-const Decompressor = @import("../util/decompressor.zig");
-const Error = Decompressor.Error;
-
-const Self = @This();
-
-const Buffer = struct {
-    node: Node,
-    buf: []u8,
-};
-
-interface: Decompressor = .{ .decomp_fn = decomp },
-
-alloc: std.mem.Allocator,
-
-block_size: u32,
-buffers: std.ArrayList(Buffer),
-buffer_queue: std.SinglyLinkedList = .{},
-
-pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self {
-    return .{
-        .alloc = alloc,
-
-        .block_size = block_size,
-        .buffers = try .initCapacity(alloc, 5),
-    };
-}
-pub fn deinit(self: *Self) void {
-    for (self.buffers.items) |buf|
-        self.alloc.free(buf.buf);
-    self.buffers.deinit(self.alloc);
-}
-
-fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    if (d == null) {
-        const buf = try alloc.alloc(u8, in.len * 2);
-        defer alloc.free(buf);
-        return zlibDecomp(buf, in, out);
-    }
-    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
-    const buf_node = self.buffer_queue.popFirst();
-    var buf: *Buffer = undefined;
-    if (buf_node == null) {
-        const new_buf = try self.buffers.addOne(self.alloc);
-        new_buf.* = .{ .node = .{}, .buf = try self.alloc.alloc(u8, self.block_size) };
-        buf = new_buf;
-    } else {
-        buf = @fieldParentPtr("node", buf_node.?);
-    }
-    defer self.buffer_queue.prepend(&buf.node);
-    return zlibDecomp(buf.buf, in, out);
-}
-
-inline fn zlibDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
-    var rdr: Reader = .fixed(in);
-    var d = flate.Decompress.init(&rdr, .zlib, buffer);
-
-    return d.reader.readSliceShort(out);
-}
-
-// Stateless
-
-pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
-
-fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    const buf = try alloc.alloc(u8, in.len * 2);
-    defer alloc.free(buf);
-    return zlibDecomp(buf, in, out);
-}

src/file.zig

@@ -10,7 +10,6 @@ const Inode = @import("inode.zig");
 const DataExtractor = @import("util/data_extractor.zig");
 const Decompressor = @import("util/decompressor.zig");
 const MetadataReader = @import("util/metadata.zig");
-const SharedCache = @import("util/shared_cache.zig");
 
 const File = @This();
 
@@ -35,24 +34,23 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, in: Inode, name: []const
         .name = new_name,
     };
 }
-pub fn fromDirEntry(alloc: std.mem.Allocator, io: Io, archive: Archive, ent: DirEntry) !File {
-    var rdr = try archive.file.readerAt(io, archive.super.inode_start + ent.block_start, &[0]u8{});
-    var meta: MetadataReader = .init(alloc, &rdr.interface, &archive.stateless_decomp);
+pub fn fromDirEntry(alloc: std.mem.Allocator, archive: *Archive, ent: DirEntry) !File {
+    var rdr = archive.file.readerAt(archive.super.inode_start + ent.block_start);
+    var meta: MetadataReader = .init(alloc, &rdr, &archive.stateless_decomp);
     try meta.interface.discardAll(ent.block_offset);
 
     var in: Inode = try .read(alloc, &meta.interface, archive.super.block_size);
     errdefer in.deinit(alloc);
-    return .init(alloc, archive, in, ent.name);
+    return .init(alloc, archive.*, in, ent.name);
 }
 pub fn deinit(self: File) void {
     self.alloc.free(self.name);
     self.inode.deinit(self.alloc);
 }
 
-pub fn open(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
+pub fn open(self: *File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
     const entries = try self.inode.readDirectory(
         alloc,
-        io,
         self.archive.file,
         &self.archive.stateless_decomp,
         self.archive.super.dir_start,
@@ -77,7 +75,7 @@ pub fn open(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8)
         }
     } else return Error.FileNotFound;
 
-    const first_elem_file = try fromDirEntry(alloc, io, self.archive, search_slice[idx]);
+    var first_elem_file = try fromDirEntry(alloc, &self.archive, search_slice[idx]);
     if (first_element.len == path.len)
         return first_elem_file;
     defer first_elem_file.deinit();

src/frag.zig

@@ -1,7 +1,85 @@
+const std = @import("std");
+const Io = std.Io;
+
 const BlockSize = @import("inode_data/file.zig").BlockSize;
+const LookupTable = @import("lookup_table.zig");
+const Decompressor = @import("util/decompressor.zig");
+const MetadataReader = @import("util/metadata.zig");
+const OffsetFile = @import("util/offset_file.zig");
+
+const FragManager = @This();
 
 pub const FragEntry = extern struct {
     start: u64,
     size: BlockSize,
     _: u32,
 };
+
+alloc: std.mem.Allocator,
+fil: OffsetFile,
+decomp: *Decompressor,
+block_size: u32,
+
+entries: []FragEntry,
+
+frag_cache: std.array_hash_map.Auto(u32, []u8),
+cache_mut: std.Io.RwLock = .init,
+
+pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, frag_start: u64, frag_num: u32, block_size: u32) !FragManager {
+    const first_offset: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[frag_start .. frag_start + 8]), .little);
+
+    var rdr = fil.readerAt(first_offset);
+    var meta: MetadataReader = .init(alloc, &rdr, decomp);
+
+    const entries = try alloc.alloc(FragEntry, frag_num);
+    errdefer alloc.free(entries);
+
+    try meta.interface.readSliceEndian(FragEntry, entries, .little);
+
+    return .{
+        .alloc = alloc,
+        .fil = fil,
+        .decomp = decomp,
+        .block_size = block_size,
+
+        .entries = entries,
+
+        .frag_cache = .empty,
+    };
+}
+pub fn deinit(self: *FragManager, io: Io) void {
+    self.cache_mut.lockUncancelable(io);
+    self.alloc.free(self.entries);
+    for (self.frag_cache.values()) |v|
+        self.alloc.free(v);
+    self.frag_cache.deinit(self.alloc);
+}
+
+pub fn get(self: *FragManager, io: Io, idx: u32) ![]u8 {
+    {
+        try self.cache_mut.lockShared(io);
+        defer self.cache_mut.unlockShared(io);
+        if (self.frag_cache.contains(idx))
+            return self.frag_cache.get(idx).?;
+    }
+
+    try self.cache_mut.lock(io);
+    defer self.cache_mut.unlock(io);
+
+    if (self.frag_cache.contains(idx))
+        return self.frag_cache.get(idx).?;
+
+    const entry = self.entries[idx];
+
+    const out = try self.alloc.alloc(u8, if (entry.size.uncompressed) entry.size.size else self.block_size);
+
+    if (entry.size.uncompressed) {
+        @memcpy(out, self.fil.map.memory[entry.start .. entry.start + entry.size.size]);
+    } else {
+        @branchHint(.likely);
+        _ = try self.decomp.Decompress(self.alloc, self.fil.map.memory[entry.start .. entry.start + entry.size.size], out);
+    }
+
+    try self.frag_cache.put(self.alloc, idx, out);
+    return out;
+}

src/inode.zig

@@ -9,6 +9,7 @@ const Decomp = @import("decomp.zig").Decomp;
 const DirEntry = @import("directory.zig");
 const ExtractionOptions = @import("options.zig");
 const FragEntry = @import("frag.zig").FragEntry;
+const FragManager = @import("frag.zig");
 const dir = @import("inode_data/dir.zig");
 const file = @import("inode_data/file.zig");
 const misc = @import("inode_data/misc.zig");
@@ -19,7 +20,6 @@ const DataReader = @import("util/data_reader.zig");
 const Decompressor = @import("util/decompressor.zig");
 const MetadataReader = @import("util/metadata.zig");
 const OffsetFile = @import("util/offset_file.zig");
-const SharedCache = @import("util/shared_cache.zig");
 const XattrTable = @import("xattr_table.zig");
 
 const Inode = @This();
@@ -63,49 +63,49 @@ pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
 // Utility Functions
 
 /// Read the directory entries
-pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64) ![]DirEntry {
+pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, dir_offset: u64) ![]DirEntry {
     return switch (self.data) {
-        .dir => |d| readDirFromData(alloc, io, fil, decomp, dir_offset, d),
-        .ext_dir => |d| readDirFromData(alloc, io, fil, decomp, dir_offset, d),
+        .dir => |d| readDirFromData(alloc, fil, decomp, dir_offset, d),
+        .ext_dir => |d| readDirFromData(alloc, fil, decomp, dir_offset, d),
         else => Error.NotDirectory,
     };
 }
-fn readDirFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64, d: anytype) ![]DirEntry {
-    var rdr = try fil.readerAt(io, dir_offset + d.block_start, &[0]u8{});
-    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+fn readDirFromData(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, dir_offset: u64, d: anytype) ![]DirEntry {
+    var rdr = fil.readerAt(dir_offset + d.block_start);
+    var meta: MetadataReader = .init(alloc, &rdr, decomp);
     try meta.interface.discardAll(d.block_offset);
 
     return DirEntry.readDirectory(alloc, &meta.interface, d.size);
 }
 /// Get a reader for a regular file's data.
-pub fn dataReader(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32) !DataReader {
+pub fn dataReader(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8) !DataReader {
     return switch (self.data) {
-        .file => |f| getReaderFromData(alloc, io, fil, cache, decomp, block_size, f),
-        .ext_file => |f| getReaderFromData(alloc, io, fil, cache, decomp, block_size, f),
+        .file => |f| getReaderFromData(alloc, io, fil, decomp, block_size, frag_block, f),
+        .ext_file => |f| getReaderFromData(alloc, io, fil, decomp, block_size, frag_block, f),
         else => Error.NotRegularFile,
     };
 }
-fn getReaderFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32, d: anytype) !DataReader {
-    const ext: DataReader = .init(alloc, io, fil, cache, decomp, block_size, d.size, d.block_start, d.blocks);
+fn getReaderFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8, d: anytype) !DataReader {
+    const ext: DataReader = .init(alloc, io, fil, decomp, block_size, d.size, d.block_start, d.blocks);
     if (d.frag_block_offset == 0xFFFFFFFF) {
-        // TODO:
-        return error.TODO;
+        if (frag_block == null) return error.FragBlockNotProvided;
+        ext.addFrag(d.frag_block_offset, frag_block.?);
     }
     return ext;
 }
 /// Get an extractor for a regular file's data.
-pub fn dataExtractor(self: Inode, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32) !DataExtractor {
+pub fn dataExtractor(self: Inode, fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8) !DataExtractor {
     return switch (self.data) {
-        .file => |f| getExtractorFromData(fil, cache, decomp, block_size, f),
-        .ext_file => |f| getExtractorFromData(fil, cache, decomp, block_size, f),
+        .file => |f| getExtractorFromData(fil, decomp, block_size, frag_block, f),
+        .ext_file => |f| getExtractorFromData(fil, decomp, block_size, frag_block, f),
         else => Error.NotRegularFile,
     };
 }
-fn getExtractorFromData(fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32, d: anytype) !DataExtractor {
-    const ext: DataExtractor = .init(fil, cache, decomp, block_size, d.size, d.block_start, d.blocks);
+fn getExtractorFromData(fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8, d: anytype) !DataExtractor {
+    const ext: DataExtractor = .init(fil, decomp, block_size, d.size, d.block_start, d.blocks);
     if (d.frag_block_offset == 0xFFFFFFFF) {
-        // TODO:
-        return error.TODO;
+        if (frag_block == null) return error.FragBlockNotProvided;
+        ext.addFrag(d.frag_block_offset, frag_block.?);
     }
     return ext;
 }
@@ -118,11 +118,11 @@ pub fn symlinkTarget(self: Inode) ![]const u8 {
     };
 }
 /// Get inode's gid
-pub fn gid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, id_table_start: u64) !u16 {
+pub fn gid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, id_table_start: u64) !u16 {
     return LookupTable.lookupValue(u16, alloc, io, decomp, fil, id_table_start, self.hdr.gid_idx);
 }
 /// Get inode's uid
-pub fn uid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, id_table_start: u64) !u16 {
+pub fn uid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, id_table_start: u64) !u16 {
     return LookupTable.lookupValue(u16, alloc, io, decomp, fil, id_table_start, self.hdr.uid_idx);
 }
 /// Get the inode's xattr values as an index into the Archive's xattr table.
@@ -136,13 +136,13 @@ pub fn xattrIndex(self: Inode) !u32 {
         .ext_char_dev => |e| e.xattr_idx,
         .ext_fifo => |e| e.xattr_idx,
         .ext_socket => |e| e.xattr_idx,
-        else => Error.NoXattr,
+        else => return error.NoXattr,
     };
-    if (idx == 0xFFFFFFFF) return Error.NoXattr;
+    if (idx == 0xFFFFFFFF) return error.NoXattr;
     return idx;
 }
 // Get an inode's xattr values. If the inode does not have xattr values (including if the inode is not an extended type), an empty slice is returned.
-pub fn xattrValues(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, xattr_table_start: u64) ![]XattrTable.XattrOwned {
+pub fn xattrValues(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, xattr_table_start: u64) ![]XattrTable.XattrOwned {
     const idx = self.xattrIndex() catch &[0]XattrTable.XattrOwned{};
     return XattrTable.statelessLookup(alloc, io, decomp, fil, xattr_table_start, idx);
 }
@@ -207,14 +207,51 @@ pub const Header = extern struct {
 
 // Extract
 
-const ExtractError = error{ MknodFailed, CannotSetXattr, ConcurrencyUnavailable } || DataExtractor.Error || Io.Dir.CreateFileAtomicError || LookupTable.Error ||
-    Io.File.Reader.SeekError || Io.File.Atomic.LinkError || Io.Dir.CreateDirError || Io.File.OpenError ||
-    Io.File.SetPermissionsError || Io.File.SetOwnerError || Io.Dir.SymLinkError || Io.Dir.CreateDirPathError;
+const ExtractError = error{ MknodFailed, CannotSetXattr } || DataExtractor.Error || DirEntry.Error ||
+    Decompressor.Error || Io.File.Atomic.InitError || Io.File.Atomic.LinkError || Io.Dir.SymLinkError;
 const PathRet = struct {
     path: []const u8,
     inode: Inode,
-    xattr_idx: ?u32 = null,
+    origin: bool,
+
+    fn deinit(self: PathRet, alloc: std.mem.Allocator) void {
+        if (self.origin) return;
+        alloc.free(self.path);
+        self.inode.deinit(alloc);
+    }
+    fn setMetadata(self: PathRet, alloc: std.mem.Allocator, io: Io, id_table: *CachedTable(u16), xattr_table: ?*XattrTable, options: ExtractionOptions) !void {
+        var fil = try Io.Dir.cwd().openFile(io, self.path, .{});
+        defer fil.close(io);
+
+        const inode = self.inode;
+
+        if (!options.ignore_permissions) {
+            try fil.setPermissions(io, @enumFromInt(inode.hdr.permissions));
+            try fil.setOwner(io, try id_table.get(io, inode.hdr.uid_idx), try id_table.get(io, inode.hdr.gid_idx));
+        }
+        if (xattr_table != null) {
+            const idx = inode.xattrIndex() catch return;
+
+            const xattrs = try xattr_table.?.get(alloc, io, idx);
+            defer {
+                for (xattrs) |x|
+                    x.deinit(alloc);
+                alloc.free(xattrs);
+            }
+
+            const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{self.path}, 0);
+            defer alloc.free(sentinel_path);
+            for (xattrs) |x| {
+                const xattr_ret = std.os.linux.fsetxattr(fil.handle, x.key, x.value.ptr, x.value.len, 0);
+                if (xattr_ret != 0)
+                    return ExtractError.CannotSetXattr;
+            }
+        }
+    }
 };
+fn DirCompare(_: void, a: PathRet, b: PathRet) std.math.Order {
+    return std.math.order(std.mem.count(u8, a.path, "/"), std.mem.count(u8, b.path, "/"));
+}
 const ExtractReturnUnion = union(enum) {
     path_ret: ExtractError!PathRet,
 };
@@ -237,276 +274,360 @@ pub fn extract(
 ) !void {
     const path = std.mem.trimEnd(u8, filepath, "/");
 
-    var decomp_base: Decomp = switch (super.compression) {
-        .gzip => .{ .gzip = try .init(alloc, super.block_size) },
-        .lzma => .{ .lzma = try .init(alloc, super.block_size) },
-        .xz => .{ .xz = try .init(alloc, super.block_size) },
-        .zstd => .{ .zstd = try .init(alloc, io, super.block_size) },
-        else => unreachable,
-    };
-    defer decomp_base.deinit();
-    const decomp = decomp_base.decompressor();
+    var decomp_base: Decompressor = try @import("decomp.zig").StatelessDecomp(super.compression); // TODO: Replace with actual Decomp value to share states & caches for efficiency.
+    const decomp = &decomp_base;
 
-    var frag_table: CachedTable(FragEntry) = .init(alloc, fil, decomp, super.frag_start, super.frag_count);
-    defer if (!options.ignore_permissions) frag_table.deinit(io);
-    try frag_table.fill(io);
+    var frag_mgr: FragManager = try .init(alloc, fil, decomp, super.frag_start, super.frag_count, super.block_size);
+    defer frag_mgr.deinit(io);
 
-    var arena: std.heap.ArenaAllocator = .init(alloc);
-    defer arena.deinit();
+    if (options.single_threaded)
+        return self.extractSinglethreaded(alloc, io, fil, super, path, options, decomp, &frag_mgr);
 
-    var sel_buf = [1]ExtractReturnUnion{undefined} ** 10;
+    var sel_buf: [10]ExtractReturnUnion = undefined;
     var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf);
     defer sel.cancelDiscard();
 
+    var loop = io.async(finishLoop, .{ alloc, io, fil, decomp, super, options, &sel });
+
+    sel.async(.path_ret, extractRealAsync, .{ self, alloc, io, fil, super, decomp, &sel, &frag_mgr, path, true });
+
+    try loop.await(io);
+}
+fn extractRealAsync(
+    self: Inode,
+    alloc: std.mem.Allocator,
+    io: Io,
+    fil: OffsetFile,
+    super: Archive.Superblock,
+    decomp: *Decompressor,
+    sel: *Io.Select(ExtractReturnUnion),
+    frag_mgr: *FragManager,
+    path: []const u8,
+    origin: bool,
+) ExtractError!PathRet {
+    errdefer {
+        if (!origin) {
+            self.deinit(alloc);
+            alloc.free(path);
+        }
+    }
     switch (self.hdr.inode_type) {
-        .dir, .ext_dir => try sel.concurrent(
-            .path_ret,
-            extractDir,
-            .{ self, alloc, io, fil, decomp, &sel, &arena, super.dir_start, super.inode_start, &frag_table, super.block_size, path },
-        ),
-        .file, .ext_file => try sel.concurrent(
-            .path_ret,
-            extractFile,
-            .{ self, alloc, io, fil, decomp, &frag_table, super.block_size, path },
-        ),
-        .symlink, .ext_symlink => try sel.concurrent(.path_ret, extractSymlink, .{ self, io, path }),
-        else => if (@hasField(std.os, "linux"))
-            try sel.concurrent(.path_ret, extractDevOrIPC, .{ self, alloc, path }),
-    }
+        .dir, .ext_dir => {
+            try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777));
 
-    var xattr_table: ?XattrTable = if (!options.ignore_xattr)
-        try .init(alloc, io, fil, decomp, super.xattr_start)
-    else
-        null;
-    defer if (!options.ignore_xattr) xattr_table.?.deinit(io);
-    if (xattr_table != null) try xattr_table.?.table.fill(io);
-
-    var id_table: ?CachedTable(u16) = if (!options.ignore_xattr)
-        .init(alloc, fil, decomp, super.id_start, super.id_count)
-    else
-        null;
-    defer if (!options.ignore_xattr) id_table.?.deinit(io);
-    if (id_table != null) try id_table.?.fill(io);
-
-    while (true) {
-        const group_token = sel.group.token.load(.acquire);
-        if (group_token == null) break;
-        // std.debug.print("{any}\n", .{sel.group.state});
-
-        // std.debug.print("Waiting for return...", .{});
-        const ret = try sel.await();
-        // std.debug.print("Got One...\n", .{});
-        const path_ret = try ret.path_ret;
-
-        if (options.ignore_permissions and options.ignore_xattr) continue;
-        if (options.ignore_permissions and path_ret.xattr_idx == null) continue;
-
-        var ret_file = try Io.Dir.cwd().openFile(io, path_ret.path, .{});
-        defer ret_file.close(io);
-
-        if (!options.ignore_permissions) {
-            try ret_file.setPermissions(io, @enumFromInt(path_ret.inode.hdr.permissions));
-            try ret_file.setOwner(
-                io,
-                try id_table.?.get(io, path_ret.inode.hdr.uid_idx),
-                try id_table.?.get(io, path_ret.inode.hdr.gid_idx),
-            );
-        }
-        if (@hasField(std.os, "linux") and !options.ignore_xattr and path_ret.xattr_idx != null) {
-            const xattrs = try xattr_table.?.get(alloc, io, path_ret.xattr_idx.?);
+            const entries = self.readDirectory(alloc, fil, decomp, super.dir_start) catch |err| switch (err) {
+                Error.NotDirectory, Error.NotExtended, Error.NotRegularFile, Error.NotSymlink => unreachable,
+                else => |e| return e,
+            };
             defer {
-                for (xattrs) |x|
-                    alloc.free(x.key);
-                alloc.free(xattrs);
+                for (entries) |e|
+                    e.deinit(alloc);
+                alloc.free(entries);
             }
 
-            for (xattrs) |x| {
-                const res = std.os.linux.fsetxattr(ret_file.handle, x.key, x.value.ptr, x.value.len, 0);
-                if (res != 0)
-                    return error.CannotSetXattr;
-            }
-        }
-    }
-}
-fn extractDir(
-    self: Inode,
-    alloc: std.mem.Allocator,
-    io: Io,
-    fil: OffsetFile,
-    decomp: *const Decompressor,
-    parent_select: *Io.Select(ExtractReturnUnion),
-    arena: *std.heap.ArenaAllocator,
-    dir_start: u64,
-    inode_start: u64,
-    frag: *CachedTable(FragEntry),
-    block_size: u32,
-    path: []const u8,
-) ExtractError!PathRet {
-    try Io.Dir.cwd().createDirPath(io, path);
+            for (entries) |e| {
+                const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", e.name });
+                errdefer alloc.free(new_path);
 
-    var sel_buf = [1]ExtractReturnUnion{undefined} ** 10;
-    var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf);
-    defer sel.cancelDiscard();
+                var rdr = fil.readerAt(super.inode_start + e.block_start);
+                var meta: MetadataReader = .init(alloc, &rdr, decomp);
+                try meta.interface.discardAll(e.block_offset);
 
-    var num: usize = 0;
-    {
-        const dir_entries = self.readDirectory(alloc, io, fil, decomp, dir_start) catch |err| switch (err) {
-            Error.NotDirectory => unreachable,
-            else => return @errorCast(err),
-        };
-        num = dir_entries.len;
-        defer {
-            for (dir_entries) |d|
-                d.deinit(alloc);
-            alloc.free(dir_entries);
-        }
+                const new_inode = try read(alloc, &meta.interface, super.block_size);
+                errdefer new_inode.deinit(alloc);
 
-        for (dir_entries) |d| {
-            var rdr = try fil.readerAt(io, d.block_start + inode_start, &[0]u8{});
-            var meta_rdr: MetadataReader = .init(alloc, &rdr.interface, decomp);
-            try meta_rdr.interface.discardAll(d.block_offset);
-            const inode = try read(arena.allocator(), &meta_rdr.interface, block_size);
-            errdefer inode.deinit(arena.allocator());
-
-            const new_path = try std.mem.concat(arena.allocator(), u8, &[_][]const u8{ path, "/", d.name });
-            errdefer arena.allocator().free(new_path);
-
-            switch (d.type) {
-                .dir => try sel.concurrent(
-                    .path_ret,
-                    extractDir,
-                    .{ inode, alloc, io, fil, decomp, parent_select, arena, dir_start, inode_start, frag, block_size, new_path },
-                ),
-                .file => try sel.concurrent(
-                    .path_ret,
-                    extractFile,
-                    .{ inode, alloc, io, fil, decomp, frag, block_size, new_path },
-                ),
-                .symlink => try sel.concurrent(.path_ret, extractSymlink, .{ inode, io, new_path }),
-                else => try sel.concurrent(.path_ret, extractDevOrIPC, .{ inode, alloc, new_path }),
+                sel.async(.path_ret, extractRealAsync, .{ new_inode, alloc, io, fil, super, decomp, sel, frag_mgr, new_path, false });
             }
-        }
-    }
-
-    while (num > 0) {
-        const ret = sel.await() catch break;
-        num -= 1;
-
-        parent_select.queue.putOne(io, ret) catch |err| switch (err) {
-            error.Canceled => return error.Canceled,
-            else => break,
-        };
-    }
-    return .{
-        .path = path,
-        .inode = self,
-    };
-}
-fn extractFile(
-    self: Inode,
-    alloc: std.mem.Allocator,
-    io: Io,
-    fil: OffsetFile,
-    decomp: *const Decompressor,
-    frag: *CachedTable(FragEntry),
-    block_size: u32,
-    path: []const u8,
-) ExtractError!PathRet {
-    var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{});
+        },
+        .file, .ext_file => {
+            var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{ .make_path = true });
             defer atomic.deinit(io);
 
-    var ret: PathRet = .{
-        .inode = self,
-        .path = path,
-    };
-    const data: DataExtractor = blk: {
-        switch (self.data) {
-            .file => |f| {
-                var data: DataExtractor = .init(fil, decomp, block_size, f.size, f.block_start, f.block_sizes);
+            var ext: DataExtractor = switch (self.data) {
+                .file => |f| blk: {
+                    var ext: DataExtractor = .init(fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
                     if (f.frag_idx != 0xFFFFFFFF)
-                    data.addFrag(f.frag_block_offset, try frag.get(io, f.frag_idx));
-
-                break :blk data;
+                        ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
+                    break :blk ext;
                 },
-            .ext_file => |f| {
-                if (f.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = f.xattr_idx;
-                var data: DataExtractor = .init(fil, decomp, block_size, f.size, f.block_start, f.block_sizes);
+                .ext_file => |f| blk: {
+                    var ext: DataExtractor = .init(fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
                     if (f.frag_idx != 0xFFFFFFFF)
-                    data.addFrag(f.frag_block_offset, try frag.get(io, f.frag_idx));
-
-                break :blk data;
+                        ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
+                    break :blk ext;
                 },
                 else => unreachable,
-        }
             };
 
-    try data.extractAsync(alloc, io, atomic.file);
+            try ext.extractAsync(alloc, io, atomic.file);
+
             try atomic.link(io);
-
-    return ret;
-}
-fn extractSymlink(self: Inode, io: Io, path: []const u8) ExtractError!PathRet {
-    const target = switch (self.data) {
-        .symlink => |s| s.target,
-        .ext_symlink => |s| s.target,
-        else => unreachable,
-    };
-
-    try Io.Dir.cwd().symLink(io, target, path, .{});
-
-    return .{
-        .path = path,
-        .inode = self,
-    };
-}
-fn extractDevOrIPC(self: Inode, alloc: std.mem.Allocator, path: []const u8) ExtractError!PathRet {
-    var dev_num: u32 = 0;
-    var mode: u32 = 0;
+        },
+        .symlink, .ext_symlink => try Io.Dir.cwd().symLink(io, self.symlinkTarget() catch unreachable, path, .{}),
+        else => {
+            var mode: u32 = undefined;
+            var dev: u32 = 0;
 
             const DT = std.posix.DT;
 
-    var ret: PathRet = .{
-        .inode = self,
-        .path = path,
-    };
-
             switch (self.data) {
-        .block_dev => |d| {
-            dev_num = d.dev;
-            mode = DT.BLK;
-        },
-        .ext_block_dev => |d| {
-            dev_num = d.dev;
-            mode = DT.BLK;
-            if (d.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = d.xattr_idx;
-        },
                 .char_dev => |d| {
-            dev_num = d.dev;
+                    dev = d.dev;
                     mode = DT.CHR;
                 },
                 .ext_char_dev => |d| {
-            dev_num = d.dev;
+                    dev = d.dev;
                     mode = DT.CHR;
-            if (d.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = d.xattr_idx;
                 },
-        .fifo => mode = DT.FIFO,
-        .ext_fifo => |f| {
-            mode = DT.FIFO;
-            if (f.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = f.xattr_idx;
+                .block_dev => |d| {
+                    dev = d.dev;
+                    mode = DT.BLK;
                 },
-        .socket => mode = DT.SOCK,
-        .ext_socket => |s| {
-            mode = DT.SOCK;
-            if (s.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = s.xattr_idx;
+                .ext_block_dev => |d| {
+                    dev = d.dev;
+                    mode = DT.BLK;
                 },
+                .fifo, .ext_fifo => mode = DT.FIFO,
+                .socket, .ext_socket => mode = DT.SOCK,
                 else => unreachable,
             }
 
             const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
-    defer alloc.free(sentinel_path);
+            const res = std.os.linux.mknod(sentinel_path, mode, dev);
+            alloc.free(sentinel_path);
+            if (res != 0)
+                return ExtractError.MknodFailed;
+        },
+    }
+    return .{
+        .path = path,
+        .inode = self,
+        .origin = origin,
+    };
+}
+fn finishLoop(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, super: Archive.Superblock, options: ExtractionOptions, sel: *Io.Select(ExtractReturnUnion)) !void {
+    var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count);
+    defer id_table.deinit(io);
+
+    var xattr_table: ?XattrTable = if (super.flags.xattr_never or options.ignore_xattr or !@hasField(std.os, "linux"))
+        null
+    else
+        try .init(alloc, fil, decomp, super.xattr_start);
+    defer if (xattr_table != null) xattr_table.?.deinit(io);
+
+    var dir_queue: std.PriorityDequeue(PathRet, void, DirCompare) = .empty;
+    defer dir_queue.deinit(alloc);
+
+    while (true) {
+        if (sel.group.token.load(.unordered) == null) break;
+
+        const ret = try sel.await();
+        const path_ret = try ret.path_ret;
+
+        if (options.ignore_permissions and xattr_table == null) {
+            path_ret.deinit(alloc);
+            continue;
+        }
+
+        if (path_ret.inode.hdr.inode_type == .dir or path_ret.inode.hdr.inode_type == .ext_dir) {
+            try dir_queue.push(alloc, path_ret);
+            continue;
+        }
+        defer path_ret.deinit(alloc);
+
+        try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options);
+    }
+
+    while (sel.cancel()) |ret| {
+        const path_ret = try ret.path_ret;
+
+        if (options.ignore_permissions and xattr_table == null) {
+            path_ret.deinit(alloc);
+            continue;
+        }
+
+        if (path_ret.inode.hdr.inode_type == .dir or path_ret.inode.hdr.inode_type == .ext_dir) {
+            try dir_queue.push(alloc, path_ret);
+            continue;
+        }
+
+        defer path_ret.deinit(alloc);
+        try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options);
+    }
+
+    var iter = dir_queue.iterator();
+    while (iter.next()) |path_ret| {
+        defer path_ret.deinit(alloc);
+        try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options);
+    }
+}
+/// Extracts the given inode to the given path. If the inode not a directory, the given path must not exist.
+/// If the inode is a directory the path must not exist or be a directory.
+fn extractSinglethreaded(
+    self: Inode,
+    alloc: std.mem.Allocator,
+    io: Io,
+    fil: OffsetFile,
+    super: Archive.Superblock,
+    path: []const u8,
+    options: ExtractionOptions,
+    decomp: *Decompressor,
+    frag: *FragManager,
+) !void {
+    var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count);
+    defer id_table.deinit(io);
+
+    var xattr_table: ?XattrTable = if (super.flags.xattr_never or options.ignore_xattr or !@hasField(std.os, "linux"))
+        null
+    else
+        try .init(alloc, fil, decomp, super.xattr_start);
+    defer if (xattr_table != null) xattr_table.?.deinit(io);
+
+    return self.extractReal(
+        alloc,
+        io,
+        fil,
+        super,
+        decomp,
+        frag,
+        &id_table,
+        if (xattr_table == null) null else &xattr_table.?,
+        path,
+        options,
+    );
+}
+fn extractReal(
+    self: Inode,
+    alloc: std.mem.Allocator,
+    io: Io,
+    fil: OffsetFile,
+    super: Archive.Superblock,
+    decomp: *Decompressor,
+    frag_mgr: *FragManager,
+    id_table: *CachedTable(u16),
+    xattr_table: ?*XattrTable,
+    path: []const u8,
+    options: ExtractionOptions,
+) !void {
+    switch (self.hdr.inode_type) {
+        .dir, .ext_dir => {
+            try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777));
+
+            const entries = self.readDirectory(alloc, fil, decomp, super.dir_start) catch |err| switch (err) {
+                Error.NotDirectory, Error.NotExtended, Error.NotRegularFile, Error.NotSymlink => unreachable,
+                else => |e| return e,
+            };
+            defer {
+                for (entries) |e|
+                    e.deinit(alloc);
+                alloc.free(entries);
+            }
+
+            for (entries) |e| {
+                const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", e.name });
+                defer alloc.free(new_path);
+
+                var rdr = fil.readerAt(super.inode_start + e.block_start);
+                var meta: MetadataReader = .init(alloc, &rdr, decomp);
+                try meta.interface.discardAll(e.block_offset);
+
+                const new_inode = try read(alloc, &meta.interface, super.block_size);
+                defer new_inode.deinit(alloc);
+
+                try new_inode.extractReal(alloc, io, fil, super, decomp, frag_mgr, id_table, xattr_table, new_path, options);
+            }
+        },
+        .file, .ext_file => {
+            var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{ .make_path = true });
+            defer atomic.deinit(io);
+
+            var rdr: DataReader = switch (self.data) {
+                .file => |f| blk: {
+                    var ext: DataReader = try .init(alloc, io, fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
+                    if (f.frag_idx != 0xFFFFFFFF)
+                        ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
+                    break :blk ext;
+                },
+                .ext_file => |f| blk: {
+                    var ext: DataReader = try .init(alloc, io, fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
+                    if (f.frag_idx != 0xFFFFFFFF)
+                        ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
+                    break :blk ext;
+                },
+                else => unreachable,
+            };
+            defer rdr.deinit();
+
+            var buf: [512 * 1024]u8 = undefined;
+            var wrt = atomic.file.writer(io, &buf);
+
+            _ = try rdr.interface.streamRemaining(&wrt.interface);
+
+            try wrt.flush();
+
+            try atomic.link(io);
+        },
+        .symlink, .ext_symlink => try Io.Dir.cwd().symLink(io, self.symlinkTarget() catch unreachable, path, .{}),
+        else => {
+            var mode: u32 = undefined;
+            var dev: u32 = 0;
+
+            const DT = std.posix.DT;
+
+            switch (self.data) {
+                .char_dev => |d| {
+                    dev = d.dev;
+                    mode = DT.CHR;
+                },
+                .ext_char_dev => |d| {
+                    dev = d.dev;
+                    mode = DT.CHR;
+                },
+                .block_dev => |d| {
+                    dev = d.dev;
+                    mode = DT.BLK;
+                },
+                .ext_block_dev => |d| {
+                    dev = d.dev;
+                    mode = DT.BLK;
+                },
+                .fifo, .ext_fifo => mode = DT.FIFO,
+                .socket, .ext_socket => mode = DT.SOCK,
+                else => unreachable,
+            }
+
+            const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
+            const res = std.os.linux.mknod(sentinel_path, mode, dev);
+            alloc.free(sentinel_path);
+            if (res != 0)
+                return ExtractError.MknodFailed;
+        },
+    }
+    if (options.ignore_permissions and options.ignore_xattr) return;
+
+    var f = try Io.Dir.cwd().openFile(io, path, .{});
+    defer f.close(io);
+
+    if (!options.ignore_permissions) {
+        try f.setPermissions(io, @enumFromInt(self.hdr.permissions));
+        try f.setOwner(io, try id_table.get(io, self.hdr.uid_idx), try id_table.get(io, self.hdr.gid_idx));
+    }
+    if (xattr_table != null) {
+        const idx = self.xattrIndex() catch return;
 
-    const res = std.os.linux.mknod(sentinel_path, mode, dev_num);
-    if (res != 0) return ExtractError.MknodFailed;
+        const xattrs = try xattr_table.?.get(alloc, io, idx);
+        defer {
+            for (xattrs) |x|
+                x.deinit(alloc);
+            alloc.free(xattrs);
+        }
 
-    return ret;
+        const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
+        defer alloc.free(sentinel_path);
+        for (xattrs) |x| {
+            const xattr_ret = std.os.linux.fsetxattr(f.handle, x.key, x.value.ptr, x.value.len, 0);
+            if (xattr_ret != 0)
+                return ExtractError.CannotSetXattr;
+        }
+    }
 }

src/lookup_table.zig

@@ -5,18 +5,17 @@ const Decompressor = @import("util/decompressor.zig");
 const MetadataReader = @import("util/metadata.zig");
 const OffsetFile = @import("util/offset_file.zig");
 
-pub fn lookupValue(comptime T: anytype, alloc: std.mem.Allocator, io: Io, decomp: *const Decompressor, file: OffsetFile, table_start: u64, idx: u32) !T {
+pub fn lookupValue(comptime T: anytype, alloc: std.mem.Allocator, decomp: *Decompressor, file: OffsetFile, table_start: u64, idx: u32) !T {
     const T_PER_BLOCK: u16 = 8192 / @sizeOf(T);
 
     const block = idx / T_PER_BLOCK;
     const block_offset = idx % T_PER_BLOCK;
 
-    var rdr = try file.readerAt(io, table_start + (8 * block), &[0]u8{});
-    var offset: u64 = undefined;
-    try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
+    const offset_pos = table_start + (8 * block);
+    const offset: u64 = std.mem.readInt(u64, @ptrCast(file.map.memory[offset_pos .. offset_pos + 8]), .little);
 
-    rdr = try file.readerAt(io, offset, &[0]u8{});
-    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+    var rdr = file.readerAt(offset);
+    var meta: MetadataReader = .init(alloc, &rdr, decomp);
 
     try meta.interface.discardAll(@sizeOf(T) * block_offset);
     var out: T = undefined;
@@ -34,16 +33,16 @@ pub fn CachedTable(comptime T: anytype) type {
 
         alloc: std.mem.Allocator,
         fil: OffsetFile,
-        decomp: *const Decompressor,
+        decomp: *Decompressor,
 
         table_start: u64,
         total_num: u32,
 
         table: std.AutoHashMap(u32, []T),
 
-        mut: Io.Mutex = .init,
+        mut: Io.RwLock = .init,
 
-        pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, offset: u64, total_num: u32) Table {
+        pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, offset: u64, total_num: u32) Table {
             return .{
                 .alloc = alloc,
                 .fil = fil,
@@ -72,16 +71,15 @@ pub fn CachedTable(comptime T: anytype) type {
                 num_blocks += 1;
 
             for (0..num_blocks) |block| {
-                var rdr = try self.fil.readerAt(io, self.table_start + (8 * block), &[0]u8{});
-                var offset: u64 = undefined;
-                try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
+                const offset_pos = self.table_start + (8 * block);
+                const offset: u64 = std.mem.readInt(u64, @ptrCast(self.fil.map.memory[offset_pos .. offset_pos + 8]), .little);
 
                 const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK)
                     @truncate(self.total_num % T_PER_BLOCK)
                 else
                     T_PER_BLOCK;
 
-                rdr = try self.fil.readerAt(io, offset, &[0]u8{});
+                var rdr = self.fil.readerAt(offset);
                 var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
 
                 const slice = try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little);
@@ -92,8 +90,14 @@ pub fn CachedTable(comptime T: anytype) type {
         pub fn get(self: *Table, io: Io, idx: u32) Error!T {
             const block = idx / T_PER_BLOCK;
             const block_offset = idx % T_PER_BLOCK;
+
+            {
+                try self.mut.lockShared(io);
+                defer self.mut.unlockShared(io);
+
                 if (self.table.contains(block))
                     return self.table.get(block).?[block_offset];
+            }
 
             try self.mut.lock(io);
             defer self.mut.unlock(io);
@@ -101,20 +105,19 @@ pub fn CachedTable(comptime T: anytype) type {
             if (self.table.contains(block))
                 return self.table.get(block).?[block_offset];
 
-            var rdr = try self.fil.readerAt(io, self.table_start + (8 * block), &[0]u8{});
-            var offset: u64 = undefined;
-            try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
+            const offset_pos = self.table_start + (8 * block);
+            const offset: u64 = std.mem.readInt(u64, @ptrCast(self.fil.map.memory[offset_pos .. offset_pos + 8]), .little);
 
             const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK)
                 @truncate(self.total_num % T_PER_BLOCK)
             else
                 T_PER_BLOCK;
 
-            rdr = try self.fil.readerAt(io, offset, &[0]u8{});
-            var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
+            var rdr = self.fil.readerAt(offset);
+            var meta: MetadataReader = .init(self.alloc, &rdr, self.decomp);
 
             const slice = try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little);
-            try self.table.put(block, slice);
+            try self.table.put(@truncate(block), slice);
 
             return slice[block_offset];
         }

src/options.zig

@@ -5,8 +5,10 @@ const Writer = std.Io.Writer;
 
 const ExtractionOptions = @This();
 
-/// The number of threads used for extraction. 0 implies single threaded.
-threads: usize = 1, // TODO: Update to better integrate with zig 0.16 Io. Maybe limit to only single or multi-threaded.
+/// Extract single-threaded only.
+/// Though not necessary if using Threaded.single_threaded,
+/// setting single_threaded is more efficient.
+single_threaded: bool = false,
 /// Don't set the file's owner & permissions after extraction
 ignore_permissions: bool = false,
 /// Don't set xattr values. Currently xattrs are never set anyway.
@@ -18,16 +20,12 @@ verbose: bool = false,
 /// Where to print verbose log.
 verbose_writer: ?*Writer = null,
 
-pub const SingleThreadedDefault: ExtractionOptions = .{};
-pub fn Default() !ExtractionOptions {
-    return .{
-        .threads = try std.Thread.getCpuCount(),
-    };
-}
+pub const default: ExtractionOptions = .{};
+pub const default_single_threaded: ExtractionOptions = .{ .single_threaded = true };
+
 pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions {
     return .{
         .verbose = true,
         .verbose_writer = wrt,
-        .threads = try std.Thread.getCpuCount(),
     };
 }

src/root.zig

@@ -1,2 +1,6 @@
 pub const Archive = @import("archive.zig");
 pub const ExtractionOptions = @import("options.zig");
+
+test {
+    @import("std").testing.refAllDecls(@This());
+}

src/test.zig

@@ -1,89 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const io = std.testing.io;
-const alloc = std.testing.allocator;
-const stuff = @import("builtin");
-
-const Archive = @import("archive.zig");
-const Superblock = Archive.Superblock;
-
-const TestArchive = "testing/LinuxPATest.sfs";
-
-test "Basics" {
-    std.debug.print("Starting test: Basics...\n", .{});
-
-    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer fil.close(io);
-    var sfs: Archive = try .init(io, fil, 0);
-    try std.testing.expectEqualDeep(sfs.super, LinuxPATestCorrectSuperblock);
-    const root_file = try sfs.root(alloc, io);
-    defer root_file.deinit();
-}
-
-const TestFile = "Start.exe";
-const TestFileExtractLocation = "testing/Start.exe";
-
-test "ExtractSingleFile" {
-    std.debug.print("Starting test: ExtractSingleFile...\n", .{});
-
-    Io.Dir.cwd().deleteFile(io, TestFileExtractLocation) catch {};
-    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer fil.close(io);
-    var sfs: Archive = try .init(io, fil, 0);
-    var test_fil = try sfs.open(alloc, io, TestFile);
-    defer test_fil.deinit();
-    try test_fil.extract(alloc, io, TestFileExtractLocation, try .Default());
-    //TODO: validate extracted file.
-}
-
-const TestFullExtractLocation = "testing/TestExtract";
-
-test "ExtractCompleteArchive" {
-    std.debug.print("Starting test: ExtractCompleteArchive...\n", .{});
-
-    Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
-    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer fil.close(io);
-    var sfs: Archive = try .init(io, fil, 0);
-    try sfs.extract(alloc, io, TestFullExtractLocation, try .Default());
-}
-
-const LinuxPATestCorrectSuperblock: Superblock = .{
-    .magic = std.mem.readInt(u32, "hsqs", .little),
-    .inode_count = 2974,
-    .mod_time = 1632696724,
-    .block_size = 131072,
-    .frag_count = 264,
-    .compression = .zstd,
-    .block_log = 17,
-    .flags = .{
-        .inode_uncompressed = false,
-        .data_uncompressed = false,
-        .check = false,
-        .frag_uncompressed = false,
-        .fragment_never = false,
-        .fragment_always = false,
-        .duplicates = true,
-        .exportable = true,
-        .xattr_uncompressed = false,
-        .xattr_never = false,
-        .compression_options = false,
-        .ids_uncompressed = false,
-        ._ = 0,
-    },
-    .id_count = 1,
-    .ver_maj = 4,
-    .ver_min = 0,
-    .root_ref = .{
-        .block_offset = 1363,
-        .block_start = 29237,
-        ._ = 0,
-    },
-    .size = 106841744,
-    .id_start = 106841632,
-    .xattr_start = 106841720,
-    .inode_start = 106778274,
-    .dir_start = 106807998,
-    .frag_start = 106837747,
-    .export_start = 106841602,
-};

src/util/data_extractor.zig

@@ -10,12 +10,12 @@ const OffsetFile = @import("offset_file.zig");
 
 // const SharedCache = @import("shared_cache.zig");
 
-pub const Error = error{OutOfMemory} || Io.File.Reader.SeekError || Io.Writer.Error;
+pub const Error = Decompressor.Error || Io.File.MemoryMap.CreateError || Io.File.WritePositionalError;
 
 const DataExtractor = @This();
 
 fil: OffsetFile,
-decomp: *const Decompressor,
+decomp: *Decompressor,
 block_size: u32,
 
 file_size: u64,
@@ -23,11 +23,11 @@ start: u64,
 blocks: []BlockSize,
 
 frag_offset: u32 = 0,
-frag_entry: ?FragEntry = null,
+frag_block: ?[]u8 = null,
 
 err: ?Error = null,
 
-pub fn init(fil: OffsetFile, decomp: *const Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) DataExtractor {
+pub fn init(fil: OffsetFile, decomp: *Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) DataExtractor {
     return .{
         .fil = fil,
         .decomp = decomp,
@@ -38,17 +38,34 @@ pub fn init(fil: OffsetFile, decomp: *const Decompressor, block_size: u32, file_
         .blocks = blocks,
     };
 }
-pub fn addFrag(self: *DataExtractor, frag_offset: u32, entry: FragEntry) void {
+pub fn addFrag(self: *DataExtractor, frag_offset: u32, block: []u8) void {
     self.frag_offset = frag_offset;
-    self.frag_entry = entry;
+    self.frag_block = block;
 }
 
 fn numBlocks(self: DataExtractor) usize {
     var num = self.blocks.len;
-    if (self.frag_entry != null) num += 1;
+    if (self.frag_block != null) num += 1;
     return num;
 }
 
+/// Starts extracting the data using the given group to spawn async tasks.
+pub fn extractConcurrent(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.File) (Error || Io.ConcurrentError)!void {
+    var group: Io.Group = .init;
+    defer group.cancel(io);
+    var err: ?Error = null;
+
+    var read_offset: u64 = self.start;
+    for (0..self.blocks.len) |idx| {
+        try group.concurrent(io, blockThread, .{ self, alloc, io, fil, read_offset, idx, &err });
+        read_offset += self.blocks[idx].size;
+    }
+    if (self.frag_block != null)
+        try group.concurrent(io, fragThread, .{ self, io, fil, &err });
+
+    group.await(io) catch |cancel| return err orelse cancel;
+}
+
 /// Starts extracting the data using the given group to spawn async tasks.
 pub fn extractAsync(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.File) Error!void {
     var group: Io.Group = .init;
@@ -60,8 +77,8 @@ pub fn extractAsync(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil:
         group.async(io, blockThread, .{ self, alloc, io, fil, read_offset, idx, &err });
         read_offset += self.blocks[idx].size;
     }
-    if (self.frag_entry != null)
-        group.async(io, fragThread, .{ self, alloc, io, fil, &err });
+    if (self.frag_block != null)
+        group.async(io, fragThread, .{ self, io, fil, &err });
 
     group.await(io) catch |cancel| return err orelse cancel;
 }
@@ -74,13 +91,14 @@ fn blockThread(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.Fi
     else
         self.block_size;
 
+    const write_offset = self.block_size * idx;
+
     var wrt = fil.writer(io, &[0]u8{});
-    wrt.seekTo(self.block_size * idx) catch |err| {
+    wrt.seekTo(write_offset) catch |err| {
         ret_err.* = err;
         if (err == error.Canceled) io.recancel();
         return Io.Cancelable.Canceled;
     };
-    defer wrt.flush() catch {};
 
     if (block.size == 0) {
         wrt.interface.splatByteAll(0, cur_block_size) catch |err| {
@@ -88,37 +106,23 @@ fn blockThread(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.Fi
             if (err == error.Canceled) io.recancel();
             return Io.Cancelable.Canceled;
         };
-        return;
-    }
-
-    var rdr = self.fil.readerAt(io, read_offset, &[0]u8{}) catch |err| {
-        ret_err.* = err;
-        if (err == error.Canceled) io.recancel();
-        return Io.Cancelable.Canceled;
-    };
+    } else {
         if (block.uncompressed) {
-        rdr.interface.streamExact(&wrt.interface, cur_block_size) catch |err| {
+            wrt.interface.writeAll(self.fil.map.memory[read_offset..][0..cur_block_size]) catch |err| {
                 ret_err.* = err;
                 if (err == error.Canceled) io.recancel();
                 return Io.Cancelable.Canceled;
             };
-        return;
         } else {
             @branchHint(.likely);
 
-        var cache: [1024 * 1024]u8 = undefined;
             var tmp: [1024 * 1024]u8 = undefined;
 
-        rdr.interface.readSliceAll(cache[0..block.size]) catch |err| {
+            _ = self.decomp.Decompress(alloc, self.fil.map.memory[read_offset..][0..block.size], tmp[0..cur_block_size]) catch |err| {
                 ret_err.* = err;
-            if (err == error.Canceled) io.recancel();
-            return Io.Cancelable.Canceled;
-        };
-        _ = self.decomp.Decompress(alloc, cache[0..block.size], tmp[0..cur_block_size]) catch |err| {
-            ret_err.* = err;
-            if (err == error.Canceled) io.recancel();
                 return Io.Cancelable.Canceled;
             };
+
             wrt.interface.writeAll(tmp[0..cur_block_size]) catch |err| {
                 ret_err.* = err;
                 if (err == error.Canceled) io.recancel();
@@ -126,55 +130,33 @@ fn blockThread(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.Fi
             };
         }
     }
-fn fragThread(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.File, ret_err: *?Error) Io.Cancelable!void {
-    const frag = self.frag_entry.?;
+    wrt.flush() catch |err| {
+        ret_err.* = err;
+        if (err == error.Canceled) io.recancel();
+        return Io.Cancelable.Canceled;
+    };
+}
+fn fragThread(self: DataExtractor, io: Io, fil: Io.File, ret_err: *?Error) Io.Cancelable!void {
     const cur_block_size = self.file_size % self.block_size;
 
+    const write_offset = self.blocks.len * self.block_size;
+
     var wrt = fil.writer(io, &[0]u8{});
-    wrt.seekTo(self.blocks.len * self.block_size) catch |err| {
+    wrt.seekTo(write_offset) catch |err| {
         ret_err.* = err;
         if (err == error.Canceled) io.recancel();
         return Io.Cancelable.Canceled;
     };
-    defer wrt.flush() catch {};
 
-    var rdr = self.fil.readerAt(io, frag.start, &[0]u8{}) catch |err| {
+    wrt.interface.writeAll(self.frag_block.?[self.frag_offset..][0..cur_block_size]) catch |err| {
         ret_err.* = err;
         if (err == error.Canceled) io.recancel();
         return Io.Cancelable.Canceled;
     };
-    if (frag.size.uncompressed) {
-        rdr.interface.discardAll(self.frag_offset) catch |err| {
-            ret_err.* = err;
-            if (err == error.Canceled) io.recancel();
-            return Io.Cancelable.Canceled;
-        };
-        rdr.interface.streamExact(&wrt.interface, cur_block_size) catch |err| {
-            ret_err.* = err;
-            if (err == error.Canceled) io.recancel();
-            return Io.Cancelable.Canceled;
-        };
-        return;
-    } else {
-        @branchHint(.likely);
 
-        var cache: [1024 * 1024]u8 = undefined;
-        var tmp: [1024 * 1024]u8 = undefined;
-
-        rdr.interface.readSliceAll(cache[0..frag.size.size]) catch |err| {
-            ret_err.* = err;
-            if (err == error.Canceled) io.recancel();
-            return Io.Cancelable.Canceled;
-        };
-        _ = self.decomp.Decompress(alloc, cache[0..frag.size.size], tmp[0..self.block_size]) catch |err| {
-            ret_err.* = err;
-            if (err == error.Canceled) io.recancel();
-            return Io.Cancelable.Canceled;
-        };
-        wrt.interface.writeAll(tmp[self.frag_offset .. self.frag_offset + cur_block_size]) catch |err| {
+    wrt.flush() catch |err| {
         ret_err.* = err;
         if (err == error.Canceled) io.recancel();
         return Io.Cancelable.Canceled;
     };
 }
-}

src/util/data_reader.zig

@@ -6,7 +6,6 @@ const Reader = Io.Reader;
 const Writer = Io.Writer;
 const Limit = Io.Limit;
 
-const FragEntry = @import("../frag.zig").FragEntry;
 const BlockSize = @import("../inode_data/file.zig").BlockSize;
 const Decompressor = @import("decompressor.zig");
 const OffsetFile = @import("offset_file.zig");
@@ -19,8 +18,7 @@ alloc: std.mem.Allocator,
 
 fil: OffsetFile,
 io: Io,
-decomp: *const Decompressor,
-cache: *Io.Queue([]u8),
+decomp: *Decompressor,
 block_size: u32,
 
 file_size: u64,
@@ -28,21 +26,20 @@ cur_offset: u64,
 blocks: []BlockSize,
 
 frag_offset: u32 = 0,
-frag_entry: ?FragEntry = null,
+frag_block: ?[]u8 = null,
 
 block_idx: usize = 0,
 sparse_block: bool = false,
 
 interface: Io.Reader,
 
-pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, cache: *Io.Queue([]u8), block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) !DataReader {
+pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) !DataReader {
     return .{
         .alloc = alloc,
 
         .fil = fil,
         .io = io,
         .decomp = decomp,
-        .cache = cache,
         .block_size = block_size,
 
         .file_size = file_size,
@@ -64,43 +61,31 @@ pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const De
 pub fn deinit(self: *DataReader) void {
     self.alloc.free(self.interface.buffer);
 }
-pub fn addFrag(self: *DataReader, frag_offset: u32, entry: FragEntry) void {
+pub fn addFrag(self: *DataReader, frag_offset: u32, block: []u8) void {
     self.frag_offset = frag_offset;
-    self.frag_entry = entry;
+    self.frag_block = block;
 }
 
 fn numBlocks(self: DataReader) usize {
     var num = self.blocks.len;
-    if (self.frag_entry != null) num += 1;
+    if (self.frag_block != null) num += 1;
     return num;
 }
 fn advanceBuffer(self: *DataReader) !void {
-    if (self.block_idx >= self.numBlocks()) {
+    if (self.block_idx >= self.numBlocks())
         return Reader.Error.EndOfStream;
-    }
+
+    errdefer self.interface.end = 0;
     defer self.block_idx += 1;
 
     self.interface.end = if (self.block_idx == self.numBlocks() - 1)
-        self.size % self.block_size
+        self.file_size % self.block_size
     else
         self.block_size;
 
     // Fragment
     if (self.block_idx == self.blocks.len) {
-        const entry = self.frag_entry.?;
-        if (entry.size.uncompressed) {
-            var rdr = try self.fil.readerAt(self.io, entry.start + self.frag_offset, &[0]u8{});
-            try rdr.interface.readSliceAll(self.interface.buffer[0..self.interface.end]);
-        } else {
-            @branchHint(.likely);
-            const tmp = try self.cache.getOne(self.io);
-            defer self.cache.putOne(tmp) catch {};
-
-            var rdr = try self.fil.readerAt(self.io, entry.start, &[0]u8{});
-            try rdr.interface.readSliceAll(tmp.cache[0..entry.size.size]);
-            _ = try self.decomp.Decompress(self.alloc, tmp.cache[0..entry.size.size], self.interface.buffer[0..self.block_size]);
-            @memmove(self.interface.buffer[0..self.interface.end], self.interface.buffer[self.frag_offset .. self.frag_offset + self.interface.end]);
-        }
+        @memcpy(self.interface.buffer[0..self.interface.end], self.frag_block.?[self.frag_offset .. self.frag_offset + self.interface.end]);
         self.interface.seek = 0;
         return;
     }
@@ -115,17 +100,13 @@ fn advanceBuffer(self: *DataReader) !void {
         self.sparse_block = false;
     }
     if (block.uncompressed) {
-        try self.fil.readAt(self.io, self.cur_offset, self.interface.buffer[0..self.interface.end]);
+        @memcpy(self.interface.buffer[0..self.interface.end], self.fil.map.memory[self.cur_offset .. self.cur_offset + self.interface.end]);
         self.cur_offset += self.interface.end;
     } else {
         @branchHint(.likely);
-        const tmp = try self.cache.getOne(self.io);
-        defer self.cache.putOne(tmp) catch {};
 
-        var rdr = try self.fil.readerAt(self.io, self.cur_offset, &[0]u8{});
-        try rdr.interface.readSliceAll(tmp.cache[0..block.size]);
+        _ = try self.decomp.Decompress(self.alloc, self.fil.map.memory[self.cur_offset .. self.cur_offset + block.size], self.interface.buffer[0..self.interface.end]);
         self.cur_offset += block.size;
-        _ = try self.decomp.Decompress(self.alloc, tmp.cache[0..block.size], self.interface.buffer[0..self.interface.end]);
     }
     self.interface.seek = 0;
 }

src/util/decompressor.zig

@@ -8,8 +8,8 @@ pub const Error = std.Io.Reader.StreamError || std.mem.Allocator.Error;
 
 /// The actual decompression function.
 /// If the given decompressor is null, then the decompression should be done "stateless" without lasting allocations.
-decomp_fn: *const fn (?*const Decompressor, std.mem.Allocator, in: []u8, out: []u8) Error!usize,
+decomp_fn: *const fn (?*Decompressor, std.mem.Allocator, in: []u8, out: []u8) Error!usize,
 
-pub fn Decompress(self: *const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+pub fn Decompress(self: *Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
     return self.decomp_fn(self, alloc, in, out);
 }

src/util/metadata.zig

@@ -15,7 +15,7 @@ const This = @This();
 
 alloc: std.mem.Allocator,
 rdr: *Reader,
-decomp: *const Decompressor,
+decomp: *Decompressor,
 
 cur_block_start: u32 = 0,
 next_start_start: u32 = 0,
@@ -34,7 +34,7 @@ interface: Reader = .{
     },
 },
 
-pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: *const Decompressor) This {
+pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: *Decompressor) This {
     return .{
         .alloc = alloc,
         .rdr = rdr,

src/util/misc.zig

@@ -16,9 +16,9 @@ pub fn pathIsSelf(path: []const u8) bool {
     return path[0] == '.';
 }
 /// Creates an Inode from an Inode.Ref.
-pub fn inodeFromRef(alloc: std.mem.Allocator, io: Io, file: OffsetFile, decomp: *const Decompressor, inode_start: u64, block_size: u32, ref: Inode.Ref) !Inode {
-    var rdr = try file.readerAt(io, inode_start + ref.block_start, &[0]u8{});
-    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+pub fn inodeFromRef(alloc: std.mem.Allocator, file: OffsetFile, decomp: *Decompressor, inode_start: u64, block_size: u32, ref: Inode.Ref) !Inode {
+    var rdr = file.readerAt(inode_start + ref.block_start);
+    var meta: MetadataReader = .init(alloc, &rdr, decomp);
     try meta.interface.discardAll(ref.block_offset);
 
     return .read(alloc, &meta.interface, block_size);

src/util/offset_file.zig

@@ -3,22 +3,25 @@
 const std = @import("std");
 const Io = std.Io;
 const File = Io.File;
-const Reader = File.Reader;
+const Reader = Io.Reader;
 
 const OffsetFile = @This();
 
-fil: File,
-offset: u64,
+map: Io.File.MemoryMap,
 
-pub fn init(fil: File, init_offset: u64) OffsetFile {
-    return .{ .fil = fil, .offset = init_offset };
+pub fn init(io: Io, fil: File, archive_size: u64, init_offset: u64) !OffsetFile {
+    return .{
+        .map = try fil.createMemoryMap(io, .{
+            .protection = .{ .read = true, .write = false, .execute = false },
+            .len = archive_size,
+            .offset = init_offset,
+        }),
+    };
+}
+pub fn deinit(self: *OffsetFile, io: Io) void {
+    self.map.destroy(io);
 }
 
-pub fn readerAt(self: OffsetFile, io: Io, offset: u64, buffer: []u8) Reader.SeekError!Reader {
-    var rdr = self.fil.reader(io, buffer);
-    try rdr.seekTo(self.offset + offset);
-    return rdr;
-}
-pub fn readAt(self: OffsetFile, io: Io, offset: u64, buf: []u8) File.ReadPositionalError!void {
-    _ = try self.fil.readPositionalAll(io, buf, self.offset + offset);
+pub fn readerAt(self: OffsetFile, offset: u64) Reader {
+    return .fixed(self.map.memory[offset..]);
 }

src/util/shared_cache.zig

@@ -1,52 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Node = std.SinglyLinkedList.Node;
-
-const SharedCache = @This();
-
-pub const CACHE_SIZE = 1024 * 1024;
-
-pub const BufferNode = struct {
-    node: Node,
-    cache: [CACHE_SIZE]u8,
-};
-
-alloc: std.mem.Allocator,
-
-caches: std.ArrayList(BufferNode),
-cache_queue: std.SinglyLinkedList,
-queue_mut: Io.Mutex,
-
-pub fn init(alloc: std.mem.Allocator, init_cache_size: u32) !SharedCache {
-    const caches: std.ArrayList(BufferNode) = try .initCapacity(alloc, init_cache_size);
-    var queue: std.SinglyLinkedList = .{};
-    for (caches.items) |item|
-        queue.prepend(&item.node);
-    return .{
-        .alloc = alloc,
-
-        .caches = caches,
-        .cache_queue = queue,
-    };
-}
-pub fn deinit(self: *SharedCache) void {
-    self.caches.deinit(self.alloc);
-}
-
-pub fn getCache(self: *SharedCache, io: Io) !*BufferNode {
-    self.queue_mut.lock(io);
-    const nxt = self.cache_queue.popFirst();
-    self.queue_mut.unlock(io);
-    if (nxt == null) {
-        const new = try self.caches.addOne(self.alloc);
-        new.* = .{
-            .node = .{},
-            .cache = undefined,
-        };
-        return new;
-    }
-    return @fieldParentPtr("node", nxt.?);
-}
-pub fn returnCache(self: *SharedCache, buf: *BufferNode) void {
-    self.cache_queue.prepend(buf);
-}

src/xattr_table.zig

@@ -12,21 +12,17 @@ const XattrCachedTable = @This();
 alloc: std.mem.Allocator,
 
 fil: OffsetFile,
-decomp: *const Decompressor,
+decomp: *Decompressor,
 
 kv_start: u64,
 
 table: LookupTable.CachedTable(TableValue),
 value_cache: std.AutoHashMap(InodeRef, []const u8),
-value_mut: Io.Mutex = .init,
+value_mut: Io.RwLock = .init,
 
-pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, xattr_start: u64) !XattrCachedTable {
-    var rdr = try fil.readerAt(io, xattr_start, &[0]u8{});
-
-    var start: u64 = undefined;
-    try rdr.interface.readSliceEndian(u64, @ptrCast(&start), .little);
-    var num: u32 = undefined;
-    try rdr.interface.readSliceEndian(u32, @ptrCast(&num), .little);
+pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, xattr_start: u64) !XattrCachedTable {
+    const start: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[xattr_start .. xattr_start + 8]), .little);
+    const num: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[xattr_start + 8 .. xattr_start + 16]), .little);
 
     return .{
         .alloc = alloc,
@@ -41,6 +37,7 @@ pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const De
     };
 }
 pub fn deinit(self: *XattrCachedTable, io: Io) void {
+    self.value_mut.lockUncancelable(io);
     self.table.deinit(io);
     self.value_cache.deinit();
 }
@@ -48,8 +45,8 @@ pub fn deinit(self: *XattrCachedTable, io: Io) void {
 pub fn get(self: *XattrCachedTable, alloc: std.mem.Allocator, io: Io, idx: u32) ![]XattrSemiOwned {
     const lookup = try self.table.get(io, idx);
 
-    var rdr = try self.fil.readerAt(io, self.kv_start + lookup.ref.block_start, &[0]u8{});
-    var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
+    var rdr = self.fil.readerAt(self.kv_start + lookup.ref.block_start);
+    var meta: MetadataReader = .init(alloc, &rdr, self.decomp);
     try meta.interface.discardAll(lookup.ref.block_offset);
 
     const out = try alloc.alloc(XattrSemiOwned, lookup.count);
@@ -99,8 +96,20 @@ pub fn get(self: *XattrCachedTable, alloc: std.mem.Allocator, io: Io, idx: u32)
         }
         const val_ref: InodeRef = .{ .block_start = meta.cur_block_start, .block_offset = @truncate(meta.interface.seek) };
 
+        {
+            try self.value_mut.lockShared(io);
+            defer self.value_mut.unlockShared(io);
+            if (self.value_cache.contains(val_ref)) {
+                out[i] = .{
+                    .key = key,
+                    .value = try self.valueAt(io, val_ref),
+                };
+                continue;
+            }
+        }
         try self.value_mut.lock(io);
         defer self.value_mut.unlock(io);
+
         if (self.value_cache.contains(val_ref)) {
             out[i] = .{
                 .key = key,
@@ -131,8 +140,8 @@ fn valueAt(self: *XattrCachedTable, io: Io, ref: InodeRef) ![]const u8 {
 
     if (self.value_cache.contains(ref)) return self.value_cache.get(ref).?;
 
-    var rdr = try self.fil.readerAt(io, self.kv_start + ref.block_start, &[0]u8{});
-    var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
+    var rdr = self.fil.readerAt(self.kv_start + ref.block_start);
+    var meta: MetadataReader = .init(self.alloc, &rdr, self.decomp);
     try meta.interface.discardAll(ref.block_offset);
 
     var val_size: u32 = undefined;
@@ -203,15 +212,12 @@ const XattrPrefix = packed struct(u16) {
 
 // Stateless
 
-pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *const Decompressor, fil: OffsetFile, table_start: u64, idx: u16) ![]XattrOwned {
-    var rdr = try fil.readerAt(io, table_start, &[0]u8{});
-
-    var kv_start: u64 = undefined;
-    try rdr.interface.readSliceEndian(u64, @ptrCast(&kv_start), .little);
+pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *Decompressor, fil: OffsetFile, table_start: u64, idx: u16) ![]XattrOwned {
+    const kv_start: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[table_start .. table_start + 8]), .little);
 
     const lookup = try LookupTable.lookupValue(TableValue, alloc, io, decomp, fil, table_start + 16, idx);
 
-    rdr = try fil.readerAt(io, kv_start + lookup.ref.block_start, &[0]u8{});
+    var rdr = fil.readerAt(kv_start + lookup.ref.block_start);
     var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
     try meta.interface.discardAll(lookup.ref.block_offset);
 
@@ -252,7 +258,7 @@ pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *const Decompre
             const value: ValueOutOfLineEntry = undefined;
             try meta.interface.readSliceEndian(ValueOutOfLineEntry, @ptrCast(&value), .little);
 
-            var ool_rdr = try fil.readerAt(io, kv_start + value.ref.block_start, &[0]u8{});
+            var ool_rdr = fil.readerAt(kv_start + value.ref.block_start);
             var ool_meta: MetadataReader = .init(alloc, &ool_rdr.interface, decomp);
             try ool_meta.interface.discardAll(value.ref.block_offset);