Further work tweaking decompression

Added FragManager so each frag block only gets decompressed once
Returned to C for decompression (only zstd stateless ATM)

.zed/debug.json

@@ -22,25 +22,4 @@
 			"testing/LinuxPATest.sfs",
 		],
 	},
-	{
-		"label": "Build & Run Single-Threaded",
-
-		"adapter": "CodeLLDB",
-		"request": "launch",
-
-		"build": {
-			"command": "zig",
-			"args": ["build", "-Ddebug=true"],
-		},
-
-		"program": "zig-out/bin/unsquashfs",
-		"args": [
-			"--force",
-			"-p",
-			"1",
-			"-d",
-			"testing/TestExtractUnsquashfs",
-			"testing/LinuxPATest.sfs",
-		],
-	},
 ]

benchmark.sh

@@ -1,22 +0,0 @@
-#! /usr/bin/env bash
-
-ARCHIVE="testing/LinuxPATest.sfs"
-
-REF_EXT_LOC="testing/LinuxPAReference"
-PROG_EXT_LOC="testing/LinuxPABinTest"
-
-echo "Building with ReleaseFast"
-echo ""
-
-zig build -Doptimize=ReleaseFast
-
-echo "Testing Multi-threaded Performance"
-echo ""
-
-hyperfine --warmup 5 --prepare "rm -rf $REF_EXT_LOC && rm -rf $PROG_EXT_LOC" "unsquashfs -d $REF_EXT_LOC $ARCHIVE" "zig-out/bin/unsquashfs -d $PROG_EXT_LOC $ARCHIVE"
-
-echo ""
-echo "Testing Single-threaded Performance"
-echo ""
-
-hyperfine --warmup 5 --prepare "rm -rf $REF_EXT_LOC && rm -rf $PROG_EXT_LOC" "unsquashfs -p 1 -d $REF_EXT_LOC $ARCHIVE" "zig-out/bin/unsquashfs -p 1 -d $PROG_EXT_LOC $ARCHIVE"

build.zig

@@ -1,125 +1,93 @@
 const std = @import("std");
 
 pub fn build(b: *std.Build) !void {
-    const optimize = b.standardOptimizeOption(.{});
+    // const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
+    // const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
+    const debug = b.option(bool, "debug", "Enable options to make debugging easier.");
+    const version_string_option = b.option([]const u8, "version", "Version of the library/binary");
+
+    // const zig_squashfs_options = b.addOptions();
+    // zig_squashfs_options.addOption(bool, "use_zig_decomp", use_zig_decomp);
+    // zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo);
+
     const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
 
-    const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
-    const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
-    const dynamic = b.option(bool, "dynamic", "Use dynamic instead of static linking.") orelse false;
-    const debug = b.option(bool, "debug", "Enable options to make debugging easier.") orelse false;
-    const version_string_option = b.option([]const u8, "version", "Version of the library/binary") orelse "0.0.0-testing";
+    const lib = b.addLibrary(.{
+        .name = "squashfs",
+        .root_module = b.createModule(.{
+            .optimize = if (debug == true) .Debug else optimize,
+            .target = target,
+            .valgrind = debug,
+            .root_source_file = b.path("src/root.zig"),
+        }),
+        .use_llvm = debug,
+    });
 
-    const version: std.SemanticVersion = try .parse(version_string_option);
-
-    const options = b.addOptions();
-    options.addOption(bool, "use_zig_decomp", use_zig_decomp);
-    options.addOption(bool, "allow_lzo", allow_lzo);
-    options.addOption(std.SemanticVersion, "version", version);
+    const zstd = b.dependency("zstd", .{ .optimize = optimize, .target = target });
+    lib.root_module.linkLibrary(zstd.artifact("zstd"));
 
     const c = b.addTranslateC(.{
         .optimize = optimize,
         .target = target,
-        .root_source_file = b.path("c.h"),
+        .root_source_file = b.path("src/c.h"),
     });
-    if (allow_lzo) c.defineCMacro("ALLOW_LZO", null);
-
-    if (!use_zig_decomp and dynamic) {
-        c.linkSystemLibrary("z", .{});
-        c.linkSystemLibrary("lzma", .{});
-        c.linkSystemLibrary("lz4", .{});
-        c.linkSystemLibrary("zstd", .{});
-        if (allow_lzo)
-            c.linkSystemLibrary("minilzo", .{});
-    }
-
-    const lib = b.addLibrary(.{
-        .name = "squashfs",
-        .use_llvm = debug,
-        .version = version,
-        .root_module = b.createModule(.{
-            .root_source_file = b.path("src/root.zig"),
-            .optimize = optimize,
-            .target = target,
-            .valgrind = debug,
-            .imports = &.{
-                .{ .name = "c", .module = c.createModule() },
-                .{ .name = "build_options", .module = options.createModule() },
-            },
-        }),
-    });
-    if (!use_zig_decomp and !dynamic) {
-        const zng = b.dependency("zlib_ng", .{ .optimize = optimize, .target = target, .zlib_compat = true });
-        lib.root_module.linkLibrary(zng.artifact("zng"));
-
-        const zstd = b.dependency("zstd", .{ .optimize = optimize, .target = target });
-        lib.root_module.linkLibrary(zstd.artifact("zstd"));
-
-        const lz4 = b.dependency("lz4", .{ .optimize = optimize, .target = target });
-        lib.root_module.linkLibrary(lz4.artifact("lz4"));
-
-        const xz = b.dependency("xz", .{ .optimize = optimize, .target = target });
-        lib.root_module.linkLibrary(xz.artifact("lzma"));
-
-        if (allow_lzo) {
-            const lzo = b.dependency("minilzo", .{ .optimize = optimize, .target = target });
-            lib.root_module.linkLibrary(lzo.artifact("minilzo"));
-        }
-    }
-
-    b.installArtifact(lib);
+    lib.root_module.addImport("c", c.createModule());
 
+    var version = version_string_option orelse "0.0.0-testing";
+    if (version[0] == 'v') version = version[1..];
     const unsquashfs_options = b.addOptions();
-    unsquashfs_options.addOption(std.SemanticVersion, "version", version);
-
+    unsquashfs_options.addOption(
+        std.SemanticVersion,
+        "version",
+        try std.SemanticVersion.parse(version),
+    );
     const exe = b.addExecutable(.{
         .name = "unsquashfs",
-        .use_llvm = debug,
-        .version = version,
         .root_module = b.createModule(.{
-            .root_source_file = b.path("src/bin/unsquashfs.zig"),
-            .optimize = optimize,
+            .optimize = if (debug == true) .Debug else optimize,
             .target = target,
             .valgrind = debug,
+            .root_source_file = b.path("src/bin/unsquashfs.zig"),
             .imports = &.{
-                .{ .name = "squashfs", .module = lib.root_module },
-                .{ .name = "build", .module = unsquashfs_options.createModule() },
+                .{ .name = "zig_squashfs", .module = lib.root_module },
             },
         }),
+        .use_llvm = debug,
     });
+    exe.root_module.addOptions("config", unsquashfs_options);
 
+    b.installArtifact(lib);
     b.installArtifact(exe);
 
-    const lib_tests = b.addTest(.{
-        .root_module = lib.root_module,
-        .use_llvm = debug,
+    const mod_tests = b.addTest(.{
+        .root_module = b.createModule(.{
+            .optimize = .Debug,
+            .target = target,
+            .root_source_file = b.path("src/test.zig"),
+            .imports = &.{
+                .{ .name = "c", .module = c.createModule() },
+            },
+            .valgrind = true,
+        }),
+        .use_llvm = true,
     });
-    const run_lib_tests = b.addRunArtifact(lib_tests);
-    const exe_tests = b.addTest(.{
-        .root_module = exe.root_module,
-        .use_llvm = debug,
-    });
-    const run_exe_tests = b.addRunArtifact(exe_tests);
+    mod_tests.root_module.linkLibrary(zstd.artifact("zstd"));
 
+    const run_mod_tests = b.addRunArtifact(mod_tests);
     const test_step = b.step("test", "Run tests");
-    test_step.dependOn(&run_lib_tests.step);
-    test_step.dependOn(&run_exe_tests.step);
+    test_step.dependOn(&run_mod_tests.step);
 
     // zls build check steps
     const lib_check = b.addLibrary(.{
         .name = "squashfs",
-        .root_module = lib.root_module,
-        .use_llvm = debug,
+        .root_module = exe.root_module,
     });
     const exe_check = b.addExecutable(.{
         .name = "unsquashfs",
-        .root_module = exe.root_module,
-        .use_llvm = debug,
+        .root_module = lib.root_module,
     });
     const check = b.step("check", "Check if unsquashfs compiles");
     check.dependOn(&lib_check.step);
     check.dependOn(&exe_check.step);
-
-    test_step.dependOn(&lib_check.step);
-    test_step.dependOn(&exe_check.step);
 }

build.zig.zon

@@ -2,11 +2,11 @@
     .name = .squashfs,
     .version = "0.0.6",
     .fingerprint = 0x37ba29474b87f145, // Changing this has security and trust implications.
-    .minimum_zig_version = "0.16.0",
+    .minimum_zig_version = "0.15.2",
     .dependencies = .{
         .zlib_ng = .{
-            .url = "git+https://github.com/CalebQ42/zig-zlib-ng#915db2dbd4a540d8a933336d67520b786f168f01",
-            .hash = "zlib_ng-2.3.3-pre1-2HYS4MlFAADc-eSZv4_xjkxZRCdGUVOYZKdtqhybstgk",
+            .url = "git+https://github.com/CalebQ42/zig-zlib-ng#5f2f02dfb28acca2517dacbbd09e9b987f57b133",
+            .hash = "zlib_ng-2.3.3-pre1-2HYS4ClFAABW8KlHMyBHtlNKE3V7kCS8wqfxawG7xeaa",
         },
         .zstd = .{
             .url = "git+https://github.com/allyourcodebase/zstd.git?ref=1.5.7-1#e1a501be57f42c541e8a5597e4b59a074dfd09a3",
@@ -16,13 +16,9 @@
             .url = "git+https://github.com/allyourcodebase/lz4.git?ref=1.10.0-6#41f52ab227caf9d48cf88c89a4d2946caa12b102",
             .hash = "lz4-1.10.0-6-ewyzw-4NAAAWDpY4xpiqr4LQhZQAC0x_rGnW2iPh6jk2",
         },
-        .xz = .{
-            .url = "git+https://github.com/akunaakwei/zig-xz.git#e2d389262c8291907e3e4c6fb119819141c16c0f",
-            .hash = "xz-5.8.2-6v47_JYeAABSL-jonprpL5-E_YaaGc4B5xrbe93WsJ3G",
-        },
         .minilzo = .{
-            .url = "git+https://github.com/CalebQ42/zig-minilzo.git#f72a0e20a30fad38cd10104f393c8f3440478697",
-            .hash = "minilzo-2.10.0-Ij7BO7YLAABd6oK2YdimXwvFTOC8wD1sNm0PHGoRNEwR",
+            .url = "git+https://github.com/CalebQ42/zig-minilzo.git#7cbae997b91a44d74b7cd6c073584dc9562a6c90",
+            .hash = "minilzo-2.10.0-Ij7BO8wLAADeWI4Pe4jp8XTDsDaquZR14oZ7_9yKKDWP",
         },
     },
     .paths = .{

c.h

@@ -1,8 +0,0 @@
-#include <zlib.h>
-#include <lzma.h>
-#include <lz4.h>
-#include <zstd.h>
-
-#ifdef ALLOW_LZO
-#include <lzo/minilzo.h>
-#endif

run_tests.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+zig test \
+	-lc \
+	-lz \
+	-llzma \
+	-lminilzo \
+	-llz4 \
+	-lzstd \
+		src/test.zig

src/archive.zig

@@ -1,73 +1,107 @@
 const std = @import("std");
 const Io = std.Io;
-const MemoryMap = Io.File.MemoryMap;
 
 const Decomp = @import("decomp.zig");
-const Inode = @import("inode.zig");
-const File = @import("file.zig");
 const ExtractionOptions = @import("options.zig");
-const Extract = @import("extract.zig");
+const File = @import("file.zig");
+const Inode = @import("inode.zig");
+const LookupTable = @import("lookup_table.zig");
+const Decompressor = @import("util/decompressor.zig");
+const MetadataReader = @import("util/metadata.zig");
+const Utils = @import("util/misc.zig");
+const OffsetFile = @import("util/offset_file.zig");
 
 const Archive = @This();
 
+file: OffsetFile,
 super: Superblock,
 
-map: MemoryMap,
-decomp: Decomp.Fn,
+stateless_decomp: *const Decompressor,
 
-pub fn init(io: Io, file: Io.File, offset: u64) !Archive {
+pub fn init(io: Io, file: std.Io.File, offset: u64) !Archive {
     var rdr = file.reader(io, &[0]u8{});
     try rdr.seekTo(offset);
-
     var super: Superblock = undefined;
     try rdr.interface.readSliceEndian(Superblock, @ptrCast(&super), .little);
-
-    try super.validate();
-
     return .{
+        .file = .init(file, offset),
         .super = super,
 
-        .map = try file.createMemoryMap(io, .{
-            .offset = offset,
-            .len = super.size,
-            .protection = .{ .read = true },
-        }),
-        .decomp = try Decomp.getFn(super.compression),
+        .stateless_decomp = try Decomp.StatelessDecomp(super.compression),
     };
 }
-pub fn deinit(self: *Archive, io: Io) void {
-    self.map.destroy(io);
-}
 
-pub fn root(self: Archive, alloc: std.mem.Allocator) !File {
-    return .initRef(alloc, self.super, self.map.memory, self.decomp, "", self.super.root_ref);
-}
-
-pub fn open(self: Archive, alloc: std.mem.Allocator, filepath: []const u8) !File {
-    const root_file = try self.root(alloc);
-
-    const path = std.mem.trim(u8, filepath, "/");
-
-    if (path.len == 0 or (path.len == 1 and path[0] == '.'))
-        return root_file;
-
-    defer root_file.deinit();
-
-    return root_file.open(alloc, filepath);
-}
-
-pub fn extract(self: Archive, alloc: std.mem.Allocator, io: Io, location: []const u8, options: ExtractionOptions) !void {
-    const root_inode: Inode = try .initRef(
+/// The root folder of the Archive. Used to open other Files.
+pub fn root(self: Archive, alloc: std.mem.Allocator, io: Io) !File {
+    const root_inode = try Utils.inodeFromRef(
         alloc,
-        self.map.memory,
-        self.decomp,
+        io,
+        self.file,
+        self.stateless_decomp,
         self.super.inode_start,
         self.super.block_size,
         self.super.root_ref,
     );
-    defer root_inode.deinit(alloc);
+    return .init(alloc, self, root_inode, "");
+}
+/// Opens a File within the archive.
+pub fn open(self: Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
+    const root_file = try self.root(alloc, io);
+    const path = std.mem.trim(u8, filepath, "/");
+    if (Utils.pathIsSelf(path))
+        return root_file;
+    defer root_file.deinit();
+    return root_file.open(alloc, io, filepath);
+}
+/// Extract the entire archive contents to the given directory.
+pub fn extract(self: Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void {
+    const root_inode = try Utils.inodeFromRef(
+        alloc,
+        io,
+        self.file,
+        self.stateless_decomp,
+        self.super.inode_start,
+        self.super.block_size,
+        self.super.root_ref,
+    );
+    return root_inode.extract(alloc, io, self.file, self.super, extract_dir, options);
+}
 
-    return root_inode.extract(alloc, io, self.super, self.map.memory, self.decomp, location, options);
+/// Returns the inode with the given inode number.
+/// Requires that the archive is exportable (has an export lookup table).
+pub fn inode(self: Archive, alloc: std.mem.Allocator, io: Io, num: u32) !Inode {
+    if (!self.super.flags.exportable)
+        return error.NotExportable;
+    const ref = try LookupTable.lookupValue(
+        Inode.Ref,
+        alloc,
+        io,
+        &self.stateless_decomp,
+        self.file,
+        self.super.export_start,
+        num + 1,
+    );
+    return Utils.inodeFromRef(
+        alloc,
+        io,
+        self.file,
+        &self.stateless_decomp,
+        self.super.inode_start,
+        self.super.block_size,
+        ref,
+    );
+}
+/// Returns a value at the given index from the Archive's id (uid/gid) table.
+pub fn idTable(self: Archive, alloc: std.mem.Allocator, io: Io, idx: u32) !u16 {
+    return LookupTable.lookupValue(
+        u16,
+        alloc,
+        io,
+        &self.stateless_decomp,
+        self.file,
+        self.super.id_start,
+        idx,
+    );
 }
 
 // Superblock

src/bin/unsquashfs.zig

@@ -3,8 +3,8 @@ const Io = std.Io;
 const Writer = Io.Writer;
 const builtin = @import("builtin");
 
-const build = @import("build");
-const squashfs = @import("squashfs");
+const config = @import("config");
+const squashfs = @import("zig_squashfs");
 
 //TODO: Add more options
 const help_mgs =
@@ -13,14 +13,12 @@ const help_mgs =
     \\
     \\Options:
     \\  -d <location>   Extract to the given location instead of "squashfs-root"
-    \\  -f <filepath>   Extract the files at the given filepath instead of the entire archive
-    \\                     Can be given multiple times.
     \\
     \\  -o <offset>     Start reading the archive at the given offset.
     \\  -dx             Don't set xattr values
     \\  -dp             Don't set permissions (includes setting uid & gid owner)
     \\
-    \\  -p <threads>    Specify how many threads to use. If no present or zero, the system's logical cores count is used.
+    \\  -p <threads>    Specify how many threads to use. If not present or zero, the system's logical cores count is used.
     \\  -v              Verbose
     \\
     \\  --force         Force extraction. If the destination already exists, it will be deleted.
@@ -33,87 +31,56 @@ const help_mgs =
 const errors = error{InvalidArguments};
 
 var archive: []const u8 = "";
-var ext_loc: []const u8 = "squashfs-root";
+var extLoc: []const u8 = "squashfs-root";
 var offset: u64 = 0;
 var threads: u32 = 0;
 var verbose: bool = false;
 var ignore_xattrs: bool = false;
 var ignore_permissions: bool = false;
 var force: bool = false;
-var files: [][:0]const u8 = &[0][:0]const u8{};
-
-var limited: ?Io.Threaded = null;
 
 pub fn main(init: std.process.Init) !void {
-    var io = init.io;
     const alloc = init.gpa;
+    const io = init.io;
 
-    var stdout = Io.File.stdout();
+    var stdout = std.Io.File.stdout();
+    defer stdout.close(io);
     var out = stdout.writer(io, &[0]u8{});
     defer out.interface.flush() catch {};
 
-    try handleArgs(alloc, init.minimal.args, &out.interface);
-    if (archive.len == 0) {
-        try out.interface.print("You must provide a squashfs archive\n", .{});
-        try out.interface.print(help_mgs, .{});
-        return;
-    }
+    // try handleArgs(init.minimal.args, &out.interface);
+    // if (archive.len == 0) {
+    //     try out.interface.print("You must provide a squashfs archive\n", .{});
+    //     try out.interface.print(help_mgs, .{});
+    //     return;
+    // }
+    archive = "testing/LinuxPATest.sfs";
+    extLoc = "testing/LinuxPABinTest";
 
     var fil = try Io.Dir.cwd().openFile(io, archive, .{}); //TODO: Handle error gracefully.
     defer fil.close(io);
 
-    var arc: squashfs.Archive = try .init(io, fil, offset); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
-    defer arc.deinit(io);
+    var arc: squashfs.Archive = try .init(io, fil, offset); //TODO: Handle error gracefully.
     const options: squashfs.ExtractionOptions = .{
-        .single_threaded = (threads == 1),
+        .threads = if (threads == 0) try std.Thread.getCpuCount() else threads,
         .verbose = verbose,
         .verbose_writer = if (verbose) &out.interface else null,
         .ignore_xattr = ignore_xattrs,
         .ignore_permissions = ignore_permissions,
     };
 
-    if (threads > 1) {
-        limited = Io.Threaded.init(alloc, .{
-            .argv0 = .init(init.minimal.args),
-            .async_limit = @enumFromInt(threads),
-            .concurrent_limit = @enumFromInt(threads),
-            .environ = init.minimal.environ,
-        });
-        io = limited.?.io();
-    }
-
     if (force)
-        try Io.Dir.cwd().deleteTree(io, ext_loc);
-
-    if (files.len > 0) {
-        for (0..files.len) |i| {
-            var sfs_fil = try arc.open(alloc, files[i]);
-            defer sfs_fil.deinit();
-
-            const last_ind = std.mem.lastIndexOf(u8, files[i], "/");
-            const file_name = if (last_ind != null) files[i][last_ind.?..] else files[i];
-
-            const fil_ext_loc = if (ext_loc.len > 0)
-                try std.mem.concat(alloc, u8, &.{ ext_loc, "/", file_name })
-            else
-                file_name;
-
-            defer if (fil_ext_loc.len != file_name.len) alloc.free(fil_ext_loc);
-
-            try sfs_fil.extract(alloc, io, fil_ext_loc, options);
-        }
-    } else {
-        try arc.extract(alloc, io, ext_loc, options); //TODO: Handle error gracefully.
-    }
+        try Io.Dir.cwd().deleteTree(io, extLoc);
+    try arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully.
 }
 
-fn handleArgs(alloc: std.mem.Allocator, main_args: std.process.Args, out: *Writer) !void {
-    var args = main_args.iterate();
-    defer args.deinit();
-    _ = args.next(); // args[0] is the application launch command.
-    while (args.next()) |arg| {
+fn handleArgs(args: std.process.Args, out: *Writer) !void {
+    var arg_iter = args.iterate();
+    defer arg_iter.deinit();
+    _ = arg_iter.next(); // args[0] is the application launch command.
+    while (arg_iter.next()) |arg| {
         if (std.mem.eql(u8, arg, "-o")) {
-            const nxt = args.next();
+            const nxt = arg_iter.next();
             if (nxt == null or nxt.?.len == 0) {
                 try out.print("-o must be followed by a number\n", .{});
                 return errors.InvalidArguments;
@@ -123,32 +90,16 @@ fn handleArgs(alloc: std.mem.Allocator, main_args: std.process.Args, out: *Write
                 return errors.InvalidArguments;
             };
             continue;
-        } else if (std.mem.eql(u8, arg, "-f")) {
-            const nxt = args.next();
-            if (nxt == null or nxt.?.len == 0) {
-                try out.print("-d must be followed by a location\n", .{});
-                return errors.InvalidArguments;
-            }
-            if (!alloc.resize(files, files.len + 1)) {
-                const new_alloc = try alloc.alloc([:0]const u8, files.len + 1);
-                @memcpy(new_alloc[0..files.len], files);
-                alloc.free(files);
-                files = new_alloc;
-            } else {
-                files.len += 1;
-            }
-            files[files.len - 1] = nxt.?;
-            continue;
         } else if (std.mem.eql(u8, arg, "-d")) {
-            const nxt = args.next();
+            const nxt = arg_iter.next();
             if (nxt == null or nxt.?.len == 0) {
                 try out.print("-d must be followed by a location\n", .{});
                 return errors.InvalidArguments;
             }
-            ext_loc = std.mem.trim(u8, nxt.?, "/");
+            extLoc = nxt.?;
             continue;
         } else if (std.mem.eql(u8, arg, "-p")) {
-            const nxt = args.next();
+            const nxt = arg_iter.next();
             if (nxt == null or nxt.?.len == 0) {
                 try out.print("-p must be followed by a number\n", .{});
                 return errors.InvalidArguments;
@@ -172,7 +123,7 @@ fn handleArgs(alloc: std.mem.Allocator, main_args: std.process.Args, out: *Write
             continue;
         } else if (std.mem.eql(u8, arg, "--version")) {
             try out.print("zig-unsquashfs v", .{});
-            try build.version.format(out);
+            try config.version.format(out);
             try out.print("\nBuilt using Zig {s} in {} mode\n", .{ builtin.zig_version_string, builtin.mode });
             std.process.exit(0);
             return;

src/c.h

@@ -0,0 +1 @@
+#include <zstd.h>

src/c_decomp.zig

@@ -1,56 +0,0 @@
-const std = @import("std");
-
-const c = @import("c");
-
-const Error = @import("decomp.zig").Error;
-
-pub fn zlib(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var stream: c.z_stream = .{
-        .next_in = in.ptr,
-        .avail_in = @truncate(in.len),
-        .next_out = out.ptr,
-        .avail_out = @truncate(out.len),
-    };
-    var res = c.inflateInit(&stream);
-    if (res != c.Z_OK)
-        return Error.DecompressionFailed;
-    res = c.inflate(&stream, c.Z_FULL_FLUSH);
-    if (res != c.Z_OK)
-        return Error.DecompressionFailed;
-    return stream.total_out;
-}
-pub fn lzma(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var stream: c.lzma_stream = .{
-        .next_in = in.ptr,
-        .avail_in = in.len,
-        .next_out = out.ptr,
-        .avail_out = out.len,
-    };
-    var res = c.lzma_auto_decoder(&stream, 0, 0);
-    if (res != c.LZMA_OK)
-        return Error.DecompressionFailed;
-    while (res == c.LZMA_OK)
-        res = c.lzma_code(&stream, c.LZMA_RUN);
-    if (res != c.LZMA_STREAM_END)
-        return Error.DecompressionFailed;
-    return stream.total_out;
-}
-pub fn lz4(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    const res = c.LZ4_decompress_safe(
-        in.ptr,
-        out.ptr,
-        @intCast(in.len),
-        @intCast(out.len),
-    );
-    if (res < 0)
-        return Error.DecompressionFailed;
-    return @abs(res);
-}
-pub fn zstd(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
-    if (c.ZSTD_isError(res) != 0) {
-        std.debug.print("decompression failed: {s}\n", .{c.ZSTD_getErrorName(res)});
-        return Error.DecompressionFailed;
-    }
-    return res;
-}

src/data/extractor.zig

@@ -1,116 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const Decomp = @import("../decomp.zig");
-const FileFinish = @import("../extract-multi.zig").FileFinish;
-const Multi = @import("../extract-multi.zig");
-const DataBlock = @import("../inode.zig").DataBlock;
-const Cache = @import("../util/cache.zig");
-
-const Extractor = @This();
-
-data: []u8,
-decomp: Decomp.Fn,
-block_size: u32,
-
-blocks: []DataBlock,
-
-start: u64,
-size: u64,
-
-frag_data: ?[]u8 = null,
-frag_offset: u32 = 0,
-
-cache: ?*Cache = null,
-
-pub fn init(data: []u8, decomp: Decomp.Fn, block_size: u32, blocks: []DataBlock, start: u64, size: u64) Extractor {
-    return .{
-        .data = data,
-        .decomp = decomp,
-        .block_size = block_size,
-
-        .blocks = blocks,
-
-        .start = start,
-        .size = size,
-    };
-}
-pub fn addFrag(self: *Extractor, frag_data: []u8, frag_offset: u32) void {
-    self.frag_data = frag_data;
-    self.frag_offset = frag_offset;
-}
-pub fn addCache(self: *Extractor, cache: *Cache) void {
-    self.cache = cache;
-}
-
-pub fn extractAsync(self: Extractor, alloc: std.mem.Allocator, io: Io, select: *Io.Select(Multi.SelectUnion), finish: *FileFinish) !void {
-    if (self.size == 0) return;
-
-    var read_offset: u64 = self.start;
-    for (0.., self.blocks) |i, block| {
-        try select.concurrent(.reg, blockThread, .{ self, alloc, io, finish.file.file, read_offset, @truncate(i), finish });
-        read_offset += block.size;
-    }
-    if (self.frag_data != null)
-        try select.concurrent(.reg, fragThread, .{ self, io, finish.file.file, finish });
-}
-
-fn blockThread(
-    self: Extractor,
-    alloc: std.mem.Allocator,
-    io: Io,
-    file: Io.File,
-    read_offset: u64,
-    block_idx: u32,
-    finish: *FileFinish,
-) Multi.Error!void {
-    const size = if (self.frag_data == null and block_idx == self.blocks.len - 1)
-        self.size % self.block_size
-    else
-        self.block_size;
-
-    const block = self.blocks[block_idx];
-
-    var wrt = file.writer(io, &[0]u8{});
-    try wrt.seekTo(block_idx * self.block_size);
-
-    if (block.size == 0) {
-        try wrt.interface.splatByteAll(0, size);
-        try finish.finish(io);
-        return;
-    }
-
-    const data = self.data[read_offset..][0..block.size];
-
-    if (block.uncompressed) {
-        try wrt.interface.writeAll(data);
-        try finish.finish(io);
-        return;
-    }
-
-    if (self.cache != null) {
-        const decomp_block = try self.cache.?.get(io, read_offset, block.size);
-        try wrt.interface.writeAll(decomp_block);
-    } else {
-        const tmp = try alloc.alloc(u8, size);
-        defer alloc.free(tmp);
-
-        _ = try self.decomp(alloc, data, tmp);
-        try wrt.interface.writeAll(tmp);
-    }
-    try finish.finish(io);
-}
-fn fragThread(
-    self: Extractor,
-    io: Io,
-    file: Io.File,
-    finish: *FileFinish,
-) Multi.Error!void {
-    const size = self.size % self.block_size;
-
-    var wrt = file.writer(io, &[0]u8{});
-    try wrt.seekTo(self.blocks.len * self.block_size);
-
-    try wrt.interface.writeAll(self.frag_data.?[self.frag_offset..][0..size]);
-    try finish.finish(io);
-}

src/data/reader.zig

@@ -1,180 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const Decomp = @import("../decomp.zig");
-const DataBlock = @import("../inode.zig").DataBlock;
-const Cache = @import("../util/cache.zig");
-
-const Reader = @This();
-
-alloc: std.mem.Allocator,
-
-data: []u8,
-decomp: Decomp.Fn,
-block_size: u32,
-
-blocks: []DataBlock,
-
-size: u64,
-
-offset: u64,
-block_idx: u32 = 0,
-sparse_block: bool = false,
-
-frag_data: ?[]u8 = null,
-frag_offset: u32 = 0,
-
-io: ?Io = null,
-cache: ?*Cache = null,
-
-block_alloc: bool = false,
-
-interface: Io.Reader = .{
-    .buffer = &[0]u8{},
-    .end = 0,
-    .seek = 0,
-    .vtable = &.{
-        .stream = stream,
-        .discard = discard,
-        .readVec = readVec,
-    },
-},
-
-pub fn init(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, block_size: u32, blocks: []DataBlock, size: u64, data_start: u64) Reader {
-    return .{
-        .alloc = alloc,
-
-        .data = data,
-        .decomp = decomp,
-        .block_size = block_size,
-
-        .blocks = blocks,
-
-        .size = size,
-
-        .offset = data_start,
-    };
-}
-pub fn deinit(self: *Reader) void {
-    if (self.block_alloc)
-        self.alloc.free(self.interface.buffer);
-}
-pub fn addFrag(self: *Reader, frag_data: []u8, frag_offset: u32) void {
-    self.frag_data = frag_data;
-    self.frag_offset = frag_offset;
-}
-pub fn addCache(self: *Reader, io: Io, cache: *Cache) void {
-    self.io = io;
-    self.cache = cache;
-}
-
-fn advance(self: *Reader) Io.Reader.Error!void {
-    if (self.block_alloc) self.alloc.free(self.interface.buffer);
-
-    if (self.block_idx > self.blocks.len) return error.EndOfStream;
-    defer self.block_idx += 1;
-
-    self.interface.seek = 0;
-    errdefer self.interface.end = 0;
-
-    if (self.block_idx == self.blocks.len) {
-        if (self.frag_data == null) return error.EndOfStream;
-
-        self.sparse_block = false;
-
-        const size = self.size % self.block_size;
-
-        self.interface.buffer = self.frag_data.?[self.frag_offset..][0..size];
-
-        self.interface.end = size;
-        return;
-    }
-
-    const size = if (self.frag_data == null and self.block_idx == self.blocks.len - 1)
-        self.size % self.block_size
-    else
-        self.block_size;
-
-    const block = self.blocks[self.block_idx];
-    defer self.offset += block.size;
-
-    if (block.size == 0) {
-        self.sparse_block = true;
-        self.interface.end = size;
-        return;
-    } else {
-        self.sparse_block = false;
-    }
-
-    if (block.uncompressed) {
-        self.interface.buffer = self.data[self.offset..][0..block.size];
-        self.interface.end = block.size;
-        return;
-    }
-
-    if (self.cache == null) {
-        self.block_alloc = true;
-        self.interface.buffer = self.alloc.alloc(u8, size) catch return error.ReadFailed;
-        errdefer {
-            self.alloc.free(self.interface.buffer);
-            self.interface.buffer = &[0]u8{};
-        }
-
-        _ = self.decomp(self.alloc, self.data[self.offset..][0..block.size], self.interface.buffer) catch return error.ReadFailed;
-        self.interface.end = size;
-    } else {
-        self.interface.buffer = self.cache.?.get(self.io.?, self.offset, block.size) catch return error.ReadFailed;
-        self.interface.end = self.interface.buffer.len;
-    }
-}
-
-fn stream(r: *Io.Reader, w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize {
-    const self: *Reader = @fieldParentPtr("interface", r);
-    if (r.seek >= r.end)
-        try self.advance();
-
-    if (limit == .nothing) return 0;
-
-    const to_write = @min(@intFromEnum(limit), r.end - r.seek);
-
-    const wrote = if (self.sparse_block)
-        try w.splatByte(0, to_write)
-    else
-        try w.write(r.buffer[r.seek..][0..to_write]);
-
-    r.seek += wrote;
-    return wrote;
-}
-fn discard(r: *Io.Reader, limit: Io.Limit) Io.Reader.Error!usize {
-    if (r.seek >= r.end) {
-        const self: *Reader = @fieldParentPtr("interface", r);
-        try self.advance();
-    }
-    if (limit == .nothing) return 0;
-
-    const to_discard = @min(@intFromEnum(limit), r.end - r.seek);
-
-    r.seek += to_discard;
-    return to_discard;
-}
-fn readVec(r: *Io.Reader, vec: [][]u8) Io.Reader.Error!usize {
-    const self: *Reader = @fieldParentPtr("interface", r);
-    if (r.seek >= r.end)
-        try self.advance();
-
-    var copied: usize = 0;
-    for (vec) |v| {
-        const to_cpy = @min(v.len, r.end - r.seek);
-
-        if (self.sparse_block)
-            @memset(v[0..to_cpy], 0)
-        else
-            @memcpy(v[0..to_cpy], r.buffer[r.seek..][0..to_cpy]);
-
-        copied += to_cpy;
-
-        if (r.seek >= r.end) break;
-    }
-
-    return copied;
-}

src/decomp.zig

@@ -1,23 +1,6 @@
 const std = @import("std");
-const build = @import("build_options");
 
-const c = @import("c_decomp.zig");
-const zig = @import("zig_decomp.zig");
-
-pub fn getFn(e: Enum) !Fn {
-    return switch (e) {
-        .gzip => if (build.use_zig_decomp) zig.zlib else c.zlib,
-        .lzma => if (build.use_zig_decomp) zig.lzma else c.lzma,
-        .lzo => if (build.use_zig_decomp or !build.allow_lzo) error.LzoUnsupported else c.lzo,
-        .xz => if (build.use_zig_decomp) zig.xz else c.lzma,
-        .lz4 => if (build.use_zig_decomp) error.Lz4Unsupported else c.lz4,
-        .zstd => if (build.use_zig_decomp) zig.zstd else c.zstd,
-    };
-}
-
-// Types
-
-pub const Fn = *const fn (std.mem.Allocator, in: []u8, out: []u8) Error!usize;
+const Decompressor = @import("util/decompressor.zig");
 
 pub const Enum = enum(u16) {
     gzip = 1,
@@ -28,7 +11,42 @@ pub const Enum = enum(u16) {
     zstd,
 };
 
-pub const Error = error{
-    OutOfMemory,
-    DecompressionFailed,
+pub fn StatelessDecomp(val: Enum) !*const Decompressor {
+    return switch (val) {
+        .gzip => &@import("decomp/zlib.zig").stateless_decompressor,
+        .lzma => &@import("decomp/lzma.zig").stateless_decompressor,
+        .lzo => error.LzoUnsupported,
+        .xz => &@import("decomp/xz.zig").stateless_decompressor,
+        .lz4 => error.Lz4Unsupported,
+        .zstd => &@import("decomp/zstd.zig").stateless_decompressor,
+    };
+}
+
+pub const Decomp = union(enum) {
+    gzip: @import("decomp/zlib.zig"),
+    lzma: @import("decomp/lzma.zig"),
+    lzo: void,
+    xz: @import("decomp/xz.zig"),
+    lz4: void,
+    zstd: @import("decomp/zstd.zig"),
+
+    pub fn deinit(self: *Decomp) void {
+        switch (self.*) {
+            .gzip => self.gzip.deinit(),
+            .lzma => self.lzma.deinit(),
+            .xz => self.xz.deinit(),
+            .zstd => self.zstd.deinit(),
+            else => unreachable,
+        }
+    }
+
+    pub fn decompressor(self: *Decomp) *Decompressor {
+        return switch (self.*) {
+            .gzip => &self.gzip.interface,
+            .lzma => &self.lzma.interface,
+            .xz => &self.xz.interface,
+            .zstd => &self.zstd.interface,
+            else => unreachable,
+        };
+    }
 };

src/decomp/lzma.zig

@@ -0,0 +1,86 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+const lzma = std.compress.lzma;
+const Node = std.SinglyLinkedList.Node;
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Self = @This();
+
+const Buffer = struct {
+    node: Node,
+    buf: []u8,
+};
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+
+block_size: u32,
+buffers: std.ArrayList(Buffer),
+buffer_queue: std.SinglyLinkedList = .{},
+
+pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self {
+    return .{
+        .alloc = alloc,
+
+        .block_size = block_size,
+        .buffers = try .initCapacity(alloc, 5),
+    };
+}
+pub fn deinit(self: *Self) void {
+    for (self.buffers.items) |buf|
+        self.alloc.free(buf.buf);
+    self.buffers.deinit(self.alloc);
+}
+
+fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        var buf = try alloc.alloc(u8, in.len * 2);
+        defer alloc.free(buf);
+        return lzmaDecomp(alloc, &buf, in, out) catch |err| return switch (err) {
+            error.OutOfMemory => Error.OutOfMemory,
+            else => Error.ReadFailed,
+        };
+    }
+    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
+    const buf_node = self.buffer_queue.popFirst();
+    var buf: *Buffer = undefined;
+    if (buf_node == null) {
+        const new_buf = try self.buffers.addOne(self.alloc);
+        new_buf.* = .{ .node = .{}, .buf = try self.alloc.alloc(u8, self.block_size) };
+        buf = new_buf;
+    } else {
+        buf = @fieldParentPtr("node", buf_node.?);
+    }
+    defer self.buffer_queue.prepend(&buf.node);
+    return lzmaDecomp(self.alloc, &buf.buf, in, out) catch |err| {
+        // self.err = err;
+        return switch (err) {
+            error.OutOfMemory => Error.OutOfMemory,
+            else => Error.ReadFailed,
+        };
+    };
+}
+
+inline fn lzmaDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = try lzma.Decompress.initOptions(&rdr, alloc, buffer.*, .{ .allow_incomplete = true }, 3 * 1024 * 1024);
+    defer {
+        buffer.* = d.takeBuffer();
+        d.deinit();
+    }
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var buf = try alloc.alloc(u8, in.len);
+    defer alloc.free(buf);
+    return lzmaDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
+}

src/decomp/xz.zig

@@ -0,0 +1,80 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+const xz = std.compress.xz;
+const Node = std.SinglyLinkedList.Node;
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Self = @This();
+
+const Buffer = struct {
+    node: Node,
+    buf: []u8,
+};
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+
+block_size: u32,
+buffers: std.ArrayList(Buffer),
+buffer_queue: std.SinglyLinkedList = .{},
+
+pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self {
+    return .{
+        .alloc = alloc,
+
+        .block_size = block_size,
+        .buffers = try .initCapacity(alloc, 5),
+    };
+}
+pub fn deinit(self: *Self) void {
+    for (self.buffers.items) |buf|
+        self.alloc.free(buf.buf);
+    self.buffers.deinit(self.alloc);
+}
+
+fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        var buf = try alloc.alloc(u8, in.len * 2);
+        defer alloc.free(buf);
+        return xzDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
+    }
+    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
+    const buf_node = self.buffer_queue.popFirst();
+    var buf: *Buffer = undefined;
+    if (buf_node == null) {
+        const new_buf = try self.buffers.addOne(self.alloc);
+        new_buf.* = .{ .node = .{}, .buf = try self.alloc.alloc(u8, self.block_size) };
+        buf = new_buf;
+    } else {
+        buf = @fieldParentPtr("node", buf_node.?);
+    }
+    defer self.buffer_queue.prepend(&buf.node);
+    return xzDecomp(self.alloc, &buf.buf, in, out) catch {
+        // self.err = err;
+        return Error.ReadFailed;
+    };
+}
+
+inline fn xzDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = try xz.Decompress.init(&rdr, alloc, buffer.*);
+    defer {
+        buffer.* = d.takeBuffer();
+        d.deinit();
+    }
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var buf = try alloc.alloc(u8, in.len);
+    defer alloc.free(buf);
+    return xzDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
+}

src/decomp/zlib.zig

@@ -0,0 +1,73 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+const flate = std.compress.flate;
+const Node = std.SinglyLinkedList.Node;
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Self = @This();
+
+const Buffer = struct {
+    node: Node,
+    buf: []u8,
+};
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+
+block_size: u32,
+buffers: std.ArrayList(Buffer),
+buffer_queue: std.SinglyLinkedList = .{},
+
+pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self {
+    return .{
+        .alloc = alloc,
+
+        .block_size = block_size,
+        .buffers = try .initCapacity(alloc, 5),
+    };
+}
+pub fn deinit(self: *Self) void {
+    for (self.buffers.items) |buf|
+        self.alloc.free(buf.buf);
+    self.buffers.deinit(self.alloc);
+}
+
+fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        const buf = try alloc.alloc(u8, in.len * 2);
+        defer alloc.free(buf);
+        return zlibDecomp(buf, in, out);
+    }
+    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
+    const buf_node = self.buffer_queue.popFirst();
+    var buf: *Buffer = undefined;
+    if (buf_node == null) {
+        const new_buf = try self.buffers.addOne(self.alloc);
+        new_buf.* = .{ .node = .{}, .buf = try self.alloc.alloc(u8, self.block_size) };
+        buf = new_buf;
+    } else {
+        buf = @fieldParentPtr("node", buf_node.?);
+    }
+    defer self.buffer_queue.prepend(&buf.node);
+    return zlibDecomp(buf.buf, in, out);
+}
+
+inline fn zlibDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = flate.Decompress.init(&rdr, .zlib, buffer);
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    const buf = try alloc.alloc(u8, in.len * 2);
+    defer alloc.free(buf);
+    return zlibDecomp(buf, in, out);
+}

src/decomp/zstd.zig

@@ -0,0 +1,82 @@
+const std = @import("std");
+const Io = std.Io;
+const Reader = std.Io.Reader;
+const zstd = std.compress.zstd;
+const Node = std.SinglyLinkedList.Node;
+
+const c = @import("c");
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Queue = std.Io.Queue([]u8);
+
+const Self = @This();
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+io: Io,
+
+block_size: u32,
+buf: [][]u8,
+buf_queue: Queue,
+
+pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self {
+    const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one.
+    var queue: Queue = .init(buf);
+    for (0..20) |_|
+        try queue.putOne(io, try alloc.alloc(u8, block_size + zstd.block_size_max));
+
+    return .{
+        .alloc = alloc,
+        .io = io,
+
+        .block_size = block_size,
+        .buf = buf,
+        .buf_queue = queue,
+    };
+}
+pub fn deinit(self: *Self) void {
+    self.buf_queue.close(self.io);
+    for (self.buf) |buf|
+        self.alloc.free(buf);
+    self.alloc.free(self.buf);
+}
+
+fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        const buf = try alloc.alloc(u8, in.len * 2);
+        defer alloc.free(buf);
+        return zstdDecomp(buf, in, out);
+    }
+    var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
+
+    const buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed;
+    defer self.buf_queue.putOne(self.io, buf) catch {};
+
+    return zstdDecomp(buf, in, out);
+}
+
+inline fn zstdDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = zstd.Decompress.init(&rdr, buffer, .{ .window_len = @truncate(out.len) });
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    _ = d;
+    _ = alloc;
+    const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
+    if (c.ZSTD_isError(res) == 1)
+        return Error.ReadFailed;
+    return res;
+    // const buf = try alloc.alloc(u8, out.len + zstd.block_size_max);
+    // defer alloc.free(buf);
+    // return zstdDecomp(buf, in, out);
+}

src/directory.zig

@@ -3,74 +3,58 @@ const Reader = std.Io.Reader;
 
 const Inode = @import("inode.zig");
 
-const Directory = @This();
+pub const Error = error{OutOfMemory} || Reader.Error;
 
-entries: []Entry,
+const DirEntry = @This();
 
-pub fn init(alloc: std.mem.Allocator, rdr: *Reader, size: u64) !Directory {
-    if (size <= 3) return Directory{ .entries = &[0]Entry{} };
+block_start: u32,
+block_offset: u16,
+type: Inode.Type,
+name: []const u8,
 
+pub fn deinit(self: DirEntry, alloc: std.mem.Allocator) void {
+    alloc.free(self.name);
+}
+
+pub fn readDirectory(alloc: std.mem.Allocator, rdr: *Reader, size: u32) Error![]DirEntry {
     var hdr: Header = undefined;
     var raw: RawEntry = undefined;
-
-    var read: u64 = 3;
-
-    var out: std.ArrayList(Entry) = try .initCapacity(alloc, 50);
+    var out: std.ArrayList(DirEntry) = try .initCapacity(alloc, 30);
     errdefer {
-        for (out.items) |entry|
-            entry.deinit(alloc);
+        for (out.items) |ent|
+            alloc.free(ent.name);
         out.deinit(alloc);
     }
 
-    while (read < size) {
+    var tot_red: u32 = 3;
+    while (tot_red < size) {
         try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
-        read += @sizeOf(Header);
-
         try out.ensureUnusedCapacity(alloc, hdr.count + 1);
 
+        tot_red += @sizeOf(Header);
+
         for (0..hdr.count + 1) |_| {
             try rdr.readSliceEndian(RawEntry, @ptrCast(&raw), .little);
 
-            const name = try alloc.alloc(u8, raw.name_size + 1);
+            const new_name = try alloc.alloc(u8, raw.name_size + 1);
+            try rdr.readSliceEndian(u8, new_name, .little);
 
-            try rdr.readSliceEndian(u8, name, .little);
-
-            const entry = out.addOneAssumeCapacity();
-
-            entry.* = .{
-                .name = name,
+            const new = out.addOneAssumeCapacity();
+            new.* = .{
                 .block_start = hdr.block_start,
                 .block_offset = raw.block_offset,
                 .type = raw.type,
+                .name = new_name,
             };
 
-            read += @sizeOf(RawEntry) + raw.name_size + 1;
+            tot_red += @sizeOf(RawEntry) + raw.name_size + 1;
         }
     }
-
-    return .{ .entries = try out.toOwnedSlice(alloc) };
-}
-pub fn deinit(self: Directory, alloc: std.mem.Allocator) void {
-    for (self.entries) |entry|
-        entry.deinit(alloc);
-    alloc.free(self.entries);
+    return out.toOwnedSlice(alloc);
 }
 
 // Types
 
-pub const Error = error{OutOfMemory} || Reader.Error;
-
-pub const Entry = struct {
-    name: []const u8,
-    block_start: u32,
-    block_offset: u16,
-    type: Inode.Type,
-
-    pub fn deinit(self: Entry, alloc: std.mem.Allocator) void {
-        alloc.free(self.name);
-    }
-};
-
 const Header = extern struct {
     count: u32,
     block_start: u32,

src/extract-multi.zig

@@ -1,503 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Atomic = std.atomic.Value;
-
-const DataExtractor = @import("data/extractor.zig");
-const Decomp = @import("decomp.zig");
-const Directory = @import("directory.zig");
-const ExtractionOption = @import("options.zig");
-const Inode = @import("inode.zig");
-const Lookup = @import("lookup.zig");
-const MetadataReader = @import("meta_rdr.zig");
-const Superblock = @import("archive.zig").Superblock;
-const Cache = @import("util/cache.zig");
-const XattrTable = @import("xattr.zig");
-
-pub fn extract(alloc: std.mem.Allocator, io: Io, super: Superblock, data: []u8, decomp: Decomp.Fn, inode: Inode, filepath: []const u8, options: ExtractionOption) !void {
-    const path = std.mem.trim(u8, filepath, "/");
-
-    var frag_table: Lookup.Table(Lookup.FragEntry) = .init(alloc, data, decomp, super.frag_start, super.frag_count);
-    defer frag_table.deinit();
-
-    var id_table: Lookup.Table(u16) = .init(alloc, data, decomp, super.id_start, super.id_count);
-    defer id_table.deinit();
-
-    var xattr_table: XattrTable = .init(alloc, data, decomp, super.xattr_start);
-    defer xattr_table.deinit();
-
-    var cache: Cache = .init(alloc, data, decomp);
-    defer cache.deinit();
-
-    var err: ?Error = null;
-    var group: Io.Group = .init;
-
-    group.async(
-        io,
-        groupExtract,
-        .{ alloc, io, data, decomp, super, &group, &frag_table, &id_table, &xattr_table, &cache, &err, inode, path, options, null },
-    );
-}
-
-fn groupExtract(
-    alloc: std.mem.Allocator,
-    io: Io,
-    data: []u8,
-    decomp: Decomp.Fn,
-    super: Superblock,
-    group: *Io.Group,
-    frag_table: *Lookup.Table(Lookup.FragEntry),
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    cache: *Cache,
-    err: *?Error,
-    inode: Inode,
-    path: []const u8,
-    options: ExtractionOption,
-    parent: ?*Parent,
-) error{Canceled}!void {
-    if (err != null) {
-        if (parent != null) {
-            alloc.free(path);
-            inode.deinit(alloc);
-            parent.?.finish(alloc, io);
-        }
-        return;
-    }
-    switch (inode.hdr.type) {
-        .dir, .ext_dir => group.async(io, extractDir, .{ alloc, io, data, decomp, super, group, frag_table, id_table, xattr_table, cache, inode, path, options, parent }),
-        .file, .ext_file => group.async(io, extractReg, .{ alloc, io, data, decomp, super.block_size, group, frag_table, cache, inode, path, options, parent }),
-        .symlink, .ext_symlink => group.async(io, extractSymlink, .{ alloc, io, inode, path, parent }),
-        else => group.async(io, extractNod, .{ alloc, io, id_table, xattr_table, inode, path, options, parent }),
-    }
-}
-
-fn extractDir(
-    alloc: std.mem.Allocator,
-    io: Io,
-    data: []u8,
-    decomp: Decomp.Fn,
-    super: Superblock,
-    group: *Io.Group,
-    frag_table: *Lookup.Table(Lookup.FragEntry),
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    cache: *Cache,
-    err: *?Error,
-    inode: Inode,
-    path: []const u8,
-    options: ExtractionOption,
-    parent: ?*Parent,
-) Error!void {
-    var xattr_idx: u32 = 0xFFFFFFFF;
-
-    const dir: Directory = switch (inode.data) {
-        .dir => |d| blk: {
-            var meta: MetadataReader = .init(alloc, data, decomp, super.dir_start + d.block_start);
-            try meta.interface.discardAll(d.block_offset);
-
-            break :blk try .init(alloc, &meta.interface, d.size);
-        },
-        .ext_dir => |d| blk: {
-            xattr_idx = d.xattr_idx;
-
-            var meta: MetadataReader = .init(alloc, data, decomp, super.dir_start + d.block_start);
-            try meta.interface.discardAll(d.block_offset);
-
-            break :blk try .init(alloc, &meta.interface, d.size);
-        },
-        else => unreachable,
-    };
-    defer dir.deinit(alloc);
-
-    try Io.Dir.cwd().createDirPath(io, path);
-
-    if (dir.entries.len == 0) {
-        try setMetadataPath(alloc, io, inode.hdr, id_table, xattr_table, path, xattr_idx, options);
-        if (parent != null) {
-            try parent.?.finish(io);
-        }
-        return;
-    }
-
-    const cur_dir = try Parent.create(alloc, id_table, xattr_table, err, inode.hdr, path, options, parent, xattr_idx, dir.entries);
-
-    for (dir.entries) |entry| {
-        const new_inode: Inode = try .initEntry(alloc, data, decomp, super.inode_start, super.block_size, entry);
-
-        const new_path = std.mem.concat(alloc, u8, &.{ path, "/", entry.name }) catch |e| {
-            new_inode.deinit(alloc);
-            return e;
-        };
-
-        group.async(
-            io,
-            groupExtract,
-            .{ alloc, io, data, decomp, super, group, frag_table, id_table, xattr_table, cache, err, new_inode, new_path, options, cur_dir },
-        );
-    }
-}
-fn extractReg(
-    alloc: std.mem.Allocator,
-    io: Io,
-    data: []u8,
-    decomp: Decomp.Fn,
-    block_size: u32,
-    group: *Io.Group,
-    frag_table: *Lookup.Table(Lookup.FragEntry),
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    cache: *Cache,
-    err: *?Error,
-    inode: Inode,
-    path: []const u8,
-    options: ExtractionOption,
-    parent: ?*Parent,
-) Error!void {
-    var blocks: usize = 0;
-    var xattr_idx: u32 = 0xFFFFFFFF;
-
-    var ext: DataExtractor = switch (inode.data) {
-        .file => |f| blk: {
-            var ext: DataExtractor = .init(data, decomp, block_size, f.blocks, f.block_start, f.size);
-            blocks = f.blocks.len;
-
-            if (f.frag_idx != 0xFFFFFFFF) {
-                blocks += 1;
-                const frag: Lookup.FragEntry = try frag_table.get(io, f.frag_idx);
-                if (frag.size.uncompressed) {
-                    ext.addFrag(data[frag.block_start..][0..frag.size.size], f.frag_offset);
-                } else {
-                    const frag_block = try cache.get(io, frag.block_start, frag.size.size);
-                    ext.addFrag(frag_block, f.frag_offset);
-                }
-            }
-            break :blk ext;
-        },
-        .ext_file => |f| blk: {
-            xattr_idx = f.xattr_idx;
-
-            var ext: DataExtractor = .init(data, decomp, block_size, f.blocks, f.block_start, f.size);
-            blocks = f.blocks.len;
-
-            if (f.frag_idx != 0xFFFFFFFF) {
-                blocks += 1;
-                const frag: Lookup.FragEntry = try frag_table.get(io, f.frag_idx);
-                if (frag.size.uncompressed) {
-                    ext.addFrag(data[frag.block_start..][0..frag.size.size], f.frag_offset);
-                } else {
-                    const frag_block = try cache.get(io, frag.block_start, frag.size.size);
-                    ext.addFrag(frag_block, f.frag_offset);
-                }
-            }
-            break :blk ext;
-        },
-        else => unreachable,
-    };
-
-    const fin = try FileFinish.create(alloc, io, id_table, xattr_table, err, inode.hdr, path, options, parent, xattr_idx, blocks);
-
-    try ext.extractAsync(alloc, io, group, fin);
-}
-fn extractSymlink(alloc: std.mem.Allocator, io: Io, inode: Inode, path: []const u8, parent: ?*Parent) Error!void {
-    defer if (parent != null) {
-        inode.deinit(alloc);
-        alloc.free(path);
-    };
-
-    const target = switch (inode.data) {
-        .symlink => |s| s.target,
-        .ext_symlink => |s| s.target,
-        else => unreachable,
-    };
-
-    try Io.Dir.cwd().symLink(io, target, path, .{});
-
-    if (parent != null)
-        parent.?.finish(io);
-}
-fn extractNod(
-    alloc: std.mem.Allocator,
-    io: Io,
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    inode: Inode,
-    path: []const u8,
-    parent: ?*Parent,
-    options: ExtractionOption,
-) Error!void {
-    var xattr_idx: u32 = 0xFFFFFFFF;
-    var device: u32 = 0;
-    var mode: u32 = undefined;
-
-    const DT = std.os.linux.DT;
-
-    switch (inode.data) {
-        .block_dev => |d| {
-            mode = DT.BLK;
-            device = d.device;
-        },
-        .ext_block_dev => |d| {
-            mode = DT.BLK;
-            device = d.device;
-            xattr_idx = d.xattr_idx;
-        },
-        .char_dev => |d| {
-            mode = DT.CHR;
-            device = d.device;
-        },
-        .ext_char_dev => |d| {
-            mode = DT.CHR;
-            device = d.device;
-            xattr_idx = d.xattr_idx;
-        },
-        .fifo => {
-            mode = DT.FIFO;
-        },
-        .ext_fifo => |f| {
-            mode = DT.FIFO;
-            xattr_idx = f.xattr_idx;
-        },
-        .socket => {
-            mode = DT.SOCK;
-        },
-        .ext_socket => |s| {
-            mode = DT.SOCK;
-            xattr_idx = s.xattr_idx;
-        },
-        else => unreachable,
-    }
-
-    const sentinel_path = try alloc.dupeSentinel(u8, path, 0);
-    defer alloc.free(sentinel_path);
-
-    const res = std.os.linux.mknod(sentinel_path, mode, device);
-    if (res != 0)
-        return Error.Mknod;
-
-    try setMetadataPath(alloc, io, inode.hdr, id_table, xattr_table, path, xattr_idx, options);
-
-    if (parent != null)
-        try parent.?.finish(io);
-}
-
-fn setMetadataPath(
-    alloc: std.mem.Allocator,
-    io: Io,
-    hdr: Inode.Header,
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    path: []const u8,
-    xattr_idx: u32,
-    options: ExtractionOption,
-) Error!void {
-    if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == 0xFFFFFFFF)) return;
-
-    var file = try Io.Dir.cwd().openFile(io, path, .{});
-    defer file.close(io);
-
-    return setMetadata(alloc, io, hdr, id_table, xattr_table, file, xattr_idx, options);
-}
-fn setMetadata(
-    alloc: std.mem.Allocator,
-    io: Io,
-    hdr: Inode.Header,
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    file: Io.File,
-    xattr_idx: u32,
-    options: ExtractionOption,
-) Error!void {
-    if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == 0xFFFFFFFF)) return;
-
-    if (!options.ignore_xattr and xattr_idx != 0xFFFFFFFF) {
-        var xattr = try xattr_table.get(alloc, io, xattr_idx);
-        defer xattr.deinit(alloc);
-
-        for (xattr.kvs) |kv| {
-            const res = std.os.linux.fsetxattr(file.handle, kv.key, kv.value.ptr, kv.value.len, 0);
-            if (res != 0)
-                return Error.SetXattr;
-        }
-    }
-
-    if (options.ignore_permissions) return;
-
-    try file.setTimestamps(io, .{
-        .modify_timestamp = .init(
-            Io.Timestamp.fromNanoseconds(@as(i96, @intCast(hdr.mod_time)) * std.time.ns_per_s),
-        ),
-    });
-    try file.setPermissions(io, @enumFromInt(hdr.permissions));
-    try file.setOwner(io, try id_table.get(io, hdr.uid_idx), try id_table.get(io, hdr.gid_idx));
-}
-
-// Types
-
-pub const Error = error{ Mknod, SetXattr } || std.mem.Allocator.Error || Io.Cancelable || Io.Reader.Error || Io.Dir.CreateDirPathError || Cache.Error ||
-    Io.File.SetPermissionsError || Io.Dir.SymLinkError || Io.File.SeekError || Io.Writer.Error || Io.File.Atomic.LinkError || Io.ConcurrentError;
-
-const FileFinish = struct {
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    err: *?Error,
-
-    inode: Inode,
-    options: ExtractionOption,
-    parent: ?*Parent,
-    xattr_idx: u32,
-
-    file: Io.File.Atomic,
-
-    atomic: Atomic(usize),
-    has_err: bool = false,
-
-    fn create(
-        alloc: std.mem.Allocator,
-        io: Io,
-        id_table: *Lookup.Table(u16),
-        xattr_table: *XattrTable,
-        err: *?Error,
-        hdr: Inode.Header,
-        path: []const u8,
-        options: ExtractionOption,
-        parent: ?*Parent,
-        xattr_idx: u32,
-        blocks: usize,
-    ) Error!*FileFinish {
-        const out = try alloc.create(FileFinish);
-        out.* = .{
-            .id_table = id_table,
-            .xattr_table = xattr_table,
-            .err = err,
-
-            .hdr = hdr,
-            .path = path,
-            .options = options,
-            .parent = parent,
-            .xattr_idx = xattr_idx,
-
-            .file = Io.Dir.cwd().createFileAtomic(io, path, .{}),
-
-            .atomic = .init(blocks),
-        };
-        return out;
-    }
-
-    fn finish(self: *FileFinish, alloc: std.mem.Allocator, io: Io) void {
-        if (self.atomic.fetchSub(1, .release) > 0) return;
-
-        defer {
-            if (self.parent != null) {
-                alloc.free(self.path);
-                if (self.has_err)
-                    self.parent.?.errFinish(alloc, io)
-                else
-                    self.parent.?.finish(alloc, io);
-            }
-            alloc.destroy(self);
-        }
-
-        if (self.has_err) return;
-
-        setMetadata(alloc, io, self.hdr, self.id_table, self.xattr_table, self.file, self.xattr_idx, self.options) catch |err| {
-            self.has_err = true;
-            self.err.* = err;
-            return;
-        };
-        self.file.link(io) catch |err| {
-            self.has_err = true;
-            self.err.* = err;
-            return;
-        };
-
-        self.file.deinit(io);
-    }
-    fn errFinish(self: *FileFinish, alloc: std.mem.Allocator, io: Io) void {
-        self.has_err = true;
-
-        if (self.atomic.fetchSub(1, .release) > 0) return;
-
-        if (self.parent != null) {
-            alloc.free(self.path);
-            self.parent.?.errFinish(alloc, io);
-        }
-        self.file.deinit(io);
-        alloc.destroy(self);
-    }
-};
-const Parent = struct {
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    err: *?Error,
-
-    hdr: Inode.Header,
-    path: []const u8,
-    options: ExtractionOption,
-    parent: ?*Parent,
-    xattr_idx: u32,
-
-    atomic: Atomic(usize),
-    has_err: bool = false,
-
-    fn create(
-        alloc: std.mem.Allocator,
-        id_table: *Lookup.Table(u16),
-        xattr_table: *XattrTable,
-        err: *?Error,
-        hdr: Inode.Header,
-        path: []const u8,
-        options: ExtractionOption,
-        parent: ?*Parent,
-        xattr_idx: u32,
-        num: usize,
-    ) Error!*Parent {
-        const out = try alloc.create(Parent);
-        out.* = .{
-            .id_table = id_table,
-            .xattr_table = xattr_table,
-            .err = err,
-
-            .hdr = hdr,
-            .path = path,
-            .options = options,
-            .parent = parent,
-            .xattr_idx = xattr_idx,
-
-            .atomic = .init(num),
-        };
-        return out;
-    }
-
-    fn finish(self: *Parent, alloc: std.mem.Allocator, io: Io) void {
-        if (self.atomic.fetchSub(1, .release) > 0) return;
-
-        defer {
-            if (self.parent != null) {
-                alloc.free(self.path);
-                if (self.has_err)
-                    self.parent.?.errFinish(alloc, io)
-                else
-                    self.parent.?.finish(alloc, io);
-            }
-            alloc.destroy(self);
-        }
-
-        if (self.has_err) return;
-
-        setMetadataPath(alloc, io, self.hdr, self.id_table, self.xattr_table, self.path, self.xattr_idx, self.options) catch |err| {
-            self.has_err = true;
-            self.err.* = err;
-        };
-    }
-    fn errFinish(self: *Parent, alloc: std.mem.Allocator, io: Io) void {
-        self.has_err = true;
-
-        if (self.atomic.fetchSub(1, .release) > 0) return;
-
-        if (self.parent != null) {
-            alloc.free(self.path);
-            self.parent.?.errFinish(alloc, io);
-        }
-        alloc.destroy(self);
-    }
-};

src/extract-single.zig

@@ -1,267 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const Archive = @import("archive.zig");
-const Superblock = Archive.Superblock;
-const DataReader = @import("data/reader.zig");
-const Decomp = @import("decomp.zig");
-const Directory = @import("directory.zig");
-const ExtractionOptions = @import("options.zig");
-const Inode = @import("inode.zig");
-const Lookup = @import("lookup.zig");
-const MetadaReader = @import("meta_rdr.zig");
-const Cache = @import("util/cache.zig");
-const XattrTable = @import("xattr.zig");
-
-pub fn extract(
-    alloc: std.mem.Allocator,
-    io: Io,
-    super: Superblock,
-    data: []u8,
-    decomp: Decomp.Fn,
-    inode: Inode,
-    ext_loc: []const u8,
-    options: ExtractionOptions,
-) !void {
-    const path = std.mem.trim(u8, ext_loc, "/");
-
-    var cache: Cache = .init(alloc, data, decomp);
-    defer cache.deinit();
-
-    var frag_table: Lookup.Table(Lookup.FragEntry) = .init(alloc, data, decomp, super.frag_start, super.frag_count);
-    defer frag_table.deinit();
-
-    var id_table: Lookup.Table(u16) = .init(alloc, data, decomp, super.id_start, super.id_count);
-    defer id_table.deinit();
-
-    var xattr_table: XattrTable = .init(alloc, data, decomp, super.xattr_start);
-    defer xattr_table.deinit();
-
-    return switch (inode.hdr.type) {
-        .file, .ext_file => try extractFile(alloc, io, super, data, decomp, &cache, &frag_table, &id_table, &xattr_table, inode, path, options),
-        .dir, .ext_dir => try extractDir(alloc, io, super, data, decomp, &cache, &frag_table, &id_table, &xattr_table, inode, path, options),
-        .symlink, .ext_symlink => try extractSymlink(io, inode, path),
-        else => try extractNod(alloc, io, &id_table, &xattr_table, inode, path, options),
-    };
-}
-
-fn setMetadata(
-    alloc: std.mem.Allocator,
-    io: Io,
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    inode: Inode,
-    path: []const u8,
-    options: ExtractionOptions,
-    xattr_idx: ?u32,
-) !void {
-    if (options.ignore_permissions and (options.ignore_xattr or xattr_idx == null)) return;
-
-    var fil: Io.File = try Io.Dir.cwd().openFile(io, path, .{});
-    defer fil.close(io);
-
-    if (!options.ignore_xattr and xattr_idx != null) {
-        const xattr = try xattr_table.get(alloc, io, xattr_idx.?);
-        defer xattr.deinit(alloc);
-
-        for (xattr.kvs) |kv| {
-            const res = std.os.linux.fsetxattr(fil.handle, kv.key, kv.value.ptr, kv.value.len, 0);
-            if (res != 0)
-                return error.SetXattrError;
-        }
-    }
-    if (!options.ignore_permissions) {
-        try fil.setTimestamps(io, .{
-            .modify_timestamp = .init(Io.Timestamp.fromNanoseconds(@as(i96, @intCast(inode.hdr.mod_time)) * std.time.ns_per_s)),
-        });
-        try fil.setPermissions(io, @enumFromInt(inode.hdr.permissions));
-        try fil.setOwner(io, try id_table.get(io, inode.hdr.uid_idx), try id_table.get(io, inode.hdr.gid_idx));
-    }
-}
-
-fn extractDir(
-    alloc: std.mem.Allocator,
-    io: Io,
-    super: Superblock,
-    data: []u8,
-    decomp: Decomp.Fn,
-    cache: *Cache,
-    frag_table: *Lookup.Table(Lookup.FragEntry),
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    inode: Inode,
-    path: []const u8,
-    options: ExtractionOptions,
-) !void {
-    var xattr_idx: ?u32 = null;
-
-    try Io.Dir.cwd().createDirPath(io, path);
-
-    var dir: Directory = switch (inode.data) {
-        .dir => |d| blk: {
-            var meta: MetadaReader = .init(alloc, data, decomp, d.block_start + super.dir_start);
-            try meta.interface.discardAll(d.block_offset);
-
-            break :blk try Directory.init(alloc, &meta.interface, d.size);
-        },
-        .ext_dir => |d| blk: {
-            if (d.xattr_idx != 0xFFFFFFFF) xattr_idx = d.xattr_idx;
-
-            var meta: MetadaReader = .init(alloc, data, decomp, d.block_start + super.dir_start);
-            try meta.interface.discardAll(d.block_offset);
-
-            break :blk try Directory.init(alloc, &meta.interface, d.size);
-        },
-        else => unreachable,
-    };
-    defer dir.deinit(alloc);
-
-    for (dir.entries) |entry| {
-        var new_inode: Inode = try .initEntry(alloc, data, decomp, super.inode_start, super.block_size, entry);
-        defer new_inode.deinit(alloc);
-
-        const new_path = try std.mem.concat(alloc, u8, &.{ path, "/", entry.name });
-        defer alloc.free(new_path);
-
-        switch (entry.type) {
-            .file => try extractFile(alloc, io, super, data, decomp, cache, frag_table, id_table, xattr_table, new_inode, new_path, options),
-            .dir => try extractDir(alloc, io, super, data, decomp, cache, frag_table, id_table, xattr_table, new_inode, new_path, options),
-            .symlink => try extractSymlink(io, new_inode, new_path),
-            else => try extractNod(alloc, io, id_table, xattr_table, new_inode, new_path, options),
-        }
-    }
-
-    try setMetadata(alloc, io, id_table, xattr_table, inode, path, options, xattr_idx);
-}
-fn extractFile(
-    alloc: std.mem.Allocator,
-    io: Io,
-    super: Superblock,
-    data: []u8,
-    decomp: Decomp.Fn,
-    cache: *Cache,
-    frag_table: *Lookup.Table(Lookup.FragEntry),
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    inode: Inode,
-    path: []const u8,
-    options: ExtractionOptions,
-) !void {
-    var xattr_idx: ?u32 = null;
-
-    var rdr: DataReader = switch (inode.data) {
-        .file => |f| blk: {
-            var rdr: DataReader = .init(alloc, data, decomp, super.block_size, f.blocks, f.size, f.block_start);
-            rdr.addCache(io, cache);
-
-            if (f.frag_idx == 0xFFFFFFFF) break :blk rdr;
-
-            const entry: Lookup.FragEntry = try frag_table.get(io, f.frag_idx);
-            if (entry.size.uncompressed) {
-                rdr.addFrag(data[entry.block_start..][0..entry.size.size], f.frag_offset);
-            } else {
-                rdr.addFrag(try cache.get(io, entry.block_start, entry.size.size), f.frag_offset);
-            }
-
-            break :blk rdr;
-        },
-        .ext_file => |f| blk: {
-            if (f.xattr_idx != 0xFFFFFFFF) xattr_idx = f.xattr_idx;
-
-            var rdr: DataReader = .init(alloc, data, decomp, super.block_size, f.blocks, f.size, f.block_start);
-            rdr.addCache(io, cache);
-
-            if (f.frag_idx == 0xFFFFFFFF) break :blk rdr;
-
-            const entry: Lookup.FragEntry = try frag_table.get(io, f.frag_idx);
-            if (entry.size.uncompressed) {
-                rdr.addFrag(data[entry.block_start..][0..entry.size.size], f.frag_offset);
-            } else {
-                rdr.addFrag(try cache.get(io, entry.block_start, entry.size.size), f.frag_offset);
-            }
-
-            break :blk rdr;
-        },
-        else => unreachable,
-    };
-    defer rdr.deinit();
-
-    var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{});
-    defer atomic.deinit(io);
-
-    var writer = atomic.file.writer(io, &[0]u8{});
-    _ = try rdr.interface.streamRemaining(&writer.interface);
-    try writer.flush();
-
-    try atomic.link(io);
-
-    try setMetadata(alloc, io, id_table, xattr_table, inode, path, options, xattr_idx);
-}
-fn extractSymlink(io: Io, inode: Inode, path: []const u8) !void {
-    return switch (inode.data) {
-        .symlink => |s| Io.Dir.cwd().symLink(io, s.target, path, .{}),
-        .ext_symlink => |s| Io.Dir.cwd().symLink(io, s.target, path, .{}),
-        else => unreachable,
-    };
-}
-fn extractNod(
-    alloc: std.mem.Allocator,
-    io: Io,
-    id_table: *Lookup.Table(u16),
-    xattr_table: *XattrTable,
-    inode: Inode,
-    path: []const u8,
-    options: ExtractionOptions,
-) !void {
-    var dev: u32 = 0;
-    var mode: u32 = undefined;
-
-    var xattr_idx: ?u32 = null;
-
-    const DT = std.posix.DT;
-
-    switch (inode.data) {
-        .char_dev => |d| {
-            dev = d.device;
-            mode = DT.CHR;
-        },
-        .block_dev => |d| {
-            dev = d.device;
-            mode = DT.BLK;
-        },
-        .ext_char_dev => |d| {
-            if (d.xattr_idx != 0xFFFFFFFF) xattr_idx = d.xattr_idx;
-
-            dev = d.device;
-            mode = DT.CHR;
-        },
-        .ext_block_dev => |d| {
-            if (d.xattr_idx != 0xFFFFFFFF) xattr_idx = d.xattr_idx;
-
-            dev = d.device;
-            mode = DT.BLK;
-        },
-        .fifo => mode = DT.FIFO,
-        .socket => mode = DT.SOCK,
-        .ext_fifo => |i| {
-            if (i.xattr_idx != 0xFFFFFFFF) xattr_idx = i.xattr_idx;
-
-            mode = DT.FIFO;
-        },
-        .ext_socket => |i| {
-            if (i.xattr_idx != 0xFFFFFFFF) xattr_idx = i.xattr_idx;
-
-            mode = DT.SOCK;
-        },
-        else => unreachable,
-    }
-
-    const sentinel_path = try alloc.dupeSentinel(u8, path, 0);
-    defer alloc.free(sentinel_path);
-
-    const res = std.os.linux.mknod(sentinel_path, mode, dev);
-    if (res != 0)
-        return error.MknodError;
-
-    try setMetadata(alloc, io, id_table, xattr_table, inode, path, options, xattr_idx);
-}

src/extract.zig

@@ -1,24 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const Decomp = @import("decomp.zig");
-const ExtractionOptions = @import("options.zig");
-const Inode = @import("inode.zig");
-const Multi = @import("extract-multi.zig");
-const Single = @import("extract-single.zig");
-const Superblock = @import("archive.zig").Superblock;
-
-pub fn extract(
-    alloc: std.mem.Allocator,
-    io: Io,
-    super: Superblock,
-    data: []u8,
-    decomp: Decomp.Fn,
-    inode: Inode,
-    ext_loc: []const u8,
-    options: ExtractionOptions,
-) !void {
-    if (options.single_threaded)
-        return Single.extract(alloc, io, super, data, decomp, inode, ext_loc, options);
-    return Multi.extract(alloc, io, super, data, decomp, inode, ext_loc, options);
-}

src/file.zig

@@ -1,144 +1,95 @@
-//! A squashfs file/directory. Basically a wrapper around an Inode.
+//! An easier to use wrapper around an inode.
 
 const std = @import("std");
 const Io = std.Io;
 
-const Directory = @import("directory.zig");
-const Inode = @import("inode.zig");
-const MetadataReader = @import("meta_rdr.zig");
-const Decomp = @import("decomp.zig");
-const Superblock = @import("archive.zig").Superblock;
+const Archive = @import("archive.zig");
+const DirEntry = @import("directory.zig");
 const ExtractionOptions = @import("options.zig");
-const Extract = @import("extract.zig");
+const Inode = @import("inode.zig");
+const DataExtractor = @import("util/data_extractor.zig");
+const Decompressor = @import("util/decompressor.zig");
+const MetadataReader = @import("util/metadata.zig");
+const SharedCache = @import("util/shared_cache.zig");
 
 const File = @This();
 
 alloc: std.mem.Allocator,
 
-super: Superblock,
-data: []u8,
-decomp: Decomp.Fn,
+archive: Archive,
 
-name: []const u8,
 inode: Inode,
+name: []const u8,
 
-/// The given allocator should have been used to create the name and inode.
-pub fn init(alloc: std.mem.Allocator, super: Superblock, data: []u8, decomp: Decomp.Fn, name: []const u8, inode: Inode) File {
+/// Creates a new File from an inode. Takes ownership of the Inode and creates a copy of the given name.
+/// Requires the given allocator was used to create the Inode.
+pub fn init(alloc: std.mem.Allocator, archive: Archive, in: Inode, name: []const u8) !File {
+    const new_name = try alloc.alloc(u8, name.len);
+    @memcpy(new_name, name);
     return .{
         .alloc = alloc,
 
-        .super = super,
-        .data = data,
-        .decomp = decomp,
+        .archive = archive,
 
-        .name = name,
-        .inode = inode,
+        .inode = in,
+        .name = new_name,
     };
 }
-/// The name from the directory entry will be duplicated so it's safe to deinit the entry.
-pub fn initEntry(alloc: std.mem.Allocator, super: Superblock, data: []u8, decomp: Decomp.Fn, entry: Directory.Entry) !File {
-    const inode: Inode = try .initEntry(alloc, data, decomp, super.inode_start, super.block_size, entry);
-    errdefer inode.deinit(alloc);
+pub fn fromDirEntry(alloc: std.mem.Allocator, io: Io, archive: Archive, ent: DirEntry) !File {
+    var rdr = try archive.file.readerAt(io, archive.super.inode_start + ent.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, archive.stateless_decomp);
+    try meta.interface.discardAll(ent.block_offset);
 
-    const new_name = try alloc.dupe(u8, entry.name);
-
-    return init(alloc, super, data, decomp, new_name, inode);
-}
-/// The given name is will be duplicated so it's safe to free it afterwards.
-pub fn initRef(alloc: std.mem.Allocator, super: Superblock, data: []u8, decomp: Decomp.Fn, name: []const u8, ref: Inode.Ref) !File {
-    const inode: Inode = try .initRef(alloc, data, decomp, super.inode_start, super.block_size, ref);
-    errdefer inode.deinit(alloc);
-
-    const new_name = try alloc.dupe(u8, name);
-
-    return init(alloc, super, data, decomp, new_name, inode);
-}
-pub fn copy(self: File, alloc: std.mem.Allocator) !File {
-    var inode = self.inode;
-    switch (inode.data) {
-        .file => |*f| f.blocks = try alloc.dupe(Inode.DataBlock, self.inode.data.file.blocks),
-        .ext_file => |*f| f.blocks = try alloc.dupe(Inode.DataBlock, self.inode.data.file.blocks),
-        .symlink => |*s| s.target = try alloc.dupe(u8, self.inode.data.symlink.target),
-        .ext_symlink => |*s| s.target = try alloc.dupe(u8, self.inode.data.symlink.target),
-        else => {},
-    }
-    errdefer inode.deinit(alloc);
-
-    const new_name = try alloc.dupe(u8, self.name);
-
-    return init(alloc, self.super, self.data, self.decomp, new_name, inode);
+    var in: Inode = try .read(alloc, &meta.interface, archive.super.block_size);
+    errdefer in.deinit(alloc);
+    return .init(alloc, archive, in, ent.name);
 }
 pub fn deinit(self: File) void {
     self.alloc.free(self.name);
     self.inode.deinit(self.alloc);
 }
 
-/// If the given File is a directory, open a sub-file at the given filepath.
-/// If the given filepath resolves to the calling File ("" or "."), a copy of the File is returned.
-pub fn open(self: File, alloc: std.mem.Allocator, filepath: []const u8) !File {
-    var size: u64 = 0;
-    var meta: MetadataReader = switch (self.inode.data) {
-        .dir => |d| blk: {
-            size = d.size;
-
-            var meta: MetadataReader = .init(alloc, self.data, self.decomp, self.super.dir_start + d.block_start);
-            try meta.interface.discardAll(d.block_offset);
-            break :blk meta;
-        },
-        .ext_dir => |d| blk: {
-            size = d.size;
-
-            var meta: MetadataReader = .init(alloc, self.data, self.decomp, self.super.dir_start + d.block_start);
-            try meta.interface.discardAll(d.block_offset);
-            break :blk meta;
-        },
-        else => return error.NotDirectory,
-    };
-
-    const path = std.mem.trim(u8, filepath, "/");
-
-    if (path.len == 0 or (path.len == 1 and path[0] == '.'))
-        return self.copy(alloc);
-
-    const first_element: []const u8 = std.mem.sliceTo(path, '/');
-
-    const file: File = blk: {
-        var directory: Directory = try .init(alloc, &meta.interface, size);
-        defer directory.deinit(alloc);
-
-        var entries = directory.entries;
-        var idx = entries.len / 2;
-
-        while (entries.len > 0) {
-            switch (std.mem.order(u8, first_element, entries[idx].name)) {
-                .eq => break,
-                .lt => entries = entries[0..idx],
-                .gt => entries = entries[idx..],
-            }
-            idx = entries.len / 2;
-        } else {
-            return error.NotFound;
-        }
-
-        break :blk try initEntry(alloc, self.super, self.data, self.decomp, entries[idx]);
-    };
-
-    if (first_element.len == path.len)
-        return file;
-    defer file.deinit();
-
-    return file.open(alloc, path[first_element.len..]);
-}
-
-pub fn extract(self: File, alloc: std.mem.Allocator, io: Io, location: []const u8, options: ExtractionOptions) !void {
-    return Extract.extract(
+pub fn open(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
+    const entries = try self.inode.readDirectory(
         alloc,
         io,
-        self.super,
-        self.data,
-        self.decomp,
-        self.inode,
-        location,
-        options,
+        self.archive.file,
+        self.archive.stateless_decomp,
+        self.archive.super.dir_start,
     );
+    defer {
+        for (entries) |ent|
+            alloc.free(ent.name);
+        alloc.free(entries);
+    }
+    const path = std.mem.trim(u8, filepath, "/");
+    const first_element: []const u8 = std.mem.sliceTo(path, '/');
+
+    var search_slice = entries;
+    var idx: usize = undefined;
+    while (search_slice.len > 0) {
+        idx = search_slice.len / 2;
+        const middle = search_slice[idx];
+        switch (std.mem.order(u8, first_element, middle.name)) {
+            .eq => break,
+            .lt => search_slice = search_slice[0..idx],
+            .gt => search_slice = search_slice[idx + 1 ..],
+        }
+    } else return Error.FileNotFound;
+
+    const first_elem_file = try fromDirEntry(alloc, io, self.archive, search_slice[idx]);
+    if (first_element.len == path.len)
+        return first_elem_file;
+    defer first_elem_file.deinit();
+    return first_elem_file.open(alloc, io, path[first_element.len + 1 ..]);
 }
+
+pub fn extract(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8, options: ExtractionOptions) !void {
+    return self.inode.extract(alloc, io, self.archive.file, self.archive.super, filepath, options);
+}
+
+// Types
+
+pub const Error = error{
+    FileNotFound,
+} || Inode.Error;

src/frag.zig

@@ -0,0 +1,84 @@
+const std = @import("std");
+const Io = std.Io;
+
+const BlockSize = @import("inode_data/file.zig").BlockSize;
+const LookupTable = @import("lookup_table.zig");
+const Decompressor = @import("util/decompressor.zig");
+const MetadataReader = @import("util/metadata.zig");
+const OffsetFile = @import("util/offset_file.zig");
+
+const FragManager = @This();
+
+pub const FragEntry = extern struct {
+    start: u64,
+    size: BlockSize,
+    _: u32,
+};
+
+alloc: std.mem.Allocator,
+fil: OffsetFile,
+decomp: *const Decompressor,
+block_size: u32,
+
+entries: []FragEntry,
+
+frag_cache: std.array_hash_map.Auto(u32, []u8),
+cache_mut: std.Io.Mutex = .init,
+
+pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, frag_start: u64, frag_num: u32, block_size: u32) !FragManager {
+    var buf: [8 * 1024]u8 = undefined;
+    var rdr = try fil.readerAt(io, frag_start, &buf);
+    var first_offset: u64 = undefined;
+    try rdr.interface.readSliceEndian(u64, @ptrCast(&first_offset), .little);
+
+    rdr = try fil.readerAt(io, first_offset, &buf);
+    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+
+    const entries = try alloc.alloc(FragEntry, frag_num);
+    errdefer alloc.free(entries);
+
+    try meta.interface.readSliceEndian(FragEntry, entries, .little);
+
+    return .{
+        .alloc = alloc,
+        .fil = fil,
+        .decomp = decomp,
+        .block_size = block_size,
+
+        .entries = entries,
+
+        .frag_cache = .empty,
+    };
+}
+pub fn deinit(self: *FragManager, io: Io) void {
+    self.cache_mut.lockUncancelable(io);
+    self.alloc.free(self.entries);
+    for (self.frag_cache.values()) |v|
+        self.alloc.free(v);
+    self.frag_cache.deinit(self.alloc);
+}
+
+pub fn get(self: *FragManager, io: Io, idx: u32) ![]u8 {
+    if (self.frag_cache.contains(idx))
+        return self.frag_cache.get(idx).?;
+
+    try self.cache_mut.lock(io);
+    defer self.cache_mut.unlock(io);
+
+    const entry = self.entries[idx];
+
+    const out = try self.alloc.alloc(u8, if (entry.size.uncompressed) entry.size.size else self.block_size);
+
+    var buf: [1024 * 1024]u8 = undefined;
+    var rdr = try self.fil.readerAt(io, entry.start, &buf);
+    if (entry.size.uncompressed) {
+        try rdr.interface.readSliceAll(out);
+    } else {
+        @branchHint(.likely);
+        try rdr.interface.fill(entry.size.size);
+        _ = try self.decomp.Decompress(self.alloc, buf[0..entry.size.size], out);
+    }
+
+    try self.frag_cache.put(self.alloc, idx, out);
+    return out;
+}

src/inode.zig

@@ -1,73 +1,166 @@
-const std = @import("std");
-const Io = std.Io;
-const Reader = Io.Reader;
+//! A file-system object. Represents a File or directory.
 
-const Decomp = @import("decomp.zig");
-const Directory = @import("directory.zig");
-const MetadataReader = @import("meta_rdr.zig");
-const Superblock = @import("archive.zig").Superblock;
-const Extract = @import("extract.zig");
+const std = @import("std");
+const Reader = std.Io.Reader;
+const Io = std.Io;
+
+const Archive = @import("archive.zig");
+const Decomp = @import("decomp.zig").Decomp;
+const DirEntry = @import("directory.zig");
 const ExtractionOptions = @import("options.zig");
+const FragEntry = @import("frag.zig").FragEntry;
+const FragManager = @import("frag.zig");
+const dir = @import("inode_data/dir.zig");
+const file = @import("inode_data/file.zig");
+const misc = @import("inode_data/misc.zig");
+const LookupTable = @import("lookup_table.zig");
+const CachedTable = LookupTable.CachedTable;
+const DataExtractor = @import("util/data_extractor.zig");
+const DataReader = @import("util/data_reader.zig");
+const Decompressor = @import("util/decompressor.zig");
+const MetadataReader = @import("util/metadata.zig");
+const OffsetFile = @import("util/offset_file.zig");
+const SharedCache = @import("util/shared_cache.zig");
+const XattrTable = @import("xattr_table.zig");
 
 const Inode = @This();
 
 hdr: Header,
 data: Data,
 
-pub fn init(alloc: std.mem.Allocator, block_size: u32, rdr: *Reader) !Inode {
+pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode {
     var hdr: Header = undefined;
     try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
-
-    const data: Data = switch (hdr.type) {
-        .dir => .{ .dir = try .init(rdr) },
-        .ext_dir => .{ .ext_dir = try .init(rdr) },
-        .file => .{ .file = try .init(alloc, rdr, block_size) },
-        .ext_file => .{ .ext_file = try .init(alloc, rdr, block_size) },
-        .symlink => .{ .symlink = try .init(alloc, rdr) },
-        .ext_symlink => .{ .ext_symlink = try .init(alloc, rdr) },
-        .block_dev => .{ .block_dev = try .init(rdr) },
-        .ext_block_dev => .{ .ext_block_dev = try .init(rdr) },
-        .char_dev => .{ .char_dev = try .init(rdr) },
-        .ext_char_dev => .{ .ext_char_dev = try .init(rdr) },
-        .fifo => .{ .fifo = try .init(rdr) },
-        .ext_fifo => .{ .ext_fifo = try .init(rdr) },
-        .socket => .{ .socket = try .init(rdr) },
-        .ext_socket => .{ .ext_socket = try .init(rdr) },
+    return .{
+        .hdr = hdr,
+        .data = switch (hdr.inode_type) {
+            .dir => .{ .dir = try .read(rdr) },
+            .file => .{ .file = try .read(alloc, rdr, block_size) },
+            .symlink => .{ .symlink = try .read(alloc, rdr) },
+            .block_dev => .{ .block_dev = try .read(rdr) },
+            .char_dev => .{ .char_dev = try .read(rdr) },
+            .fifo => .{ .fifo = try .read(rdr) },
+            .socket => .{ .socket = try .read(rdr) },
+            .ext_dir => .{ .ext_dir = try .read(rdr) },
+            .ext_file => .{ .ext_file = try .read(alloc, rdr, block_size) },
+            .ext_symlink => .{ .ext_symlink = try .read(alloc, rdr) },
+            .ext_block_dev => .{ .ext_block_dev = try .read(rdr) },
+            .ext_char_dev => .{ .ext_char_dev = try .read(rdr) },
+            .ext_fifo => .{ .ext_fifo = try .read(rdr) },
+            .ext_socket => .{ .ext_socket = try .read(rdr) },
+        },
     };
-    return .{ .hdr = hdr, .data = data };
-}
-pub fn initRef(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, inode_start: u64, block_size: u32, ref: Ref) !Inode {
-    var meta: MetadataReader = .init(alloc, data, decomp, inode_start + ref.block_start);
-    try meta.interface.discardAll(ref.block_offset);
-
-    return .init(alloc, block_size, &meta.interface);
-}
-pub fn initEntry(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, inode_start: u64, block_size: u32, entry: Directory.Entry) !Inode {
-    var meta: MetadataReader = .init(alloc, data, decomp, inode_start + entry.block_start);
-    try meta.interface.discardAll(entry.block_offset);
-
-    return .init(alloc, block_size, &meta.interface);
 }
 pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
     switch (self.data) {
-        .file => |f| alloc.free(f.blocks),
-        .ext_file => |f| alloc.free(f.blocks),
-        .symlink => |s| alloc.free(s.target),
-        .ext_symlink => |s| alloc.free(s.target),
+        .file => |d| d.deinit(alloc),
+        .symlink => |d| d.deinit(alloc),
+        .ext_file => |d| d.deinit(alloc),
+        .ext_symlink => |d| d.deinit(alloc),
         else => {},
     }
 }
 
-pub fn extract(self: Inode, alloc: std.mem.Allocator, io: Io, super: Superblock, data: []u8, decomp: Decomp.Fn, location: []const u8, options: ExtractionOptions) !void {
-    return Extract.extract(alloc, io, super, data, decomp, self, location, options);
+// Utility Functions
+
+/// Read the directory entries
+pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64) ![]DirEntry {
+    return switch (self.data) {
+        .dir => |d| readDirFromData(alloc, io, fil, decomp, dir_offset, d),
+        .ext_dir => |d| readDirFromData(alloc, io, fil, decomp, dir_offset, d),
+        else => Error.NotDirectory,
+    };
+}
+fn readDirFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64, d: anytype) ![]DirEntry {
+    var rdr = try fil.readerAt(io, dir_offset + d.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+    try meta.interface.discardAll(d.block_offset);
+
+    return DirEntry.readDirectory(alloc, &meta.interface, d.size);
+}
+/// Get a reader for a regular file's data.
+pub fn dataReader(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32) !DataReader {
+    return switch (self.data) {
+        .file => |f| getReaderFromData(alloc, io, fil, cache, decomp, block_size, f),
+        .ext_file => |f| getReaderFromData(alloc, io, fil, cache, decomp, block_size, f),
+        else => Error.NotRegularFile,
+    };
+}
+fn getReaderFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32, d: anytype) !DataReader {
+    const ext: DataReader = .init(alloc, io, fil, cache, decomp, block_size, d.size, d.block_start, d.blocks);
+    if (d.frag_block_offset == 0xFFFFFFFF) {
+        // TODO:
+        return error.TODO;
+    }
+    return ext;
+}
+/// Get an extractor for a regular file's data.
+pub fn dataExtractor(self: Inode, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32) !DataExtractor {
+    return switch (self.data) {
+        .file => |f| getExtractorFromData(fil, cache, decomp, block_size, f),
+        .ext_file => |f| getExtractorFromData(fil, cache, decomp, block_size, f),
+        else => Error.NotRegularFile,
+    };
+}
+fn getExtractorFromData(fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32, d: anytype) !DataExtractor {
+    const ext: DataExtractor = .init(fil, cache, decomp, block_size, d.size, d.block_start, d.blocks);
+    if (d.frag_block_offset == 0xFFFFFFFF) {
+        // TODO:
+        return error.TODO;
+    }
+    return ext;
+}
+/// Get a symlink's target path
+pub fn symlinkTarget(self: Inode) ![]const u8 {
+    return switch (self.data) {
+        .symlink => |s| s.target,
+        .ext_symlink => |s| s.target,
+        else => Error.NotSymlink,
+    };
+}
+/// Get inode's gid
+pub fn gid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, id_table_start: u64) !u16 {
+    return LookupTable.lookupValue(u16, alloc, io, decomp, fil, id_table_start, self.hdr.gid_idx);
+}
+/// Get inode's uid
+pub fn uid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, id_table_start: u64) !u16 {
+    return LookupTable.lookupValue(u16, alloc, io, decomp, fil, id_table_start, self.hdr.uid_idx);
+}
+/// Get the inode's xattr values as an index into the Archive's xattr table.
+/// Returns error.NoXattr if the inode doesn't have extended attributes.
+pub fn xattrIndex(self: Inode) !u32 {
+    const idx = switch (self.data) {
+        .ext_dir => |e| e.xattr_idx,
+        .ext_file => |e| e.xattr_idx,
+        .ext_symlink => |e| e.xattr_idx,
+        .ext_block_dev => |e| e.xattr_idx,
+        .ext_char_dev => |e| e.xattr_idx,
+        .ext_fifo => |e| e.xattr_idx,
+        .ext_socket => |e| e.xattr_idx,
+        else => Error.NoXattr,
+    };
+    if (idx == 0xFFFFFFFF) return Error.NoXattr;
+    return idx;
+}
+// Get an inode's xattr values. If the inode does not have xattr values (including if the inode is not an extended type), an empty slice is returned.
+pub fn xattrValues(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, xattr_table_start: u64) ![]XattrTable.XattrOwned {
+    const idx = self.xattrIndex() catch &[0]XattrTable.XattrOwned{};
+    return XattrTable.statelessLookup(alloc, io, decomp, fil, xattr_table_start, idx);
 }
 
 // Types
 
+pub const Error = error{
+    NotDirectory,
+    NotRegularFile,
+    NotSymlink,
+    NotExtended,
+};
+
 pub const Ref = packed struct(u64) {
     block_offset: u16,
     block_start: u32,
-    _: u16,
+    _: u16 = 0,
 };
 
 pub const Type = enum(u16) {
@@ -87,8 +180,25 @@ pub const Type = enum(u16) {
     ext_socket,
 };
 
+pub const Data = union(Type) {
+    dir: dir.Dir,
+    file: file.File,
+    symlink: misc.Symlink,
+    block_dev: misc.Dev,
+    char_dev: misc.Dev,
+    fifo: misc.IPC,
+    socket: misc.IPC,
+    ext_dir: dir.ExtDir,
+    ext_file: file.ExtFile,
+    ext_symlink: misc.ExtSymlink,
+    ext_block_dev: misc.ExtDev,
+    ext_char_dev: misc.ExtDev,
+    ext_fifo: misc.ExtIPC,
+    ext_socket: misc.ExtIPC,
+};
+
 pub const Header = extern struct {
-    type: Type,
+    inode_type: Type,
     permissions: u16,
     uid_idx: u16,
     gid_idx: u16,
@@ -96,201 +206,298 @@ pub const Header = extern struct {
     num: u32,
 };
 
-pub const Data = union(Type) {
-    dir: Dir,
-    file: File,
-    symlink: Symlink,
-    block_dev: Dev,
-    char_dev: Dev,
-    fifo: IPC,
-    socket: IPC,
-    ext_dir: ExtDir,
-    ext_file: ExtFile,
-    ext_symlink: ExtSymlink,
-    ext_block_dev: ExtDev,
-    ext_char_dev: ExtDev,
-    ext_fifo: ExtIPC,
-    ext_socket: ExtIPC,
+// Extract
+
+const ExtractError = error{ MknodFailed, CannotSetXattr, ConcurrencyUnavailable } || DataExtractor.Error || Io.Dir.CreateFileAtomicError || LookupTable.Error ||
+    Io.File.Reader.SeekError || Io.File.Atomic.LinkError || Io.Dir.CreateDirError || Io.File.OpenError ||
+    Io.File.SetPermissionsError || Io.File.SetOwnerError || Io.Dir.SymLinkError || Io.Dir.CreateDirPathError;
+const PathRet = struct {
+    path: []const u8,
+    inode: Inode,
+    xattr_idx: ?u32 = null,
+};
+fn DirCompare(_: void, a: PathRet, b: PathRet) std.math.Order{
+
+}
+const ExtractReturnUnion = union(enum) {
+    path_ret: ExtractError!PathRet,
+};
+const Tables = struct {
+    id: LookupTable.CachedTable(u16),
+    frag: LookupTable.CachedTable(FragEntry),
+    xattr: XattrTable,
 };
 
-pub const DataBlock = packed struct(u32) {
-    size: u24,
-    uncompressed: bool,
-    _: u7,
-};
+/// Extracts the given inode to the given path. If the inode not a directory, the given path must not exist.
+/// If the inode is a directory the path must not exist or be a directory.
+pub fn extract(
+    self: Inode,
+    alloc: std.mem.Allocator,
+    io: Io,
+    fil: OffsetFile,
+    super: Archive.Superblock,
+    filepath: []const u8,
+    options: ExtractionOptions,
+) !void {
+    const path = std.mem.trimEnd(u8, filepath, "/");
 
-pub const Dir = extern struct {
-    block_start: u32,
-    hard_links: u32,
-    size: u16,
-    block_offset: u16,
-    parent_num: u32,
+    var decomp_base: Decomp = switch (super.compression) {
+        .gzip => .{ .gzip = try .init(alloc, super.block_size) },
+        .lzma => .{ .lzma = try .init(alloc, super.block_size) },
+        .xz => .{ .xz = try .init(alloc, super.block_size) },
+        .zstd => .{ .zstd = try .init(alloc, io, super.block_size) },
+        else => unreachable,
+    };
+    defer decomp_base.deinit();
+    const decomp = decomp_base.decompressor();
 
-    fn init(rdr: *Reader) !Dir {
-        var new: Dir = undefined;
-        try rdr.readSliceEndian(Dir, @ptrCast(&new), .little);
-        return new;
+    var frags: FragManager = try .init(alloc, io, fil, decomp, super.frag_start, super.frag_count, super.block_size);
+    defer frags.deinit(io);
+
+    var sel_buf = [1]ExtractReturnUnion{undefined} ** 10;
+    var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf);
+    defer sel.cancelDiscard();
+
+    var fold_queu: std.PriorityDequeue(PathRet, void, comptime compareFn: fn (Context, T, T) Order)
+
+    try sel.concurrent(.path_ret, extractReal, .{ self, alloc, io, fil, decomp, super, &frags, &sel, path });
+
+    var xattr_table: ?XattrTable = if (!options.ignore_xattr)
+        try .init(alloc, io, fil, decomp, super.xattr_start)
+    else
+        null;
+    defer if (!options.ignore_xattr) xattr_table.?.deinit(io);
+    if (xattr_table != null) try xattr_table.?.table.fill(io);
+
+    var id_table: ?CachedTable(u16) = if (!options.ignore_xattr)
+        .init(alloc, fil, decomp, super.id_start, super.id_count)
+    else
+        null;
+    defer if (!options.ignore_xattr) id_table.?.deinit(io);
+    if (id_table != null) try id_table.?.fill(io);
+
+    while (true) {
+        const group_token = sel.group.token.load(.acquire);
+        if (group_token == null) break;
+        // std.debug.print("{any}\n", .{sel.group.state});
+
+        // std.debug.print("Waiting for return...", .{});
+        const ret = try sel.await();
+        defer sel.queue.putOneUncancelable(io, ret) catch {};
+        // std.debug.print("Got One...\n", .{});
+        const path_ret = try ret.path_ret;
+
+        if (options.ignore_permissions and options.ignore_xattr) continue;
+        if (options.ignore_permissions and path_ret.xattr_idx == null) continue;
+
+        var ret_file = try Io.Dir.cwd().openFile(io, path_ret.path, .{});
+        defer ret_file.close(io);
+
+        if (!options.ignore_permissions) {
+            try ret_file.setPermissions(io, @enumFromInt(path_ret.inode.hdr.permissions));
+            try ret_file.setOwner(
+                io,
+                try id_table.?.get(io, path_ret.inode.hdr.uid_idx),
+                try id_table.?.get(io, path_ret.inode.hdr.gid_idx),
+            );
+        }
+        if (@hasField(std.os, "linux") and !options.ignore_xattr and path_ret.xattr_idx != null) {
+            const xattrs = try xattr_table.?.get(alloc, io, path_ret.xattr_idx.?);
+            defer {
+                for (xattrs) |x|
+                    alloc.free(x.key);
+                alloc.free(xattrs);
+            }
+
+            for (xattrs) |x| {
+                const res = std.os.linux.fsetxattr(ret_file.handle, x.key, x.value.ptr, x.value.len, 0);
+                if (res != 0)
+                    return error.CannotSetXattr;
+            }
+        }
     }
-};
-pub const ExtDir = extern struct {
-    hard_links: u32,
-    size: u32,
-    block_start: u32,
-    parent_num: u32,
-    idx_count: u16,
-    block_offset: u16,
-    xattr_idx: u32,
+}
+pub fn extractReal(
+    self: Inode,
+    alloc: std.mem.Allocator,
+    io: Io,
+    fil: OffsetFile,
+    decomp: *const Decompressor,
+    super: Archive.Superblock,
+    frags: *FragManager,
+    sel: *Io.Select(ExtractReturnUnion),
+    path: []const u8,
+) ExtractError!PathRet {
+    return switch (self.hdr.inode_type) {
+        .dir, .ext_dir => extractDir(self, alloc, io, fil, decomp, super, sel, frags, path),
+        .file, .ext_file => extractFile(self, alloc, io, fil, decomp, frags, super.block_size, path),
+        .symlink, .ext_symlink => extractSymlink(self, io, path),
+        else => extractDevOrIPC(self, alloc, path),
+    };
+}
+fn extractDir(
+    self: Inode,
+    alloc: std.mem.Allocator,
+    io: Io,
+    fil: OffsetFile,
+    decomp: *const Decompressor,
+    super: Archive.Superblock,
+    parent_select: *Io.Select(ExtractReturnUnion),
+    frags: *FragManager,
+    path: []const u8,
+) ExtractError!PathRet {
+    try Io.Dir.cwd().createDirPath(io, path);
 
-    fn init(rdr: *Reader) !ExtDir {
-        var new: ExtDir = undefined;
-        try rdr.readSliceEndian(ExtDir, @ptrCast(&new), .little);
-        return new;
+    var sel_buf = [1]ExtractReturnUnion{undefined} ** 10;
+    var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf);
+    defer sel.cancelDiscard();
+
+    var num: usize = 0;
+    {
+        const dir_entries = self.readDirectory(alloc, io, fil, decomp, super.dir_start) catch |err| switch (err) {
+            Error.NotDirectory => unreachable,
+            else => return @errorCast(err),
+        };
+        num = dir_entries.len;
+        defer {
+            for (dir_entries) |d|
+                d.deinit(alloc);
+            alloc.free(dir_entries);
+        }
+
+        for (dir_entries) |d| {
+            var rdr = try fil.readerAt(io, d.block_start + super.inode_start, &[0]u8{});
+            var meta_rdr: MetadataReader = .init(alloc, &rdr.interface, decomp);
+            try meta_rdr.interface.discardAll(d.block_offset);
+            const inode = try read(alloc, &meta_rdr.interface, super.block_size);
+            errdefer inode.deinit(alloc);
+
+            const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", d.name });
+            errdefer alloc.free(new_path);
+
+            try sel.concurrent(.path_ret, extractReal, .{ self, alloc, io, fil, decomp, super, frags, &sel, new_path });
+        }
     }
-};
-pub const File = struct {
-    block_start: u32,
-    frag_idx: u32,
-    frag_offset: u32,
-    size: u32,
-    blocks: []DataBlock,
 
-    fn init(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !File {
-        var data: [16]u8 = undefined;
-        try rdr.readSliceAll(&data);
+    while (num > 0) {
+        const ret = sel.await() catch break;
+        num -= 1;
 
-        const frag_idx = std.mem.readInt(u32, data[4..8], .little);
-        const size = std.mem.readInt(u32, data[12..], .little);
-
-        var blocks_num = size / block_size;
-        if (size % block_size != 0 and frag_idx == 0xFFFFFFFF)
-            blocks_num += 1;
-
-        const blocks = try alloc.alloc(DataBlock, blocks_num);
-        errdefer alloc.free(blocks);
-        try rdr.readSliceEndian(DataBlock, blocks, .little);
-
-        return .{
-            .block_start = std.mem.readInt(u32, data[0..4], .little),
-            .frag_idx = frag_idx,
-            .frag_offset = std.mem.readInt(u32, data[8..12], .little),
-            .size = size,
-            .blocks = blocks,
+        parent_select.queue.putOne(io, ret) catch |err| switch (err) {
+            error.Canceled => return error.Canceled,
+            else => break,
         };
     }
-};
-pub const ExtFile = struct {
-    block_start: u64,
-    size: u64,
-    sparse: u64,
-    hard_links: u32,
-    frag_idx: u32,
-    frag_offset: u32,
-    xattr_idx: u32,
-    blocks: []DataBlock,
+    return .{
+        .path = path,
+        .inode = self,
+    };
+}
+fn extractFile(
+    self: Inode,
+    alloc: std.mem.Allocator,
+    io: Io,
+    fil: OffsetFile,
+    decomp: *const Decompressor,
+    frag: *FragManager,
+    block_size: u32,
+    path: []const u8,
+) ExtractError!PathRet {
+    var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{});
+    defer atomic.deinit(io);
 
-    fn init(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !ExtFile {
-        var data: [40]u8 = undefined;
-        try rdr.readSliceAll(&data);
+    var ret: PathRet = .{
+        .inode = self,
+        .path = path,
+    };
+    const data: DataExtractor = blk: {
+        switch (self.data) {
+            .file => |f| {
+                var data: DataExtractor = .init(fil, decomp, block_size, f.size, f.block_start, f.block_sizes);
+                if (f.frag_idx != 0xFFFFFFFF)
+                    data.addFrag(f.frag_block_offset, try frag.get(io, f.frag_idx));
 
-        const frag_idx = std.mem.readInt(u32, data[28..32], .little);
-        const size = std.mem.readInt(u64, data[8..16], .little);
+                break :blk data;
+            },
+            .ext_file => |f| {
+                if (f.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = f.xattr_idx;
+                var data: DataExtractor = .init(fil, decomp, block_size, f.size, f.block_start, f.block_sizes);
+                if (f.frag_idx != 0xFFFFFFFF)
+                    data.addFrag(f.frag_block_offset, try frag.get(io, f.frag_idx));
 
-        var blocks_num = size / block_size;
-        if (size % block_size != 0 and frag_idx == 0xFFFFFFFF)
-            blocks_num += 1;
+                break :blk data;
+            },
+            else => unreachable,
+        }
+    };
 
-        const blocks = try alloc.alloc(DataBlock, blocks_num);
-        errdefer alloc.free(blocks);
-        try rdr.readSliceEndian(DataBlock, blocks, .little);
+    try data.extractAsync(alloc, io, atomic.file);
+    try atomic.link(io);
 
-        return .{
-            .block_start = std.mem.readInt(u64, data[0..8], .little),
-            .size = size,
-            .sparse = std.mem.readInt(u64, data[16..24], .little),
-            .hard_links = std.mem.readInt(u32, data[24..28], .little),
-            .frag_idx = frag_idx,
-            .frag_offset = std.mem.readInt(u32, data[32..36], .little),
-            .xattr_idx = std.mem.readInt(u32, data[36..], .little),
-            .blocks = blocks,
-        };
+    return ret;
+}
+fn extractSymlink(self: Inode, io: Io, path: []const u8) ExtractError!PathRet {
+    const target = switch (self.data) {
+        .symlink => |s| s.target,
+        .ext_symlink => |s| s.target,
+        else => unreachable,
+    };
+
+    try Io.Dir.cwd().symLink(io, target, path, .{});
+
+    return .{
+        .path = path,
+        .inode = self,
+    };
+}
+fn extractDevOrIPC(self: Inode, alloc: std.mem.Allocator, path: []const u8) ExtractError!PathRet {
+    var dev_num: u32 = 0;
+    var mode: u32 = 0;
+
+    const DT = std.posix.DT;
+
+    var ret: PathRet = .{
+        .inode = self,
+        .path = path,
+    };
+
+    switch (self.data) {
+        .block_dev => |d| {
+            dev_num = d.dev;
+            mode = DT.BLK;
+        },
+        .ext_block_dev => |d| {
+            dev_num = d.dev;
+            mode = DT.BLK;
+            if (d.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = d.xattr_idx;
+        },
+        .char_dev => |d| {
+            dev_num = d.dev;
+            mode = DT.CHR;
+        },
+        .ext_char_dev => |d| {
+            dev_num = d.dev;
+            mode = DT.CHR;
+            if (d.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = d.xattr_idx;
+        },
+        .fifo => mode = DT.FIFO,
+        .ext_fifo => |f| {
+            mode = DT.FIFO;
+            if (f.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = f.xattr_idx;
+        },
+        .socket => mode = DT.SOCK,
+        .ext_socket => |s| {
+            mode = DT.SOCK;
+            if (s.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = s.xattr_idx;
+        },
+        else => unreachable,
     }
-};
-pub const Symlink = struct {
-    hard_links: u32,
-    target: []const u8,
 
-    fn init(alloc: std.mem.Allocator, rdr: *Reader) !Symlink {
-        var data: [8]u8 = undefined;
-        try rdr.readSliceAll(&data);
+    const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
+    defer alloc.free(sentinel_path);
 
-        const target_size = std.mem.readInt(u32, data[4..], .little);
+    const res = std.os.linux.mknod(sentinel_path, mode, dev_num);
+    if (res != 0) return ExtractError.MknodFailed;
 
-        const target = try alloc.alloc(u8, target_size);
-        errdefer alloc.free(target);
-        try rdr.readSliceEndian(u8, target, .little);
-
-        return .{
-            .hard_links = std.mem.readInt(u32, data[0..4], .little),
-            .target = target,
-        };
-    }
-};
-pub const ExtSymlink = struct {
-    hard_links: u32,
-    target: []const u8,
-    xattr_idx: u32,
-
-    fn init(alloc: std.mem.Allocator, rdr: *Reader) !ExtSymlink {
-        const sym: Symlink = try .init(alloc, rdr);
-        errdefer alloc.free(sym.target);
-
-        var xattr_idx: u32 = undefined;
-        try rdr.readSliceEndian(u32, @ptrCast(&xattr_idx), .little);
-
-        return .{
-            .hard_links = sym.hard_links,
-            .target = sym.target,
-            .xattr_idx = xattr_idx,
-        };
-    }
-};
-pub const Dev = extern struct {
-    hard_links: u32,
-    device: u32,
-
-    fn init(rdr: *Reader) !Dev {
-        var new: Dev = undefined;
-        try rdr.readSliceEndian(Dev, @ptrCast(&new), .little);
-        return new;
-    }
-};
-pub const ExtDev = extern struct {
-    hard_links: u32,
-    device: u32,
-    xattr_idx: u32,
-
-    fn init(rdr: *Reader) !ExtDev {
-        var new: ExtDev = undefined;
-        try rdr.readSliceEndian(ExtDev, @ptrCast(&new), .little);
-        return new;
-    }
-};
-pub const IPC = extern struct {
-    hard_links: u32,
-
-    fn init(rdr: *Reader) !IPC {
-        var new: IPC = undefined;
-        try rdr.readSliceEndian(IPC, @ptrCast(&new), .little);
-        return new;
-    }
-};
-pub const ExtIPC = extern struct {
-    hard_links: u32,
-    xattr_idx: u32,
-
-    fn init(rdr: *Reader) !ExtIPC {
-        var new: ExtIPC = undefined;
-        try rdr.readSliceEndian(ExtIPC, @ptrCast(&new), .little);
-        return new;
-    }
-};
+    return ret;
+}

src/inode_data/dir.zig

@@ -0,0 +1,32 @@
+const Reader = @import("std").Io.Reader;
+
+pub const Dir = extern struct {
+    block_start: u32,
+    hard_links: u32,
+    size: u16,
+    block_offset: u16,
+    parent_num: u32,
+
+    pub fn read(rdr: *Reader) !Dir {
+        var d: Dir = undefined;
+        try rdr.readSliceEndian(Dir, @ptrCast(&d), .little);
+        return d;
+    }
+};
+
+pub const ExtDir = extern struct {
+    hard_links: u32,
+    size: u32,
+    block_start: u32,
+    parent_num: u32,
+    idx_count: u16,
+    block_offset: u16,
+    xattr_idx: u32,
+    // index: []DirIndex
+
+    pub fn read(rdr: *Reader) !ExtDir {
+        var d: ExtDir = undefined;
+        try rdr.readSliceEndian(ExtDir, @ptrCast(&d), .little);
+        return d;
+    }
+};

src/inode_data/file.zig

@@ -0,0 +1,97 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+
+pub const BlockSize = packed struct(u32) {
+    size: u24,
+    uncompressed: bool,
+    _: u7,
+};
+
+const FileRawRead = extern struct {
+    block_start: u32,
+    frag_idx: u32,
+    frag_block_offset: u32,
+    size: u32,
+};
+
+pub const File = struct {
+    block_start: u32,
+    frag_idx: u32,
+    frag_block_offset: u32,
+    size: u32,
+    block_sizes: []BlockSize,
+
+    pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !File {
+        var raw: FileRawRead = undefined;
+        try rdr.readSliceEndian(FileRawRead, @ptrCast(&raw), .little);
+
+        var num_blocks: u32 = raw.size / block_size;
+        if (raw.size % block_size != 0 and raw.frag_idx == 0xFFFFFFFF)
+            num_blocks += 1;
+
+        const sizes = try alloc.alloc(BlockSize, num_blocks);
+        errdefer alloc.free(sizes);
+        try rdr.readSliceEndian(BlockSize, sizes, .little);
+
+        return .{
+            .block_start = raw.block_start,
+            .frag_idx = raw.frag_idx,
+            .frag_block_offset = raw.frag_block_offset,
+            .size = raw.size,
+            .block_sizes = sizes,
+        };
+    }
+
+    pub fn deinit(self: File, alloc: std.mem.Allocator) void {
+        alloc.free(self.block_sizes);
+    }
+};
+
+const ExtFileRawRead = extern struct {
+    block_start: u64,
+    size: u64,
+    sparse: u64,
+    hard_links: u32,
+    frag_idx: u32,
+    frag_block_offset: u32,
+    xattr_idx: u32,
+};
+
+pub const ExtFile = struct {
+    block_start: u64,
+    size: u64,
+    sparse: u64,
+    hard_links: u32,
+    frag_idx: u32,
+    frag_block_offset: u32,
+    xattr_idx: u32,
+    block_sizes: []BlockSize,
+
+    pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !ExtFile {
+        var raw: ExtFileRawRead = undefined;
+        try rdr.readSliceEndian(ExtFileRawRead, @ptrCast(&raw), .little);
+
+        var num_blocks: u32 = @truncate(raw.size / block_size);
+        if (raw.size % block_size != 0 and raw.frag_idx == 0xFFFFFFFF)
+            num_blocks += 1;
+
+        const sizes = try alloc.alloc(BlockSize, num_blocks);
+        errdefer alloc.free(sizes);
+        try rdr.readSliceEndian(BlockSize, sizes, .little);
+
+        return .{
+            .block_start = raw.block_start,
+            .size = raw.size,
+            .sparse = raw.sparse,
+            .hard_links = raw.hard_links,
+            .frag_idx = raw.frag_idx,
+            .frag_block_offset = raw.frag_block_offset,
+            .xattr_idx = raw.xattr_idx,
+            .block_sizes = sizes,
+        };
+    }
+
+    pub fn deinit(self: ExtFile, alloc: std.mem.Allocator) void {
+        alloc.free(self.block_sizes);
+    }
+};

src/inode_data/misc.zig

@@ -0,0 +1,98 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+
+pub const Symlink = struct {
+    hard_links: u32,
+    target: []const u8,
+
+    pub fn read(alloc: std.mem.Allocator, rdr: *Reader) !Symlink {
+        var start: [8]u8 = undefined;
+        try rdr.readSliceAll(&start);
+        const target_size = std.mem.readInt(u32, start[4..8], .little);
+        const target = try alloc.alloc(u8, target_size + 1);
+        errdefer alloc.free(target);
+        try rdr.readSliceEndian(u8, target, .little);
+        return .{
+            .hard_links = std.mem.readInt(u32, start[0..4], .little),
+            .target = target,
+        };
+    }
+
+    pub fn deinit(self: Symlink, alloc: std.mem.Allocator) void {
+        alloc.free(self.target);
+    }
+};
+
+pub const ExtSymlink = struct {
+    hard_links: u32,
+    target: []const u8,
+    xattr_idx: u32,
+
+    pub fn read(alloc: std.mem.Allocator, rdr: *Reader) !ExtSymlink {
+        var start: [8]u8 = undefined;
+        try rdr.readSliceAll(&start);
+        const target_size = std.mem.readInt(u32, start[4..8], .little);
+        const target = try alloc.alloc(u8, target_size + 1);
+        errdefer alloc.free(target);
+        try rdr.readSliceEndian(u8, target, .little);
+        var xattr_idx: u32 = undefined;
+        try rdr.readSliceEndian(u32, @ptrCast(&xattr_idx), .little);
+        return .{
+            .hard_links = std.mem.readInt(u32, start[0..4], .little),
+            .target = target,
+            .xattr_idx = xattr_idx,
+        };
+    }
+
+    pub fn deinit(self: ExtSymlink, alloc: std.mem.Allocator) void {
+        alloc.free(self.target);
+    }
+};
+
+/// A block or character device.
+pub const Dev = extern struct {
+    hard_links: u32,
+    dev: u32,
+
+    pub fn read(rdr: *Reader) !Dev {
+        var d: Dev = undefined;
+        try rdr.readSliceEndian(Dev, @ptrCast(&d), .little);
+        return d;
+    }
+};
+
+/// An extended block or character device.
+pub const ExtDev = extern struct {
+    hard_links: u32,
+    dev: u32,
+    xattr_idx: u32,
+
+    pub fn read(rdr: *Reader) !ExtDev {
+        var d: ExtDev = undefined;
+        try rdr.readSliceEndian(ExtDev, @ptrCast(&d), .little);
+        return d;
+    }
+};
+
+/// A socket or FIFO file.
+pub const IPC = extern struct {
+    hard_links: u32,
+
+    pub fn read(rdr: *Reader) !IPC {
+        var d: IPC = undefined;
+        try rdr.readSliceEndian(IPC, @ptrCast(&d), .little);
+        return d;
+    }
+};
+
+/// An extended socket or FIFO file.
+pub const ExtIPC = extern struct {
+    hard_links: u32,
+    xattr_idx: u32,
+
+    pub fn read(rdr: *Reader) !ExtIPC {
+        var d: ExtIPC = undefined;
+        try rdr.readSliceEndian(ExtIPC, @ptrCast(&d), .little);
+        return d;
+    }
+};

src/lookup.zig

@@ -1,83 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const DataBlock = @import("inode.zig").DataBlock;
-const Decomp = @import("decomp.zig");
-const MetadataReader = @import("meta_rdr.zig");
-const ProtectedMap = @import("util/protected_map.zig").ProtectedMap;
-
-pub fn Table(comptime T: anytype) type {
-    return struct {
-        const VALUES_PER_BLOCK = 8192 / @sizeOf(T);
-
-        const Self = @This();
-
-        data: []u8,
-        decomp: Decomp.Fn,
-
-        table_start: u64,
-        table_num: u32,
-
-        table: ProtectedMap(u32, []T, getBlock),
-
-        pub fn init(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, table_start: u64, table_num: u32) Self {
-            return .{
-                .data = data,
-                .decomp = decomp,
-
-                .table_start = table_start,
-                .table_num = table_num,
-
-                .table = .init(alloc),
-            };
-        }
-        pub fn deinit(self: *Self) void {
-            var iter = self.table.map.valueIterator();
-
-            while (iter.next()) |val| {
-                if (val.err != null) continue;
-
-                self.table.alloc.free(val.value);
-            }
-            self.table.deinit();
-        }
-
-        pub fn get(self: *Self, io: Io, idx: u32) !T {
-            const block = idx / VALUES_PER_BLOCK;
-            const block_idx = idx % VALUES_PER_BLOCK;
-
-            const values = try self.table.getOrPut(io, block, .{
-                self.table.alloc,
-                self.data,
-                self.decomp,
-                self.table_start,
-                self.table_num,
-                block,
-            });
-
-            return values.*[block_idx];
-        }
-
-        fn getBlock(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, table_start: u64, table_num: u32, block_idx: u32) ![]T {
-            const offset: u64 = std.mem.readInt(u64, data[table_start + (block_idx * 8) ..][0..8], .little);
-
-            const block = try alloc.alloc(T, if (block_idx == (table_num - 1) / VALUES_PER_BLOCK)
-                table_num % VALUES_PER_BLOCK
-            else
-                VALUES_PER_BLOCK);
-
-            var meta: MetadataReader = .init(alloc, data, decomp, offset);
-            try meta.interface.readSliceEndian(T, block, .little);
-
-            return block;
-        }
-    };
-}
-
-// Types
-
-pub const FragEntry = extern struct {
-    block_start: u64,
-    size: DataBlock,
-    _: u32,
-};

src/lookup_table.zig

@@ -0,0 +1,122 @@
+const std = @import("std");
+const Io = std.Io;
+
+const Decompressor = @import("util/decompressor.zig");
+const MetadataReader = @import("util/metadata.zig");
+const OffsetFile = @import("util/offset_file.zig");
+
+pub fn lookupValue(comptime T: anytype, alloc: std.mem.Allocator, io: Io, decomp: *const Decompressor, file: OffsetFile, table_start: u64, idx: u32) !T {
+    const T_PER_BLOCK: u16 = 8192 / @sizeOf(T);
+
+    const block = idx / T_PER_BLOCK;
+    const block_offset = idx % T_PER_BLOCK;
+
+    var rdr = try file.readerAt(io, table_start + (8 * block), &[0]u8{});
+    var offset: u64 = undefined;
+    try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
+
+    rdr = try file.readerAt(io, offset, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+
+    try meta.interface.discardAll(@sizeOf(T) * block_offset);
+    var out: T = undefined;
+    try meta.interface.readSliceEndian(T, @ptrCast(&out), .little);
+    return out;
+}
+
+pub const Error = Io.Cancelable || Io.File.Reader.SeekError || Io.Reader.ReadAllocError;
+
+pub fn CachedTable(comptime T: anytype) type {
+    return struct {
+        const T_PER_BLOCK: u16 = 8192 / @sizeOf(T);
+
+        const Table = @This();
+
+        alloc: std.mem.Allocator,
+        fil: OffsetFile,
+        decomp: *const Decompressor,
+
+        table_start: u64,
+        total_num: u32,
+
+        table: std.AutoHashMap(u32, []T),
+
+        mut: Io.Mutex = .init,
+
+        pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, offset: u64, total_num: u32) Table {
+            return .{
+                .alloc = alloc,
+                .fil = fil,
+                .decomp = decomp,
+
+                .table_start = offset,
+                .total_num = total_num,
+
+                .table = .init(alloc),
+            };
+        }
+        pub fn deinit(self: *Table, io: Io) void {
+            self.mut.lockUncancelable(io);
+            var iter = self.table.valueIterator();
+            while (iter.next()) |val|
+                self.alloc.free(val.*);
+            self.table.deinit();
+        }
+
+        pub fn fill(self: *Table, io: Io) Error!void {
+            try self.mut.lock(io);
+            defer self.mut.unlock(io);
+
+            var num_blocks = self.total_num / T_PER_BLOCK;
+            if (self.total_num % T_PER_BLOCK > 0)
+                num_blocks += 1;
+
+            for (0..num_blocks) |block| {
+                var rdr = try self.fil.readerAt(io, self.table_start + (8 * block), &[0]u8{});
+                var offset: u64 = undefined;
+                try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
+
+                const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK)
+                    @truncate(self.total_num % T_PER_BLOCK)
+                else
+                    T_PER_BLOCK;
+
+                rdr = try self.fil.readerAt(io, offset, &[0]u8{});
+                var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
+
+                const slice = try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little);
+                try self.table.put(@truncate(block), slice);
+            }
+        }
+
+        pub fn get(self: *Table, io: Io, idx: u32) Error!T {
+            const block = idx / T_PER_BLOCK;
+            const block_offset = idx % T_PER_BLOCK;
+            if (self.table.contains(block))
+                return self.table.get(block).?[block_offset];
+
+            try self.mut.lock(io);
+            defer self.mut.unlock(io);
+
+            if (self.table.contains(block))
+                return self.table.get(block).?[block_offset];
+
+            var rdr = try self.fil.readerAt(io, self.table_start + (8 * block), &[0]u8{});
+            var offset: u64 = undefined;
+            try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
+
+            const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK)
+                @truncate(self.total_num % T_PER_BLOCK)
+            else
+                T_PER_BLOCK;
+
+            rdr = try self.fil.readerAt(io, offset, &[0]u8{});
+            var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
+
+            const slice = try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little);
+            try self.table.put(block, slice);
+
+            return slice[block_offset];
+        }
+    };
+}

src/meta_rdr.zig

@@ -1,114 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const Reader = Io.Reader;
-const Writer = Io.Writer;
-const Limit = Io.Limit;
-
-const Decomp = @import("decomp.zig");
-
-const MetadataReader = @This();
-
-alloc: std.mem.Allocator,
-
-data: []u8,
-decomp: Decomp.Fn,
-
-cur_offset: u64,
-
-block: [8192]u8 = undefined,
-
-interface: Reader = .{
-    .buffer = &[0]u8{},
-    .end = 0,
-    .seek = 0,
-    .vtable = &.{
-        .stream = stream,
-        .discard = discard,
-        .readVec = readVec,
-    },
-},
-
-pub fn init(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, offset: u64) MetadataReader {
-    return .{
-        .alloc = alloc,
-
-        .data = data,
-        .decomp = decomp,
-
-        .cur_offset = offset,
-    };
-}
-
-fn advance(self: *MetadataReader) Reader.Error!void {
-    self.interface.seek = 0;
-    errdefer self.interface.end = 0;
-
-    const hdr: Header = @bitCast(std.mem.readInt(u16, self.data[self.cur_offset..][0..2], .little));
-    if (hdr.size == 0 or hdr.size > 8192) return Reader.Error.ReadFailed;
-    defer self.cur_offset += hdr.size + 2;
-
-    const block = self.data[self.cur_offset + 2 ..][0..hdr.size];
-
-    if (hdr.uncompressed) {
-        self.interface.end = hdr.size;
-        self.interface.buffer = block;
-        return;
-    }
-    const decomp_size = self.decomp(self.alloc, block, &self.block) catch return Reader.Error.ReadFailed;
-    self.interface.buffer = self.block[0..decomp_size];
-    self.interface.end = decomp_size;
-}
-
-fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {
-    if (r.seek >= r.end) {
-        const self: *MetadataReader = @fieldParentPtr("interface", r);
-        try self.advance();
-    }
-    if (limit == .nothing) return 0;
-
-    const to_write = @min(@intFromEnum(limit), r.end - r.seek);
-
-    const wrote = try w.write(r.buffer[r.seek..][0..to_write]);
-    r.seek += wrote;
-
-    return wrote;
-}
-fn discard(r: *Reader, limit: Limit) Reader.Error!usize {
-    if (r.seek >= r.end) {
-        const self: *MetadataReader = @fieldParentPtr("interface", r);
-        try self.advance();
-    }
-    if (limit == .nothing) return 0;
-
-    const to_discard = @min(@intFromEnum(limit), r.end - r.seek);
-    r.seek += to_discard;
-
-    return to_discard;
-}
-fn readVec(r: *Reader, vec: [][]u8) Reader.Error!usize {
-    if (r.seek >= r.end) {
-        const self: *MetadataReader = @fieldParentPtr("interface", r);
-        try self.advance();
-    }
-
-    var wrote: usize = 0;
-    for (vec) |v| {
-        const to_cpy = @min(v.len, r.end - r.seek);
-
-        @memcpy(v[0..to_cpy], r.buffer[r.seek..][0..to_cpy]);
-
-        wrote += to_cpy;
-        r.seek += to_cpy;
-
-        if (r.seek >= r.end) break;
-    }
-
-    return wrote;
-}
-
-// Types
-
-const Header = packed struct(u16) {
-    size: u15,
-    uncompressed: bool,
-};

src/options.zig

@@ -5,8 +5,8 @@ const Writer = std.Io.Writer;
 
 const ExtractionOptions = @This();
 
-/// Whether to use single threaded extraction. For finer control, create an std.Io instance.
-single_threaded: bool = false,
+/// The number of threads used for extraction. 0 implies single threaded.
+threads: usize = 1, // TODO: Update to better integrate with zig 0.16 Io. Maybe limit to only single or multi-threaded.
 /// Don't set the file's owner & permissions after extraction
 ignore_permissions: bool = false,
 /// Don't set xattr values. Currently xattrs are never set anyway.
@@ -18,9 +18,12 @@ verbose: bool = false,
 /// Where to print verbose log.
 verbose_writer: ?*Writer = null,
 
-pub const default: ExtractionOptions = .{};
-pub const single_threaded_default: ExtractionOptions = .{ .single_threaded = true };
-
+pub const SingleThreadedDefault: ExtractionOptions = .{};
+pub fn Default() !ExtractionOptions {
+    return .{
+        .threads = try std.Thread.getCpuCount(),
+    };
+}
 pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions {
     return .{
         .verbose = true,

src/root.zig

@@ -1,7 +1,2 @@
 pub const Archive = @import("archive.zig");
-pub const File = @import("file.zig");
 pub const ExtractionOptions = @import("options.zig");
-
-test {
-    @import("std").testing.refAllDecls(@import("test.zig"));
-}

src/test.zig

@@ -1,118 +1,54 @@
 const std = @import("std");
 const Io = std.Io;
-const testing = std.testing;
+const io = std.testing.io;
+const alloc = std.testing.allocator;
+const stuff = @import("builtin");
 
 const Archive = @import("archive.zig");
+const Superblock = Archive.Superblock;
 
 const TestArchive = "testing/LinuxPATest.sfs";
 
 test "Basics" {
-    const io = testing.io;
-    const alloc = testing.allocator;
+    std.debug.print("Starting test: Basics...\n", .{});
 
-    var archive_file = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer archive_file.close(io);
-
-    var archive: Archive = try .init(io, archive_file, 0);
-    defer archive.deinit(io);
-
-    try testing.expectEqualDeep(archive.super, LinuxPATestCorrectSuperblock);
-
-    var root = try archive.root(alloc);
-    defer root.deinit();
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    var sfs: Archive = try .init(io, fil, 0);
+    try std.testing.expectEqualDeep(sfs.super, LinuxPATestCorrectSuperblock);
+    const root_file = try sfs.root(alloc, io);
+    defer root_file.deinit();
 }
 
 const TestFile = "Start.exe";
 const TestFileExtractLocation = "testing/Start.exe";
 
-test "ExtractSingleFileMT" {
-    const io = testing.io;
-    const alloc = testing.allocator;
+test "ExtractSingleFile" {
+    std.debug.print("Starting test: ExtractSingleFile...\n", .{});
 
     Io.Dir.cwd().deleteFile(io, TestFileExtractLocation) catch {};
-
-    var archive_file = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer archive_file.close(io);
-
-    var archive: Archive = try .init(io, archive_file, 0);
-    defer archive.deinit(io);
-
-    var start_exe = try archive.open(alloc, TestFile);
-    defer start_exe.deinit();
-
-    try start_exe.extract(alloc, io, TestFileExtractLocation, .default);
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    var sfs: Archive = try .init(io, fil, 0);
+    var test_fil = try sfs.open(alloc, io, TestFile);
+    defer test_fil.deinit();
+    try test_fil.extract(alloc, io, TestFileExtractLocation, try .Default());
+    //TODO: validate extracted file.
 }
 
-test "ExtractSingleFileST" {
-    const io = testing.io;
-    const alloc = testing.allocator;
+const TestFullExtractLocation = "testing/TestExtract";
 
-    Io.Dir.cwd().deleteFile(io, TestFileExtractLocation) catch {};
+test "ExtractCompleteArchive" {
+    std.debug.print("Starting test: ExtractCompleteArchive...\n", .{});
 
-    var archive_file = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer archive_file.close(io);
-
-    var archive: Archive = try .init(io, archive_file, 0);
-    defer archive.deinit(io);
-
-    var start_exe = try archive.open(alloc, TestFile);
-    defer start_exe.deinit();
-
-    try start_exe.extract(alloc, io, TestFileExtractLocation, .single_threaded_default);
+    Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
+    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
+    defer fil.close(io);
+    var sfs: Archive = try .init(io, fil, 0);
+    try sfs.extract(alloc, io, TestFullExtractLocation, try .Default());
 }
 
-const TestFullExtractLocationSTOption = "testing/TestExtractSTOption";
-
-test "ExtractCompleteArchiveSingleThreadedOption" {
-    const io = testing.io;
-    const alloc = testing.allocator;
-
-    Io.Dir.cwd().deleteTree(io, TestFullExtractLocationSTOption) catch {};
-
-    var archive_file = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer archive_file.close(io);
-
-    var archive: Archive = try .init(io, archive_file, 0);
-    defer archive.deinit(io);
-
-    try archive.extract(alloc, io, TestFullExtractLocationSTOption, .single_threaded_default);
-}
-
-const TestFullExtractLocationMT = "testing/TestExtractMT";
-
-test "ExtractCompleteArchiveMultiThreaded" {
-    const io = testing.io;
-    const alloc = testing.allocator;
-
-    Io.Dir.cwd().deleteTree(io, TestFullExtractLocationMT) catch {};
-
-    var archive_file = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer archive_file.close(io);
-
-    var archive: Archive = try .init(io, archive_file, 0);
-    defer archive.deinit(io);
-
-    try archive.extract(alloc, io, TestFullExtractLocationMT, .default);
-}
-
-const TestFullExtractLocationSTIo = "testing/TestExtractSTIo";
-
-// test "ExtractCompleteArchiveSingleThreadedIo" {
-//     const io = Io.Threaded.global_single_threaded.io();
-//     const alloc = testing.allocator;
-
-//     Io.Dir.cwd().deleteFile(io, TestFullExtractLocationSTIo) catch {};
-
-//     var archive_file = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-//     defer archive_file.close(io);
-
-//     var archive: Archive = try .init(io, archive_file, 0);
-//     defer archive.deinit(io);
-
-//     try archive.extract(alloc, io, TestFullExtractLocationSTIo, .default);
-// }
-
-const LinuxPATestCorrectSuperblock: Archive.Superblock = .{
+const LinuxPATestCorrectSuperblock: Superblock = .{
     .magic = std.mem.readInt(u32, "hsqs", .little),
     .inode_count = 2974,
     .mod_time = 1632696724,

src/util/cache.zig

@@ -1,52 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const Decomp = @import("../decomp.zig");
-
-const CacheMap = @import("protected_map.zig").ProtectedMap(u64, CachedBlock, decompressBlock);
-
-const Cache = @This();
-
-alloc: std.mem.Allocator,
-
-data: []u8,
-decomp: Decomp.Fn,
-
-cache: CacheMap,
-
-pub fn init(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn) Cache {
-    return .{
-        .alloc = alloc,
-
-        .data = data,
-        .decomp = decomp,
-
-        .cache = .init(alloc),
-    };
-}
-pub fn deinit(self: *Cache) void {
-    self.cache.deinit();
-}
-
-pub fn get(self: *Cache, io: Io, offset: u64, compressed_size: u32) Error![]u8 {
-    const res: *CachedBlock = try self.cache.getOrPut(io, offset, .{ self.alloc, self.data, self.decomp, offset, compressed_size });
-    return res.block[0..res.size];
-}
-
-fn decompressBlock(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, offset: u64, compressed_size: u32) !CachedBlock {
-    const block = data[offset..][0..compressed_size];
-    var new_cache: CachedBlock = undefined;
-
-    new_cache.size = @truncate(try decomp(alloc, block, &new_cache.block));
-
-    return new_cache;
-}
-
-// Types
-
-pub const Error = CacheMap.Error;
-
-const CachedBlock = struct {
-    block: [1024 * 1024]u8,
-    size: u32,
-};

src/util/data_extractor.zig

@@ -0,0 +1,151 @@
+//! The DataExtractor is meant to extract a regular file's data to a given file asyncronously.
+
+const std = @import("std");
+const Io = std.Io;
+
+const FragEntry = @import("../frag.zig").FragEntry;
+const BlockSize = @import("../inode_data/file.zig").BlockSize;
+const Decompressor = @import("decompressor.zig");
+const OffsetFile = @import("offset_file.zig");
+
+// const SharedCache = @import("shared_cache.zig");
+
+pub const Error = error{OutOfMemory} || Io.File.Reader.SeekError || Io.Writer.Error || Io.File.Writer.Error;
+
+const DataExtractor = @This();
+
+fil: OffsetFile,
+decomp: *const Decompressor,
+block_size: u32,
+
+file_size: u64,
+start: u64,
+blocks: []BlockSize,
+
+frag_offset: u32 = 0,
+frag_block: ?[]u8 = null,
+
+err: ?Error = null,
+
+pub fn init(fil: OffsetFile, decomp: *const Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) DataExtractor {
+    return .{
+        .fil = fil,
+        .decomp = decomp,
+        .block_size = block_size,
+
+        .file_size = file_size,
+        .start = data_start,
+        .blocks = blocks,
+    };
+}
+pub fn addFrag(self: *DataExtractor, frag_offset: u32, block: []u8) void {
+    self.frag_offset = frag_offset;
+    self.frag_block = block;
+}
+
+fn numBlocks(self: DataExtractor) usize {
+    var num = self.blocks.len;
+    if (self.frag_block != null) num += 1;
+    return num;
+}
+
+/// Starts extracting the data using the given group to spawn async tasks.
+pub fn extractAsync(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.File) Error!void {
+    var group: Io.Group = .init;
+    defer group.cancel(io);
+    var err: ?Error = null;
+
+    var read_offset: u64 = self.start;
+    for (0..self.blocks.len) |idx| {
+        group.async(io, blockThread, .{ self, alloc, io, fil, read_offset, idx, &err });
+        read_offset += self.blocks[idx].size;
+    }
+    if (self.frag_block != null)
+        group.async(io, fragThread, .{ self, io, fil, &err });
+
+    group.await(io) catch |cancel| return err orelse cancel;
+}
+
+fn blockThread(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.File, read_offset: u64, idx: usize, ret_err: *?Error) Io.Cancelable!void {
+    const block = self.blocks[idx];
+
+    const cur_block_size = if (idx == self.numBlocks() - 1)
+        self.file_size % self.block_size
+    else
+        self.block_size;
+
+    var wrt = fil.writer(io, &[0]u8{});
+    wrt.seekTo(self.block_size * idx) catch |err| {
+        ret_err.* = err;
+        if (err == error.Canceled) io.recancel();
+        return Io.Cancelable.Canceled;
+    };
+    defer wrt.flush() catch {};
+
+    if (block.size == 0) {
+        wrt.interface.splatByteAll(0, cur_block_size) catch |err| {
+            ret_err.* = err;
+            if (err == error.Canceled) io.recancel();
+            return Io.Cancelable.Canceled;
+        };
+        return;
+    }
+
+    var rdr = self.fil.readerAt(io, read_offset, &[0]u8{}) catch |err| {
+        ret_err.* = err;
+        if (err == error.Canceled) io.recancel();
+        return Io.Cancelable.Canceled;
+    };
+    if (block.uncompressed) {
+        rdr.interface.streamExact(&wrt.interface, cur_block_size) catch |err| {
+            ret_err.* = err;
+            if (err == error.Canceled) io.recancel();
+            return Io.Cancelable.Canceled;
+        };
+        return;
+    } else {
+        @branchHint(.likely);
+
+        var cache: [1024 * 1024]u8 = undefined;
+        var tmp: [1024 * 1024]u8 = undefined;
+
+        rdr.interface.readSliceAll(cache[0..block.size]) catch |err| {
+            ret_err.* = err;
+            if (err == error.Canceled) io.recancel();
+            return Io.Cancelable.Canceled;
+        };
+        _ = self.decomp.Decompress(alloc, cache[0..block.size], tmp[0..cur_block_size]) catch |err| {
+            ret_err.* = err;
+            if (err == error.Canceled) io.recancel();
+            return Io.Cancelable.Canceled;
+        };
+        wrt.interface.writeAll(tmp[0..cur_block_size]) catch |err| {
+            ret_err.* = err;
+            if (err == error.Canceled) io.recancel();
+            return Io.Cancelable.Canceled;
+        };
+    }
+}
+fn fragThread(self: DataExtractor, io: Io, fil: Io.File, ret_err: *?Error) Io.Cancelable!void {
+    const cur_block_size = self.file_size % self.block_size;
+
+    var write_buf: [10 * 1024]u8 = undefined;
+    var wrt = fil.writer(io, &write_buf);
+    wrt.seekTo(self.blocks.len * self.block_size) catch |err| {
+        ret_err.* = err;
+        if (err == error.Canceled) io.recancel();
+        return Io.Cancelable.Canceled;
+    };
+
+    wrt.interface.writeAll(self.frag_block.?[self.frag_offset .. self.frag_offset + cur_block_size]) catch |err| {
+        ret_err.* = err;
+        if (err == error.Canceled) io.recancel();
+        return Io.Cancelable.Canceled;
+    };
+
+    wrt.flush() catch |err| {
+        ret_err.* = err;
+        if (err == error.Canceled) io.recancel();
+        return Io.Cancelable.Canceled;
+    };
+}

src/util/data_reader.zig

@@ -0,0 +1,195 @@
+//! DataReader reads a regular file's data linearly from start to finish using Io.Reader interface.
+
+const std = @import("std");
+const Io = std.Io;
+const Reader = Io.Reader;
+const Writer = Io.Writer;
+const Limit = Io.Limit;
+
+const BlockSize = @import("../inode_data/file.zig").BlockSize;
+const Decompressor = @import("decompressor.zig");
+const OffsetFile = @import("offset_file.zig");
+
+// const SharedCache = @import("shared_cache.zig");
+
+const DataReader = @This();
+
+alloc: std.mem.Allocator,
+
+fil: OffsetFile,
+io: Io,
+decomp: *const Decompressor,
+cache: *Io.Queue([]u8),
+block_size: u32,
+
+file_size: u64,
+cur_offset: u64,
+blocks: []BlockSize,
+
+frag_offset: u32 = 0,
+frag_block: ?[]u8 = null,
+
+block_idx: usize = 0,
+sparse_block: bool = false,
+
+interface: Io.Reader,
+
+pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, cache: *Io.Queue([]u8), block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) !DataReader {
+    return .{
+        .alloc = alloc,
+
+        .fil = fil,
+        .io = io,
+        .decomp = decomp,
+        .cache = cache,
+        .block_size = block_size,
+
+        .file_size = file_size,
+        .cur_offset = data_start,
+        .blocks = blocks,
+
+        .interface = .{
+            .buffer = try alloc.alloc(u8, block_size),
+            .seek = 0,
+            .end = 0,
+            .vtable = &.{
+                .stream = stream,
+                .discard = discard,
+                .readVec = readVec,
+            },
+        },
+    };
+}
+pub fn deinit(self: *DataReader) void {
+    self.alloc.free(self.interface.buffer);
+}
+pub fn addFrag(self: *DataReader, frag_offset: u32, block: []u8) void {
+    self.frag_offset = frag_offset;
+    self.frag_block = block;
+}
+
+fn numBlocks(self: DataReader) usize {
+    var num = self.blocks.len;
+    if (self.frag_block != null) num += 1;
+    return num;
+}
+fn advanceBuffer(self: *DataReader) !void {
+    if (self.block_idx >= self.numBlocks()) {
+        return Reader.Error.EndOfStream;
+    }
+    defer self.block_idx += 1;
+
+    self.interface.end = if (self.block_idx == self.numBlocks() - 1)
+        self.file_size % self.block_size
+    else
+        self.block_size;
+
+    // Fragment
+    if (self.block_idx == self.blocks.len) {
+        @memcpy(self.interface.buffer[0..self.interface.end], self.frag_block.?[self.frag_offset .. self.frag_offset + self.interface.end]);
+        self.interface.seek = 0;
+        return;
+    }
+
+    // Normal Block
+    const block = self.blocks[self.block_idx];
+    if (block.size == 0) {
+        self.interface.seek = 0;
+        self.sparse_block = true;
+        return;
+    } else {
+        self.sparse_block = false;
+    }
+    if (block.uncompressed) {
+        try self.fil.readAt(self.io, self.cur_offset, self.interface.buffer[0..self.interface.end]);
+        self.cur_offset += self.interface.end;
+    } else {
+        @branchHint(.likely);
+        const tmp = try self.cache.getOne(self.io);
+        defer self.cache.putOne(self.io, tmp) catch {};
+
+        var rdr_buf: [50 * 1024]u8 = undefined;
+        var rdr = try self.fil.readerAt(self.io, self.cur_offset, &rdr_buf);
+        try rdr.interface.readSliceAll(tmp[0..block.size]);
+        self.cur_offset += block.size;
+        _ = try self.decomp.Decompress(self.alloc, tmp[0..block.size], self.interface.buffer[0..self.interface.end]);
+    }
+    self.interface.seek = 0;
+}
+
+fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) Reader.StreamError!usize {
+    var data: *DataReader = @fieldParentPtr("interface", rdr);
+    if (rdr.seek == rdr.end)
+        data.advanceBuffer() catch |err| return switch (err) {
+            error.ReadFailed => error.ReadFailed,
+            error.EndOfStream => error.EndOfStream,
+            else => error.ReadFailed,
+        };
+
+    switch (limit) {
+        .nothing => return 0,
+        .unlimited => {
+            const wrote = if (data.sparse_block)
+                try wrt.splatByte(0, rdr.end - rdr.seek)
+            else
+                try wrt.write(rdr.buffer[rdr.seek..rdr.end]);
+            rdr.seek += wrote;
+            return wrote;
+        },
+        else => {
+            const to_read = @min(rdr.end - rdr.seek, @intFromEnum(limit));
+            const wrote = if (data.sparse_block)
+                try wrt.splatByte(0, to_read)
+            else
+                try wrt.write(rdr.buffer[rdr.seek .. rdr.seek + to_read]);
+            rdr.seek += wrote;
+            return wrote;
+        },
+    }
+}
+fn discard(rdr: *Reader, limit: Limit) Reader.Error!usize {
+    var data: *DataReader = @fieldParentPtr("interface", rdr);
+    if (rdr.seek == rdr.end)
+        data.advanceBuffer() catch |err| return switch (err) {
+            error.ReadFailed => error.ReadFailed,
+            error.EndOfStream => error.EndOfStream,
+            else => error.ReadFailed,
+        };
+
+    switch (limit) {
+        .nothing => return 0,
+        .unlimited => {
+            const adv = rdr.end - rdr.seek;
+            rdr.seek = rdr.end;
+            return adv;
+        },
+        else => {
+            const adv = @min(rdr.end - rdr.seek, @intFromEnum(limit));
+            rdr.seek += adv;
+            return adv;
+        },
+    }
+}
+fn readVec(rdr: *Reader, vec: [][]u8) Reader.Error!usize {
+    var data: *DataReader = @fieldParentPtr("interface", rdr);
+    if (rdr.seek == rdr.end)
+        data.advanceBuffer() catch |err| return switch (err) {
+            error.ReadFailed => error.ReadFailed,
+            error.EndOfStream => error.EndOfStream,
+            else => error.ReadFailed,
+        };
+
+    var wrote: usize = 0;
+    for (vec) |buf| {
+        if (rdr.seek == rdr.end) break;
+
+        const to_copy = @min(rdr.end - rdr.seek, buf.len);
+        if (data.sparse_block)
+            @memset(buf[0..to_copy], 0)
+        else
+            @memcpy(buf[0..to_copy], rdr.buffer[rdr.seek .. rdr.seek + to_copy]);
+        rdr.seek += to_copy;
+        wrote += to_copy;
+    }
+    return wrote;
+}

src/util/decompressor.zig

@@ -0,0 +1,15 @@
+//! A decompression interface
+
+const std = @import("std");
+
+const Decompressor = @This();
+
+pub const Error = std.Io.Reader.StreamError || std.mem.Allocator.Error;
+
+/// The actual decompression function.
+/// If the given decompressor is null, then the decompression should be done "stateless" without lasting allocations.
+decomp_fn: *const fn (?*const Decompressor, std.mem.Allocator, in: []u8, out: []u8) Error!usize,
+
+pub fn Decompress(self: *const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    return self.decomp_fn(self, alloc, in, out);
+}

src/util/metadata.zig

@@ -0,0 +1,103 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+const Writer = std.Io.Writer;
+const Limit = std.Io.Limit;
+const StreamError = std.Io.Reader.StreamError;
+
+const Decompressor = @import("decompressor.zig");
+
+const BlockHeader = packed struct(u16) {
+    size: u15,
+    uncompressed: bool,
+};
+
+const This = @This();
+
+alloc: std.mem.Allocator,
+rdr: *Reader,
+decomp: *const Decompressor,
+
+cur_block_start: u32 = 0,
+next_start_start: u32 = 0,
+buf: [8192]u8 = undefined,
+
+err: ?anyerror = null,
+interface: Reader = .{
+    .buffer = &[0]u8{},
+    .end = 0,
+    .seek = 0,
+    .vtable = &.{
+        .stream = stream,
+        .discard = discard,
+        .readVec = readVec,
+        // TODO: Potentially add rebase so that we can guarentee that self.block_start & interface.seek is correct.
+    },
+},
+
+pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: *const Decompressor) This {
+    return .{
+        .alloc = alloc,
+        .rdr = rdr,
+        .decomp = decomp,
+    };
+}
+
+fn advance(self: *This) !void {
+    self.interface.seek = 0;
+    var hdr: BlockHeader = undefined;
+    try self.rdr.readSliceEndian(BlockHeader, @ptrCast(&hdr), .little);
+    self.cur_block_start = self.next_start_start;
+    self.next_start_start += hdr.size;
+    if (hdr.uncompressed) {
+        try self.rdr.readSliceEndian(u8, self.buf[0..hdr.size], .little);
+        self.interface.end = hdr.size;
+        self.interface.buffer = self.buf[0..hdr.size];
+        return;
+    } else {
+        @branchHint(.likely);
+        var tmp_buf: [8192]u8 = undefined;
+        try self.rdr.readSliceAll(tmp_buf[0..hdr.size]);
+        self.interface.end = try self.decomp.Decompress(self.alloc, tmp_buf[0..hdr.size], &self.buf);
+        self.interface.buffer = self.buf[0..self.interface.end];
+    }
+}
+
+fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) StreamError!usize {
+    const self: *This = @fieldParentPtr("interface", rdr);
+    if (rdr.end == rdr.seek) self.advance() catch |err| {
+        self.err = err;
+        return StreamError.ReadFailed;
+    };
+    if (@intFromEnum(limit) == 0) return 0;
+    const to_write = @min(rdr.end - rdr.seek, @intFromEnum(limit));
+    const wrote = try wrt.write(self.buf[rdr.seek .. rdr.seek + to_write]);
+    self.interface.seek += wrote;
+    return wrote;
+}
+fn discard(rdr: *Reader, limit: Limit) Reader.Error!usize {
+    const self: *This = @fieldParentPtr("interface", rdr);
+    if (rdr.end == rdr.seek) self.advance() catch |err| {
+        self.err = err;
+        return error.ReadFailed;
+    };
+    if (@intFromEnum(limit) == 0) return 0;
+    const to_skip = @min(rdr.end - rdr.seek, @intFromEnum(limit));
+    rdr.seek += to_skip;
+    return to_skip;
+}
+fn readVec(rdr: *Reader, vec: [][]u8) Reader.Error!usize {
+    const self: *This = @fieldParentPtr("interface", rdr);
+    if (rdr.end == rdr.seek) self.advance() catch |err| {
+        self.err = err;
+        return error.ReadFailed;
+    };
+    var cur_red: usize = 0;
+    for (vec) |s| {
+        const to_copy: usize = @min(rdr.end - rdr.seek, s.len);
+        @memcpy(s[0..to_copy], self.buf[rdr.seek .. rdr.seek + to_copy]);
+        rdr.seek += to_copy;
+        cur_red += to_copy;
+        if (rdr.end == rdr.seek) break;
+    }
+    return cur_red;
+}

src/util/misc.zig

@@ -0,0 +1,25 @@
+//! Miscellaneous utility functions.
+
+const std = @import("std");
+const Io = std.Io;
+
+const Inode = @import("../inode.zig");
+const Decompressor = @import("decompressor.zig");
+const MetadataReader = @import("metadata.zig");
+const OffsetFile = @import("offset_file.zig");
+
+/// check is the path is referencing itself ("" or ".").
+/// separators must be trimmed before calling this function for it to work properly.
+pub fn pathIsSelf(path: []const u8) bool {
+    if (path.len == 0) return true;
+    if (path.len > 1) return false;
+    return path[0] == '.';
+}
+/// Creates an Inode from an Inode.Ref.
+pub fn inodeFromRef(alloc: std.mem.Allocator, io: Io, file: OffsetFile, decomp: *const Decompressor, inode_start: u64, block_size: u32, ref: Inode.Ref) !Inode {
+    var rdr = try file.readerAt(io, inode_start + ref.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+    try meta.interface.discardAll(ref.block_offset);
+
+    return .read(alloc, &meta.interface, block_size);
+}

src/util/offset_file.zig

@@ -0,0 +1,24 @@
+//! A File where it's meaningful (to us) content starts at a given offset.
+
+const std = @import("std");
+const Io = std.Io;
+const File = Io.File;
+const Reader = File.Reader;
+
+const OffsetFile = @This();
+
+fil: File,
+offset: u64,
+
+pub fn init(fil: File, init_offset: u64) OffsetFile {
+    return .{ .fil = fil, .offset = init_offset };
+}
+
+pub fn readerAt(self: OffsetFile, io: Io, offset: u64, buffer: []u8) Reader.SeekError!Reader {
+    var rdr = self.fil.reader(io, buffer);
+    try rdr.seekTo(self.offset + offset);
+    return rdr;
+}
+pub fn readAt(self: OffsetFile, io: Io, offset: u64, buf: []u8) File.ReadPositionalError!void {
+    _ = try self.fil.readPositionalAll(io, buf, self.offset + offset);
+}

src/util/protected_map.zig

@@ -1,85 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-pub fn ProtectedMap(comptime K: anytype, comptime T: anytype, comptime create_fn: anytype) type {
-    const fn_info = @typeInfo(@TypeOf(create_fn));
-    std.debug.assert(std.meta.activeTag(fn_info) == .@"fn");
-
-    const ret_info = fn_info.@"fn".return_type;
-
-    std.debug.assert(ret_info != null);
-    std.debug.assert(ret_info == T or
-        (std.meta.activeTag(@typeInfo(ret_info.?)) == .error_union and @typeInfo(ret_info.?).error_union.payload == T));
-
-    return struct {
-        const Map = @This();
-
-        alloc: std.mem.Allocator,
-
-        map: std.AutoHashMap(K, ProtectedValue),
-        mut: Io.RwLock = .init,
-
-        pub fn init(alloc: std.mem.Allocator) Map {
-            return .{
-                .alloc = alloc,
-
-                .map = .init(alloc),
-            };
-        }
-        pub fn deinit(self: *Map) void {
-            self.map.deinit();
-        }
-
-        pub fn getOrPut(self: *Map, io: Io, key: K, create_fn_args: std.meta.ArgsTuple(@TypeOf(create_fn))) Error!*T {
-            {
-                try self.mut.lockShared(io);
-                const value = self.map.getPtr(key);
-                self.mut.unlockShared(io);
-                if (value != null) {
-                    if (!value.?.filled.isSet())
-                        try value.?.filled.wait(io);
-
-                    if (value.?.err != null) return value.?.err.?;
-                    return &value.?.value;
-                }
-            }
-            var value: *ProtectedValue = blk: {
-                try self.mut.lock(io);
-                defer self.mut.unlock(io);
-
-                const res = try self.map.getOrPut(key);
-                if (res.found_existing)
-                    return self.getOrPut(io, key, create_fn_args);
-                res.value_ptr.* = .{};
-                break :blk res.value_ptr;
-            };
-            defer value.filled.set(io);
-
-            value.value = if (@TypeOf(CreateError) == void)
-                @call(.auto, create_fn, create_fn_args)
-            else
-                @call(.auto, create_fn, create_fn_args) catch |err| {
-                    value.err = err;
-                    return err;
-                };
-
-            return &value.value;
-        }
-
-        // Map Types
-
-        pub const Error = error{ Canceled, OutOfMemory } ||
-            if (@TypeOf(CreateError) == void) error{} else CreateError;
-
-        const CreateError: type = switch (@typeInfo(ret_info.?)) {
-            .error_union => |e| e.error_set,
-            else => void,
-        };
-
-        const ProtectedValue = struct {
-            value: T = undefined,
-            err: ?CreateError = null,
-            filled: Io.Event = .unset,
-        };
-    };
-}

src/util/shared_cache.zig

@@ -0,0 +1,52 @@
+const std = @import("std");
+const Io = std.Io;
+const Node = std.SinglyLinkedList.Node;
+
+const SharedCache = @This();
+
+pub const CACHE_SIZE = 1024 * 1024;
+
+pub const BufferNode = struct {
+    node: Node,
+    cache: [CACHE_SIZE]u8,
+};
+
+alloc: std.mem.Allocator,
+
+caches: std.ArrayList(BufferNode),
+cache_queue: std.SinglyLinkedList,
+queue_mut: Io.Mutex,
+
+pub fn init(alloc: std.mem.Allocator, init_cache_size: u32) !SharedCache {
+    const caches: std.ArrayList(BufferNode) = try .initCapacity(alloc, init_cache_size);
+    var queue: std.SinglyLinkedList = .{};
+    for (caches.items) |item|
+        queue.prepend(&item.node);
+    return .{
+        .alloc = alloc,
+
+        .caches = caches,
+        .cache_queue = queue,
+    };
+}
+pub fn deinit(self: *SharedCache) void {
+    self.caches.deinit(self.alloc);
+}
+
+pub fn getCache(self: *SharedCache, io: Io) !*BufferNode {
+    self.queue_mut.lock(io);
+    const nxt = self.cache_queue.popFirst();
+    self.queue_mut.unlock(io);
+    if (nxt == null) {
+        const new = try self.caches.addOne(self.alloc);
+        new.* = .{
+            .node = .{},
+            .cache = undefined,
+        };
+        return new;
+    }
+    return @fieldParentPtr("node", nxt.?);
+}
+pub fn returnCache(self: *SharedCache, buf: *BufferNode) void {
+    self.cache_queue.prepend(buf);
+}

src/xattr.zig

@@ -1,134 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const Decomp = @import("decomp.zig");
-const Lookup = @import("lookup.zig");
-const MetadataReader = @import("meta_rdr.zig");
-const Ref = @import("inode.zig").Ref;
-
-const XattrTable = @This();
-
-table_start: u64,
-
-table: Lookup.Table(Entry),
-
-pub fn init(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, table_start: u64) XattrTable {
-    const start = std.mem.readInt(u64, data[table_start..][0..8], .little);
-    const table_num = std.mem.readInt(u32, data[table_start + 8 ..][0..4], .little);
-
-    return .{
-        .table_start = start,
-        .table = .init(alloc, data, decomp, table_start + 16, table_num),
-    };
-}
-pub fn deinit(self: *XattrTable) void {
-    self.table.deinit();
-}
-
-pub fn get(self: *XattrTable, alloc: std.mem.Allocator, io: Io, idx: u32) !Xattr {
-    const entry: Entry = try self.table.get(io, idx);
-
-    var out: std.ArrayList(KeyValue) = try .initCapacity(alloc, entry.count);
-    errdefer {
-        for (out.items) |kv|
-            kv.deinit(alloc);
-        out.deinit(alloc);
-    }
-
-    var meta: MetadataReader = .init(alloc, self.table.data, self.table.decomp, self.table_start + entry.ref.block_start);
-    try meta.interface.discardAll(entry.ref.block_offset);
-
-    for (0..entry.count) |_| {
-        var key_entry: KeyEntry = undefined;
-
-        try meta.interface.readSliceEndian(KeyEntry, @ptrCast(&key_entry), .little);
-
-        const prefix = switch (key_entry.type.type) {
-            .user => "user.",
-            .trusted => "trusted.",
-            .security => "security.",
-        };
-
-        const key = try alloc.allocSentinel(u8, key_entry.name_size + prefix.len, 0);
-        errdefer alloc.free(key);
-
-        @memcpy(key[0..prefix.len], prefix);
-
-        try meta.interface.readSliceEndian(u8, key[prefix.len..][0..key_entry.name_size], .little);
-
-        const value = if (key_entry.type.out_of_line) blk: {
-            try meta.interface.discardAll(4);
-
-            var ref: Ref = undefined;
-            try meta.interface.readSliceEndian(Ref, @ptrCast(&ref), .little);
-
-            var value_meta: MetadataReader = .init(alloc, self.table.data, self.table.decomp, self.table_start + ref.block_start);
-            try value_meta.interface.discardAll(ref.block_offset);
-
-            break :blk try readValue(alloc, &value_meta.interface);
-        } else try readValue(alloc, &meta.interface);
-
-        const ptr = out.addOneAssumeCapacity();
-
-        ptr.* = .{
-            .key = key,
-            .value = value,
-        };
-    }
-
-    return .{ .kvs = try out.toOwnedSlice(alloc) };
-}
-
-fn readValue(alloc: std.mem.Allocator, rdr: *Io.Reader) ![]u8 {
-    var size: u32 = undefined;
-    try rdr.readSliceEndian(u32, @ptrCast(&size), .little);
-
-    const value = try alloc.alloc(u8, size);
-    errdefer alloc.free(value);
-
-    try rdr.readSliceEndian(u8, value, .little);
-
-    return value;
-}
-
-// Types
-
-const Xattr = struct {
-    kvs: []KeyValue,
-
-    pub fn deinit(self: Xattr, alloc: std.mem.Allocator) void {
-        for (self.kvs) |kv|
-            kv.deinit(alloc);
-
-        alloc.free(self.kvs);
-    }
-};
-
-const KeyValue = struct {
-    key: [:0]const u8,
-    value: []const u8,
-
-    pub fn deinit(self: KeyValue, alloc: std.mem.Allocator) void {
-        alloc.free(self.key);
-        alloc.free(self.value);
-    }
-};
-
-const KeyEntry = extern struct {
-    type: packed struct(u16) {
-        type: enum(u8) {
-            user,
-            trusted,
-            security,
-        },
-        out_of_line: bool,
-        _: u7,
-    },
-    name_size: u16,
-};
-
-const Entry = extern struct {
-    ref: Ref,
-    count: u32,
-    size: u32,
-};

src/xattr_table.zig

@@ -0,0 +1,286 @@
+const std = @import("std");
+const Io = std.Io;
+
+const InodeRef = @import("inode.zig").Ref;
+const LookupTable = @import("lookup_table.zig");
+const Decompressor = @import("util/decompressor.zig");
+const MetadataReader = @import("util/metadata.zig");
+const OffsetFile = @import("util/offset_file.zig");
+
+const XattrCachedTable = @This();
+
+alloc: std.mem.Allocator,
+
+fil: OffsetFile,
+decomp: *const Decompressor,
+
+kv_start: u64,
+
+table: LookupTable.CachedTable(TableValue),
+value_cache: std.AutoHashMap(InodeRef, []const u8),
+value_mut: Io.Mutex = .init,
+
+pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, xattr_start: u64) !XattrCachedTable {
+    var rdr = try fil.readerAt(io, xattr_start, &[0]u8{});
+
+    var start: u64 = undefined;
+    try rdr.interface.readSliceEndian(u64, @ptrCast(&start), .little);
+    var num: u32 = undefined;
+    try rdr.interface.readSliceEndian(u32, @ptrCast(&num), .little);
+
+    return .{
+        .alloc = alloc,
+
+        .fil = fil,
+        .decomp = decomp,
+
+        .kv_start = start,
+
+        .table = .init(alloc, fil, decomp, xattr_start + 16, num),
+        .value_cache = .init(alloc),
+    };
+}
+pub fn deinit(self: *XattrCachedTable, io: Io) void {
+    self.table.deinit(io);
+    self.value_cache.deinit();
+}
+
+pub fn get(self: *XattrCachedTable, alloc: std.mem.Allocator, io: Io, idx: u32) ![]XattrSemiOwned {
+    const lookup = try self.table.get(io, idx);
+
+    var rdr = try self.fil.readerAt(io, self.kv_start + lookup.ref.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
+    try meta.interface.discardAll(lookup.ref.block_offset);
+
+    const out = try alloc.alloc(XattrSemiOwned, lookup.count);
+    errdefer alloc.free(out);
+
+    for (0..lookup.count) |i| {
+        var key_entry: KeyEntry = undefined;
+        try meta.interface.readSliceEndian(KeyEntry, @ptrCast(&key_entry), .little);
+
+        const key: [:0]u8 = switch (key_entry.type.namespace) {
+            .user => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 5);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[5 .. tmp.len - 1], .little);
+                @memcpy(tmp[0..5], "user.");
+                tmp[tmp.len - 1] = 0;
+                break :blk @ptrCast(tmp);
+            },
+            .trusted => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 8);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[8 .. tmp.len - 1], .little);
+                @memcpy(tmp[0..8], "trusted.");
+                tmp[tmp.len - 1] = 0;
+                break :blk @ptrCast(tmp);
+            },
+            .security => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 9);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[9 .. tmp.len - 1], .little);
+                @memcpy(tmp[0..9], "security.");
+                tmp[tmp.len - 1] = 0;
+                break :blk @ptrCast(tmp);
+            },
+        };
+        errdefer alloc.free(key);
+
+        if (key_entry.type.out_of_line) {
+            var value: ValueOutOfLineEntry = undefined;
+            try meta.interface.readSliceEndian(ValueOutOfLineEntry, @ptrCast(&value), .little);
+
+            out[i] = .{
+                .key = key,
+                .value = try self.valueAt(io, value.ref),
+            };
+            continue;
+        }
+        const val_ref: InodeRef = .{ .block_start = meta.cur_block_start, .block_offset = @truncate(meta.interface.seek) };
+
+        try self.value_mut.lock(io);
+        defer self.value_mut.unlock(io);
+        if (self.value_cache.contains(val_ref)) {
+            out[i] = .{
+                .key = key,
+                .value = try self.valueAt(io, val_ref),
+            };
+            continue;
+        }
+
+        var val_size: u32 = undefined;
+        try meta.interface.readSliceEndian(u32, @ptrCast(&val_size), .little);
+
+        const val = try self.alloc.alloc(u8, val_size);
+        errdefer alloc.free(val);
+        try meta.interface.readSliceEndian(u8, val, .little);
+
+        try self.value_cache.put(val_ref, val);
+        out[i] = .{
+            .key = key,
+            .value = val,
+        };
+    }
+    return out;
+}
+
+fn valueAt(self: *XattrCachedTable, io: Io, ref: InodeRef) ![]const u8 {
+    try self.value_mut.lock(io);
+    defer self.value_mut.unlock(io);
+
+    if (self.value_cache.contains(ref)) return self.value_cache.get(ref).?;
+
+    var rdr = try self.fil.readerAt(io, self.kv_start + ref.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
+    try meta.interface.discardAll(ref.block_offset);
+
+    var val_size: u32 = undefined;
+    try meta.interface.readSliceEndian(u32, @ptrCast(&val_size), .little);
+
+    const val = try self.alloc.alloc(u8, val_size);
+    errdefer self.alloc.free(val);
+    try meta.interface.readSliceEndian(u8, val, .little);
+
+    try self.value_cache.put(ref, val);
+    return val;
+}
+
+// Types
+
+/// An Xattr return value where the reciever only owns the key value.
+pub const XattrSemiOwned = struct {
+    key: [:0]const u8,
+    value: []const u8,
+
+    pub fn deinit(self: XattrSemiOwned, alloc: std.mem.Allocator) void {
+        alloc.free(self.key);
+    }
+};
+/// An Xattr return value where the reciever owns both the key & value.
+pub const XattrOwned = struct {
+    key: [:0]const u8,
+    value: []const u8,
+
+    pub fn deinit(self: XattrSemiOwned, alloc: std.mem.Allocator) void {
+        alloc.free(self.key);
+        alloc.free(self.value);
+    }
+};
+
+const TableValue = extern struct {
+    ref: InodeRef,
+    count: u32,
+    size: u32,
+};
+
+const KeyEntry = extern struct {
+    type: XattrPrefix,
+    name_size: u16,
+};
+const ValueOutOfLineEntry = extern struct {
+    _: u32,
+    ref: InodeRef,
+};
+
+const XattrPrefix = packed struct(u16) {
+    namespace: enum(u8) {
+        user,
+        trusted,
+        security,
+
+        fn prefixSize(self: @This()) u16 {
+            return switch (self) {
+                .user => 5,
+                .trusted => 8,
+                .security => 9,
+            };
+        }
+    },
+    out_of_line: bool,
+    _: u7,
+};
+
+// Stateless
+
+pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *const Decompressor, fil: OffsetFile, table_start: u64, idx: u16) ![]XattrOwned {
+    var rdr = try fil.readerAt(io, table_start, &[0]u8{});
+
+    var kv_start: u64 = undefined;
+    try rdr.interface.readSliceEndian(u64, @ptrCast(&kv_start), .little);
+
+    const lookup = try LookupTable.lookupValue(TableValue, alloc, io, decomp, fil, table_start + 16, idx);
+
+    rdr = try fil.readerAt(io, kv_start + lookup.ref.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+    try meta.interface.discardAll(lookup.ref.block_offset);
+
+    const out = try alloc.alloc(XattrOwned, lookup.count);
+    errdefer alloc.free(out);
+
+    for (0..lookup.count) |i| {
+        const key_entry: KeyEntry = undefined;
+        try meta.interface.readSliceEndian(KeyEntry, @ptrCast(&key_entry), .little);
+
+        const key = switch (key_entry.type.namespace) {
+            .user => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 5);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[5 .. tmp.len - 1], .little);
+                @memset(tmp[0..5], "user.");
+                break :blk tmp;
+            },
+            .trusted => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 8);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[8 .. tmp.len - 1], .little);
+                @memset(tmp[0..8], "trusted.");
+                break :blk tmp;
+            },
+            .security => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 9);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[9 .. tmp.len - 1], .little);
+                @memset(tmp[0..9], "security.");
+                break :blk tmp;
+            },
+        };
+        key[key.len - 1] = 0;
+        errdefer alloc.free(key);
+
+        if (key_entry.type.out_of_line) {
+            const value: ValueOutOfLineEntry = undefined;
+            try meta.interface.readSliceEndian(ValueOutOfLineEntry, @ptrCast(&value), .little);
+
+            var ool_rdr = try fil.readerAt(io, kv_start + value.ref.block_start, &[0]u8{});
+            var ool_meta: MetadataReader = .init(alloc, &ool_rdr.interface, decomp);
+            try ool_meta.interface.discardAll(value.ref.block_offset);
+
+            var val_size: u32 = undefined;
+            try ool_meta.interface.readSliceEndian(val_size, @ptrCast(&val_size), .little);
+
+            const val = try alloc.alloc(u8, val_size);
+            errdefer alloc.free(val);
+            try ool_meta.interface.readSliceEndian(u8, val, .little);
+
+            out[i] = .{
+                .key = key,
+                .value = val,
+            };
+            continue;
+        }
+
+        var val_size: u32 = undefined;
+        try meta.interface.readSliceEndian(val_size, @ptrCast(&val_size), .little);
+
+        const val = try alloc.alloc(u8, val_size);
+        errdefer alloc.free(val);
+        try meta.interface.readSliceEndian(u8, val, .little);
+
+        out[i] = .{
+            .key = key,
+            .value = val,
+        };
+    }
+    return out;
+}