Finished (?) Data extractor

Added more utility to Inode's

.zed/debug.json

@@ -11,7 +11,7 @@
 
 		"build": {
 			"command": "zig",
-			"args": ["build", "-Duse_c_libs=true", "-Ddebug=true"],
+			"args": ["build", "-Ddebug=true"],
 		},
 
 		"program": "zig-out/bin/unsquashfs",

.zed/settings.json

@@ -1,18 +0,0 @@
-// Folder-specific settings
-//
-// For a full list of overridable settings, and general information on folder-specific settings,
-// see the documentation: https://zed.dev/docs/configuring-zed#settings-files
-{
-	"lsp": {
-		"zls": {
-			"initialization_options": {
-				"usePlaceholders": false,
-			},
-			"settings": {
-				"build_on_save": true,
-				"use_placeholders": false,
-				"build_on_save_args": ["-fincremental", "-Dallow_lzo=true"],
-			},
-		},
-	},
-}

build.zig

@@ -1,118 +1,62 @@
 const std = @import("std");
-const Build = std.Build;
-const Step = Build.Step;
 
 pub fn build(b: *std.Build) !void {
-    const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
-    const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
-    var debug = b.option(bool, "debug", "Enable options to make debugging easier.") orelse false;
-    const dynamic = b.option(bool, "dynamic", "Use dynamic linking for C libraries (if used).") orelse false;
-    var version_string = b.option([]const u8, "version", "Version of the library/binary") orelse "0.0.0-testing";
+    // const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
+    // const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
+    const debug = b.option(bool, "debug", "Enable options to make debugging easier.");
+    const version_string_option = b.option([]const u8, "version", "Version of the library/binary");
+
+    // const zig_squashfs_options = b.addOptions();
+    // zig_squashfs_options.addOption(bool, "use_zig_decomp", use_zig_decomp);
+    // zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo);
 
     const target = b.standardTargetOptions(.{});
-    var optimize = b.standardOptimizeOption(.{});
+    const optimize = b.standardOptimizeOption(.{});
 
-    const zig_squashfs_options = b.addOptions();
-    zig_squashfs_options.addOption(bool, "use_zig_decomp", use_zig_decomp);
-    zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo);
+    const lib = b.addLibrary(.{
+        .name = "squashfs",
+        .root_module = b.createModule(.{
+            .optimize = if (debug == true) .Debug else optimize,
+            .target = target,
+            .valgrind = debug,
+            .root_source_file = b.path("src/root.zig"),
+        }),
+        .use_llvm = debug,
+    });
 
-    version_string = std.mem.trimStart(u8, version_string, "v");
-    const version = try std.SemanticVersion.parse(version_string);
+    var version = version_string_option orelse "0.0.0-testing";
+    if (version[0] == 'v') version = version[1..];
     const unsquashfs_options = b.addOptions();
     unsquashfs_options.addOption(
         std.SemanticVersion,
         "version",
-        version,
+        try std.SemanticVersion.parse(version),
     );
-
-    if (debug) optimize = .Debug;
-    if (optimize == .Debug) debug = true;
-
-    const c_import = b.addTranslateC(.{
-        .root_source_file = b.path("src/c.h"),
-        .target = target,
-        .optimize = optimize,
-    });
-    if (allow_lzo) c_import.defineCMacro("ALLOW_LZO", null);
-    if (dynamic) {
-        c_import.linkSystemLibrary("zlib-ng", .{});
-        c_import.linkSystemLibrary("lzma", .{});
-        if (allow_lzo)
-            c_import.linkSystemLibrary("minilzo", .{});
-        c_import.linkSystemLibrary("lz4", .{});
-        c_import.linkSystemLibrary("zstd", .{});
-    }
-
-    var lib = b.addLibrary(.{
-        .name = "squashfs",
-        .root_module = b.addModule("squashfs", .{
-            .root_source_file = b.path("src/root.zig"),
-            .target = target,
-            .optimize = optimize,
-            .valgrind = debug,
-            .error_tracing = debug,
-            .strip = !debug,
-            .imports = &.{
-                .{ .name = "config", .module = zig_squashfs_options.createModule() },
-                .{ .name = "c", .module = c_import.createModule() },
-            },
-        }),
-        .use_llvm = debug,
-        .version = version,
-    });
-
-    const deps = try getDependencies(b, target, optimize, allow_lzo, dynamic);
-
-    for (deps) |d|
-        lib.root_module.linkLibrary(d);
-
     const exe = b.addExecutable(.{
         .name = "unsquashfs",
         .root_module = b.createModule(.{
-            .root_source_file = b.path("src/bin/unsquashfs.zig"),
+            .optimize = if (debug == true) .Debug else optimize,
             .target = target,
-            .optimize = optimize,
-            .imports = &.{
-                .{ .name = "config", .module = unsquashfs_options.createModule() },
-                .{ .name = "squashfs", .module = lib.root_module },
-            },
             .valgrind = debug,
-            .error_tracing = debug,
-            .strip = !debug,
+            .root_source_file = b.path("src/bin/unsquashfs.zig"),
+            .imports = &.{
+                .{ .name = "zig_squashfs", .module = lib.root_module },
+            },
         }),
         .use_llvm = debug,
-        .version = version,
     });
+    exe.root_module.addOptions("config", unsquashfs_options);
 
     b.installArtifact(lib);
     b.installArtifact(exe);
 
     const mod_tests = b.addTest(.{
         .root_module = b.createModule(.{
-            .root_source_file = b.path("src/root.zig"),
-            .target = target,
             .optimize = optimize,
-            .valgrind = true,
-            .error_tracing = true,
-            .strip = false,
-            .imports = &.{
-                .{ .name = "config", .module = zig_squashfs_options.createModule() },
-                .{ .name = "c", .module = c_import.createModule() },
-            },
+            .target = target,
+            .root_source_file = b.path("src/test.zig"),
         }),
-        .use_llvm = debug, // Helps with lldb degugging
     });
-
-    for (deps) |d|
-        mod_tests.root_module.linkLibrary(d);
-
-    if (dynamic) {
-        mod_tests.root_module.linkSystemLibrary("zlib-ng", .{});
-        mod_tests.root_module.linkSystemLibrary("lzma", .{});
-        mod_tests.root_module.linkSystemLibrary("minilzo", .{});
-        mod_tests.root_module.linkSystemLibrary("lz4", .{});
-    }
-
     const run_mod_tests = b.addRunArtifact(mod_tests);
     const test_step = b.step("test", "Run tests");
     test_step.dependOn(&run_mod_tests.step);
@@ -120,54 +64,13 @@ pub fn build(b: *std.Build) !void {
     // zls build check steps
     const lib_check = b.addLibrary(.{
         .name = "squashfs",
-        .root_module = lib.root_module,
+        .root_module = exe.root_module,
     });
     const exe_check = b.addExecutable(.{
         .name = "unsquashfs",
-        .root_module = exe.root_module,
+        .root_module = lib.root_module,
     });
     const check = b.step("check", "Check if unsquashfs compiles");
     check.dependOn(&lib_check.step);
     check.dependOn(&exe_check.step);
 }
-
-fn getDependencies(b: *Build, target: Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, allow_lzo: bool, dynamic: bool) ![]*Step.Compile {
-    if (dynamic) return &.{};
-
-    var list: std.ArrayList(*Step.Compile) = .empty;
-    errdefer list.clearAndFree(b.allocator);
-
-    var zlib_ng = b.dependency("zlib_ng", .{
-        .target = target,
-        .optimize = optimize,
-    });
-    try list.append(b.allocator, zlib_ng.artifact("zng"));
-
-    var xz = b.dependency("xz", .{
-        .target = target,
-        .optimize = optimize,
-    });
-    try list.append(b.allocator, xz.artifact("lzma"));
-
-    if (allow_lzo) {
-        var minilzo = b.dependency("minilzo", .{
-            .target = target,
-            .optimize = optimize,
-        });
-        try list.append(b.allocator, minilzo.artifact("minilzo"));
-    }
-
-    var lz4 = b.dependency("lz4", .{
-        .target = target,
-        .optimize = optimize,
-    });
-    try list.append(b.allocator, lz4.artifact("lz4"));
-
-    var zstd = b.dependency("zstd", .{
-        .target = target,
-        .optimize = optimize,
-    });
-    try list.append(b.allocator, zstd.artifact("zstd"));
-
-    return list.toOwnedSlice(b.allocator);
-}

build.zig.zon

@@ -2,7 +2,7 @@
     .name = .squashfs,
     .version = "0.0.6",
     .fingerprint = 0x37ba29474b87f145, // Changing this has security and trust implications.
-    .minimum_zig_version = "0.16.1",
+    .minimum_zig_version = "0.15.2",
     .dependencies = .{
         .zlib_ng = .{
             .url = "git+https://github.com/CalebQ42/zig-zlib-ng#5f2f02dfb28acca2517dacbbd09e9b987f57b133",
@@ -20,10 +20,6 @@
             .url = "git+https://github.com/CalebQ42/zig-minilzo.git#7cbae997b91a44d74b7cd6c073584dc9562a6c90",
             .hash = "minilzo-2.10.0-Ij7BO8wLAADeWI4Pe4jp8XTDsDaquZR14oZ7_9yKKDWP",
         },
-        .xz = .{
-            .url = "git+https://github.com/akunaakwei/zig-xz.git#e2d389262c8291907e3e4c6fb119819141c16c0f",
-            .hash = "xz-5.8.2-6v47_JYeAABSL-jonprpL5-E_YaaGc4B5xrbe93WsJ3G",
-        },
     },
     .paths = .{
         "build.zig",

mise.toml

@@ -1,2 +0,0 @@
-[tools]
-zig = "0.16.0"

src/archive.zig

@@ -1,100 +1,181 @@
 const std = @import("std");
 const Io = std.Io;
-const MemoryMap = Io.File.MemoryMap;
-
-const c = @import("c");
-const config = @import("config");
 
 const ExtractionOptions = @import("options.zig");
 const File = @import("file.zig");
 const Inode = @import("inode.zig");
-const Superblock = @import("super.zig").Superblock;
-const DecompCache = @import("util/decomp_cache.zig");
-const CompressionType = @import("util/decompress.zig").CompressionType;
+const LookupTable = @import("lookup_table.zig");
+const Decompressor = @import("util/decompressor.zig");
+const MetadataReader = @import("util/metadata.zig");
+const Utils = @import("util/misc.zig");
+const OffsetFile = @import("util/offset_file.zig");
 
 const Archive = @This();
 
-const CACHE_MIN = 16 * 1024 * 1024;
-const CACHE_MAX = 1 * 1024 * 1024 * 1024;
-
-cache: DecompCache,
-
+file: OffsetFile,
 super: Superblock,
 
-/// Open a squashfs archive from an Io.File.
-pub fn init(alloc: std.mem.Allocator, io: Io, fil: Io.File) !Archive {
-    return initAdvanced(alloc, io, fil, 0, 0);
-}
-/// If max_cache_size is zero, a size is selected based on system ram, up to 1GB with a minimum of 16MB.
-pub fn initAdvanced(alloc: std.mem.Allocator, io: Io, file: Io.File, offset: u64, max_cache_size: u64) !Archive {
+stateless_decomp: Decompressor,
+
+pub fn init(io: Io, file: std.Io.File, offset: u64) !Archive {
     var rdr = file.reader(io, &[0]u8{});
     try rdr.seekTo(offset);
-
     var super: Superblock = undefined;
     try rdr.interface.readSliceEndian(Superblock, @ptrCast(&super), .little);
-    try super.validate();
-
-    if (!config.use_zig_decomp and config.allow_lzo)
-        _ = c.lzo_init();
-
-    const cache_size = blk: {
-        if (max_cache_size > CACHE_MIN) break :blk CACHE_MIN;
-        const sys_mem = std.process.totalSystemMemory() catch break :blk CACHE_MIN;
-        var min = @min(CACHE_MAX, sys_mem / 4);
-        if (min < CACHE_MIN and sys_mem > CACHE_MIN)
-            min = CACHE_MIN;
-        break :blk min;
-    };
     return .{
-        .cache = try .init(
-            alloc,
-            try file.createMemoryMap(
-                io,
-                .{
-                    .offset = offset,
-                    .len = super.size,
-                    .protection = .{ .read = true },
-                },
-            ),
-            super.compression,
-            cache_size,
-        ),
-
+        .file = .init(file, offset),
         .super = super,
+
+        .stateless_decomp = switch (super.compression) {
+            .gzip => @import("decomp/zlib.zig").stateless_decompressor,
+            .lzma => @import("decomp/lzma.zig").stateless_decompressor,
+            .lzo => return error.LzoUnsupported,
+            .xz => @import("decomp/xz.zig").stateless_decompressor,
+            .lz4 => return error.Lz4Unsupported,
+            .zstd => @import("decomp/zstd.zig").stateless_decompressor,
+        },
     };
 }
-pub fn deinit(self: *Archive, io: Io) void {
-    self.cache.deinit(io);
-}
 
-pub fn root(self: *Archive, alloc: std.mem.Allocator, io: Io) !File {
-    return .fromRef(alloc, io, self, "", self.super.root_ref);
-}
-
-pub fn open(self: *Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
-    const path = std.mem.trim(u8, filepath, "/");
-
-    var root_file = try self.root(alloc, io);
-
-    if (path.len == 0 or std.mem.eql(u8, path, ".")) return root_file;
-    defer root_file.deinit();
-
-    return root_file.open(alloc, io, path);
-}
-
-pub fn extract(self: *Archive, alloc: std.mem.Allocator, io: Io, ext_dir: []const u8, options: ExtractionOptions) !void {
-    const root_inode: Inode = try .fromRef(alloc, io, &self.cache, self.super.inode_start, self.super.block_size, self.super.root_ref);
-    return root_inode.extract(
+/// The root folder of the Archive. Used to open other Files.
+pub fn root(self: Archive, alloc: std.mem.Allocator, io: Io) !File {
+    const root_inode = try Utils.inodeFromRef(
         alloc,
         io,
-        &self.cache,
-        self.super.dir_start,
+        self.file,
+        &self.stateless_decomp,
         self.super.inode_start,
-        self.super.frag_start,
         self.super.block_size,
-        self.super.id_start,
-        self.super.xattr_start,
-        ext_dir,
-        options,
+        self.super.root_ref,
+    );
+    return .init(alloc, self, root_inode, "");
+}
+/// Opens a File within the archive.
+pub fn open(self: Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
+    const root_file = try self.root(alloc, io);
+    const path = std.mem.trim(u8, filepath, "/");
+    if (Utils.pathIsSelf(path))
+        return root_file;
+    defer root_file.deinit();
+    return root_file.open(alloc, io, filepath);
+}
+/// Extract the entire archive contents to the given directory.
+pub fn extract(self: Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void {
+    const root_inode = try Utils.inodeFromRef(
+        alloc,
+        io,
+        self.file,
+        &self.stateless_decomp,
+        self.super.inode_start,
+        self.super.block_size,
+        self.super.root_ref,
+    );
+    _ = root_inode;
+    _ = extract_dir;
+    _ = options;
+    return error.TODO;
+}
+
+/// Returns the inode with the given inode number.
+/// Requires that the archive is exportable (has an export lookup table).
+pub fn inode(self: Archive, alloc: std.mem.Allocator, io: Io, num: u32) !Inode {
+    if (!self.super.flags.exportable)
+        return error.NotExportable;
+    const ref = try LookupTable.lookupValue(
+        Inode.Ref,
+        alloc,
+        io,
+        &self.stateless_decomp,
+        self.file,
+        self.super.export_start,
+        num + 1,
+    );
+    return Utils.inodeFromRef(
+        alloc,
+        io,
+        self.file,
+        &self.stateless_decomp,
+        self.super.inode_start,
+        self.super.block_size,
+        ref,
     );
 }
+/// Returns a value at the given index from the Archive's id (uid/gid) table.
+pub fn idTable(self: Archive, alloc: std.mem.Allocator, io: Io, idx: u32) !u16 {
+    return LookupTable.lookupValue(
+        u16,
+        alloc,
+        io,
+        &self.stateless_decomp,
+        self.file,
+        self.super.id_start,
+        idx,
+    );
+}
+
+// Superblock
+
+const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little);
+
+const SuperblockError = error{
+    InvalidMagic,
+    InvalidBlockLog,
+    InvalidVersion,
+    InvalidCheck,
+};
+
+/// A squashfs Superblock
+pub const Superblock = extern struct {
+    magic: u32,
+    inode_count: u32,
+    mod_time: u32,
+    block_size: u32,
+    frag_count: u32,
+    compression: enum(u16) {
+        gzip = 1,
+        lzma,
+        lzo,
+        xz,
+        lz4,
+        zstd,
+    },
+    block_log: u16,
+    flags: packed struct(u16) {
+        inode_uncompressed: bool,
+        data_uncompressed: bool,
+        check: bool,
+        frag_uncompressed: bool,
+        fragment_never: bool,
+        fragment_always: bool,
+        duplicates: bool,
+        exportable: bool,
+        xattr_uncompressed: bool,
+        xattr_never: bool,
+        compression_options: bool,
+        ids_uncompressed: bool,
+        _: u4,
+    },
+    id_count: u16,
+    ver_maj: u16,
+    ver_min: u16,
+    root_ref: Inode.Ref,
+    size: u64,
+    id_start: u64,
+    xattr_start: u64,
+    inode_start: u64,
+    dir_start: u64,
+    frag_start: u64,
+    export_start: u64,
+
+    /// Validate the Superblock. If an error is returned, it's likely the archive is corrupted or not a squashfs archive.
+    pub fn validate(self: Superblock) !void {
+        if (self.magic != SQUASHFS_MAGIC)
+            return SuperblockError.InvalidMagic;
+        if (self.flags.check)
+            return SuperblockError.InvalidCheck;
+        if (self.ver_maj != 4 or self.ver_min != 0)
+            return SuperblockError.InvalidVersion;
+        if (std.math.log2(self.block_size) != self.block_log)
+            return SuperblockError.InvalidBlockLog;
+    }
+};

src/bin/unsquashfs.zig

@@ -1,10 +1,10 @@
 const std = @import("std");
 const Io = std.Io;
-const Writer = std.Io.Writer;
+const Writer = Io.Writer;
 const builtin = @import("builtin");
 
 const config = @import("config");
-const squashfs = @import("squashfs");
+const squashfs = @import("zig_squashfs");
 
 //TODO: Add more options
 const help_mgs =
@@ -42,24 +42,20 @@ var force: bool = false;
 pub fn main(init: std.process.Init) !void {
     const alloc = init.gpa;
     const io = init.io;
-
-    var stdout = Io.File.stdout();
+    var stdout = std.Io.File.stdout();
     var out = stdout.writer(io, &[0]u8{});
     defer out.interface.flush() catch {};
-
-    try handleArgs(&out.interface, init.minimal.args);
+    try handleArgs(init.minimal.args, &out.interface);
     if (archive.len == 0) {
         try out.interface.print("You must provide a squashfs archive\n", .{});
         try out.interface.print(help_mgs, .{});
         return;
     }
-
-    var fil: std.Io.File = try Io.Dir.cwd().openFile(io, archive, .{}); //TODO: Handle error gracefully.
+    var fil = try Io.Dir.cwd().openFile(io, archive, .{}); //TODO: Handle error gracefully.
     defer fil.close(io);
-    var arc: squashfs.Archive = try .initAdvanced(alloc, io, fil, offset, 0); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
-    defer arc.deinit(io);
+    var arc: squashfs.Archive = try .init(io, fil, offset); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
     const options: squashfs.ExtractionOptions = .{
-        .single_threaded = threads == 1,
+        .threads = if (threads == 0) try std.Thread.getCpuCount() else threads,
         .verbose = verbose,
         .verbose_writer = if (verbose) &out.interface else null,
         .ignore_xattr = ignore_xattrs,
@@ -70,8 +66,9 @@ pub fn main(init: std.process.Init) !void {
     try arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully.
 }
 
-fn handleArgs(out: *Writer, args: std.process.Args) !void {
+fn handleArgs(args: std.process.Args, out: *Writer) !void {
     var arg_iter = args.iterate();
+    defer arg_iter.deinit();
     _ = arg_iter.next(); // args[0] is the application launch command.
     while (arg_iter.next()) |arg| {
         if (std.mem.eql(u8, arg, "-o")) {
@@ -135,7 +132,3 @@ fn handleArgs(out: *Writer, args: std.process.Args) !void {
         archive = arg;
     }
 }
-
-test {
-    std.testing.refAllDecls(squashfs.Archive);
-}

src/c.h

@@ -1,7 +0,0 @@
-#ifdef ALLOW_LZO
-#include <lzo/minilzo.h>
-#endif
-#include <zlib-ng.h>
-#include <zstd.h>
-#include <lz4.h>
-#include <lzma.h>

src/decomp/lzma.zig

@@ -0,0 +1,77 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+const lzma = std.compress.lzma;
+const Node = std.SinglyLinkedList.Node;
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Self = @This();
+
+const Buffer = struct {
+    node: Node,
+    buf: []u8,
+};
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+
+block_size: u32,
+buffers: std.ArrayList(Buffer),
+buffer_queue: std.SinglyLinkedList,
+
+pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self {
+    return .{
+        .alloc = alloc,
+
+        .block_size = block_size,
+        .buffers = try .initCapacity(alloc, 5),
+    };
+}
+pub fn deinit(self: Self) void {
+    for (self.buffers) |buf|
+        self.alloc.free(buf);
+    self.buffers.deinit(self.alloc);
+}
+
+fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        const buf = try alloc.alloc(u8, in.len * 2);
+        defer alloc.free(buf);
+        return lzmaDecomp(buf, in, out);
+    }
+    var self: Self = @fieldParentPtr("interface", d.?);
+    const buf_node = self.buffer_queue.popFirst();
+    var buf: *Buffer = undefined;
+    if (buf_node == null) {
+        const new_buf = try self.buffers.addOne(self.alloc);
+        new_buf.* = .{ .{}, try self.alloc.alloc(u8, self.block_size + lzma.block_size_max) };
+        buf = new_buf;
+    } else {
+        buf = @fieldParentPtr("node", buf_node);
+    }
+    defer self.buffer_queue.prepend(&buf.node);
+    return lzmaDecomp(self.alloc, &buf.buf, in, out);
+}
+
+inline fn lzmaDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = try lzma.Decompress.initOptions(&rdr, alloc, buffer.*, .{ .allow_incomplete = true }, 3 * 1024 * 1024);
+    defer {
+        buffer.* = d.takeBuffer();
+        d.deinit();
+    }
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var buf = try alloc.alloc(u8, in.len);
+    defer alloc.free(buf);
+    return lzmaDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
+}

src/decomp/xz.zig

@@ -0,0 +1,77 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+const xz = std.compress.xz;
+const Node = std.SinglyLinkedList.Node;
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Self = @This();
+
+const Buffer = struct {
+    node: Node,
+    buf: []u8,
+};
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+
+block_size: u32,
+buffers: std.ArrayList(Buffer),
+buffer_queue: std.SinglyLinkedList,
+
+pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self {
+    return .{
+        .alloc = alloc,
+
+        .block_size = block_size,
+        .buffers = try .initCapacity(alloc, 5),
+    };
+}
+pub fn deinit(self: Self) void {
+    for (self.buffers) |buf|
+        self.alloc.free(buf);
+    self.buffers.deinit(self.alloc);
+}
+
+fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        const buf = try alloc.alloc(u8, in.len * 2);
+        defer alloc.free(buf);
+        return lzmaDecomp(buf, in, out);
+    }
+    var self: Self = @fieldParentPtr("interface", d.?);
+    const buf_node = self.buffer_queue.popFirst();
+    var buf: *Buffer = undefined;
+    if (buf_node == null) {
+        const new_buf = try self.buffers.addOne(self.alloc);
+        new_buf.* = .{ .{}, try self.alloc.alloc(u8, self.block_size + xz.block_size_max) };
+        buf = new_buf;
+    } else {
+        buf = @fieldParentPtr("node", buf_node);
+    }
+    defer self.buffer_queue.prepend(&buf.node);
+    return lzmaDecomp(self.alloc, &buf.buf, in, out);
+}
+
+inline fn lzmaDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = try xz.Decompress.init(&rdr, alloc, buffer.*);
+    defer {
+        buffer.* = d.takeBuffer();
+        d.deinit();
+    }
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    var buf = try alloc.alloc(u8, in.len);
+    defer alloc.free(buf);
+    return lzmaDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
+}

src/decomp/zlib.zig

@@ -0,0 +1,73 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+const flate = std.compress.flate;
+const Node = std.SinglyLinkedList.Node;
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Self = @This();
+
+const Buffer = struct {
+    node: Node,
+    buf: []u8,
+};
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+
+block_size: u32,
+buffers: std.ArrayList(Buffer),
+buffer_queue: std.SinglyLinkedList,
+
+pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self {
+    return .{
+        .alloc = alloc,
+
+        .block_size = block_size,
+        .buffers = try .initCapacity(alloc, 5),
+    };
+}
+pub fn deinit(self: Self) void {
+    for (self.buffers) |buf|
+        self.alloc.free(buf);
+    self.buffers.deinit(self.alloc);
+}
+
+fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        const buf = try alloc.alloc(u8, in.len * 2);
+        defer alloc.free(buf);
+        return zlibDecomp(buf, in, out);
+    }
+    var self: Self = @fieldParentPtr("interface", d.?);
+    const buf_node = self.buffer_queue.popFirst();
+    var buf: *Buffer = undefined;
+    if (buf_node == null) {
+        const new_buf = try self.buffers.addOne(self.alloc);
+        new_buf.* = .{ .{}, try self.alloc.alloc(u8, self.block_size) };
+        buf = new_buf;
+    } else {
+        buf = @fieldParentPtr("node", buf_node);
+    }
+    defer self.buffer_queue.prepend(&buf.node);
+    return zlibDecomp(buf.buf, in, out);
+}
+
+inline fn zlibDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = flate.Decompress.init(&rdr, .zlib, buffer);
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    const buf = try alloc.alloc(u8, in.len * 2);
+    defer alloc.free(buf);
+    return zlibDecomp(buf, in, out);
+}

src/decomp/zstd.zig

@@ -0,0 +1,73 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+const zstd = std.compress.zstd;
+const Node = std.SinglyLinkedList.Node;
+
+const Decompressor = @import("../util/decompressor.zig");
+const Error = Decompressor.Error;
+
+const Self = @This();
+
+const Buffer = struct {
+    node: Node,
+    buf: []u8,
+};
+
+interface: Decompressor = .{ .decomp_fn = decomp },
+
+alloc: std.mem.Allocator,
+
+block_size: u32,
+buffers: std.ArrayList(Buffer),
+buffer_queue: std.SinglyLinkedList,
+
+pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self {
+    return .{
+        .alloc = alloc,
+
+        .block_size = block_size,
+        .buffers = try .initCapacity(alloc, 5),
+    };
+}
+pub fn deinit(self: Self) void {
+    for (self.buffers) |buf|
+        self.alloc.free(buf);
+    self.buffers.deinit(self.alloc);
+}
+
+fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    if (d == null) {
+        const buf = try alloc.alloc(u8, in.len * 2);
+        defer alloc.free(buf);
+        return zstdDecomp(buf, in, out);
+    }
+    var self: Self = @fieldParentPtr("interface", d.?);
+    const buf_node = self.buffer_queue.popFirst();
+    var buf: *Buffer = undefined;
+    if (buf_node == null) {
+        const new_buf = try self.buffers.addOne(self.alloc);
+        new_buf.* = .{ .{}, try self.alloc.alloc(u8, self.block_size + zstd.block_size_max) };
+        buf = new_buf;
+    } else {
+        buf = @fieldParentPtr("node", buf_node);
+    }
+    defer self.buffer_queue.prepend(&buf.node);
+    return zstdDecomp(buf.buf, in, out);
+}
+
+inline fn zstdDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
+    var rdr: Reader = .fixed(in);
+    var d = zstd.Decompress.init(&rdr, buffer, .{ .window_len = @truncate(out.len) });
+
+    return d.reader.readSliceShort(out);
+}
+
+// Stateless
+
+pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
+
+fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    const buf = try alloc.alloc(u8, out.len + zstd.block_size_max);
+    defer alloc.free(buf);
+    return zstdDecomp(buf, in, out);
+}

src/dir_entry.zig

@@ -1,67 +0,0 @@
-const std = @import("std");
-const Reader = std.Io.Reader;
-
-const Inode = @import("inode.zig");
-
-const Header = extern struct {
-    count: u32,
-    block_start: u32,
-    num: u32,
-};
-const Entry = extern struct {
-    block_offset: u16,
-    num_offset: i16,
-    inode_type: Inode.Type,
-    name_size: u16,
-};
-
-pub const Error = error{OutOfMemory} || std.Io.Reader.Error;
-
-const DirEntry = @This();
-
-inode_type: Inode.Type,
-name: []const u8,
-
-block_start: u32,
-block_offset: u32,
-num: u32,
-
-pub fn deinit(self: DirEntry, alloc: std.mem.Allocator) void {
-    alloc.free(self.name);
-}
-
-pub fn readEntries(alloc: std.mem.Allocator, rdr: *Reader, size: u32) Error![]DirEntry {
-    var out: std.ArrayList(DirEntry) = try .initCapacity(alloc, 50);
-    errdefer out.deinit(alloc);
-
-    var tot_read: u32 = 3;
-    while (tot_read < size) {
-        var hdr: Header = undefined;
-        try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
-        tot_read += @sizeOf(Header);
-
-        try out.ensureUnusedCapacity(alloc, hdr.count + 1);
-
-        for (0..hdr.count + 1) |_| {
-            var ent: Entry = undefined;
-            try rdr.readSliceEndian(Entry, @ptrCast(&ent), .little);
-            tot_read += @sizeOf(Entry) + ent.name_size + 1;
-
-            const name = try alloc.alloc(u8, ent.name_size + 1);
-            errdefer alloc.free(name);
-
-            try rdr.readSliceEndian(u8, name, .little);
-
-            out.appendAssumeCapacity(.{
-                .inode_type = ent.inode_type,
-                .name = name,
-
-                .block_offset = ent.block_offset,
-                .block_start = hdr.block_start,
-                .num = @intCast(@as(i64, @intCast(hdr.num)) + ent.num_offset),
-            });
-        }
-    }
-
-    return out.toOwnedSlice(alloc);
-}

src/directory.zig

@@ -0,0 +1,67 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+
+const Inode = @import("inode.zig");
+
+const DirEntry = @This();
+
+block_start: u32,
+block_offset: u16,
+type: Inode.Type,
+name: []const u8,
+
+pub fn deinit(self: DirEntry, alloc: std.mem.Allocator) void {
+    alloc.free(self.name);
+}
+
+pub fn readDirectory(alloc: std.mem.Allocator, rdr: *Reader, size: u32) ![]DirEntry {
+    var hdr: Header = undefined;
+    var raw: RawEntry = undefined;
+    var out: std.ArrayList(DirEntry) = try .initCapacity(alloc, 30);
+    errdefer {
+        for (out.items) |ent|
+            alloc.free(ent.name);
+        out.deinit(alloc);
+    }
+
+    var tot_red: u32 = 3;
+    while (tot_red < size) {
+        try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
+        try out.ensureUnusedCapacity(alloc, hdr.count + 1);
+
+        tot_red += @sizeOf(Header);
+
+        for (hdr.count + 1) |_| {
+            try rdr.readSliceEndian(RawEntry, @ptrCast(&raw), .little);
+
+            const new_name = try alloc.alloc(u8, raw.name_size + 1);
+            try rdr.readSliceEndian(u8, new_name, .little);
+
+            const new = out.addOneAssumeCapacity();
+            new.* = .{
+                .block_start = hdr.block_start,
+                .block_offset = raw.block_offset,
+                .type = raw.type,
+                .name = new_name,
+            };
+
+            tot_red += @sizeOf(RawEntry) + raw.name_size + 1;
+        }
+    }
+    return out.toOwnedSlice(alloc);
+}
+
+// Types
+
+const Header = extern struct {
+    count: u32,
+    block_start: u32,
+    num: u32,
+};
+
+const RawEntry = extern struct {
+    block_offset: u16,
+    num_offset: i16,
+    type: Inode.Type,
+    name_size: u16,
+};

src/file.zig

@@ -1,126 +1,124 @@
-//! A wrapper around an Inode to make common activities easier.
+//! An easier to use wrapper around an inode.
 
 const std = @import("std");
 const Io = std.Io;
 
 const Archive = @import("archive.zig");
-const DirEntry = @import("dir_entry.zig");
+const DirEntry = @import("directory.zig");
 const ExtractionOptions = @import("options.zig");
 const Inode = @import("inode.zig");
-const LookupTable = @import("lookup_table.zig");
+const DataExtractor = @import("util/data_extractor.zig");
+const Decompressor = @import("util/decompressor.zig");
 const MetadataReader = @import("util/metadata.zig");
-
-pub const Error = error{
-    NotFound,
-};
+const SharedCache = @import("util/shared_cache.zig");
 
 const File = @This();
 
 alloc: std.mem.Allocator,
-archive: *Archive,
 
-name: []const u8,
+archive: Archive,
+
 inode: Inode,
+name: []const u8,
 
-pub fn fromEntry(alloc: std.mem.Allocator, io: Io, archive: *Archive, entry: DirEntry) !File {
-    var meta: MetadataReader = .init(io, &archive.cache, archive.super.inode_start + entry.block_start);
-    defer meta.deinit();
-    try meta.interface.discardAll(entry.block_offset);
-
-    const new_name = try alloc.alloc(u8, entry.name.len);
-    errdefer alloc.free(new_name);
-    @memcpy(new_name, entry.name);
-
+/// Creates a new File from an inode. Takes ownership of the Inode and creates a copy of the given name.
+/// Requires the given allocator was used to create the Inode.
+pub fn init(alloc: std.mem.Allocator, archive: Archive, in: Inode, name: []const u8) !File {
+    const new_name = try alloc.alloc(u8, name.len);
+    @memcpy(new_name, name);
     return .{
         .alloc = alloc,
+
         .archive = archive,
 
-        .name = new_name,
-        .inode = try .fromReader(alloc, &meta.interface, archive.super.block_size),
-    };
-}
-/// Create a File from an Inode.Ref. name should be created using the alloc given.
-pub fn fromRef(alloc: std.mem.Allocator, io: Io, archive: *Archive, name: []const u8, ref: Inode.Ref) !File {
-    return .{
-        .alloc = alloc,
-        .archive = archive,
-
-        .name = name,
-        .inode = try .fromRef(
-            alloc,
-            io,
-            &archive.cache,
-            archive.super.inode_start,
-            archive.super.block_size,
-            ref,
-        ),
-    };
-}
-pub fn copy(alloc: std.mem.Allocator, from: File) !File {
-    const new_name = try alloc.alloc(u8, from.name.len);
-    errdefer alloc.free(new_name);
-    @memcpy(new_name, from.name);
-
-    return .{
-        .alloc = alloc,
-        .archive = from.archive,
-
-        .inode = try .copy(alloc, from.inode),
+        .inode = in,
         .name = new_name,
     };
 }
+pub fn fromDirEntry(alloc: std.mem.Allocator, io: Io, archive: Archive, ent: DirEntry) !File {
+    var rdr = try archive.file.readerAt(io, archive.super.inode_start + ent.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, &archive.stateless_decomp);
+    try meta.interface.discardAll(ent.block_offset);
+
+    var in: Inode = try .read(alloc, &meta.interface, archive.super.block_size);
+    errdefer in.deinit(alloc);
+    return .init(alloc, archive, in, ent.name);
+}
 pub fn deinit(self: File) void {
     self.alloc.free(self.name);
     self.inode.deinit(self.alloc);
 }
 
 pub fn open(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
-    const path = std.mem.trim(u8, filepath, "/");
-
-    if (path.len == 0 or std.mem.eql(u8, path, ".")) return .copy(alloc, self);
-
-    const first_element = std.mem.sliceTo(path, '/');
-
-    const entries = try self.inode.readDirectory(alloc, io, &self.archive.cache, self.archive.super.dir_start);
-    defer {
-        for (entries) |entry|
-            entry.deinit(alloc);
-        alloc.free(entries);
-    }
-
-    // Potentially I could use linear searching on small dir tables...
-    var search_slice = entries;
-    var idx = search_slice.len / 2;
-    while (search_slice.len > 0) {
-        const order = std.mem.order(u8, first_element, search_slice[idx].name);
-        switch (order) {
-            .eq => break,
-            .gt => search_slice = search_slice[idx..],
-            .lt => search_slice = search_slice[0..idx],
-        }
-        idx = search_slice.len / 2;
-    }
-    if (search_slice.len == 0) return Error.NotFound;
-
-    var fil: File = try .fromEntry(alloc, io, self.archive, search_slice[idx]);
-    if (path.len == first_element.len) return fil;
-    defer fil.deinit();
-
-    return fil.open(alloc, io, filepath[first_element.len..]);
-}
-
-pub fn extract(self: File, alloc: std.mem.Allocator, io: Io, path: []const u8, options: ExtractionOptions) !void {
-    return self.inode.extract(
+    const entries = try self.inode.readDirectory(
         alloc,
         io,
-        &self.archive.cache,
+        self.archive.file,
+        &self.archive.stateless_decomp,
         self.archive.super.dir_start,
-        self.archive.super.inode_start,
-        self.archive.super.frag_start,
-        self.archive.super.block_size,
-        self.archive.super.id_start,
-        self.archive.super.xattr_start,
-        path,
-        options,
     );
+    defer {
+        for (entries) |ent|
+            alloc.free(ent.name);
+        alloc.free(entries);
+    }
+    const path = std.mem.trim(u8, filepath, "/");
+    const first_element: []u8 = std.mem.sliceTo(path, "/");
+
+    var search_slice = entries;
+    var idx: usize = undefined;
+    while (search_slice.len > 0) {
+        idx = search_slice / 2;
+        const middle = search_slice[idx];
+        switch (std.mem.order(u8, first_element, middle.name)) {
+            .eq => break,
+            .lt => search_slice = search_slice[0..idx],
+            .gt => search_slice = search_slice[idx + 1 ..],
+        }
+    } else return Error.FileNotFound;
+
+    const first_elem_file = try fromDirEntry(alloc, io, self.archive, search_slice[idx]);
+    if (first_element.len == path.len)
+        return first_elem_file;
+    defer first_elem_file.deinit();
+    return first_elem_file.open(alloc, io, path[first_element.len + 1 ..]);
 }
+
+pub fn extract(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8, options: ExtractionOptions) !void {
+    var cache: SharedCache = try .init(alloc, 10); // TODO: calculate a good initial cache size.
+    defer cache.deinit();
+    var decomp = switch (self.archive.super.compression) {
+        .gzip => {},
+        .lzma => {},
+        .xz => {},
+        .zstd => {},
+        else => unreachable,
+    };
+    return self.extractReal(alloc, io, &cache, &decomp.interface, filepath, options);
+}
+fn extractReal(self: File, alloc: std.mem.Allocator, io: Io, cache: *SharedCache, decomp: *const Decompressor, filepath: []const u8, options: ExtractionOptions) !void {
+    _ = options;
+    switch (self.inode.hdr.inode_type) {
+        .file, .ext_file => {
+            var ext = try self.inode.dataExtractor(
+                self.archive.file,
+                cache,
+                decomp,
+                self.archive.super.block_size,
+            );
+
+            var atomic_file = try Io.Dir.cwd().createFileAtomic(io, filepath, .{});
+            defer atomic_file.deinit(io);
+
+            try ext.extract(alloc, io, atomic_file.file);
+            try atomic_file.link(io);
+        },
+        else => return error.TODO,
+    }
+}
+
+// Types
+
+pub const Error = error{
+    FileNotFound,
+} || Inode.Error;

src/inode.zig

@@ -1,19 +1,151 @@
 //! A file-system object. Represents a File or directory.
 
 const std = @import("std");
+const Reader = std.Io.Reader;
 const Io = std.Io;
-const Reader = Io.Reader;
 
-const DirEntry = @import("dir_entry.zig");
+const Archive = @import("archive.zig");
+const DirEntry = @import("directory.zig");
 const ExtractionOptions = @import("options.zig");
-const FragEntry = @import("frag.zig").FragEntry;
-const DirTypes = @import("inode_data/dir.zig");
-const FileTypes = @import("inode_data/file.zig");
-const MiscTypes = @import("inode_data/misc.zig");
+const dir = @import("inode_data/dir.zig");
+const file = @import("inode_data/file.zig");
+const misc = @import("inode_data/misc.zig");
 const LookupTable = @import("lookup_table.zig");
-const DataExtract = @import("util/data_extract.zig");
-const DecompCache = @import("util/decomp_cache.zig");
+const DataExtractor = @import("util/data_extractor.zig");
+const DataReader = @import("util/data_reader.zig");
+const Decompressor = @import("util/decompressor.zig");
 const MetadataReader = @import("util/metadata.zig");
+const OffsetFile = @import("util/offset_file.zig");
+const SharedCache = @import("util/shared_cache.zig");
+const XattrTable = @import("xattr_table.zig");
+
+const Inode = @This();
+
+hdr: Header,
+data: Data,
+
+pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode {
+    var hdr: Header = undefined;
+    try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
+    return .{
+        .hdr = hdr,
+        .data = switch (hdr.inode_type) {
+            .dir => .{ .dir = try .read(rdr) },
+            .file => .{ .file = try .read(alloc, rdr, block_size) },
+            .symlink => .{ .symlink = try .read(alloc, rdr) },
+            .block_dev => .{ .block_dev = try .read(rdr) },
+            .char_dev => .{ .char_dev = try .read(rdr) },
+            .fifo => .{ .fifo = try .read(rdr) },
+            .socket => .{ .socket = try .read(rdr) },
+            .ext_dir => .{ .ext_dir = try .read(rdr) },
+            .ext_file => .{ .ext_file = try .read(alloc, rdr, block_size) },
+            .ext_symlink => .{ .ext_symlink = try .read(alloc, rdr) },
+            .ext_block_dev => .{ .ext_block_dev = try .read(rdr) },
+            .ext_char_dev => .{ .ext_char_dev = try .read(rdr) },
+            .ext_fifo => .{ .ext_fifo = try .read(rdr) },
+            .ext_socket => .{ .ext_socket = try .read(rdr) },
+        },
+    };
+}
+pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
+    switch (self.data) {
+        .file => |d| d.deinit(alloc),
+        .symlink => |d| d.deinit(alloc),
+        .ext_file => |d| d.deinit(alloc),
+        .ext_symlink => |d| d.deinit(alloc),
+        else => {},
+    }
+}
+
+// Utility Functions
+
+/// Read the directory entries
+pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64) ![]DirEntry {
+    return switch (self.data) {
+        .dir => |d| readDirFromData(alloc, io, fil, decomp, dir_offset, d),
+        .ext_dir => |d| readDirFromData(alloc, io, fil, decomp, dir_offset, d),
+        else => Error.NotDirectory,
+    };
+}
+fn readDirFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64, d: anytype) ![]DirEntry {
+    var rdr = try fil.readerAt(io, dir_offset + d.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+    try meta.interface.discardAll(d.block_offset);
+
+    return DirEntry.readDirectory(alloc, &meta.interface, d.size);
+}
+/// Get a reader for a regular file's data.
+pub fn dataReader(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32) !DataReader {
+    return switch (self.data) {
+        .file => |f| getReaderFromData(alloc, io, fil, cache, decomp, block_size, f),
+        .ext_file => |f| getReaderFromData(alloc, io, fil, cache, decomp, block_size, f),
+        else => Error.NotRegularFile,
+    };
+}
+fn getReaderFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32, d: anytype) !DataReader {
+    const ext: DataReader = .init(alloc, io, fil, cache, decomp, block_size, d.size, d.block_start, d.blocks);
+    if (d.frag_block_offset == 0xFFFFFFFF) {
+        // TODO:
+        return error.TODO;
+    }
+    return ext;
+}
+/// Get an extractor for a regular file's data.
+pub fn dataExtractor(self: Inode, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32) !DataExtractor {
+    return switch (self.data) {
+        .file => |f| getExtractorFromData(fil, cache, decomp, block_size, f),
+        .ext_file => |f| getExtractorFromData(fil, cache, decomp, block_size, f),
+        else => Error.NotRegularFile,
+    };
+}
+fn getExtractorFromData(fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32, d: anytype) !DataExtractor {
+    const ext: DataExtractor = .init(fil, cache, decomp, block_size, d.size, d.block_start, d.blocks);
+    if (d.frag_block_offset == 0xFFFFFFFF) {
+        // TODO:
+        return error.TODO;
+    }
+    return ext;
+}
+// Get a symlink's target path
+pub fn symlinkTarget(self: Inode) ![]const u8 {
+    return switch (self.data) {
+        .symlink => |s| s.target,
+        .ext_symlink => |s| s.target,
+        else => Error.NotSymlink,
+    };
+}
+// Get inode's gid
+pub fn gid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, id_table_start: u64) !u16 {
+    return LookupTable.lookupValue(u16, alloc, io, decomp, fil, id_table_start, self.hdr.gid_idx);
+}
+// Get inode's uid
+pub fn uid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, id_table_start: u64) !u16 {
+    return LookupTable.lookupValue(u16, alloc, io, decomp, fil, id_table_start, self.hdr.uid_idx);
+}
+// Get an inode's xattr values. If the inode does not have xattr values (including if the inode is not an extended type), an empty slice is returned.
+pub fn xattrValues(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, xattr_table_start: u64) ![]XattrTable.XattrOwned {
+    const idx = switch (self.data) {
+        .ext_dir => |e| e.xattr_idx,
+        .ext_file => |e| e.xattr_idx,
+        .ext_symlink => |e| e.xattr_idx,
+        .ext_block_dev => |e| e.xattr_idx,
+        .ext_char_dev => |e| e.xattr_idx,
+        .ext_fifo => |e| e.xattr_idx,
+        .ext_socket => |e| e.xattr_idx,
+        else => return &[0]XattrTable.XattrOwned{},
+    };
+    if (idx == 0xFFFFFFFF) return &[0]XattrTable.XattrOwned{};
+    return XattrTable.statelessLookup(alloc, io, decomp, fil, xattr_table_start, idx);
+}
+
+// Types
+
+pub const Error = error{
+    NotDirectory,
+    NotRegularFile,
+    NotSymlink,
+    NotExtended,
+};
 
 pub const Ref = packed struct(u64) {
     block_offset: u16,
@@ -39,23 +171,23 @@ pub const Type = enum(u16) {
 };
 
 pub const Data = union(Type) {
-    dir: DirTypes.Dir,
-    file: FileTypes.File,
-    symlink: MiscTypes.Symlink,
-    block_dev: MiscTypes.Dev,
-    char_dev: MiscTypes.Dev,
-    fifo: MiscTypes.IPC,
-    socket: MiscTypes.IPC,
-    ext_dir: DirTypes.ExtDir,
-    ext_file: FileTypes.ExtFile,
-    ext_symlink: MiscTypes.ExtSymlink,
-    ext_block_dev: MiscTypes.ExtDev,
-    ext_char_dev: MiscTypes.ExtDev,
-    ext_fifo: MiscTypes.ExtIPC,
-    ext_socket: MiscTypes.ExtIPC,
+    dir: dir.Dir,
+    file: file.File,
+    symlink: misc.Symlink,
+    block_dev: misc.Dev,
+    char_dev: misc.Dev,
+    fifo: misc.IPC,
+    socket: misc.IPC,
+    ext_dir: dir.ExtDir,
+    ext_file: file.ExtFile,
+    ext_symlink: misc.ExtSymlink,
+    ext_block_dev: misc.ExtDev,
+    ext_char_dev: misc.ExtDev,
+    ext_fifo: misc.ExtIPC,
+    ext_socket: misc.ExtIPC,
 };
 
-pub const Header = packed struct {
+pub const Header = extern struct {
     inode_type: Type,
     permissions: u16,
     uid_idx: u16,
@@ -64,371 +196,11 @@ pub const Header = packed struct {
     num: u32,
 };
 
-pub const Error = error{
-    NotDirectory,
-};
+// Extract
 
-const Inode = @This();
-
-hdr: Header,
-data: Data,
-
-pub fn fromRef(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, inode_start: u64, block_size: u32, ref: Ref) !Inode {
-    var meta: MetadataReader = .init(io, cache, ref.block_start + inode_start);
-    defer meta.deinit();
-    try meta.interface.discardAll(ref.block_offset);
-    return fromReader(alloc, &meta.interface, block_size);
-}
-pub fn fromReader(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode {
-    var hdr: Header = undefined;
-    try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
-    return .{
-        .hdr = hdr,
-        .data = switch (hdr.inode_type) {
-            .dir => .{ .dir = try .read(rdr) },
-            .file => .{ .file = try .read(alloc, rdr, block_size) },
-            .symlink => .{ .symlink = try .read(alloc, rdr) },
-            .block_dev => .{ .block_dev = try .read(rdr) },
-            .char_dev => .{ .char_dev = try .read(rdr) },
-            .fifo => .{ .fifo = try .read(rdr) },
-            .socket => .{ .socket = try .read(rdr) },
-            .ext_dir => .{ .ext_dir = try .read(rdr) },
-            .ext_file => .{ .ext_file = try .read(alloc, rdr, block_size) },
-            .ext_symlink => .{ .ext_symlink = try .read(alloc, rdr) },
-            .ext_block_dev => .{ .ext_block_dev = try .read(rdr) },
-            .ext_char_dev => .{ .ext_char_dev = try .read(rdr) },
-            .ext_fifo => .{ .ext_fifo = try .read(rdr) },
-            .ext_socket => .{ .ext_socket = try .read(rdr) },
-        },
-    };
-}
-pub fn copy(alloc: std.mem.Allocator, from: Inode) !Inode {
-    var new = from;
-    switch (from.data) {
-        .file => |f| {
-            new.data.file.block_sizes = try alloc.alloc(FileTypes.BlockSize, f.block_sizes.len);
-            @memcpy(new.data.file.block_sizes, f.block_sizes);
-        },
-        .ext_file => |f| {
-            new.data.ext_file.block_sizes = try alloc.alloc(FileTypes.BlockSize, f.block_sizes.len);
-            @memcpy(new.data.ext_file.block_sizes, f.block_sizes);
-        },
-        .symlink => |s| {
-            const new_target = try alloc.alloc(u8, s.target.len);
-            @memcpy(new_target, s.target);
-            new.data.symlink.target = new_target;
-        },
-        .ext_symlink => |s| {
-            const new_target = try alloc.alloc(u8, s.target.len);
-            @memcpy(new_target, s.target);
-            new.data.ext_symlink.target = new_target;
-        },
-        else => {},
-    }
-    return new;
-}
-pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
-    switch (self.data) {
-        .file => |f| alloc.free(f.block_sizes),
-        .ext_file => |f| alloc.free(f.block_sizes),
-        .symlink => |s| alloc.free(s.target),
-        .ext_symlink => |s| alloc.free(s.target),
-        else => {},
-    }
-}
-
-pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, io: Io, cache: *DecompCache, dir_start: u64) ![]DirEntry {
-    return switch (self.data) {
-        .dir => |d| readDirectoryFromData(alloc, io, cache, dir_start, d),
-        .ext_dir => |d| readDirectoryFromData(alloc, io, cache, dir_start, d),
-        else => Error.NotDirectory,
-    };
-}
-fn readDirectoryFromData(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, dir_start: u64, d: anytype) ![]DirEntry {
-    var meta: MetadataReader = .init(io, cache, dir_start + d.block_start);
-    defer meta.deinit();
-    try meta.interface.discardAll(d.block_offset);
-
-    return DirEntry.readEntries(alloc, &meta.interface, d.size);
-}
-
-// Extraction
-
-pub fn extract(
-    self: Inode,
-    alloc: std.mem.Allocator,
-    io: Io,
-    cache: *DecompCache,
-    dir_start: u64,
-    inode_start: u64,
-    frag_start: u64,
-    block_size: u32,
-    id_start: u64,
-    xattr_start: u64,
-    ext_loc: []const u8,
-    options: ExtractionOptions,
-) !void {
-    const path = std.mem.trimEnd(u8, ext_loc, "/");
-
-    var sel_val: std.atomic.Value(usize) = .init(1);
-
-    var sel_buf: [5]ExtractUnion = undefined;
-    var sel: Io.Select(ExtractUnion) = .init(io, &sel_buf);
-    defer sel.cancelDiscard();
-
-    var meta_loop = io.async(metadataLoop, .{ alloc, io, cache, id_start, xattr_start, &sel, &sel_val, options });
-    defer _ = meta_loop.cancel(io) catch {};
-
-    sel.async(.ret, extractReal, .{ self, alloc, io, cache, dir_start, inode_start, frag_start, block_size, &sel, &sel_val, path, true });
-
-    try meta_loop.await(io);
-}
-fn extractReal(
-    self: Inode,
-    alloc: std.mem.Allocator,
-    io: Io,
-    cache: *DecompCache,
-    dir_start: u64,
-    inode_start: u64,
-    frag_start: u64,
-    block_size: u32,
-    sel: *Io.Select(ExtractUnion),
-    sel_val: *std.atomic.Value(usize),
-    path: []const u8,
-    origin: bool,
-) ExtractionError!ExtractReturn {
-    errdefer if (!origin) {
-        self.deinit(alloc);
-        alloc.free(path);
-    };
-    switch (self.hdr.inode_type) {
-        .dir, .ext_dir => {
-            try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777));
-
-            const entries = self.readDirectory(alloc, io, cache, dir_start) catch |err| switch (err) {
-                error.NotDirectory => unreachable,
-                else => |e| return e,
-            };
-            defer {
-                for (entries) |entry|
-                    entry.deinit(alloc);
-                alloc.free(entries);
-            }
-
-            if (entries.len != 0) {
-                _ = sel_val.fetchAdd(entries.len, .acq_rel);
-
-                for (entries) |entry| {
-                    var meta: MetadataReader = .init(io, cache, inode_start + entry.block_start);
-                    defer meta.deinit();
-                    try meta.interface.discardAll(entry.block_offset);
-
-                    var new_inode: Inode = try .fromReader(alloc, &meta.interface, block_size);
-                    errdefer new_inode.deinit(alloc);
-
-                    const new_path = try std.mem.concat(alloc, u8, &.{ path, "/", entry.name });
-                    errdefer alloc.free(new_path);
-                    sel.async(
-                        .ret,
-                        extractReal,
-                        .{ new_inode, alloc, io, cache, dir_start, inode_start, frag_start, block_size, sel, sel_val, new_path, false },
-                    );
-                }
-            }
-        },
-        .file, .ext_file => {
-            std.debug.print("{s} {}\n", .{ path, self.data });
-            // var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{});
-            // defer atomic.deinit(io);
-
-            // var data: DataExtract = undefined;
-            // var frag_offset: ?u64 = null;
-            // switch (self.data) {
-            //     .file => |f| {
-            //         data = .init(cache.decomp, cache.map, block_size, f.block_start, f.size, f.block_sizes);
-            //         if (f.frag_idx != 0xFFFFFFFF) {
-            //             const entry: FragEntry = try LookupTable.lookup(FragEntry, io, cache, frag_start, f.frag_idx);
-            //             if (entry.size.uncompressed) {
-            //                 data.addFrag(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset);
-            //             } else {
-            //                 frag_offset = entry.start;
-            //                 const block = try cache.checkoutBlock(io, entry.start, entry.size.size, block_size);
-            //                 data.addFrag(block, f.frag_offset);
-            //             }
-            //         }
-            //     },
-            //     .ext_file => |f| {
-            //         data = .init(cache.decomp, cache.map, block_size, f.block_start, f.size, f.block_sizes);
-            //         if (f.frag_idx != 0xFFFFFFFF) {
-            //             const entry: FragEntry = try LookupTable.lookup(FragEntry, io, cache, frag_start, f.frag_idx);
-            //             if (entry.size.uncompressed) {
-            //                 data.addFrag(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset);
-            //             } else {
-            //                 frag_offset = entry.start;
-            //                 const block = try cache.checkoutBlock(io, entry.start, entry.size.size, block_size);
-            //                 data.addFrag(block, f.frag_offset);
-            //             }
-            //         }
-            //     },
-            //     else => unreachable,
-            // }
-            // defer if (frag_offset != null) cache.checkinBlock(io, frag_offset.?) catch {};
-            // try data.asyncExtract(alloc, io, atomic.file);
-
-            // try atomic.link(io);
-        },
-        .symlink, .ext_symlink => {
-            const target = switch (self.data) {
-                .symlink => |s| s.target,
-                .ext_symlink => |s| s.target,
-                else => unreachable,
-            };
-            try Io.Dir.cwd().symLink(io, target, path, .{});
-        },
-        else => {
-            var dev: u32 = 0;
-            var mode: u32 = undefined;
-
-            const DT = std.os.linux.DT;
-
-            switch (self.data) {
-                .block_dev => |d| {
-                    mode = DT.BLK;
-                    dev = d.dev;
-                },
-                .ext_block_dev => |d| {
-                    mode = DT.BLK;
-                    dev = d.dev;
-                },
-                .char_dev => |d| {
-                    mode = DT.CHR;
-                    dev = d.dev;
-                },
-                .ext_char_dev => |d| {
-                    mode = DT.CHR;
-                    dev = d.dev;
-                },
-                .fifo, .ext_fifo => mode = DT.FIFO,
-                .socket, .ext_socket => mode = DT.SOCK,
-                else => unreachable,
-            }
-
-            const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &.{path}, 0);
-            defer alloc.free(sentinel_path);
-            const res = std.os.linux.mknod(sentinel_path, mode, dev);
-            if (res != 0)
-                return ExtractionError.Mknod;
-        },
-    }
-
-    return .{
-        .path = path,
-        .inode = self,
-        .origin = origin,
-    };
-}
-
-const ExtractUnion = union { ret: ExtractionError!ExtractReturn };
-pub const ExtractionError = error{ SetXattr, Mknod, Canceled } || DirEntry.Error || Io.Dir.CreateFileAtomicError || DataExtract.Error || Io.File.Atomic.LinkError ||
-    Io.Dir.SymLinkError;
-
-const ExtractReturn = struct {
-    path: []const u8,
-    inode: Inode,
-    origin: bool,
-
-    fn deinit(self: ExtractReturn, alloc: std.mem.Allocator) void {
-        if (self.origin) return;
-        alloc.free(self.path);
-        self.inode.deinit(alloc);
-    }
-    fn setMetadata(self: ExtractReturn, alloc: std.mem.Allocator, io: Io, cache: *DecompCache, id_start: u64, xattr_start: u64, options: ExtractionOptions) !void {
-        if (options.ignore_permissions and options.ignore_xattr) return;
-
-        var fil = try Io.Dir.cwd().openFile(io, self.path, .{});
-        defer fil.close(io);
-
-        if (!options.ignore_permissions) {
-            try fil.setTimestamps(io, .{ .modify_timestamp = .{
-                .new = .{ .nanoseconds = @as(i96, @intCast(self.inode.hdr.mod_time)) * std.time.ns_per_s },
-            } });
-            try fil.setPermissions(io, @enumFromInt(self.inode.hdr.permissions));
-            try fil.setOwner(
-                io,
-                try LookupTable.lookup(u16, io, cache, id_start, self.inode.hdr.uid_idx),
-                try LookupTable.lookup(u16, io, cache, id_start, self.inode.hdr.gid_idx),
-            );
-        }
-        if (options.ignore_xattr) return;
-        const xattr_idx: u32 = switch (self.inode.data) {
-            .ext_dir => |d| d.xattr_idx,
-            .ext_file => |f| f.xattr_idx,
-            .ext_symlink => |s| s.xattr_idx,
-            .ext_block_dev, .ext_char_dev => |d| d.xattr_idx,
-            .ext_fifo, .ext_socket => |i| i.xattr_idx,
-            else => return,
-        };
-        if (xattr_idx == 0xFFFFFFFF) return;
-        const xattrs = try LookupTable.xattrLookup(alloc, io, cache, xattr_start, xattr_idx);
-        defer {
-            for (xattrs) |kv|
-                kv.deinit(alloc);
-            alloc.free(xattrs);
-        }
-
-        for (xattrs) |kv| {
-            const res = std.os.linux.fsetxattr(fil.handle, kv.key.ptr, kv.value.ptr, kv.value.len, 0);
-            if (res != 0)
-                return ExtractionError.SetXattr;
-        }
-    }
-};
-
-fn metadataLoop(
-    alloc: std.mem.Allocator,
-    io: Io,
-    cache: *DecompCache,
-    id_start: u64,
-    xattr_start: u64,
-    sel: *Io.Select(ExtractUnion),
-    sel_val: *std.atomic.Value(usize),
-    options: ExtractionOptions,
-) !void {
-    errdefer {
-        while (sel.group.token.load(.unordered) != null) {
-            const ret = sel.queue.getOne(io) catch break;
-
-            const res = ret.ret catch continue;
-            res.deinit(alloc);
-        }
-    }
-    var dir_queue: std.PriorityDequeue(ExtractReturn, void, dirReturnQueueOrder) = .empty;
-    defer {
-        while (dir_queue.popMax()) |ret|
-            ret.deinit(alloc);
-        dir_queue.deinit(alloc);
-    }
-
-    while (sel_val.load(.unordered) > 0) {
-        defer _ = sel_val.fetchSub(1, .acq_rel);
-        const ret = try sel.queue.getOne(io);
-
-        const res = try ret.ret;
-
-        if (res.inode.hdr.inode_type == .dir or res.inode.hdr.inode_type == .ext_dir) {
-            try dir_queue.push(alloc, res);
-        } else {
-            defer res.deinit(alloc);
-            try res.setMetadata(alloc, io, cache, id_start, xattr_start, options);
-        }
-    }
-
-    while (dir_queue.popMax()) |res| {
-        defer res.deinit(alloc);
-        try res.setMetadata(alloc, io, cache, id_start, xattr_start, options);
-    }
-}
-
-fn dirReturnQueueOrder(_: void, a: ExtractReturn, b: ExtractReturn) std.math.Order {
-    return std.math.order(std.mem.count(u8, a.path, "/"), std.mem.count(u8, b.path, "/"));
-}
+pub fn extract(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, super: Archive.Superblock, path: []const u8, options: ExtractionOptions) !void {}
+pub fn extractDir(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, super: Archive.Superblock, path: []const u8, options: ExtractionOptions) !void {}
+pub fn extractRegFile(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, super: Archive.Superblock, path: []const u8, options: ExtractionOptions) !void {}
+pub fn extractSymlink(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, super: Archive.Superblock, path: []const u8, options: ExtractionOptions) !void {}
+pub fn extractDevice(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, super: Archive.Superblock, path: []const u8, options: ExtractionOptions) !void {}
+pub fn extractIPC(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, super: Archive.Superblock, path: []const u8, options: ExtractionOptions) !void {}

src/inode_data/dir.zig

@@ -21,7 +21,7 @@ pub const ExtDir = extern struct {
     parent_num: u32,
     idx_count: u16,
     block_offset: u16,
-    xattr_idx: u32,
+    xattr_id: u32,
     // index: []DirIndex
 
     pub fn read(rdr: *Reader) !ExtDir {

src/inode_data/file.zig

@@ -7,34 +7,37 @@ pub const BlockSize = packed struct(u32) {
     _: u7,
 };
 
+const FileRawRead = extern struct {
+    block_start: u32,
+    frag_idx: u32,
+    frag_block_offset: u32,
+    size: u32,
+};
+
 pub const File = struct {
-    block_start: u32, // bytes 0-3
-    frag_idx: u32, // bytes 4-7
-    frag_offset: u32, // bytes 8-11
-    size: u32, // bytes 12-15
+    block_start: u32,
+    frag_idx: u32,
+    frag_block_offset: u32,
+    size: u32,
     block_sizes: []BlockSize,
 
     pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !File {
-        const raw_values = extern struct {
-            block_start: u32, // bytes 0-3
-            frag_idx: u32, // bytes 4-7
-            frag_offset: u32, // bytes 8-11
-            size: u32, // bytes 12-15
-        };
-        var values: raw_values = undefined;
-        try rdr.readSliceEndian(@TypeOf(values), @ptrCast(&values), .little);
+        var raw: FileRawRead = undefined;
+        try rdr.readSliceEndian(FileRawRead, @ptrCast(&raw), .little);
+
+        var num_blocks: u32 = raw.size / block_size;
+        if (raw.size % block_size != 0 and raw.frag_idx == 0xFFFFFFFF)
+            num_blocks += 1;
 
-        var num_blocks: u32 = values.size / block_size;
-        if (values.size % block_size != 0 and values.frag_idx == 0xFFFFFFFF) num_blocks += 1;
         const sizes = try alloc.alloc(BlockSize, num_blocks);
         errdefer alloc.free(sizes);
         try rdr.readSliceEndian(BlockSize, sizes, .little);
 
         return .{
-            .block_start = values.block_start,
-            .frag_idx = values.frag_idx,
-            .frag_offset = values.frag_offset,
-            .size = values.size,
+            .block_start = raw.block_start,
+            .frag_idx = raw.frag_idx,
+            .frag_block_offset = raw.frag_block_offset,
+            .size = raw.size,
             .block_sizes = sizes,
         };
     }
@@ -44,43 +47,46 @@ pub const File = struct {
     }
 };
 
+const ExtFileRawRead = extern struct {
+    block_start: u64,
+    size: u64,
+    sparse: u64,
+    hard_links: u32,
+    frag_idx: u32,
+    frag_block_offset: u32,
+    xattr_idx: u32,
+};
+
 pub const ExtFile = struct {
-    block_start: u64, // bytes 0-7
-    size: u64, // bytes 8-15
-    sparse: u64, // bytes 16-23
-    hard_links: u32, // bytes 24-27
-    frag_idx: u32, // bytes 28-31
-    frag_offset: u32, // bytes 32-35
-    xattr_idx: u32, // bytes 36-39
+    block_start: u64,
+    size: u64,
+    sparse: u64,
+    hard_links: u32,
+    frag_idx: u32,
+    frag_block_offset: u32,
+    xattr_idx: u32,
     block_sizes: []BlockSize,
 
     pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !ExtFile {
-        const raw_values = extern struct {
-            block_start: u64, // bytes 0-7
-            size: u64, // bytes 8-15
-            sparse: u64, // bytes 16-23
-            hard_links: u32, // bytes 24-27
-            frag_idx: u32, // bytes 28-31
-            frag_offset: u32, // bytes 32-35
-            xattr_idx: u32, // bytes 36-39
-        };
-        var values: raw_values = undefined;
-        try rdr.readSliceEndian(@TypeOf(values), @ptrCast(&values), .little);
+        var raw: ExtFileRawRead = undefined;
+        try rdr.readSliceEndian(ExtFileRawRead, @ptrCast(&raw), .little);
+
+        var num_blocks: u32 = @truncate(raw.size / block_size);
+        if (raw.size % block_size != 0 and raw.frag_idx == 0xFFFFFFFF)
+            num_blocks += 1;
 
-        var num_blocks: u32 = @truncate(values.size / block_size);
-        if (values.size % block_size != 0 and values.frag_idx == 0xFFFFFFFF) num_blocks += 1;
         const sizes = try alloc.alloc(BlockSize, num_blocks);
         errdefer alloc.free(sizes);
         try rdr.readSliceEndian(BlockSize, sizes, .little);
 
         return .{
-            .block_start = values.block_start,
-            .size = values.size,
-            .sparse = values.sparse,
-            .hard_links = values.hard_links,
-            .frag_idx = values.frag_idx,
-            .frag_offset = values.frag_offset,
-            .xattr_idx = values.xattr_idx,
+            .block_start = raw.block_start,
+            .size = raw.size,
+            .sparse = raw.sparse,
+            .hard_links = raw.hard_links,
+            .frag_idx = raw.frag_idx,
+            .frag_block_offset = raw.frag_block_offset,
+            .xattr_idx = raw.xattr_idx,
             .block_sizes = sizes,
         };
     }

src/lookup_table.zig

@@ -1,124 +1,84 @@
 const std = @import("std");
 const Io = std.Io;
 
-const Inode = @import("inode.zig");
-const DecompCache = @import("util/decomp_cache.zig");
+const Decompressor = @import("util/decompressor.zig");
 const MetadataReader = @import("util/metadata.zig");
+const OffsetFile = @import("util/offset_file.zig");
 
-pub fn lookup(comptime T: anytype, io: Io, cache: *DecompCache, table_start: u64, idx: u32) !T {
-    const PER_BLOCK = 8192 / @sizeOf(T);
+pub fn lookupValue(comptime T: anytype, alloc: std.mem.Allocator, io: Io, decomp: *Decompressor, file: OffsetFile, table_start: u64, idx: u16) !T {
+    const T_PER_BLOCK: u16 = 8192 / @sizeOf(T);
 
-    const block_idx = idx / PER_BLOCK;
-    const block_offset = idx % PER_BLOCK;
+    const block = idx / T_PER_BLOCK;
+    const block_offset = idx % T_PER_BLOCK;
 
-    if (table_start + (block_idx * 8) > cache.map.memory.len) return error.ReadFailed;
-    const offset: u64 = std.mem.readInt(u64, cache.map.memory[table_start + (block_idx * 8) ..][0..8], .little);
+    var rdr = try file.readerAt(io, table_start + (8 * block), &[0]u8{});
+    var offset: u64 = undefined;
+    try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
 
-    var meta: MetadataReader = .init(io, cache, offset);
-    defer meta.deinit();
-    try meta.interface.discardAll(block_offset * @sizeOf(T));
-
-    var new: T = undefined;
-    try meta.interface.readSliceEndian(T, @ptrCast(&new), .little);
-    return new;
-}
-
-pub const XattrKV = struct {
-    key: [:0]u8,
-    value: []u8,
-
-    pub fn deinit(self: XattrKV, alloc: std.mem.Allocator) void {
-        alloc.free(self.key);
-        alloc.free(self.value);
-    }
-};
-
-const LookupValue = extern struct {
-    ref: Inode.Ref,
-    count: u32,
-    size: u32,
-};
-const KeyEntry = extern struct {
-    prefix: packed struct(u16) {
-        prefix: enum(u8) {
-            user,
-            trusted,
-            security,
-        },
-        out_of_line: bool,
-        _: u7,
-    },
-    name_size: u16,
-};
-
-pub fn xattrLookup(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, xattr_start: u64, idx: u32) ![]XattrKV {
-    const table_start = std.mem.readInt(u64, cache.map.memory[xattr_start..][0..8], .little);
-
-    const val: LookupValue = try lookup(
-        LookupValue,
-        io,
-        cache,
-        xattr_start + 16,
-        idx,
-    );
-
-    const out = try alloc.alloc(XattrKV, val.count);
-    errdefer alloc.free(out);
-
-    var meta: MetadataReader = .init(io, cache, table_start + val.ref.block_start);
-    defer meta.deinit();
-    try meta.interface.discardAll(val.ref.block_offset);
-
-    for (out) |*kv| {
-        var key_entry: KeyEntry = undefined;
-        try meta.interface.readSliceEndian(KeyEntry, @ptrCast(&key_entry), .little);
-
-        const prefix_len: u16 = switch (key_entry.prefix.prefix) {
-            .user => 5,
-            .trusted => 8,
-            .security => 9,
-        };
-        var key_len = key_entry.name_size;
-        key_len += prefix_len;
-
-        kv.key = try alloc.allocSentinel(u8, key_len, 0);
-        errdefer alloc.free(kv.key);
-
-        try meta.interface.readSliceEndian(u8, kv.key[prefix_len..], .little);
-        switch (key_entry.prefix.prefix) {
-            .user => @memcpy(kv.key[0..prefix_len], "user."),
-            .trusted => @memcpy(kv.key[0..prefix_len], "trusted."),
-            .security => @memcpy(kv.key[0..prefix_len], "security."),
-        }
-
-        if (key_entry.prefix.out_of_line) {
-            try meta.interface.discardAll(8);
-
-            var ool_ref: Inode.Ref = undefined;
-            try meta.interface.readSliceEndian(Inode.Ref, @ptrCast(&ool_ref), .little);
-
-            var ool_meta: MetadataReader = .init(io, cache, table_start + ool_ref.block_start);
-            defer ool_meta.deinit();
-            try ool_meta.interface.discardAll(ool_ref.block_offset);
-
-            kv.value = try readValue(alloc, &ool_meta.interface);
-            errdefer alloc.free(kv.value);
-        } else {
-            kv.value = try readValue(alloc, &meta.interface);
-        }
-    }
+    rdr = try file.readerAt(io, offset, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr, decomp);
 
+    try meta.interface.discardAll(@sizeOf(T) * block_offset);
+    var out: T = undefined;
+    try meta.interface.readSliceEndian(T, @ptrCast(&out), .little);
     return out;
 }
 
-fn readValue(alloc: std.mem.Allocator, rdr: *Io.Reader) ![]u8 {
-    var val_size: u32 = undefined;
-    try rdr.readSliceEndian(u32, @ptrCast(&val_size), .little);
+pub fn CachedTable(comptime T: anytype) type {
+    return struct {
+        const T_PER_BLOCK: u16 = 8192 / @sizeOf(T);
 
-    const val = try alloc.alloc(u8, val_size);
-    errdefer alloc.free(val);
+        const Table = @This();
 
-    try rdr.readSliceEndian(u8, val, .little);
+        alloc: std.mem.Allocator,
+        fil: OffsetFile,
+        table_start: u64,
+        total_num: u32,
 
-    return val;
+        table: std.AutoHashMap(u32, []T),
+
+        mut: Io.Mutex = .init,
+
+        pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, offset: u64, total_num: u32) Table {
+            return .{
+                .alloc = alloc,
+                .fil = fil,
+                .table_start = offset,
+                .total_num = total_num,
+
+                .table = .init(alloc),
+            };
+        }
+        pub fn deinit(self: *Table, io: Io) void {
+            self.mut.lockUncancelable(io);
+            self.table.deinit();
+        }
+
+        pub fn get(self: *Table, io: Io, idx: u32) !T {
+            const block = idx / T_PER_BLOCK;
+            const block_offset = idx % T_PER_BLOCK;
+            if (self.table.contains(block))
+                return self.table.get(block).?[block_offset];
+
+            try self.mut.lock(io);
+            defer self.mut.unlock(io);
+
+            if (self.table.contains(block))
+                return self.table.get(block).?[block_offset];
+
+            var rdr = try self.fil.readerAt(io, self.table_start + (8 * block), &[0]u8{});
+            var offset: u64 = undefined;
+            try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
+
+            const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK)
+                self.total_num % T_PER_BLOCK
+            else
+                T_PER_BLOCK;
+
+            rdr = try self.fil.readerAt(io, offset, &[0]u8{});
+            var meta: MetadataReader = .init(self.alloc, &rdr, self.decomp);
+
+            try self.table.put(block, try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little));
+        }
+    };
 }

src/options.zig

@@ -5,22 +5,25 @@ const Writer = std.Io.Writer;
 
 const ExtractionOptions = @This();
 
-/// Force single-threaded extraction. Io.Threaded.global_single_threaded also works.
-single_threaded: bool = false,
-/// Don't set the file's owner, permissions, & modify time after extraction.
+/// The number of threads used for extraction. 0 implies single threaded.
+threads: usize = 1,
+/// Don't set the file's owner & permissions after extraction
 ignore_permissions: bool = false,
-/// Don't set xattr values.
+/// Don't set xattr values. Currently xattrs are never set anyway.
 ignore_xattr: bool = false,
-/// Replace symlinks with their target. Currently doesn't do anything.
+/// Replace symlinks with their target.
 dereference_symlinks: bool = false,
 /// Verbose logging. If true, verbose_writer must be set
 verbose: bool = false,
 /// Where to print verbose log.
 verbose_writer: ?*Writer = null,
 
-pub const defaultSingleThreaded: ExtractionOptions = .{ .single_threaded = true };
-pub const default: ExtractionOptions = .{};
-
+pub const SingleThreadedDefault: ExtractionOptions = .{};
+pub fn Default() !ExtractionOptions {
+    return .{
+        .threads = try std.Thread.getCpuCount(),
+    };
+}
 pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions {
     return .{
         .verbose = true,

src/root.zig

@@ -1,7 +1,2 @@
 pub const Archive = @import("archive.zig");
 pub const ExtractionOptions = @import("options.zig");
-const Test = @import("test.zig");
-
-test {
-    @import("std").testing.refAllDecls(Test);
-}

src/super.zig

@@ -1,63 +0,0 @@
-const std = @import("std");
-const math = std.math;
-
-const InodeRef = @import("inode.zig").Ref;
-const CompressionType = @import("util/decompress.zig").CompressionType;
-
-const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little);
-
-const SuperblockError = error{
-    InvalidMagic,
-    InvalidBlockLog,
-    InvalidVersion,
-    InvalidCheck,
-};
-
-/// A squashfs Superblock
-pub const Superblock = extern struct {
-    magic: u32,
-    inode_count: u32,
-    mod_time: u32,
-    block_size: u32,
-    frag_count: u32,
-    compression: CompressionType,
-    block_log: u16,
-    flags: packed struct(u16) {
-        inode_uncompressed: bool,
-        data_uncompressed: bool,
-        check: bool,
-        frag_uncompressed: bool,
-        fragment_never: bool,
-        fragment_always: bool,
-        duplicates: bool,
-        exportable: bool,
-        xattr_uncompressed: bool,
-        xattr_never: bool,
-        compression_options: bool,
-        ids_uncompressed: bool,
-        _: u4,
-    },
-    id_count: u16,
-    ver_maj: u16,
-    ver_min: u16,
-    root_ref: InodeRef,
-    size: u64,
-    id_start: u64,
-    xattr_start: u64,
-    inode_start: u64,
-    dir_start: u64,
-    frag_start: u64,
-    export_start: u64,
-
-    /// Validate the Superblock. If an error is returned, it's likely the archive is corrupted or not a squashfs archive.
-    pub fn validate(self: Superblock) !void {
-        if (self.magic != SQUASHFS_MAGIC)
-            return SuperblockError.InvalidMagic;
-        if (self.flags.check)
-            return SuperblockError.InvalidCheck;
-        if (self.ver_maj != 4 or self.ver_min != 0)
-            return SuperblockError.InvalidVersion;
-        if (math.log2(self.block_size) != self.block_log)
-            return SuperblockError.InvalidBlockLog;
-    }
-};

src/test.zig

@@ -1,71 +1,45 @@
 const std = @import("std");
 const Io = std.Io;
+const io = std.testing.io;
+const alloc = std.testing.allocator;
 const stuff = @import("builtin");
 
 const Archive = @import("archive.zig");
-const Superblock = @import("super.zig").Superblock;
+const Superblock = Archive.Superblock;
 
 const TestArchive = "testing/LinuxPATest.sfs";
 
 test "Basics" {
-    const io = std.testing.io;
-    const alloc = std.testing.allocator;
-
     var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
     defer fil.close(io);
-    var sfs: Archive = try .init(alloc, io, fil);
-    defer sfs.deinit(io);
+    var sfs: Archive = try .init(io, fil, 0);
     try std.testing.expectEqualDeep(sfs.super, LinuxPATestCorrectSuperblock);
+    const root_file = try sfs.root(alloc, io);
+    defer root_file.deinit();
 }
 
 const TestFile = "Start.exe";
 const TestFileExtractLocation = "testing/Start.exe";
 
 test "ExtractSingleFile" {
-    const io = std.testing.io;
-    const alloc = std.testing.allocator;
-
     Io.Dir.cwd().deleteFile(io, TestFileExtractLocation) catch {};
     var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
     defer fil.close(io);
-    var sfs: Archive = try .init(alloc, io, fil);
-    defer sfs.deinit(io);
+    var sfs: Archive = try .init(io, fil, 0);
     var test_fil = try sfs.open(alloc, io, TestFile);
     defer test_fil.deinit();
-    try test_fil.extract(alloc, io, TestFileExtractLocation, .default);
-    //TODO: validate extracted file.
-}
-
-const TestDir = "Documents";
-const TestDirExtractLocation = "testing/Documents";
-
-test "ExtractSmallDir" {
-    const io = std.testing.io;
-    const alloc = std.testing.allocator;
-
-    Io.Dir.cwd().deleteTree(io, TestDirExtractLocation) catch {};
-    var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
-    defer fil.close(io);
-    var sfs: Archive = try .init(alloc, io, fil);
-    defer sfs.deinit(io);
-    var test_fil = try sfs.open(alloc, io, TestDir);
-    defer test_fil.deinit();
-    try test_fil.extract(alloc, io, TestDirExtractLocation, .default);
+    try test_fil.extract(alloc, io, TestFileExtractLocation, try .Default());
     //TODO: validate extracted file.
 }
 
 const TestFullExtractLocation = "testing/TestExtract";
 
 test "ExtractCompleteArchive" {
-    const io = std.testing.io;
-    const alloc = std.testing.allocator;
-
     Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {};
     var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{});
     defer fil.close(io);
-    var sfs: Archive = try .init(alloc, io, fil);
-    defer sfs.deinit(io);
-    try sfs.extract(alloc, io, TestFullExtractLocation, .default);
+    var sfs: Archive = try .init(io, fil, 0);
+    try sfs.extract(alloc, io, TestFullExtractLocation, try .Default());
 }
 
 const LinuxPATestCorrectSuperblock: Superblock = .{

src/util/c_decomp.zig

@@ -1,61 +0,0 @@
-const std = @import("std");
-
-const c = @import("c");
-
-const Error = @import("decompress.zig").Error;
-
-pub fn zlibDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var strem: c.zng_stream = .{
-        .next_in = in.ptr,
-        .avail_in = @truncate(in.len),
-        .next_out = out.ptr,
-        .avail_out = @truncate(out.len),
-    };
-    var res = c.zng_inflateInit(&strem);
-    if (res != c.Z_OK) return Error.ReadFailed;
-    defer _ = c.zng_inflateEnd(&strem);
-
-    res = c.zng_inflate(&strem, c.Z_FULL_FLUSH);
-    if (res != c.Z_OK) return Error.ReadFailed;
-
-    return strem.total_out;
-}
-pub fn lzmaDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var strem: c.lzma_stream = .{
-        .next_in = in.ptr,
-        .avail_in = in.len,
-        .next_out = out.ptr,
-        .avail_out = out.len,
-    };
-    var res = c.lzma_auto_decoder(&strem, out.len * 2, 0);
-    if (res != c.LZMA_OK) return Error.ReadFailed;
-    defer c.lzma_end(&strem);
-
-    while (res == c.LZMA_OK)
-        res = c.lzma_code(&strem, c.LZMA_RUN);
-    if (res != c.LZMA_FINISH) return Error.ReadFailed;
-
-    return strem.total_out;
-}
-pub fn lzoDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var out_len = out.len;
-    const res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &out_len, null);
-    if (res != c.LZO_E_OK) return Error.ReadFailed;
-    return out_len;
-}
-pub fn lz4Decompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    const res = c.LZ4_decompress_safe(
-        in.ptr,
-        out.ptr,
-        @bitCast(@as(u32, @truncate(in.len))),
-        @bitCast(@as(u32, @truncate(out.len))),
-    );
-    if (res < 0) return Error.ReadFailed;
-    return @abs(res);
-}
-pub fn zstdDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
-    if (c.ZSTD_isError(res) != 0)
-        return Error.ReadFailed;
-    return res;
-}

src/util/data_extract.zig

@@ -1,85 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const BlockSize = @import("../inode_data/file.zig").BlockSize;
-const Decompress = @import("decompress.zig");
-
-const DataExtract = @This();
-
-decomp: Decompress.Fn,
-map: Io.File.MemoryMap,
-
-block_size: u32,
-block_start: u64,
-size: u64,
-blocks: []BlockSize,
-
-frag_data: ?[]u8 = null,
-frag_offset: u32 = undefined,
-
-pub fn init(decomp: Decompress.Fn, map: Io.File.MemoryMap, block_size: u32, block_start: u64, size: u64, blocks: []BlockSize) DataExtract {
-    return .{
-        .decomp = decomp,
-        .map = map,
-
-        .block_size = block_size,
-        .block_start = block_start,
-        .size = size,
-        .blocks = blocks,
-    };
-}
-pub fn addFrag(self: *DataExtract, frag_block: []u8, frag_offset: u32) void {
-    self.frag_data = frag_block;
-    self.frag_offset = frag_offset;
-}
-
-pub const Error = error{} || Io.File.MemoryMap.CreateError || Io.File.WritePositionalError || Decompress.Error || Io.File.MemoryMap.SetLengthError;
-
-pub fn asyncExtract(self: DataExtract, alloc: std.mem.Allocator, io: Io, fil: Io.File) Error!void {
-    if (self.size == 0) return;
-
-    try fil.writePositionalAll(io, &.{0}, self.size - 1);
-    var map = try fil.createMemoryMap(io, .{ .len = self.size, .protection = .{ .write = true }, .undefined_contents = true });
-    defer map.destroy(io);
-
-    var group: Io.Group = .init;
-    defer group.cancel(io);
-
-    var ret_err: ?Error = null;
-
-    var offset: u64 = self.block_start;
-    for (0..self.blocks.len) |i| {
-        group.async(io, blockThread, .{ self, alloc, map, offset, i, &ret_err });
-        offset += self.blocks[i].size;
-    }
-    if (self.frag_data != null)
-        group.async(io, fragThread, .{ self, map });
-
-    try group.await(io);
-    if (ret_err != null) return ret_err.?;
-    return map.write(io);
-}
-fn blockThread(self: DataExtract, alloc: std.mem.Allocator, map: Io.File.MemoryMap, read_offset: u64, idx: usize, ret_err: *?Error) error{Canceled}!void {
-    const block = self.blocks[idx];
-    const write_offset = idx * self.block_size;
-
-    const size = if (self.frag_data == null and idx == self.blocks.len - 1)
-        self.size % self.block_size
-    else
-        self.block_size;
-
-    if (block.size == 0) {
-        @memset(map.memory[write_offset..][0..size], 0);
-        return;
-    } else if (block.uncompressed) {
-        @memcpy(map.memory[write_offset..][0..size], self.map.memory[read_offset..][0..block.size]);
-    }
-    _ = self.decomp(alloc, self.map.memory[read_offset..][0..block.size], map.memory[write_offset..][0..size]) catch |err| {
-        ret_err.* = err;
-        return error.Canceled;
-    };
-}
-fn fragThread(self: DataExtract, map: Io.File.MemoryMap) error{Canceled}!void {
-    const size = self.size % self.block_size;
-    @memcpy(map.memory[self.blocks.len * self.block_size ..][0..size], self.frag_data.?[self.frag_offset..][0..size]);
-}

src/util/data_extractor.zig

@@ -0,0 +1,119 @@
+//! The DataExtractor is meant to extract a regular file's data to a given file asyncronously.
+
+const std = @import("std");
+const Io = std.Io;
+
+const FragEntry = @import("../frag.zig").FragEntry;
+const BlockSize = @import("../inode_data/file.zig").BlockSize;
+const Decompressor = @import("decompressor.zig");
+const OffsetFile = @import("offset_file.zig");
+const SharedCache = @import("shared_cache.zig");
+
+const DataExtractor = @This();
+
+fil: OffsetFile,
+cache: *SharedCache,
+decomp: *const Decompressor,
+block_size: u32,
+
+file_size: u64,
+start: u64,
+blocks: []BlockSize,
+
+frag_offset: u32 = 0,
+frag_entry: ?FragEntry = null,
+
+pub fn init(fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) DataExtractor {
+    return .{
+        .fil = fil,
+        .cache = cache,
+        .decomp = decomp,
+        .block_size = block_size,
+
+        .file_size = file_size,
+        .start = data_start,
+        .blocks = blocks,
+    };
+}
+pub fn addFrag(self: *DataExtractor, frag_offset: u32, entry: FragEntry) void {
+    self.frag_offset = frag_offset;
+    self.frag_entry = entry;
+}
+
+fn numBlocks(self: DataExtractor) usize {
+    var num = self.blocks.len;
+    if (self.frag_entry != null) num += 1;
+    return num;
+}
+
+/// Starts extracting the data using the given group to spawn async tasks.
+pub fn extractAsync(self: DataExtractor, alloc: std.mem.Allocator, io: Io, group: *Io.Group, fil: Io.File) void {
+    var read_offset: u64 = self.start;
+    for (0..self.blocks.len) |idx| {
+        group.async(io, blockThread, .{ self, alloc, io, fil, read_offset, idx });
+        read_offset += self.blocks[idx].size;
+    }
+    if (self.frag_entry != null)
+        group.async(io, fragThread, .{ self, alloc, io, fil });
+}
+
+fn blockThread(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.File, read_offset: u64, idx: u32) !void {
+    const block = self.blocks[idx];
+
+    const cur_block_size = if (idx == self.numBlocks() - 1)
+        self.file_size % self.block_size
+    else
+        self.block_size;
+
+    var wrt = fil.writer(io, &[0]u8{});
+    try wrt.seekTo(self.block_size * idx);
+    defer wrt.flush() catch {};
+
+    if (block.size == 0) {
+        try wrt.interface.splatByteAll(0, cur_block_size);
+        return;
+    }
+
+    var rdr = try self.fil.readerAt(io, read_offset, &[0]u8{});
+    if (block.uncompressed) {
+        try rdr.interface.streamExact(&wrt, cur_block_size);
+        return;
+    } else {
+        @branchHint(.likely);
+        var cache = try self.cache.getCache(io);
+        defer self.cache.returnCache(cache);
+
+        var tmp = try self.cache.getCache(io);
+        defer self.cache.returnCache(tmp);
+
+        try rdr.interface.readSliceAll(cache.cache[0..block.size]);
+        _ = try self.decomp.Decompress(alloc, cache.cache[0..block.size], tmp.cache[0..cur_block_size]);
+        try wrt.interface.writeAll(tmp.cache[0..cur_block_size]);
+    }
+}
+fn fragThread(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.File) !void {
+    const frag = self.frag_entry.?;
+    const cur_block_size = self.file_size % self.block_size;
+
+    var wrt = fil.writer(io, &[0]u8{});
+    try wrt.seekTo(self.blocks.len * self.block_size);
+    defer wrt.flush() catch {};
+
+    var rdr = try self.fil.readerAt(io, frag.start, &[0]u8{});
+    if (frag.size.uncompressed) {
+        try rdr.interface.discardAll(self.frag_offset);
+        try rdr.interface.streamExact(&wrt, cur_block_size);
+        return;
+    } else {
+        @branchHint(.likely);
+        var cache = try self.cache.getCache(io);
+        defer self.cache.returnCache(cache);
+
+        var tmp = try self.cache.getCache(io);
+        defer self.cache.returnCache(tmp);
+
+        try rdr.interface.readSliceAll(cache.cache[0..frag.size.size]);
+        _ = try self.decomp.Decompress(alloc, cache.cache[0..frag.size.size], tmp.cache[0..self.block_size]);
+        try wrt.interface.writeAll(tmp.cache[0..cur_block_size]);
+    }
+}

src/util/data_reader.zig

@@ -0,0 +1,209 @@
+//! DataReader reads a regular file's data linearly from start to finish using Io.Reader interface.
+
+const std = @import("std");
+const Io = std.Io;
+const Reader = Io.Reader;
+const Writer = Io.Writer;
+const Limit = Io.Limit;
+
+const FragEntry = @import("../frag.zig").FragEntry;
+const BlockSize = @import("../inode_data/file.zig").BlockSize;
+const Decompressor = @import("decompressor.zig");
+const OffsetFile = @import("offset_file.zig");
+const SharedCache = @import("shared_cache.zig");
+
+const DataReader = @This();
+
+alloc: std.mem.Allocator,
+
+fil: OffsetFile,
+io: Io,
+cache: *SharedCache,
+decomp: *const Decompressor,
+block_size: u32,
+
+file_size: u64,
+cur_offset: u64,
+blocks: []BlockSize,
+
+frag_offset: u32 = 0,
+frag_entry: ?FragEntry = null,
+
+block_idx: usize = 0,
+sparse_block: bool = false,
+
+interface: Io.Reader,
+
+pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) !DataReader {
+    return .{
+        .alloc = alloc,
+
+        .fil = fil,
+        .io = io,
+        .decomp = decomp,
+        .block_size = block_size,
+
+        .file_size = file_size,
+        .cur_offset = data_start,
+        .blocks = blocks,
+
+        .interface = .{
+            .buffer = try cache.getCache(io),
+            .seek = 0,
+            .end = 0,
+            .vtable = &.{
+                .stream = stream,
+                .discard = discard,
+                .readVec = readVec,
+            },
+        },
+    };
+}
+pub fn deinit(self: *DataReader) void {
+    if (self.interface.buffer.len > 0) {
+        const buf_nod: *SharedCache.BufferNode = @fieldParentPtr("cache", self.interface.buffer);
+        self.cache.returnCache(buf_nod);
+    }
+}
+pub fn addFrag(self: *DataReader, frag_offset: u32, entry: FragEntry) void {
+    self.frag_offset = frag_offset;
+    self.frag_entry = entry;
+}
+
+fn numBlocks(self: DataReader) usize {
+    var num = self.blocks.len;
+    if (self.frag_entry != null) num += 1;
+    return num;
+}
+fn advanceBuffer(self: *DataReader) !void {
+    if (self.block_idx >= self.numBlocks()) {
+        return Reader.Error.EndOfStream;
+    }
+    defer self.block_idx += 1;
+
+    self.interface.end = if (self.block_idx == self.numBlocks() - 1)
+        self.size % self.block_size
+    else
+        self.block_size;
+
+    // Fragment
+    if (self.block_idx == self.blocks.len) {
+        const entry = self.frag_entry.?;
+        if (entry.size.uncompressed) {
+            var rdr = try self.fil.readerAt(self.io, entry.start + self.frag_offset, &[0]u8{});
+            try rdr.interface.readSliceAll(self.interface.buffer[0..self.interface.end]);
+        } else {
+            @branchHint(.likely);
+            const tmp = try self.cache.getCache(self.io);
+            defer self.cache.returnCache(tmp);
+
+            var rdr = try self.fil.readerAt(self.io, entry.start, &[0]u8{});
+            try rdr.interface.readSliceAll(tmp.cache[0..entry.size.size]);
+            _ = try self.decomp.Decompress(self.alloc, tmp.cache[0..entry.size.size], self.interface.buffer[0..self.block_size]);
+            @memmove(self.interface.buffer[0..self.interface.end], self.interface.buffer[self.frag_offset .. self.frag_offset + self.interface.end]);
+        }
+        self.interface.seek = 0;
+        return;
+    }
+
+    // Normal Block
+    const block = self.blocks[self.block_idx];
+    if (block.size == 0) {
+        self.interface.seek = 0;
+        self.sparse_block = true;
+        return;
+    } else {
+        self.sparse_block = false;
+    }
+    if (block.uncompressed) {
+        try self.fil.readAt(self.io, self.cur_offset, self.interface.buffer[0..self.interface.end]);
+        self.cur_offset += self.interface.end;
+    } else {
+        @branchHint(.likely);
+        const tmp = try self.cache.getCache(self.io);
+        defer self.cache.returnCache(tmp);
+
+        var rdr = try self.fil.readerAt(self.io, self.cur_offset, &[0]u8{});
+        try rdr.interface.readSliceAll(tmp.cache[0..block.size]);
+        self.cur_offset += block.size;
+        _ = try self.decomp.Decompress(self.alloc, tmp.cache[0..block.size], self.interface.buffer[0..self.interface.end]);
+    }
+    self.interface.seek = 0;
+}
+
+fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) Reader.StreamError!usize {
+    var data: *DataReader = @fieldParentPtr("interface", rdr);
+    if (rdr.seek == rdr.end)
+        data.advanceBuffer() catch |err| return switch (err) {
+            error.ReadFailed => error.ReadFailed,
+            error.EndOfStream => error.EndOfStream,
+            else => error.ReadFailed,
+        };
+
+    switch (limit) {
+        .nothing => return 0,
+        .unlimited => {
+            const wrote = if (data.sparse_block)
+                try wrt.splatByte(0, rdr.end - rdr.seek)
+            else
+                try wrt.write(rdr.buffer[rdr.seek..rdr.end]);
+            rdr.seek += wrote;
+            return wrote;
+        },
+        else => {
+            const to_read = @min(rdr.end - rdr.seek, @intFromEnum(limit));
+            const wrote = if (data.sparse_block)
+                try wrt.splatByte(0, to_read)
+            else
+                try wrt.write(rdr.buffer[rdr.seek .. rdr.seek + to_read]);
+            rdr.seek += wrote;
+            return wrote;
+        },
+    }
+}
+fn discard(rdr: *Reader, limit: Limit) Reader.Error!usize {
+    var data: *DataReader = @fieldParentPtr("interface", rdr);
+    if (rdr.seek == rdr.end)
+        data.advanceBuffer() catch |err| return switch (err) {
+            error.ReadFailed => error.ReadFailed,
+            error.EndOfStream => error.EndOfStream,
+            else => error.ReadFailed,
+        };
+
+    switch (limit) {
+        .nothing => return 0,
+        .unlimited => {
+            const adv = rdr.end - rdr.seek;
+            rdr.seek = rdr.end;
+            return adv;
+        },
+        else => {
+            const adv = @min(rdr.end - rdr.seek, @intFromEnum(limit));
+            rdr.seek += adv;
+            return adv;
+        },
+    }
+}
+fn readVec(rdr: *Reader, vec: [][]u8) Reader.Error!usize {
+    var data: *DataReader = @fieldParentPtr("interface", rdr);
+    if (rdr.seek == rdr.end)
+        data.advanceBuffer() catch |err| return switch (err) {
+            error.ReadFailed => error.ReadFailed,
+            error.EndOfStream => error.EndOfStream,
+            else => error.ReadFailed,
+        };
+
+    var wrote: usize = 0;
+    for (vec) |buf| {
+        if (rdr.seek == rdr.end) break;
+
+        const to_copy = @min(rdr.end - rdr.seek, buf.len);
+        if (data.sparse_block)
+            @memset(buf[0..to_copy], 0)
+        else
+            @memcpy(buf[0..to_copy], rdr.buffer[rdr.seek .. rdr.seek + to_copy]);
+        rdr.seek += to_copy;
+        wrote += to_copy;
+    }
+    return wrote;
+}

src/util/decomp_cache.zig

@@ -1,112 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-const ArrayHashMap = std.array_hash_map.Auto;
-const Atomic = std.atomic.Value;
-
-const Decompress = @import("decompress.zig");
-const Fn = Decompress.Fn;
-const DecompressType = Decompress.CompressionType;
-
-const DecompCache = @This();
-
-const Cache = struct {
-    cache: []u8,
-    usage: Atomic(u32),
-};
-
-arena: std.heap.ArenaAllocator,
-decomp: Fn,
-
-map: Io.File.MemoryMap,
-
-cache: ArrayHashMap(u64, Cache),
-mut: Io.RwLock = .init,
-cond: Io.Condition = .init,
-
-max_size: u64,
-cur_size: u64 = 0,
-
-pub fn init(alloc: std.mem.Allocator, map: Io.File.MemoryMap, decomp_type: DecompressType, max_size: u64) !DecompCache {
-    return .{
-        .arena = .init(alloc),
-        .decomp = try Decompress.getDecompressFn(decomp_type),
-
-        .map = map,
-
-        .cache = .empty,
-
-        .max_size = max_size,
-    };
-}
-pub fn deinit(self: *DecompCache, io: Io) void {
-    self.mut.lockUncancelable(io);
-    self.cache.deinit(self.arena.child_allocator);
-    self.arena.deinit();
-    self.map.destroy(io);
-}
-
-fn makeRoom(self: *DecompCache, io: Io, size: u32) !void {
-    if (size + self.cur_size < self.max_size) return;
-    var iter = self.cache.iterator();
-    while (iter.next()) |ent| {
-        const val = ent.value_ptr;
-        if (val.usage.load(.unordered) == 0) {
-            self.cur_size -= val.cache.len;
-            _ = self.cache.orderedRemove(ent.key_ptr.*);
-        }
-        if (size + self.cur_size < self.max_size) return;
-    }
-    try self.cond.wait(io, &self.mut.mutex);
-    return self.makeRoom(io, size);
-}
-
-pub fn checkinBlock(self: *DecompCache, io: Io, offset: u64) !void {
-    self.mut.lockSharedUncancelable(io);
-    defer self.mut.unlockShared(io);
-
-    const get = self.cache.getPtr(offset);
-    if (get == null) return error.NotACachedBlock;
-    const res = get.?.usage.fetchSub(1, .acq_rel);
-    if (res == 0) self.cond.broadcast(io);
-}
-pub fn checkoutBlock(self: *DecompCache, io: Io, offset: u64, data_size: u32, max_result_size: u32) ![]u8 {
-    {
-        try self.mut.lockShared(io);
-        defer self.mut.unlockShared(io);
-
-        const get = self.cache.getPtr(offset);
-        if (get != null) {
-            _ = get.?.usage.fetchAdd(1, .acq_rel);
-            return get.?.cache;
-        }
-    }
-    try self.mut.lock(io);
-    defer self.mut.unlock(io);
-
-    try self.makeRoom(io, max_result_size);
-
-    var alloc = self.arena.allocator();
-    const buf_alloc = self.arena.child_allocator;
-
-    var out = try alloc.alloc(u8, max_result_size);
-    errdefer alloc.free(out);
-
-    const out_size = try self.decomp(buf_alloc, self.map.memory[offset..][0..data_size], out);
-    if (out_size != max_result_size) {
-        if (alloc.resize(out, out_size)) {
-            out.len = out_size;
-        } else {
-            const new_out = try alloc.alloc(u8, out_size);
-            @memcpy(new_out, out[0..out_size]);
-            alloc.free(out);
-            out = new_out;
-        }
-    }
-
-    try self.cache.put(buf_alloc, offset, .{
-        .cache = out,
-        .usage = .init(1),
-    });
-
-    return out;
-}

src/util/decompress.zig

@@ -1,38 +0,0 @@
-const std = @import("std");
-const Io = std.Io;
-
-const config = @import("config");
-
-const c_decomp = @import("c_decomp.zig");
-const zig_decomp = @import("zig_decomp.zig");
-
-pub const Error = Io.Reader.Error || std.mem.Allocator.Error;
-
-pub const Fn = *const fn (alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize;
-
-pub const CompressionType = enum(u16) {
-    gzip = 1,
-    lzma,
-    lzo,
-    xz,
-    lz4,
-    zstd,
-};
-
-pub fn getDecompressFn(t: CompressionType) !Fn {
-    return if (config.use_zig_decomp) switch (t) {
-        .lzo => error.LzoUnsupported,
-        .lz4 => error.Lz4Unsupported,
-        .gzip => zig_decomp.zlibDecompress,
-        .lzma => zig_decomp.lzmaDecompress,
-        .xz => zig_decomp.xzDecompress,
-        .zstd => zig_decomp.zstdDecompress,
-    } else switch (t) {
-        .gzip => c_decomp.zlibDecompress,
-        .lzma => c_decomp.lzmaDecompress,
-        .lzo => if (config.allow_lzo) c_decomp.lzoDecompress else error.LzoUnsupported,
-        .xz => c_decomp.lzmaDecompress,
-        .lz4 => c_decomp.lz4Decompress,
-        .zstd => c_decomp.zstdDecompress,
-    };
-}

src/util/decompressor.zig

@@ -0,0 +1,15 @@
+//! A decompression interface
+
+const std = @import("std");
+
+const Decompressor = @This();
+
+pub const Error = std.Io.Reader.StreamError || std.mem.Allocator.Error;
+
+/// The actual decompression function.
+/// If the given decompressor is null, then the decompression should be done "stateless" without lasting allocations.
+decomp_fn: *const fn (?*const Decompressor, std.mem.Allocator, in: []u8, out: []u8) Error!usize,
+
+pub fn Decompress(self: *const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
+    return self.decomp_fn(self, alloc, in, out);
+}

src/util/metadata.zig

@@ -1,29 +1,27 @@
-//! A cache for decompressed blocks. Used for Metadata & fragments.
-
 const std = @import("std");
-const Io = std.Io;
-const Reader = Io.Reader;
-const Writer = Io.Writer;
-const Limit = Io.Limit;
+const Reader = std.Io.Reader;
+const Writer = std.Io.Writer;
+const Limit = std.Io.Limit;
+const StreamError = std.Io.Reader.StreamError;
 
-const DecompCache = @import("decomp_cache.zig");
-
-const MetadataReader = @This();
+const Decompressor = @import("decompressor.zig");
 
 const BlockHeader = packed struct(u16) {
     size: u15,
     uncompressed: bool,
 };
 
-io: Io,
+const This = @This();
 
-cur_offset: u64 = 0,
-next_offset: u64,
+alloc: std.mem.Allocator,
+rdr: *Reader,
+decomp: *const Decompressor,
 
-cache: *DecompCache,
-
-buf_uncompress: bool = false,
+cur_block_start: u32 = 0,
+next_start_start: u32 = 0,
+buf: [8192]u8 = undefined,
 
+err: ?anyerror = null,
 interface: Reader = .{
     .buffer = &[0]u8{},
     .end = 0,
@@ -32,79 +30,74 @@ interface: Reader = .{
         .stream = stream,
         .discard = discard,
         .readVec = readVec,
+        // TODO: Potentially add rebase so that we can guarentee that self.block_start & interface.seek is correct.
     },
 },
 
-pub fn init(io: Io, cache: *DecompCache, offset: u64) MetadataReader {
+pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: *const Decompressor) This {
     return .{
-        .io = io,
-
-        .next_offset = offset,
-
-        .cache = cache,
+        .alloc = alloc,
+        .rdr = rdr,
+        .decomp = decomp,
     };
 }
-pub fn deinit(self: *MetadataReader) void {
-    if (self.cur_offset != 0 and !self.buf_uncompress)
-        self.cache.checkinBlock(self.io, self.cur_offset) catch {};
-}
 
-fn advance(self: *MetadataReader) !void {
-    if (self.interface.buffer.len > 0 and !self.buf_uncompress)
-        self.cache.checkinBlock(self.io, self.cur_offset) catch |err| {
-            std.debug.print("UH OH! {}\n", .{err});
-            return error.ReadFailed;
-        };
-    const hdr: BlockHeader = @bitCast(std.mem.readInt(u16, self.cache.map.memory[self.next_offset..][0..2], .little));
-    self.cur_offset = self.next_offset + 2;
-    self.next_offset = self.cur_offset + hdr.size;
-
-    self.buf_uncompress = hdr.uncompressed;
-    if (hdr.uncompressed) {
-        self.interface.buffer = self.cache.map.memory[self.cur_offset..][0..hdr.size];
-        self.interface.end = hdr.size;
-        self.interface.seek = 0;
-        return;
-    }
-    self.interface.buffer = try self.cache.checkoutBlock(self.io, self.cur_offset, hdr.size, 8192);
-    self.interface.end = self.interface.buffer.len;
+fn advance(self: *This) !void {
     self.interface.seek = 0;
+    var hdr: BlockHeader = undefined;
+    try self.rdr.readSliceEndian(BlockHeader, @ptrCast(&hdr), .little);
+    self.cur_block_start = self.next_start_start;
+    self.next_start_start += hdr.size;
+    if (hdr.uncompressed) {
+        try self.rdr.readSliceEndian(u8, self.buf[0..hdr.size], .little);
+        self.interface.end = hdr.size;
+        self.interface.buffer = self.buf[0..hdr.size];
+        return;
+    } else {
+        @branchHint(.likely);
+        var tmp_buf: [8192]u8 = undefined;
+        try self.rdr.readSliceAll(tmp_buf[0..hdr.size]);
+        self.interface.end = try self.decomp.Decompress(self.alloc, tmp_buf[0..hdr.size], &self.buf);
+        self.interface.buffer = self.buf[0..self.interface.end];
+    }
 }
 
-fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {
-    if (r.seek == r.end) {
-        var self: *MetadataReader = @fieldParentPtr("interface", r);
-        self.advance() catch return Reader.Error.ReadFailed;
-    }
-    if (limit == .nothing) return 0;
-    const to_write = @min(r.end - r.seek, @intFromEnum(limit));
-    const wrote = try w.write(r.buffer[r.seek..][0..to_write]);
-    r.seek += wrote;
+fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) StreamError!usize {
+    const self: *This = @fieldParentPtr("interface", rdr);
+    if (rdr.end == rdr.seek) self.advance() catch |err| {
+        self.err = err;
+        return StreamError.ReadFailed;
+    };
+    if (@intFromEnum(limit) == 0) return 0;
+    const to_write = @min(rdr.end - rdr.seek, @intFromEnum(limit));
+    const wrote = try wrt.write(self.buf[rdr.seek .. rdr.seek + to_write]);
+    self.interface.seek += wrote;
     return wrote;
 }
-fn discard(r: *Reader, limit: Limit) Reader.Error!usize {
-    if (r.seek == r.end) {
-        var self: *MetadataReader = @fieldParentPtr("interface", r);
-        self.advance() catch return Reader.Error.ReadFailed;
-    }
-    if (limit == .nothing) return 0;
-    const to_skip = @min(r.end - r.seek, @intFromEnum(limit));
-    r.seek += to_skip;
+fn discard(rdr: *Reader, limit: Limit) Reader.Error!usize {
+    const self: *This = @fieldParentPtr("interface", rdr);
+    if (rdr.end == rdr.seek) self.advance() catch |err| {
+        self.err = err;
+        return error.ReadFailed;
+    };
+    if (@intFromEnum(limit) == 0) return 0;
+    const to_skip = @min(rdr.end - rdr.seek, @intFromEnum(limit));
+    rdr.seek += to_skip;
     return to_skip;
 }
-fn readVec(r: *Reader, vec: [][]u8) Reader.Error!usize {
-    if (r.seek == r.end) {
-        var self: *MetadataReader = @fieldParentPtr("interface", r);
-        self.advance() catch return Reader.Error.ReadFailed;
+fn readVec(rdr: *Reader, vec: [][]u8) Reader.Error!usize {
+    const self: *This = @fieldParentPtr("interface", rdr);
+    if (rdr.end == rdr.seek) self.advance() catch |err| {
+        self.err = err;
+        return error.ReadFailed;
+    };
+    var cur_red: usize = 0;
+    for (vec) |s| {
+        const to_copy: usize = @min(rdr.end - rdr.seek, s.len);
+        @memcpy(s[0..to_copy], self.buf[rdr.seek .. rdr.seek + to_copy]);
+        rdr.seek += to_copy;
+        cur_red += to_copy;
+        if (rdr.end == rdr.seek) break;
     }
-    if (vec.len == 0) return 0;
-    var total_copied: usize = 0;
-    for (vec) |v| {
-        const to_cpy = @min(r.end - r.seek, v.len);
-        @memcpy(v[0..to_cpy], r.buffer[r.seek..][0..to_cpy]);
-        r.seek += to_cpy;
-        total_copied += to_cpy;
-        if (r.seek == r.end) break;
-    }
-    return total_copied;
+    return cur_red;
 }

src/util/misc.zig

@@ -0,0 +1,25 @@
+//! Miscellaneous utility functions.
+
+const std = @import("std");
+const Io = std.Io;
+
+const Inode = @import("../inode.zig");
+const Decompressor = @import("decompressor.zig");
+const MetadataReader = @import("metadata.zig");
+const OffsetFile = @import("offset_file.zig");
+
+/// check is the path is referencing itself ("" or ".").
+/// separators must be trimmed before calling this function for it to work properly.
+pub fn pathIsSelf(path: []const u8) bool {
+    if (path.len == 0) return true;
+    if (path.len > 1) return false;
+    return path[0] == '.';
+}
+/// Creates an Inode from an Inode.Ref.
+pub fn inodeFromRef(alloc: std.mem.Allocator, io: Io, file: OffsetFile, decomp: *const Decompressor, inode_start: u64, block_size: u32, ref: Inode.Ref) !Inode {
+    var rdr = try file.readerAt(io, inode_start + ref.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+    try meta.interface.discardAll(ref.block_offset);
+
+    return .read(alloc, &meta.interface, block_size);
+}

src/util/offset_file.zig

@@ -0,0 +1,29 @@
+//! A File where it's meaningful (to us) content starts at a given offset.
+
+const std = @import("std");
+const Io = std.Io;
+const File = Io.File;
+const Reader = File.Reader;
+
+const OffsetFile = @This();
+
+fil: File,
+offset: u64,
+
+pub fn init(fil: File, init_offset: u64) OffsetFile {
+    return .{ .fil = fil, .offset = init_offset };
+}
+
+pub fn readerAt(self: OffsetFile, io: Io, offset: u64, buffer: []u8) !Reader {
+    var rdr = self.fil.reader(io, buffer);
+    try rdr.seekTo(self.offset + offset);
+    return rdr;
+}
+pub fn readAt(self: OffsetFile, io: Io, offset: u64, buf: []u8) !void {
+    _ = try self.fil.readPositionalAll(io, buf, self.offset + offset);
+}
+pub fn readValueAt(self: OffsetFile, comptime T: anytype, io: Io, offset: u64) !void {
+    //TODO: check for endianess and decode accordingly.
+    var new: T = undefined;
+    _ = try self.fil.readPositionalAll(io, @ptrCast(&new), self.offset + offset);
+}

src/util/shared_cache.zig

@@ -0,0 +1,52 @@
+const std = @import("std");
+const Io = std.Io;
+const Node = std.SinglyLinkedList.Node;
+
+const SharedCache = @This();
+
+pub const CACHE_SIZE = 1024 * 1024;
+
+pub const BufferNode = struct {
+    node: Node,
+    cache: [CACHE_SIZE]u8,
+};
+
+alloc: std.mem.Allocator,
+
+caches: std.ArrayList(BufferNode),
+cache_queue: std.SinglyLinkedList,
+queue_mut: Io.Mutex,
+
+pub fn init(alloc: std.mem.Allocator, init_cache_size: u32) !SharedCache {
+    const caches: std.ArrayList(BufferNode) = try .initCapacity(alloc, init_cache_size);
+    var queue: std.SinglyLinkedList = .{};
+    for (caches.items) |item|
+        queue.prepend(&item.node);
+    return .{
+        .alloc = alloc,
+
+        .caches = caches,
+        .cache_queue = queue,
+    };
+}
+pub fn deinit(self: *SharedCache) void {
+    self.caches.deinit(self.alloc);
+}
+
+pub fn getCache(self: *SharedCache, io: Io) !*BufferNode {
+    self.queue_mut.lock(io);
+    const nxt = self.cache_queue.popFirst();
+    self.queue_mut.unlock(io);
+    if (nxt == null) {
+        const new = try self.caches.addOne(self.alloc);
+        new.* = .{
+            .node = .{},
+            .cache = undefined,
+        };
+        return new;
+    }
+    return @fieldParentPtr("node", nxt.?);
+}
+pub fn returnCache(self: *SharedCache, buf: *BufferNode) void {
+    self.cache_queue.prepend(buf);
+}

src/util/zig_decomp.zig

@@ -1,40 +0,0 @@
-const std = @import("std");
-const Reader = std.Io.Reader;
-const flate = std.compress.flate;
-const zstd = std.compress.zstd;
-const xz = std.compress.xz;
-const lzma = std.compress.lzma;
-
-const Error = @import("decompress.zig").Error;
-
-pub fn zlibDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var buf: [flate.max_window_len]u8 = undefined;
-
-    var rdr: Reader = .fixed(in);
-    var decomp: flate.Decompress = .init(&rdr, .zlib, &buf);
-
-    return decomp.reader.readSliceShort(out);
-}
-pub fn zstdDecompress(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    const buf = try alloc.alloc(u8, in.len + zstd.block_size_max);
-    defer alloc.free(buf);
-
-    var rdr: Reader = .fixed(in);
-    var decomp: zstd.Decompress = .init(&rdr, buf, .{ .window_len = in.len });
-
-    return decomp.reader.readSliceShort(out);
-}
-pub fn lzmaDecompress(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var rdr: Reader = .fixed(in);
-    var decomp: lzma.Decompress = .initOptions(&rdr, alloc, &[0]u8{}, .{}, 2 * out.len);
-    defer decomp.deinit();
-
-    return decomp.reader.readSliceShort(out);
-}
-pub fn xzDecompress(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
-    var rdr: Reader = .fixed(in);
-    var decomp: xz.Decompress = .init(&rdr, alloc, &[0]u8{});
-    defer decomp.deinit();
-
-    return decomp.reader.readSliceShort(out);
-}

src/xattr_table.zig

@@ -0,0 +1,284 @@
+const std = @import("std");
+const Io = std.Io;
+
+const InodeRef = @import("inode.zig").Ref;
+const LookupTable = @import("lookup_table.zig");
+const Decompressor = @import("util/decompressor.zig");
+const MetadataReader = @import("util/metadata.zig");
+const OffsetFile = @import("util/offset_file.zig");
+
+const XattrCachedTable = @This();
+
+alloc: std.mem.Allocator,
+
+fil: OffsetFile,
+decomp: *const Decompressor,
+
+kv_start: u64,
+
+table: LookupTable.CachedTable(TableValue),
+value_cache: std.AutoHashMap(InodeRef, []const u8),
+value_mut: Io.Mutex,
+
+pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, xattr_start: u64) !XattrCachedTable {
+    var rdr = try fil.readerAt(io, xattr_start, &[0]u8{});
+
+    var start: u64 = undefined;
+    try rdr.interface.readSliceEndian(u64, @ptrCast(&start), .little);
+    var num: u32 = undefined;
+    try rdr.interface.readSliceEndian(u32, @ptrCast(&num), .little);
+
+    return .{
+        .alloc = alloc,
+
+        .fil = fil,
+        .decomp = decomp,
+
+        .kv_start = start,
+
+        .table = .init(alloc, fil, xattr_start + 16, num),
+        .value_cache = .init(alloc),
+    };
+}
+pub fn deinit(self: *XattrCachedTable) void {
+    self.table.deinit();
+    self.value_cache.deinit();
+}
+
+pub fn get(self: *XattrCachedTable, alloc: std.mem.Allocator, io: Io, idx: u32) ![]XattrSemiOwned {
+    const lookup = try self.table.get(io, idx);
+
+    var rdr = try self.fil.readerAt(io, self.kv_start + lookup.ref.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
+    try meta.interface.discardAll(lookup.ref.block_offset);
+
+    const out = try alloc.alloc(XattrSemiOwned, lookup.count);
+    errdefer alloc.free(out);
+
+    for (0..lookup.count) |i| {
+        const key_entry: KeyEntry = undefined;
+        try meta.interface.readSliceEndian(KeyEntry, @ptrCast(&key_entry), .little);
+
+        const key = switch (key_entry.type.namespace) {
+            .user => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 5);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[5 .. tmp.len - 1], .little);
+                @memset(tmp[0..5], "user.");
+                break :blk tmp;
+            },
+            .trusted => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 8);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[8 .. tmp.len - 1], .little);
+                @memset(tmp[0..8], "trusted.");
+                break :blk tmp;
+            },
+            .security => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 9);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[9 .. tmp.len - 1], .little);
+                @memset(tmp[0..9], "security.");
+                break :blk tmp;
+            },
+        };
+        key[key.len - 1] = 0;
+        errdefer alloc.free(key);
+
+        if (key_entry.type.out_of_line) {
+            const value: ValueOutOfLineEntry = undefined;
+            try meta.interface.readSliceEndian(ValueOutOfLineEntry, @ptrCast(&value), .little);
+
+            out[i] = .{
+                .key = key,
+                .value = try self.valueAt(io, value.ref),
+            };
+            continue;
+        }
+        const val_ref: InodeRef = .{ .block_start = meta.cur_block_start, .block_offset = meta.interface.seek };
+
+        try self.value_mut.lock(io);
+        defer self.value_mut.unlock(io);
+        if (self.value_cache.contains(val_ref)) {
+            out[i] = .{
+                .key = key,
+                .value = try self.valueAt(io, val_ref),
+            };
+            continue;
+        }
+
+        var val_size: u32 = undefined;
+        try meta.interface.readSliceEndian(val_size, @ptrCast(&val_size), .little);
+
+        const val = try self.alloc.alloc(u8, val_size);
+        errdefer alloc.free(val);
+        try meta.interface.readSliceEndian(u8, val, .little);
+
+        try self.value_cache.put(val_ref, val);
+        out[i] = .{
+            .key = key,
+            .value = val,
+        };
+    }
+    return out;
+}
+
+fn valueAt(self: *XattrCachedTable, io: Io, ref: InodeRef) ![]const u8 {
+    try self.value_mut.lock(io);
+    defer self.value_mut.unlock(io);
+
+    if (self.value_cache.contains(ref)) return self.value_cache.get(ref).?;
+
+    var rdr = try self.fil.readerAt(io, self.kv_start + ref.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
+    try meta.interface.discardAll(ref.block_offset);
+
+    var val_size: u32 = undefined;
+    try meta.interface.readSliceEndian(val_size, @ptrCast(&val_size), .little);
+
+    const val = try self.alloc.alloc(u8, val_size);
+    errdefer self.alloc.free(val);
+    try meta.interface.readSliceEndian(u8, val, .little);
+
+    try self.value_cache.put(ref, val);
+    return val;
+}
+
+// Types
+
+/// An Xattr return value where the reciever only owns the key value.
+pub const XattrSemiOwned = struct {
+    key: [:0]const u8,
+    value: []const u8,
+
+    pub fn deinit(self: XattrSemiOwned, alloc: std.mem.Allocator) void {
+        alloc.free(self.key);
+    }
+};
+/// An Xattr return value where the reciever owns both the key & value.
+pub const XattrOwned = struct {
+    key: [:0]const u8,
+    value: []const u8,
+
+    pub fn deinit(self: XattrSemiOwned, alloc: std.mem.Allocator) void {
+        alloc.free(self.key);
+        alloc.free(self.value);
+    }
+};
+
+const TableValue = extern struct {
+    ref: InodeRef,
+    count: u32,
+    size: u32,
+};
+
+const KeyEntry = extern struct {
+    type: XattrPrefix,
+    name_size: u16,
+};
+const ValueOutOfLineEntry = extern struct {
+    _: u32,
+    ref: InodeRef,
+};
+
+const XattrPrefix = packed struct(u16) {
+    namespace: enum(u8) {
+        user,
+        trusted,
+        security,
+
+        fn prefixSize(self: @This()) u16 {
+            return switch (self) {
+                .user => 5,
+                .trusted => 8,
+                .security => 9,
+            };
+        }
+    },
+    out_of_line: bool,
+    _: u7,
+};
+
+// Stateless
+
+pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *const Decompressor, fil: OffsetFile, table_start: u64, idx: u16) ![]XattrOwned {
+    var rdr = try fil.readerAt(io, table_start, &[0]u8{});
+
+    var kv_start: u64 = undefined;
+    try rdr.interface.readSliceEndian(u64, @ptrCast(&kv_start), .little);
+
+    const lookup = try LookupTable.lookupValue(TableValue, alloc, io, decomp, fil, table_start + 16, idx);
+
+    rdr = try fil.readerAt(io, kv_start + lookup.ref.block_start, &[0]u8{});
+    var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
+    try meta.interface.discardAll(lookup.ref.block_offset);
+
+    const out = try alloc.alloc(XattrOwned, lookup.count);
+    errdefer alloc.free(out);
+
+    for (0..lookup.count) |i| {
+        const key_entry: KeyEntry = undefined;
+        try meta.interface.readSliceEndian(KeyEntry, @ptrCast(&key_entry), .little);
+
+        const key = switch (key_entry.type.namespace) {
+            .user => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 5);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[5 .. tmp.len - 1], .little);
+                @memset(tmp[0..5], "user.");
+                break :blk tmp;
+            },
+            .trusted => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 8);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[8 .. tmp.len - 1], .little);
+                @memset(tmp[0..8], "trusted.");
+                break :blk tmp;
+            },
+            .security => blk: {
+                const tmp = try alloc.alloc(u8, key_entry.name_size + 1 + 9);
+                errdefer alloc.free(tmp);
+                try meta.interface.readSliceEndian(u8, tmp[9 .. tmp.len - 1], .little);
+                @memset(tmp[0..9], "security.");
+                break :blk tmp;
+            },
+        };
+        key[key.len - 1] = 0;
+        errdefer alloc.free(key);
+
+        if (key_entry.type.out_of_line) {
+            const value: ValueOutOfLineEntry = undefined;
+            try meta.interface.readSliceEndian(ValueOutOfLineEntry, @ptrCast(&value), .little);
+
+            var ool_rdr = try fil.readerAt(io, kv_start + value.ref.block_start, &[0]u8{});
+            var ool_meta: MetadataReader = .init(alloc, &ool_rdr.interface, decomp);
+            try ool_meta.interface.discardAll(value.ref.block_offset);
+
+            var val_size: u32 = undefined;
+            try ool_meta.interface.readSliceEndian(val_size, @ptrCast(&val_size), .little);
+
+            const val = try alloc.alloc(u8, val_size);
+            errdefer alloc.free(val);
+            try ool_meta.interface.readSliceEndian(u8, val, .little);
+
+            out[i] = .{
+                .key = key,
+                .value = val,
+            };
+            continue;
+        }
+
+        var val_size: u32 = undefined;
+        try meta.interface.readSliceEndian(val_size, @ptrCast(&val_size), .little);
+
+        const val = try alloc.alloc(u8, val_size);
+        errdefer alloc.free(val);
+        try meta.interface.readSliceEndian(u8, val, .little);
+
+        out[i] = .{
+            .key = key,
+            .value = val,
+        };
+    }
+    return out;
+}