More foundational work on extraction

src/data/extractor.zig

@@ -0,0 +1,103 @@
+const std = @import("std");
+const Io = std.Io;
+
+const Decomp = @import("../decomp.zig");
+const DataBlock = @import("../inode.zig").DataBlock;
+
+const Extractor = @This();
+
+data: []u8,
+decomp: Decomp.Fn,
+block_size: u32,
+
+blocks: []DataBlock,
+
+start: u64,
+size: u64,
+
+frag_data: ?[]u8 = null,
+frag_offset: u32 = 0,
+
+pub fn init(data: []u8, decomp: Decomp.Fn, block_size: u32, blocks: []DataBlock, start: u64, size: u64) Extractor {
+    return .{
+        .data = data,
+        .decomp = decomp,
+        .block_size = block_size,
+
+        .blocks = blocks,
+
+        .start = start,
+        .size = size,
+    };
+}
+pub fn addFrag(self: *Extractor, frag_data: []u8, frag_offset: u32) void {
+    self.frag_data = frag_data;
+    self.frag_offset = frag_offset;
+}
+
+pub fn extractAsync(self: Extractor, alloc: std.mem.Allocator, io: Io, file: Io.File) Error!void {
+    if (self.size == 0) return;
+
+    try file.writePositionalAll(io, &[0]u8{}, self.size);
+
+    var map = try file.createMemoryMap(io, .{
+        .len = self.size,
+        .protection = .{ .write = true },
+    });
+    defer map.destroy(io);
+
+    var err: ?Error = null;
+    var group: Io.Group = .init;
+
+    var read_offset: u64 = self.start;
+    for (0.., self.blocks) |i, block| {
+        group.async(io, blockThread, .{ self, alloc, map.memory, read_offset, i, &err });
+        read_offset += block.size;
+    }
+    if (self.frag_data != null)
+        group.async(io, fragThread, .{ self, map.memory });
+
+    try group.await(io);
+
+    if (err != null)
+        return err;
+
+    try map.write(io);
+}
+
+fn blockThread(self: Extractor, alloc: std.mem.Allocator, map_data: []u8, read_offset: u64, block_idx: u32, err: *?Error) error{Canceled}!void {
+    const size = if (self.frag_data == null and block_idx == (self.size - 1 / self.block_size))
+        self.size % self.block_size
+    else
+        self.block_size;
+    const offset = self.block_size * block_idx;
+
+    const block = self.blocks[block_idx];
+
+    if (block.size == 0) {
+        @memset(map_data[offset..][0..size], 0);
+        return;
+    }
+
+    const data = self.data[read_offset..][0..block.size];
+
+    if (block.uncompressed) {
+        @memcpy(map_data[offset..][0..block.size], data);
+        return;
+    }
+
+    _ = self.decomp(alloc, data, map_data[offset..][0..size]) catch |inner_err| {
+        err.* = inner_err;
+        return;
+    };
+}
+fn fragThread(self: Extractor, map_data: []u8) error{Canceled}!void {
+    const size = self.size % self.block_size;
+    const offset = self.blocks.len * self.block_size;
+
+    @memcpy(map_data[offset..][0..size], self.frag_data.?[self.frag_offset..][0..size]);
+}
+
+// Types
+
+const Error = Io.File.WritePositionalError || Io.File.MemoryMap.CreateError || Decomp.Error;

src/data/reader.zig

@@ -0,0 +1,155 @@
+const std = @import("std");
+const Io = std.Io;
+
+const Decomp = @import("../decomp.zig");
+const DataBlock = @import("../inode.zig").DataBlock;
+
+const Reader = @This();
+
+alloc: std.mem.Allocator,
+
+data: []u8,
+decomp: Decomp.Fn,
+block_size: u32,
+
+blocks: []DataBlock,
+
+size: u64,
+
+offset: u64,
+block_idx: u32 = 0,
+sparse_block: bool = false,
+
+frag_data: ?[]u8 = null,
+frag_offset: u32 = 0,
+
+block: [1024 * 1024]u8 = undefined,
+
+interface: Io.Reader = .{
+    .buffer = &[0]u8{},
+    .end = 0,
+    .seek = 0,
+    .vtable = &.{
+        .stream = stream,
+        .discard = discard,
+        .readVec = readVec,
+    },
+},
+
+pub fn init(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, block_size: u32, blocks: []DataBlock, size: u64, data_start: u64) Reader {
+    return .{
+        .alloc = alloc,
+
+        .data = data,
+        .decomp = decomp,
+        .block_size = block_size,
+
+        .blocks = blocks,
+
+        .size = size,
+
+        .offset = data_start,
+    };
+}
+pub fn addFrag(self: *Reader, frag_data: []u8, frag_offset: u32) void {
+    self.frag_data = frag_data;
+    self.frag_offset = frag_offset;
+}
+
+fn advance(self: *Reader) Io.Reader.Error!void {
+    if (self.block_idx > self.blocks.len) return error.EndOfStream;
+    defer self.block_idx += 1;
+
+    self.interface.seek = 0;
+    errdefer self.interface.end = 0;
+
+    if (self.block_idx == self.blocks.len) {
+        if (self.frag_data == null) return error.EndOfStream;
+
+        self.sparse_block = false;
+
+        const size = self.size % self.block_size;
+
+        self.interface.buffer = self.frag_data.?[self.frag_offset..][0..size];
+
+        self.interface.end = size;
+        return;
+    }
+
+    const size = if (self.frag_data == null and self.block_idx == self.blocks.len - 1)
+        self.size % self.block_size
+    else
+        self.block_size;
+
+    const block = self.blocks[self.block_idx];
+    defer self.offset += block.size;
+
+    if (block.size == 0) {
+        self.sparse_block = true;
+        self.end = size;
+        return;
+    } else {
+        self.sparse_block = false;
+    }
+
+    if (block.uncompressed) {
+        self.interface.buffer = self.data[self.offset..][0..block.size];
+        self.interface.end = block.size;
+        return;
+    }
+
+    _ = self.decomp(self.alloc, self.data[self.offset..][0..block.size], self.block[0..size]) catch return error.ReadFailed;
+    self.interface.buffer = self.block[0..size];
+    self.interface.end = size;
+}
+
+fn stream(r: *Io.Reader, w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize {
+    const self: *Reader = @fieldParentPtr("interface", r);
+    if (r.seek >= r.end)
+        try self.advance();
+
+    if (limit == .nothing) return;
+
+    const to_write = @min(@intFromEnum(limit), r.end - r.seek);
+
+    const wrote = if (self.sparse_block)
+        try w.splatByte(0, to_write)
+    else
+        try w.write(r.buffer[r.seek..][0..to_write]);
+
+    r.seek += wrote;
+    return wrote;
+}
+fn discard(r: *Io.Reader, limit: Io.Limit) Io.Reader.Error!usize {
+    if (r.seek >= r.end) {
+        const self: *Reader = @fieldParentPtr("interface", r);
+        try self.advance();
+    }
+    if (limit == .nothing) return;
+
+    const to_discard = @min(@intFromEnum(limit), r.end - r.seek);
+
+    r.seek += to_discard;
+    return to_discard;
+}
+fn readVec(r: *Io.Reader, vec: [][]u8) Io.Reader.Error!usize {
+    const self: *Reader = @fieldParentPtr("interface", r);
+    if (r.seek >= r.end)
+        try self.advance();
+
+    var copied: usize = 0;
+    for (vec) |v| {
+        const to_cpy = @min(v.len, r.end - r.seek);
+
+        if (self.sparse_block)
+            @memset(v[0..to_cpy], 0)
+        else
+            @memcpy(v[0..to_cpy], r.buffer[r.seek..][0..to_cpy]);
+
+        copied += to_cpy;
+
+        if (r.seek >= r.end) break;
+    }
+
+    return copied;
+}

src/extract-multi.zig

@@ -1,10 +1,12 @@
 const std = @import("std");
 const Io = std.Io;
 
-const Superblock = @import("archive.zig").Superblock;
-const Inode = @import("inode.zig");
 const Decomp = @import("decomp.zig");
 const ExtractionOptions = @import("options.zig");
+const Inode = @import("inode.zig");
+const Lookup = @import("lookup.zig");
+const Superblock = @import("archive.zig").Superblock;
+const XattrTable = @import("xattr.zig");
 
 pub fn extract(
     alloc: std.mem.Allocator,
@@ -16,3 +18,15 @@ pub fn extract(
     ext_loc: []const u8,
     options: ExtractionOptions,
 ) !void {}
+
+pub fn extractReal(
+    alloc: std.mem.Allocator,
+    io: Io,
+    super: Superblock,
+    data: []u8,
+    decomp: Decomp.Fn,
+    frag_table: Lookup.Table(u64),
+    inode: Inode,
+    ext_loc: []const u8,
+    options: ExtractionOptions,
+) !void {}

src/extract-single.zig

@@ -1,10 +1,12 @@
 const std = @import("std");
 const Io = std.Io;
 
-const Superblock = @import("archive.zig").Superblock;
-const Inode = @import("inode.zig");
 const Decomp = @import("decomp.zig");
 const ExtractionOptions = @import("options.zig");
+const Inode = @import("inode.zig");
+const Lookup = @import("lookup.zig");
+const Superblock = @import("archive.zig").Superblock;
+const XattrTable = @import("xattr.zig");
 
 pub fn extract(
     alloc: std.mem.Allocator,
@@ -16,3 +18,17 @@ pub fn extract(
     ext_loc: []const u8,
     options: ExtractionOptions,
 ) !void {}
+
+pub fn extractReal(
+    alloc: std.mem.Allocator,
+    io: Io,
+    super: Superblock,
+    data: []u8,
+    decomp: Decomp.Fn,
+    frag_table: Lookup.Table(u64),
+    id_table: Lookup.Table(u16),
+    xattr_table: XattrTable,
+    inode: Inode,
+    ext_loc: []const u8,
+    options: ExtractionOptions,
+) !void {}

src/lookup.zig

@@ -2,6 +2,8 @@ const std = @import("std");
 const Io = std.Io;
 
 const Decomp = @import("decomp.zig");
+const MetadataReader = @import("meta_rdr.zig");
+const ProtectedMap = @import("util/protected_map.zig");
 
 pub fn Table(comptime T: anytype) type {
     return struct {
@@ -15,91 +17,43 @@ pub fn Table(comptime T: anytype) type {
         table_start: u64,
         table_num: u32,
 
-        pub fn init(alloc: std.mem.Allocator, table_start: u64, table_num: u32) Self {}
+        table: ProtectedMap(u32, []T, getBlock),
-        pub fn deinit(self: *Self) void {}
 
-        pub fn get(self: *Self, io: Io, idx: u32) !T {}
+        pub fn init(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, table_start: u64, table_num: u32) Self {
-    };
+            return .{ .data = data, .decomp = decomp, .table_start = table_start, .table_num = table_num, .table = .init(alloc) };
-}
-
-pub fn ProtectedMap(comptime K: anytype, comptime T: anytype, comptime create_fn: anytype) type {
-    std.debug.assert(std.meta.activeTag(@typeInfo(create_fn)) == .@"fn");
-
-    const ret_info = @typeInfo(create_fn).@"fn".return_type;
-
-    std.debug.assert(ret_info == T or
-        (std.meta.activeTag(@typeInfo(ret_info)) == .error_union and @typeInfo(ret_info).error_union.payload == T));
-
-    return struct {
-        const Map = @This();
-
-        alloc: std.mem.Allocator,
-
-        map: std.AutoHashMap(K, ProtectedValue),
-        mut: Io.RwLock = .init,
-
-        pub fn init(alloc: std.mem.Allocator, default_value: T) Map {
-            return .{
-                .alloc = alloc,
-
-                .map = .init(alloc),
-
-                .default_value = default_value,
-            };
         }
-        pub fn deinit(self: *Map) void {}
+        pub fn deinit(self: *Self) void {
+            var iter = self.table.map.valueIterator();
 
-        pub fn getOrPut(self: *Map, io: Io, key: K, create_fn_args: std.meta.ArgsTuple(create_fn)) !*T {
+            while (iter.next()) |val| {
-            {
+                if (val.err != null) continue;
-                try self.mut.lockShared(io);
-                defer self.mut.unlockShared(io);
 
-                const value = self.map.getPtr(key);
+                self.table.alloc.free(val.value);
-                if (value != null) {
-                    if (!value.?.filled.isSet())
-                        value.?.filled.wait(io);
-                    if (value.?.err != null) return value.?.err != null;
-                    return value.?.value;
             }
-            }
+            self.table.deinit();
-            try self.mut.lock(io);
-
-            const res = self.map.getOrPut(key) catch |err| {
-                self.mut.unlock(io);
-                return err;
-            };
-            if (res.found_existing) {
-                self.mut.unlock(io);
-                return self.getOrPut(io, key, create_fn_args);
         }
-            res.value_ptr.* = .{};
-            defer res.value_ptr.filled.set(io);
 
-            res.mut.unlock(io);
+        pub fn get(self: *Self, io: Io, idx: u32) !T {
+            const block = idx / VALUES_PER_BLOCK;
+            const block_idx = idx % VALUES_PER_BLOCK;
 
-            self.mut.lockSharedUncancelable(io);
+            const values = try self.table.getOrPut(io, block, .{ self.*, block });
-            defer self.mut.unlockShared(io);
 
-            if (@TypeOf(CreateError) == void) {
+            return values[block_idx];
-                res.value_ptr.value = @call(.auto, create_fn, create_fn_args);
-            } else {
-                res.value_ptr.value = @call(.auto, create_fn, create_fn_args) catch |err| {
-                    res.value_ptr.err = err;
-                };
         }
-        }
 
-        // Map Types
+        pub fn getBlock(self: Self, block_idx: u32) ![]T {
+            const offset: u64 = std.mem.readInt(u64, self.data[self.table_start + (block_idx * 8) ..][0..8], .little);
+
+            const block = try self.table.alloc(T, if (block_idx == (self.table_num - 1 / VALUES_PER_BLOCK))
+                self.table_num % VALUES_PER_BLOCK
+            else
+                VALUES_PER_BLOCK);
 
-        const CreateError: type = switch (@typeInfo(@typeInfo(create_fn).@"fn".return_type)) {
+            var meta: MetadataReader = .init(self.table.alloc, self.data, self.decomp, offset);
-            .error_union => |e| e.error_set,
+            try meta.interface.readSliceEndian(T, block, .little);
-            else => void,
-        };
 
-        const ProtectedValue = struct {
+            return block;
-            value: T = undefined,
+        }
-            err: ?CreateError = null,
-            filled: Io.Event = .unset,
-        };
     };
 }

src/util/protected_map.zig

@@ -0,0 +1,84 @@
+const std = @import("std");
+const Io = std.Io;
+
+pub fn ProtectedMap(comptime K: anytype, comptime T: anytype, comptime create_fn: anytype) type {
+    std.debug.assert(std.meta.activeTag(@typeInfo(create_fn)) == .@"fn");
+
+    const ret_info = @typeInfo(create_fn).@"fn".return_type;
+
+    std.debug.assert(ret_info == T or
+        (std.meta.activeTag(@typeInfo(ret_info)) == .error_union and @typeInfo(ret_info).error_union.payload == T));
+
+    return struct {
+        const Map = @This();
+
+        alloc: std.mem.Allocator,
+
+        map: std.AutoHashMap(K, ProtectedValue),
+        mut: Io.RwLock = .init,
+
+        pub fn init(alloc: std.mem.Allocator) Map {
+            return .{
+                .alloc = alloc,
+
+                .map = .init(alloc),
+            };
+        }
+        pub fn deinit(self: *Map) void {
+            self.map.deinit();
+        }
+
+        pub fn getOrPut(self: *Map, io: Io, key: K, create_fn_args: std.meta.ArgsTuple(create_fn)) !*T {
+            {
+                try self.mut.lockShared(io);
+                defer self.mut.unlockShared(io);
+
+                const value = self.map.getPtr(key);
+                if (value != null) {
+                    if (!value.?.filled.isSet())
+                        value.?.filled.wait(io);
+                    if (value.?.err != null) return value.?.err != null;
+                    return value.?.value;
+                }
+            }
+            try self.mut.lock(io);
+
+            const res = self.map.getOrPut(key) catch |err| {
+                self.mut.unlock(io);
+                return err;
+            };
+            if (res.found_existing) {
+                self.mut.unlock(io);
+                return self.getOrPut(io, key, create_fn_args);
+            }
+            res.value_ptr.* = .{};
+            defer res.value_ptr.filled.set(io);
+
+            res.mut.unlock(io);
+
+            self.mut.lockSharedUncancelable(io);
+            defer self.mut.unlockShared(io);
+
+            if (@TypeOf(CreateError) == void) {
+                res.value_ptr.value = @call(.auto, create_fn, create_fn_args);
+            } else {
+                res.value_ptr.value = @call(.auto, create_fn, create_fn_args) catch |err| {
+                    res.value_ptr.err = err;
+                };
+            }
+        }
+
+        // Map Types
+
+        const CreateError: type = switch (@typeInfo(@typeInfo(create_fn).@"fn".return_type)) {
+            .error_union => |e| e.error_set,
+            else => void,
+        };
+
+        const ProtectedValue = struct {
+            value: T = undefined,
+            err: ?CreateError = null,
+            filled: Io.Event = .unset,
+        };
+    };
+}

src/xattr.zig

@@ -0,0 +1,3 @@
+const std = @import("std");
+
+const XattrTable = @This();