From 274d0884900eac51f35725952545fb3664a0ef61 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Thu, 30 Apr 2026 07:00:46 -0500 Subject: [PATCH] Further work on getting everything working again Mainly working on decompression interface --- src/decomp/zlib.zig | 55 +++++++++++++++++++++++++++++++++++++++ src/lookup_table.zig | 41 ++++++++++++++++++++--------- src/util/decompressor.zig | 19 ++++++++++++++ src/util/metadata.zig | 8 +++--- 4 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 src/decomp/zlib.zig create mode 100644 src/util/decompressor.zig diff --git a/src/decomp/zlib.zig b/src/decomp/zlib.zig new file mode 100644 index 0000000..a8f123f --- /dev/null +++ b/src/decomp/zlib.zig @@ -0,0 +1,55 @@ +const std = @import("std"); +const Reader = std.Io.Reader; +const flate = std.compress.flate; + +const Decompressor = @import("../util/decompressor.zig"); +const Error = Decompressor.Error; + +pub fn Zlib(stateless: bool) type { + return if (stateless) + struct { + const Self = @This(); + + interface: Decompressor = .{ .decomp_fn = decomp }, + + const init: Self = .{}; + + fn decomp(_: *?Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + const buf = try alloc.alloc(u8, in.len * 2); + defer alloc.free(buf); + return zlibDecomp(buf, in, out); + } + } + else + struct { + const Self = @This(); + + interface: Decompressor = .{ .decomp_fn = decomp }, + + alloc: std.mem.Allocator, + + block_size: u32, + buffers: std.ArrayList([]u8), + buffer_queue: std.SinglyLinkedList, + + pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self { + return .{ + .alloc = alloc, + + .block_size = block_size, + .buffers = try .initCapacity(alloc, 20), + }; + } + pub fn deinit(self: Self) void { + for (self.buffers) |buf| + self.alloc.free(buf); + } + }; +} + +inline fn zlibDecomp(buffer: []u8, in: []u8, out: []u8) !usize { + var rdr: Reader = .fixed(in); + var decomp = flate.Decompress.init(&rdr, .zlib, buffer); + + return decomp.reader.readSliceShort(out); +} diff --git a/src/lookup_table.zig b/src/lookup_table.zig index efcb225..3147ced 100644 --- a/src/lookup_table.zig +++ b/src/lookup_table.zig @@ -1,9 +1,29 @@ const std = @import("std"); const Io = std.Io; +const Decompressor = @import("util/decompressor.zig"); const OffsetFile = @import("util/offset_file.zig"); const MetadataReader = @import("util/metadata.zig"); +pub fn lookupValue(comptime T: anytype, alloc: std.mem.Allocator, io: Io, decomp: *Decompressor, file: OffsetFile, table_start: u64, idx: u16) !T { + const T_PER_BLOCK: u16 = 8192 / @sizeOf(T); + + const block = idx / T_PER_BLOCK; + const block_offset = idx % T_PER_BLOCK; + + var rdr = try file.readerAt(io, table_start + (8 * block), &[0]u8{}); + var offset: u64 = undefined; + try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little); + + rdr = try file.readerAt(io, offset, &[0]u8{}); + var meta: MetadataReader = .init(alloc, &rdr, decomp); + + try meta.interface.discardAll(@sizeOf(T) * block_offset); + var out: T = undefined; + try meta.interface.readSliceEndian(T, @ptrCast(&out), .little); + return out; +} + pub fn CachedTable(comptime T: anytype) type { return struct { const T_PER_BLOCK: u16 = 8192 / @sizeOf(T); @@ -11,7 +31,6 @@ pub fn CachedTable(comptime T: anytype) type { const Table = @This(); alloc: std.mem.Allocator, - io: Io, fil: OffsetFile, table_start: u64, total_num: u32, @@ -20,10 +39,9 @@ pub fn CachedTable(comptime T: anytype) type { mut: Io.Mutex = .init, - pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, offset: u64, total_num: u32) Table { + pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, offset: u64, total_num: u32) Table { return .{ .alloc = alloc, - .io = io, .fil = fil, .table_start = offset, .total_num = total_num, @@ -32,34 +50,31 @@ pub fn CachedTable(comptime T: anytype) type { }; } - pub fn get(self: *Table, idx: u32) !T { + pub fn get(self: *Table, io: Io, idx: u32) !T { const block = idx / T_PER_BLOCK; const block_offset = idx % T_PER_BLOCK; if (self.table.contains(block)) return self.table.get(block).?[block_offset]; - try self.mut.lock(self.io); - defer self.mut.unlock(self.io); + try self.mut.lock(io); + defer self.mut.unlock(io); if (self.table.contains(block)) return self.table.get(block).?[block_offset]; - var rdr = try self.fil.readerAt(self.io, self.table_start + (8 * block), &[0]u8{}); + var rdr = try self.fil.readerAt(io, self.table_start + (8 * block), &[0]u8{}); var offset: u64 = undefined; try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little); - const arr_num: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK) + const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK) self.total_num % T_PER_BLOCK else T_PER_BLOCK; - rdr = try self.fil.readerAt(self.io, offset, &[0]u8{}); + rdr = try self.fil.readerAt(io, offset, &[0]u8{}); var meta: MetadataReader = .init(self.alloc, &rdr, self.decomp); - try self.table.put( - block, - try meta.interface.readSliceEndianAlloc(self.alloc, T, arr_num, .little), - ); + try self.table.put(block, try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little)); } }; } diff --git a/src/util/decompressor.zig b/src/util/decompressor.zig new file mode 100644 index 0000000..8b0ad00 --- /dev/null +++ b/src/util/decompressor.zig @@ -0,0 +1,19 @@ +//! A decompression interface + +const std = @import("std"); + +const Decompressor = @This(); + +pub const Error = std.Io.Reader.StreamError || std.mem.Allocator.Error; + +/// The actual decompression function. +/// If the given decompressor is null, then the decompression should be done "stateless" without lasting allocations. +decomp_fn: *fn (?*Decompressor, std.mem.Allocator, in: []u8, out: []u8) Error!usize, + +pub fn Decompress(self: *Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + return self.decomp_fn(self, alloc, in, out); +} + +pub fn StatelessDecompression(self: Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + return self.decomp_fn(null, alloc, in, out); +} diff --git a/src/util/metadata.zig b/src/util/metadata.zig index 2984c7c..56e22b0 100644 --- a/src/util/metadata.zig +++ b/src/util/metadata.zig @@ -4,7 +4,7 @@ const Writer = std.Io.Writer; const Limit = std.Io.Limit; const StreamError = std.Io.Reader.StreamError; -const DecompFn = @import("../decomp.zig").DecompFn; +const Decompressor = @import("decompressor.zig"); const BlockHeader = packed struct { size: u15, @@ -15,14 +15,14 @@ const This = @This(); alloc: std.mem.Allocator, rdr: *Reader, -decomp: DecompFn, +decomp: *Decompressor, buf: [8192]u8 = undefined, interface: Reader, err: ?anyerror = null, -pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: DecompFn) This { +pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: *Decompressor) This { return .{ .alloc = alloc, .rdr = rdr, @@ -52,7 +52,7 @@ fn advance(self: *This) !void { } var tmp_buf: [8192]u8 = undefined; try self.rdr.readSliceAll(tmp_buf[0..hdr.size]); - self.interface.end = try self.decomp(self.alloc, tmp_buf[0..hdr.size], &self.buf); + self.interface.end = try self.decomp.Decompress(self.alloc, tmp_buf[0..hdr.size], &self.buf); self.interface.buffer = self.buf[0..self.interface.end]; }