Started work on stateful decompression

2026-03-20 01:55:00 -05:00
parent 6b5c830234
commit df22cf6529
5 changed files with 238 additions and 77 deletions
@@ -23,17 +23,26 @@ pub fn build(b: *std.Build) !void {
    });
    mod.addOptions("config", zig_squashfs_options);
    if (!use_zig_decomp) {
-        var zlib_ng = b.dependency("zlib_ng", .{});
+        var zlib_ng = b.dependency("zlib_ng", .{
+            .target = target,
+            .optimize = optimize,
+        });
        mod.linkLibrary(zlib_ng.artifact("zng"));

        mod.linkSystemLibrary("lzma", .{ .preferred_link_mode = .static });
        if (allow_lzo == true)
            mod.linkSystemLibrary("minilzo", .{ .preferred_link_mode = .static });

-        var lz4 = b.dependency("lz4", .{});
+        var lz4 = b.dependency("lz4", .{
+            .target = target,
+            .optimize = optimize,
+        });
        mod.linkLibrary(lz4.artifact("lz4"));

-        var zstd = b.dependency("zstd", .{});
+        var zstd = b.dependency("zstd", .{
+            .target = target,
+            .optimize = optimize,
+        });
        mod.linkLibrary(zstd.artifact("zstd"));
    }

@@ -176,8 +176,6 @@ pub fn cLz4(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
    return error.Lz4DecompressFailed;
 }

-pub const zstdDecompress = if (!config.use_zig_decomp) cZstd else zigZstd;
-
 pub fn zigZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
    var rdr: Reader = .fixed(in);
    const buf = try alloc.alloc(u8, 1024 * 1024);
@@ -187,75 +185,3 @@ pub fn zigZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
        return decomp.err orelse err;
    };
 }
-fn cZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
-    _ = alloc;
-    const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
-    if (c.ZSTD_isError(res) == 0) return res;
-    return switch (c.ZSTD_getErrorCode(res)) {
-        c.ZSTD_error_prefix_unknown => cZstdError.PrefixUnknown,
-        c.ZSTD_error_version_unsupported => cZstdError.VersionUnsupported,
-        c.ZSTD_error_frameParameter_unsupported => cZstdError.FrameParameterUnsupported,
-        c.ZSTD_error_frameParameter_windowTooLarge => cZstdError.FrameParameterWindowTooLarge,
-        c.ZSTD_error_corruption_detected => cZstdError.CorruptionDetected,
-        c.ZSTD_error_checksum_wrong => cZstdError.ChecksumWrong,
-        c.ZSTD_error_literals_headerWrong => cZstdError.LiteralsHeaderWrong,
-        c.ZSTD_error_dictionary_corrupted => cZstdError.DictionaryCorrupted,
-        c.ZSTD_error_dictionary_wrong => cZstdError.DictionaryWrong,
-        c.ZSTD_error_dictionaryCreation_failed => cZstdError.DictionaryCreationFailed,
-        c.ZSTD_error_parameter_unsupported => cZstdError.ParameterUnsupported,
-        c.ZSTD_error_parameter_combination_unsupported => cZstdError.ParameterCombinationUnsupported,
-        c.ZSTD_error_parameter_outOfBound => cZstdError.ParameterOutOfBound,
-        c.ZSTD_error_tableLog_tooLarge => cZstdError.TableLogTooLarge,
-        c.ZSTD_error_maxSymbolValue_tooLarge => cZstdError.MaxSymbolValueTooLarge,
-        c.ZSTD_error_maxSymbolValue_tooSmall => cZstdError.MaxSymbolValueTooSmall,
-        c.ZSTD_error_stabilityCondition_notRespected => cZstdError.StabilityConditionNotRespected,
-        c.ZSTD_error_stage_wrong => cZstdError.StageWrong,
-        c.ZSTD_error_init_missing => cZstdError.InitMissing,
-        c.ZSTD_error_memory_allocation => cZstdError.MemoryAllocation,
-        c.ZSTD_error_workSpace_tooSmall => cZstdError.WorkSpaceTooSmall,
-        c.ZSTD_error_dstSize_tooSmall => cZstdError.DstSizeTooSmall,
-        c.ZSTD_error_srcSize_wrong => cZstdError.SrcSizeWrong,
-        c.ZSTD_error_dstBuffer_null => cZstdError.DstBufferNull,
-        c.ZSTD_error_noForwardProgress_destFull => cZstdError.NoForwardProgressDestFull,
-        c.ZSTD_error_noForwardProgress_inputEmpty => cZstdError.NoForwardProgressInputEmpty,
-        else => cZstdError.Generic,
-    };
-}
-
-pub const cZstdError = error{
-    Generic,
-    PrefixUnknown,
-    VersionUnsupported,
-    FrameParameterUnsupported,
-    FrameParameterWindowTooLarge,
-    CorruptionDetected,
-    ChecksumWrong,
-    LiteralsHeaderWrong,
-    DictionaryCorrupted,
-    DictionaryWrong,
-    DictionaryCreationFailed,
-    ParameterUnsupported,
-    ParameterCombinationUnsupported,
-    ParameterOutOfBound,
-    TableLogTooLarge,
-    MaxSymbolValueTooLarge,
-    MaxSymbolValueTooSmall,
-    CannotProduceUncompressedBlock,
-    StabilityConditionNotRespected,
-    StageWrong,
-    InitMissing,
-    MemoryAllocation,
-    WorkSpaceTooSmall,
-    DstSizeTooSmall,
-    SrcSizeWrong,
-    DstBufferNull,
-    NoForwardProgressDestFull,
-    NoForwardProgressInputEmpty,
-    FrameIndexTooLarge,
-    SeekableIo,
-    DstBufferWrong,
-    SrcBufferWrong,
-    SequenceProducerFailed,
-    ExternalSequencesInvalid,
-    MaxCode,
-};
@@ -0,0 +1,18 @@
+const std = @import("std");
+const Reader = std.Io.Reader;
+const builtin = @import("builtin");
+
+const config = if (builtin.is_test) .{
+    .use_zig_decomp = !builtin.link_libc,
+    .allow_lzo = false, // Change once LZO compilation is fixed
+} else @import("config");
+
+pub const c = @cImport({
+    @cInclude("zlib-ng.h");
+    @cInclude("lzma.h");
+    @cInclude("lz4.h");
+    @cInclude("zstd.h");
+    @cInclude("zstd_errors.h");
+    if (config.allow_lzo)
+        @cInclude("lzo/minilzo.h");
+});
@@ -0,0 +1,89 @@
+const std = @import("std");
+const builtin = @import("builtin");
+
+const c = @import("c.zig").c;
+const zng_stream = c.zng_stream;
+
+const ZlibErrors = error{
+    OutOfMemory,
+    OutputBufferTooSmall,
+    BadData,
+    StreamError,
+    Unknown,
+};
+
+const Self = @This();
+
+alloc: std.mem.Allocator,
+window_size: i16,
+
+streams: std.AutoHashMap(std.Thread.Id, zng_stream),
+
+pub fn init(alloc: std.mem.Allocator, window_size: i16) !Self {
+    return .{
+        .alloc = alloc,
+        .window_size = window_size,
+        .streams = .init(alloc),
+    };
+}
+pub fn deinit(self: Self) void {
+    var iter = self.streams.keyIterator();
+    while (iter.next()) |key| {
+        _ = c.inflateEnd(self.streams.getPtr(key).?);
+    }
+    self.streams.deinit(self.alloc);
+}
+
+pub fn decompress(self: *Self, in: []u8, out: []u8) ZlibErrors!usize {
+    var stream = try self.getOrCreate();
+    stream.next_in = in.ptr;
+    stream.avail_in = in.len;
+    stream.next_out = out.ptr;
+    stream.avail_out = out.len;
+    var res = c.zng_inflateReset2(stream, self.window_size);
+    switch (res) {
+        c.Z_OK => {},
+        c.Z_STREAM_ERROR => return ZlibErrors.StreamError,
+        else => return ZlibErrors.Unknown,
+    }
+    res = c.zng_inflate(stream, c.Z_FINISH);
+    return switch (res) {
+        c.Z_OK => stream.total_out,
+        c.Z_MEM_ERROR => ZlibErrors.NotEnoughMemory,
+        c.Z_BUF_ERROR => ZlibErrors.OutputBufferTooSmall,
+        c.Z_DATA_ERROR => ZlibErrors.BadData,
+        else => ZlibErrors.Unknown,
+    };
+}
+inline fn getOrCreate(self: *Self) ZlibErrors!*zng_stream {
+    const res = try self.streams.getOrPut(std.Thread.getCurrentId());
+    if (res.found_existing) return res.value_ptr;
+    res.value_ptr.* = .{
+        .zalloc = zalloc,
+        .zfree = zfree,
+        .@"opaque" = @ptrCast(self),
+    };
+    return res.value_ptr;
+}
+
+fn zalloc(self_ptr: ?*anyopaque, items: c_uint, size: c_uint) ?*anyopaque {
+    var self: *Self = @ptrCast(self_ptr);
+    return self.alloc.rawAlloc(items * size, .@"1", 0);
+}
+fn zfree(self_ptr: ?*anyopaque, alloc_ptr: ?*anyopaque) ?*anyopaque {
+    var self: *Self = @ptrCast(self_ptr);
+    self.alloc.rawFree(@ptrCast(alloc_ptr), .@"1", 0);
+}
+
+pub fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
+    _ = alloc;
+    var out_len: usize = out.len;
+    const res = c.zng_uncompress(out.ptr, &out_len, in.ptr, in.len);
+    return switch (res) {
+        c.Z_OK => out_len,
+        c.Z_MEM_ERROR => ZlibErrors.NotEnoughMemory,
+        c.Z_BUF_ERROR => ZlibErrors.OutputBufferTooSmall,
+        c.Z_DATA_ERROR => ZlibErrors.BadData,
+        else => ZlibErrors.Unknown,
+    };
+}
@@ -0,0 +1,119 @@
+const std = @import("std");
+const builtin = @import("builtin");
+
+const c = @import("c.zig").c;
+const DCtx = c.ZSTD_DCtx;
+
+const Self = @This();
+
+alloc: std.mem.Allocator,
+ctx: std.AutoHashMap(std.Thread.Id, DCtx),
+
+pub fn init(alloc: std.mem.Allocator) !Self {
+    return .{
+        .alloc = alloc,
+        .ctx = .init(alloc),
+    };
+}
+pub fn deinit(self: Self) void {
+    var iter = self.ctx.keyIterator();
+    while (iter.next()) |key| {
+        _ = c.ZSTD_freeDCtx(self.ctx.getPtr(key));
+    }
+    self.ctx.deinit(self.alloc);
+}
+
+pub fn decompress(self: *Self, in: []u8, out: []u8) ZstdError!usize {
+    const ctx = try self.getOrCreate();
+    const res = c.ZSTD_decompressDCtx(ctx, out.ptr, out.len, in.ptr, in.len);
+    try checkError(res);
+    return res;
+}
+inline fn getOrCreate(self: *Self) ZstdError!*DCtx {
+    const res = try self.ctx.getOrPut(std.Thread.getCurrentId());
+    if (res.found_existing) {
+        try checkError(c.ZSTD_DCtx_reset(res.value_ptr, c.ZSTD_reset_session_only));
+        return res.value_ptr;
+    }
+    res.value_ptr.* = c.ZSTD_createDCtx() orelse return ZstdError.OutOfMemory;
+    return res.value_ptr;
+}
+
+fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
+    _ = alloc;
+    const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
+    try checkError(res);
+    return res;
+}
+
+inline fn checkError(res: usize) !void {
+    if (res == 0) return;
+    if (c.ZSTD_isError(res) == 0) return;
+    return switch (c.ZSTD_getErrorCode(res)) {
+        c.ZSTD_error_prefix_unknown => ZstdError.PrefixUnknown,
+        c.ZSTD_error_version_unsupported => ZstdError.VersionUnsupported,
+        c.ZSTD_error_frameParameter_unsupported => ZstdError.FrameParameterUnsupported,
+        c.ZSTD_error_frameParameter_windowTooLarge => ZstdError.FrameParameterWindowTooLarge,
+        c.ZSTD_error_corruption_detected => ZstdError.CorruptionDetected,
+        c.ZSTD_error_checksum_wrong => ZstdError.ChecksumWrong,
+        c.ZSTD_error_literals_headerWrong => ZstdError.LiteralsHeaderWrong,
+        c.ZSTD_error_dictionary_corrupted => ZstdError.DictionaryCorrupted,
+        c.ZSTD_error_dictionary_wrong => ZstdError.DictionaryWrong,
+        c.ZSTD_error_dictionaryCreation_failed => ZstdError.DictionaryCreationFailed,
+        c.ZSTD_error_parameter_unsupported => ZstdError.ParameterUnsupported,
+        c.ZSTD_error_parameter_combination_unsupported => ZstdError.ParameterCombinationUnsupported,
+        c.ZSTD_error_parameter_outOfBound => ZstdError.ParameterOutOfBound,
+        c.ZSTD_error_tableLog_tooLarge => ZstdError.TableLogTooLarge,
+        c.ZSTD_error_maxSymbolValue_tooLarge => ZstdError.MaxSymbolValueTooLarge,
+        c.ZSTD_error_maxSymbolValue_tooSmall => ZstdError.MaxSymbolValueTooSmall,
+        c.ZSTD_error_stabilityCondition_notRespected => ZstdError.StabilityConditionNotRespected,
+        c.ZSTD_error_stage_wrong => ZstdError.StageWrong,
+        c.ZSTD_error_init_missing => ZstdError.InitMissing,
+        c.ZSTD_error_memory_allocation => ZstdError.MemoryAllocation,
+        c.ZSTD_error_workSpace_tooSmall => ZstdError.WorkSpaceTooSmall,
+        c.ZSTD_error_dstSize_tooSmall => ZstdError.DstSizeTooSmall,
+        c.ZSTD_error_srcSize_wrong => ZstdError.SrcSizeWrong,
+        c.ZSTD_error_dstBuffer_null => ZstdError.DstBufferNull,
+        c.ZSTD_error_noForwardProgress_destFull => ZstdError.NoForwardProgressDestFull,
+        c.ZSTD_error_noForwardProgress_inputEmpty => ZstdError.NoForwardProgressInputEmpty,
+        else => ZstdError.Generic,
+    };
+}
+pub const ZstdError = error{
+    OutOfMemory,
+    Generic,
+    PrefixUnknown,
+    VersionUnsupported,
+    FrameParameterUnsupported,
+    FrameParameterWindowTooLarge,
+    CorruptionDetected,
+    ChecksumWrong,
+    LiteralsHeaderWrong,
+    DictionaryCorrupted,
+    DictionaryWrong,
+    DictionaryCreationFailed,
+    ParameterUnsupported,
+    ParameterCombinationUnsupported,
+    ParameterOutOfBound,
+    TableLogTooLarge,
+    MaxSymbolValueTooLarge,
+    MaxSymbolValueTooSmall,
+    CannotProduceUncompressedBlock,
+    StabilityConditionNotRespected,
+    StageWrong,
+    InitMissing,
+    MemoryAllocation,
+    WorkSpaceTooSmall,
+    DstSizeTooSmall,
+    SrcSizeWrong,
+    DstBufferNull,
+    NoForwardProgressDestFull,
+    NoForwardProgressInputEmpty,
+    FrameIndexTooLarge,
+    SeekableIo,
+    DstBufferWrong,
+    SrcBufferWrong,
+    SequenceProducerFailed,
+    ExternalSequencesInvalid,
+    MaxCode,
+};