Moving decompression to a vtable interface

This commit is contained in:
Caleb J. Gardner
2026-03-22 06:35:25 -05:00
parent 54aaf30ea5
commit 8e4661c4c6
6 changed files with 311 additions and 79 deletions
+2 -1
View File
@@ -17,7 +17,8 @@
.hash = "lz4-1.10.0-6-ewyzw-4NAAAWDpY4xpiqr4LQhZQAC0x_rGnW2iPh6jk2",
},
.minilzo = .{
.path = "../zig-minilzo",
.url = "git+https://github.com/CalebQ42/zig-minilzo.git#7cbae997b91a44d74b7cd6c073584dc9562a6c90",
.hash = "minilzo-2.10.0-Ij7BO8wLAADeWI4Pe4jp8XTDsDaquZR14oZ7_9yKKDWP",
},
},
.paths = .{
+22
View File
@@ -0,0 +1,22 @@
const std = @import("std");
const Decompressor = @This();
pub const Error = error{
OutOfMemory,
BadInput,
OutputTooSmall,
};
vtable: *const struct {
decompress: *const fn (*Decompressor, []u8, []u8) Error!usize = DefaultDecompress,
stateless: *const fn (std.mem.Allocator, []u8, []u8) Error!usize,
},
pub fn decompress(self: *Decompressor, in: []u8, out: []u8) Error!usize {
return self.vtable.decompress(self, in, out);
}
fn DefaultDecompress(self: *Decompressor, in: []u8, out: []u8) Error!usize {
return self.vtable.stateless(std.heap.smp_allocator, in, out);
}
+56 -17
View File
@@ -1,6 +1,7 @@
const std = @import("std");
const builtin = @import("builtin");
const Decompressor = @import("../decomp.zig");
const c = @import("c.zig").c;
const zng_stream = c.zng_stream;
@@ -17,6 +18,12 @@ const Self = @This();
alloc: std.mem.Allocator,
streams: std.AutoHashMap(std.Thread.Id, zng_stream),
interface: Decompressor = .{ .vtable = &.{
.decompress = decompress,
.stateless = stateless,
} },
err: ?ZlibErrors = null,
pub fn init(alloc: std.mem.Allocator) !Self {
return .{
@@ -32,28 +39,48 @@ pub fn deinit(self: Self) void {
self.streams.deinit(self.alloc);
}
pub fn decompress(self: *Self, in: []u8, out: []u8) ZlibErrors!usize {
fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
var self: *Self = @fieldParentPtr("interface", decomp);
var stream = try self.getOrCreate();
stream.next_in = in.ptr;
stream.avail_in = in.len;
stream.avail_in = @truncate(in.len);
stream.next_out = out.ptr;
stream.avail_out = out.len;
stream.avail_out = @truncate(out.len);
var res = c.zng_inflateReset(stream);
switch (res) {
c.Z_OK => {},
c.Z_STREAM_ERROR => return ZlibErrors.StreamError,
else => return ZlibErrors.Unknown,
c.Z_STREAM_ERROR => {
self.err = ZlibErrors.StreamError;
return Decompressor.Error.BadInput;
},
else => {
self.err = ZlibErrors.Unknown;
return Decompressor.Error.BadInput;
},
}
res = c.zng_inflate(stream, c.Z_FINISH);
return switch (res) {
c.Z_OK => stream.total_out,
c.Z_MEM_ERROR => ZlibErrors.NotEnoughMemory,
c.Z_BUF_ERROR => ZlibErrors.OutputBufferTooSmall,
c.Z_DATA_ERROR => ZlibErrors.BadData,
else => ZlibErrors.Unknown,
};
switch (res) {
c.Z_OK => return stream.total_out,
c.Z_MEM_ERROR => {
self.err = ZlibErrors.OutOfMemory;
return Decompressor.Error.OutOfMemory;
},
c.Z_BUF_ERROR => {
self.err = ZlibErrors.OutputBufferTooSmall;
return Decompressor.Error.OutputTooSmall;
},
c.Z_DATA_ERROR => {
self.err = ZlibErrors.BadData;
return Decompressor.Error.BadInput;
},
else => {
self.err = ZlibErrors.Unknown;
return Decompressor.Error.BadInput;
},
}
inline fn getOrCreate(self: *Self) ZlibErrors!*zng_stream {
}
inline fn getOrCreate(self: *Self) Decompressor.Error!*zng_stream {
const res = try self.streams.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) return res.value_ptr;
res.value_ptr.* = .{
@@ -64,11 +91,23 @@ inline fn getOrCreate(self: *Self) ZlibErrors!*zng_stream {
return res.value_ptr;
}
fn zalloc(self_ptr: ?*anyopaque, items: c_uint, size: c_uint) ?*anyopaque {
var self: *Self = @ptrCast(self_ptr);
fn zalloc(self_ptr: ?*anyopaque, items: c_uint, size: c_uint) callconv(.c) ?*anyopaque {
var self: *Self = @ptrCast(@alignCast(self_ptr));
return self.alloc.rawAlloc(items * size, .@"1", 0);
}
fn zfree(self_ptr: ?*anyopaque, alloc_ptr: ?*anyopaque) ?*anyopaque {
var self: *Self = @ptrCast(self_ptr);
fn zfree(self_ptr: ?*anyopaque, alloc_ptr: ?*anyopaque) callconv(.c) void {
var self: *Self = @ptrCast(@alignCast(self_ptr));
self.alloc.rawFree(@ptrCast(alloc_ptr), .@"1", 0);
}
fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var out_len = out.len;
const res = c.zng_uncompress(out.ptr, &out_len, in.ptr, in.len);
return switch (res) {
c.Z_OK => out_len,
c.Z_MEM_ERROR => Decompressor.Error.OutOfMemory,
c.Z_BUF_ERROR => Decompressor.Error.OutputTooSmall,
c.Z_DATA_ERROR => Decompressor.Error.BadInput,
else => Decompressor.Error.BadInput,
};
}
+79 -12
View File
@@ -1,6 +1,7 @@
const std = @import("std");
const builtin = @import("builtin");
const Decompressor = @import("../decomp.zig");
const c = @import("c.zig").c;
const stream = c.lzma_stream;
@@ -10,6 +11,12 @@ alloc: std.mem.Allocator,
streams: std.AutoHashMap(std.Thread.Id, stream),
xz: bool,
interface: Decompressor = .{ .vtable = &.{
.decompress = decompress,
.stateless = stateless,
} },
err: ?LzmaError = null,
pub fn init(alloc: std.mem.Allocator, xz: bool) !Self {
return .{
@@ -25,7 +32,9 @@ pub fn deinit(self: Self) void {
self.streams.deinit();
}
pub fn decompress(self: *Self, in: []u8, out: []u8) LzmaError!usize {
pub fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
var self: *Self = @fieldParentPtr("interface", decomp);
var strm = try self.getOrCreate();
strm.next_in = in.ptr;
strm.avail_in = in.len;
@@ -38,25 +47,83 @@ pub fn decompress(self: *Self, in: []u8, out: []u8) LzmaError!usize {
c.lzma_alone_decoder(strm, in.len * 2);
switch (res) {
c.LZMA_OK => {},
c.LZMA_MEM_ERROR => return LzmaError.LzmaMemoryError,
c.LZMA_PROG_ERROR => return LzmaError.LzmaProgramError,
else => return LzmaError.UnknownResult,
c.LZMA_MEM_ERROR => {
self.err = LzmaError.LzmaMemoryError;
return Decompressor.Error.OutOfMemory;
},
c.LZMA_PROG_ERROR => {
self.err = LzmaError.LzmaProgramError;
return Decompressor.Error.BadInput;
},
else => {
self.err = LzmaError.Unknown;
return Decompressor.Error.BadInput;
},
}
while (res == c.LZMA_OK)
res = c.lzma_code(strm, c.LZMA_RUN);
switch (res) {
c.LZMA_STREAM_END => return strm.total_out,
c.LZMA_MEM_ERROR => {
self.err = LzmaError.LzmaMemoryError;
return Decompressor.Error.OutOfMemory;
},
c.LZMA_MEMLIMIT_ERROR => {
self.err = LzmaError.LzmaMemoryLimit;
return Decompressor.Error.OutOfMemory;
},
c.LZMA_FORMAT_ERROR => {
self.err = LzmaError.LzmaBadFormat;
return Decompressor.Error.BadInput;
},
c.LZMA_DATA_ERROR => {
self.err = LzmaError.LzmaDataCorrupt;
return Decompressor.Error.BadInput;
},
c.LZMA_BUF_ERROR => {
self.err = LzmaError.LzmaCannotProgress;
return Decompressor.Error.BadInput;
},
c.LZMA_PROG_ERROR => {
self.err = LzmaError.LzmaProgramError;
return Decompressor.Error.BadInput;
},
else => {
self.err = LzmaError.Unknown;
return Decompressor.Error.BadInput;
},
}
}
pub fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var strm = c.lzma_stream{
.next_in = in.ptr,
.avail_in = in.len,
.next_out = out.ptr,
.avail_out = out.len,
};
var res = c.lzma_auto_decoder(&strm, out.len * 2, 0);
switch (res) {
c.LZMA_OK => {},
c.LZMA_MEM_ERROR => return Decompressor.Error.OutOfMemory,
c.LZMA_PROG_ERROR => return Decompressor.Error.BadInput,
else => return Decompressor.Error.BadInput,
}
while (res == c.LZMA_OK)
res = c.lzma_code(&strm, c.LZMA_RUN);
return switch (res) {
c.LZMA_STREAM_END => strm.total_out,
c.LZMA_MEM_ERROR => LzmaError.LzmaMemoryError,
c.LZMA_MEMLIMIT_ERROR => LzmaError.LzmaMemoryLimit,
c.LZMA_FORMAT_ERROR => LzmaError.LzmaBadFormat,
c.LZMA_DATA_ERROR => LzmaError.LzmaDataCorrupt,
c.LZMA_BUF_ERROR => LzmaError.LzmaCannotProgress,
c.LZMA_PROG_ERROR => LzmaError.LzmaProgramError,
else => LzmaError.UnknownResult,
c.LZMA_MEM_ERROR => Decompressor.Error.OutOfMemory,
c.LZMA_MEMLIMIT_ERROR => Decompressor.Error.OutOfMemory,
c.LZMA_FORMAT_ERROR => Decompressor.Error.BadInput,
c.LZMA_DATA_ERROR => Decompressor.Error.BadInput,
c.LZMA_BUF_ERROR => Decompressor.Error.BadInput,
c.LZMA_PROG_ERROR => Decompressor.Error.BadInput,
else => Decompressor.Error.BadInput,
};
}
inline fn getOrCreate(self: *Self) LzmaError!*stream {
inline fn getOrCreate(self: *Self) Decompressor.Error!*stream {
const res = try self.streams.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) return res.value_ptr;
+13 -7
View File
@@ -1,31 +1,37 @@
const std = @import("std");
const Decompressor = @import("../decomp.zig");
const c = @import("c.zig").c;
pub fn lzo(in: []u8, out: []u8) anyerror!usize {
pub const LzoDecompressor = struct { interface: Decompressor = .{ .vtable = .{ .stateless = lzo } } };
fn lzo(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var res = c.lzo_init();
if (res != 0) return error.LzoInitFailed;
if (res != 0) return Decompressor.Error.BadInput;
var out_len: usize = out.len;
res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &out_len, null);
return switch (res) {
c.LZO_E_OK => out_len,
c.LZO_E_ERROR => error.LzoError,
c.LZO_E_OUT_OF_MEMORY => error.LzoOutOfMemory,
c.LZO_E_OUT_OF_MEMORY => Decompressor.Error.OutOfMemory,
c.LZO_E_NOT_COMPRESSIBLE => error.LzoNotCompressible,
c.LZO_E_INPUT_OVERRUN => error.LzoInputOverrun,
c.LZO_E_OUTPUT_OVERRUN => error.LzoOutputOverrun,
c.LZO_E_LOOKBEHIND_OVERRUN => error.LzoLookbehindOverrun,
c.LZO_E_EOF_NOT_FOUND => error.LzoEofNotFound,
c.LZO_E_INPUT_NOT_CONSUMED => error.LzoInputNotConsumed,
c.LZO_E_INPUT_NOT_CONSUMED => Decompressor.Error.OutputTooSmall,
c.LZO_E_NOT_YET_IMPLEMENTED => error.LzoNotYetImplemented,
c.LZO_E_INVALID_ARGUMENT => error.LzoInvalidArgument,
c.LZO_E_INVALID_ALIGNMENT => error.LzoInvalidAlignment,
c.LZO_E_OUTPUT_NOT_CONSUMED => error.LzoOutputNotConsumed,
c.LZO_E_INTERNAL_ERROR => error.LzoInternalError,
c.LZO_E_OUTPUT_NOT_CONSUMED, c.LZO_E_OUTPUT_OVERRUN => Decompressor.Error.OutputTooSmall,
else => error.UnknownResult,
};
}
pub fn lz4(in: []u8, out: []u8) anyerror!usize {
pub const Lz4Decompressor = struct { interface: Decompressor = .{ .vtable = .{ .stateless = lz4 } } };
fn lz4(_: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
const res = c.LZ4_decompress_safe(in.ptr, out.ptr, @intCast(in.len), @intCast(out.len));
if (res > 0) return @abs(res); // TODO: Find out what error values it can return.
return error.Lz4DecompressFailed;
+139 -42
View File
@@ -1,13 +1,20 @@
const std = @import("std");
const builtin = @import("builtin");
const Decompressor = @import("../decomp.zig");
const c = @import("c.zig").c;
const DCtx = c.ZSTD_DCtx;
const Self = @This();
alloc: std.mem.Allocator,
ctx: std.AutoHashMap(std.Thread.Id, DCtx),
ctx: std.AutoHashMap(std.Thread.Id, *DCtx),
interface: Decompressor = .{ .vtable = &.{
.decompress = decompress,
.stateless = stateless,
} },
err: ?ZstdError = null,
pub fn init(alloc: std.mem.Allocator) !Self {
return .{
@@ -23,61 +30,151 @@ pub fn deinit(self: Self) void {
self.ctx.deinit(self.alloc);
}
pub fn decompress(self: *Self, in: []u8, out: []u8) ZstdError!usize {
pub fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
var self: *Self = @fieldParentPtr("interface", decomp);
const ctx = try self.getOrCreate();
const res = c.ZSTD_decompressDCtx(ctx, out.ptr, out.len, in.ptr, in.len);
try checkError(res);
try self.checkError(res);
return res;
}
inline fn getOrCreate(self: *Self) ZstdError!*DCtx {
inline fn getOrCreate(self: *Self) Decompressor.Error!*DCtx {
const res = try self.ctx.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) {
try checkError(c.ZSTD_DCtx_reset(res.value_ptr, c.ZSTD_reset_session_only));
return res.value_ptr;
try self.checkError(c.ZSTD_DCtx_reset(res.value_ptr.*, c.ZSTD_reset_session_only));
return res.value_ptr.*;
}
res.value_ptr.* = c.ZSTD_createDCtx() orelse return ZstdError.OutOfMemory;
return res.value_ptr;
res.value_ptr.* = c.ZSTD_createDCtx() orelse return Decompressor.Error.OutOfMemory;
return res.value_ptr.*;
}
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
_ = alloc;
fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
try checkError(res);
return res;
if (c.ZSTD_isError(res) == 0) return res;
return switch (c.ZSTD_getErrorCode(res)) {
c.ZSTD_error_memory_allocation => Decompressor.Error.OutOfMemory,
c.ZSTD_error_workSpace_tooSmall,
c.ZSTD_error_dstSize_tooSmall,
c.ZSTD_error_dstBuffer_null,
c.ZSTD_error_noForwardProgress_destFull,
=> Decompressor.Error.OutputTooSmall,
else => Decompressor.Error.BadInput,
};
}
inline fn checkError(res: usize) !void {
inline fn checkError(self: *Self, res: usize) Decompressor.Error!void {
if (res == 0) return;
if (c.ZSTD_isError(res) == 0) return;
return switch (c.ZSTD_getErrorCode(res)) {
c.ZSTD_error_prefix_unknown => ZstdError.PrefixUnknown,
c.ZSTD_error_version_unsupported => ZstdError.VersionUnsupported,
c.ZSTD_error_frameParameter_unsupported => ZstdError.FrameParameterUnsupported,
c.ZSTD_error_frameParameter_windowTooLarge => ZstdError.FrameParameterWindowTooLarge,
c.ZSTD_error_corruption_detected => ZstdError.CorruptionDetected,
c.ZSTD_error_checksum_wrong => ZstdError.ChecksumWrong,
c.ZSTD_error_literals_headerWrong => ZstdError.LiteralsHeaderWrong,
c.ZSTD_error_dictionary_corrupted => ZstdError.DictionaryCorrupted,
c.ZSTD_error_dictionary_wrong => ZstdError.DictionaryWrong,
c.ZSTD_error_dictionaryCreation_failed => ZstdError.DictionaryCreationFailed,
c.ZSTD_error_parameter_unsupported => ZstdError.ParameterUnsupported,
c.ZSTD_error_parameter_combination_unsupported => ZstdError.ParameterCombinationUnsupported,
c.ZSTD_error_parameter_outOfBound => ZstdError.ParameterOutOfBound,
c.ZSTD_error_tableLog_tooLarge => ZstdError.TableLogTooLarge,
c.ZSTD_error_maxSymbolValue_tooLarge => ZstdError.MaxSymbolValueTooLarge,
c.ZSTD_error_maxSymbolValue_tooSmall => ZstdError.MaxSymbolValueTooSmall,
c.ZSTD_error_stabilityCondition_notRespected => ZstdError.StabilityConditionNotRespected,
c.ZSTD_error_stage_wrong => ZstdError.StageWrong,
c.ZSTD_error_init_missing => ZstdError.InitMissing,
c.ZSTD_error_memory_allocation => ZstdError.MemoryAllocation,
c.ZSTD_error_workSpace_tooSmall => ZstdError.WorkSpaceTooSmall,
c.ZSTD_error_dstSize_tooSmall => ZstdError.DstSizeTooSmall,
c.ZSTD_error_srcSize_wrong => ZstdError.SrcSizeWrong,
c.ZSTD_error_dstBuffer_null => ZstdError.DstBufferNull,
c.ZSTD_error_noForwardProgress_destFull => ZstdError.NoForwardProgressDestFull,
c.ZSTD_error_noForwardProgress_inputEmpty => ZstdError.NoForwardProgressInputEmpty,
else => ZstdError.Generic,
};
switch (c.ZSTD_getErrorCode(res)) {
c.ZSTD_error_prefix_unknown => {
self.err = ZstdError.PrefixUnknown;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_version_unsupported => {
self.err = ZstdError.VersionUnsupported;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_frameParameter_unsupported => {
self.err = ZstdError.FrameParameterUnsupported;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_frameParameter_windowTooLarge => {
self.err = ZstdError.FrameParameterWindowTooLarge;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_corruption_detected => {
self.err = ZstdError.CorruptionDetected;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_checksum_wrong => {
self.err = ZstdError.ChecksumWrong;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_literals_headerWrong => {
self.err = ZstdError.LiteralsHeaderWrong;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_dictionary_corrupted => {
self.err = ZstdError.DictionaryCorrupted;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_dictionary_wrong => {
self.err = ZstdError.DictionaryWrong;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_dictionaryCreation_failed => {
self.err = ZstdError.DictionaryCreationFailed;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_parameter_unsupported => {
self.err = ZstdError.ParameterUnsupported;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_parameter_combination_unsupported => {
self.err = ZstdError.ParameterCombinationUnsupported;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_parameter_outOfBound => {
self.err = ZstdError.ParameterOutOfBound;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_tableLog_tooLarge => {
self.err = ZstdError.TableLogTooLarge;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_maxSymbolValue_tooLarge => {
self.err = ZstdError.MaxSymbolValueTooLarge;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_maxSymbolValue_tooSmall => {
self.err = ZstdError.MaxSymbolValueTooSmall;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_stabilityCondition_notRespected => {
self.err = ZstdError.StabilityConditionNotRespected;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_stage_wrong => {
self.err = ZstdError.StageWrong;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_init_missing => {
self.err = ZstdError.InitMissing;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_memory_allocation => {
self.err = ZstdError.MemoryAllocation;
return Decompressor.Error.OutOfMemory;
},
c.ZSTD_error_workSpace_tooSmall => {
self.err = ZstdError.WorkSpaceTooSmall;
return Decompressor.Error.OutputTooSmall;
},
c.ZSTD_error_dstSize_tooSmall => {
self.err = ZstdError.DstSizeTooSmall;
return Decompressor.Error.OutputTooSmall;
},
c.ZSTD_error_srcSize_wrong => {
self.err = ZstdError.SrcSizeWrong;
return Decompressor.Error.BadInput;
},
c.ZSTD_error_dstBuffer_null => {
self.err = ZstdError.DstBufferNull;
return Decompressor.Error.OutputTooSmall;
},
c.ZSTD_error_noForwardProgress_destFull => {
self.err = ZstdError.NoForwardProgressDestFull;
return Decompressor.Error.OutputTooSmall;
},
c.ZSTD_error_noForwardProgress_inputEmpty => {
self.err = ZstdError.NoForwardProgressInputEmpty;
return Decompressor.Error.BadInput;
},
else => {
self.err = ZstdError.Generic;
return Decompressor.Error.BadInput;
},
}
}
pub const ZstdError = error{
OutOfMemory,