From 5f1089406e18d3ff2f23eab0733e4446d48ffac5 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sat, 23 May 2026 06:37:34 -0500 Subject: [PATCH] Re-added all C decompressors Some cleanup Remove inode arena Added deinit to Archive to destroy the File.MemoryMap --- build.zig.zon | 4 ++ src/archive.zig | 3 + src/bin/unsquashfs.zig | 2 +- src/c.h | 6 ++ src/decomp.zig | 62 +++++++++++------ src/decomp/c_lz4.zig | 25 +++++++ src/decomp/c_lzma.zig | 48 +++++++++++++ src/decomp/c_lzo.zig | 35 ++++++++++ src/decomp/c_xz.zig | 48 +++++++++++++ src/decomp/c_zlib.zig | 98 +++++++++++++++++++++++++++ src/decomp/c_zstd.zig | 71 +++++++++++++++++++ src/decomp/{lzma.zig => zig_lzma.zig} | 57 +++++++--------- src/decomp/{xz.zig => zig_xz.zig} | 49 +++++++------- src/decomp/{zlib.zig => zig_zlib.zig} | 48 +++++++------ src/decomp/{zstd.zig => zig_zstd.zig} | 14 +--- src/inode.zig | 78 ++++++++++++++------- src/test.zig | 3 + 17 files changed, 520 insertions(+), 131 deletions(-) create mode 100644 src/decomp/c_lz4.zig create mode 100644 src/decomp/c_lzma.zig create mode 100644 src/decomp/c_lzo.zig create mode 100644 src/decomp/c_xz.zig create mode 100644 src/decomp/c_zlib.zig create mode 100644 src/decomp/c_zstd.zig rename src/decomp/{lzma.zig => zig_lzma.zig} (52%) rename src/decomp/{xz.zig => zig_xz.zig} (58%) rename src/decomp/{zlib.zig => zig_zlib.zig} (57%) rename src/decomp/{zstd.zig => zig_zstd.zig} (83%) diff --git a/build.zig.zon b/build.zig.zon index c4c13be..4242f9c 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -20,6 +20,10 @@ .url = "git+https://github.com/CalebQ42/zig-minilzo.git#7cbae997b91a44d74b7cd6c073584dc9562a6c90", .hash = "minilzo-2.10.0-Ij7BO8wLAADeWI4Pe4jp8XTDsDaquZR14oZ7_9yKKDWP", }, + .xz = .{ + .url = "git+https://github.com/akunaakwei/zig-xz.git#e2d389262c8291907e3e4c6fb119819141c16c0f", + .hash = "xz-5.8.2-6v47_JYeAABSL-jonprpL5-E_YaaGc4B5xrbe93WsJ3G", + }, }, .paths = .{ "build.zig", diff --git a/src/archive.zig b/src/archive.zig index 25e6ff6..33588b9 100644 --- a/src/archive.zig +++ b/src/archive.zig @@ -31,6 +31,9 @@ pub fn init(io: Io, file: std.Io.File, offset: u64) !Archive { .stateless_decomp = try Decomp.StatelessDecomp(super.compression), }; } +pub fn deinit(self: Archive, io: Io) void { + self.file.deinit(io); +} /// The root folder of the Archive. Used to open other Files. pub fn root(self: Archive, alloc: std.mem.Allocator, io: Io) !File { diff --git a/src/bin/unsquashfs.zig b/src/bin/unsquashfs.zig index c278ec3..2d11698 100644 --- a/src/bin/unsquashfs.zig +++ b/src/bin/unsquashfs.zig @@ -4,7 +4,7 @@ const Writer = Io.Writer; const builtin = @import("builtin"); const config = @import("config"); -const squashfs = @import("zig_squashfs"); +const squashfs = @import("squashfs"); //TODO: Add more options const help_mgs = diff --git a/src/c.h b/src/c.h index b1152a8..2069065 100644 --- a/src/c.h +++ b/src/c.h @@ -1 +1,7 @@ #include +#include +#include +#ifdef ALLOW_LZO +#include +#endif +#include diff --git a/src/decomp.zig b/src/decomp.zig index 7441044..2ffbd08 100644 --- a/src/decomp.zig +++ b/src/decomp.zig @@ -1,7 +1,16 @@ const std = @import("std"); +const options = @import("options"); + const Decompressor = @import("util/decompressor.zig"); +const zlib = if (options.use_zig_decomp) @import("decomp/zig_zlib.zig") else @import("decomp/c_zlib.zig"); +const lzma = if (options.use_zig_decomp) @import("decomp/zig_lzma.zig") else @import("decomp/c_lzma.zig"); +const lzo = if (options.use_zig_decomp or !options.allow_lzo) void else @import("decomp/c_lzo.zig"); +const xz = if (options.use_zig_decomp) @import("decomp/zig_xz.zig") else @import("decomp/c_xz.zig"); +const lz4 = if (options.use_zig_decomp) void else @import("decomp/c_lz4.zig"); +const zstd = if (options.use_zig_decomp) @import("decomp/zig_zstd.zig") else @import("decomp/c_zstd.zig"); + pub const Enum = enum(u16) { gzip = 1, lzma, @@ -13,40 +22,55 @@ pub const Enum = enum(u16) { pub fn StatelessDecomp(val: Enum) !*const Decompressor { return switch (val) { - .gzip => &@import("decomp/zlib.zig").stateless_decompressor, - .lzma => &@import("decomp/lzma.zig").stateless_decompressor, - .lzo => error.LzoUnsupported, - .xz => &@import("decomp/xz.zig").stateless_decompressor, - .lz4 => error.Lz4Unsupported, - .zstd => &@import("decomp/zstd.zig").stateless_decompressor, + .gzip => &zlib.stateless_decompressor, + .lzma => &lzma.stateless_decompressor, + .lzo => if (options.use_zig_decomp or !options.allow_lzo) + error.LzoUnsupported + else + &lzo.stateless_decompressor, + .xz => &xz.stateless_decompressor, + .lz4 => if (options.use_zig_decomp) + error.Lz4Unsupported + else + &lz4.stateless_decompressor, + .zstd => &zstd.stateless_decompressor, }; } pub const Decomp = union(enum) { - gzip: @import("decomp/zlib.zig"), - lzma: @import("decomp/lzma.zig"), - lzo: void, - xz: @import("decomp/xz.zig"), - lz4: void, - zstd: @import("decomp/zstd.zig"), + gzip: zlib, + lzma: lzma, + lzo: lzo, + xz: xz, + lz4: lz4, + zstd: zstd, + pub fn init(val: Enum, alloc: std.mem.Allocator) !Decomp { + return switch (val) { + .gzip => .{ .gzip = zlib.init(alloc) }, + .lzma => .{ .lzma = .{} }, + .lzo => .{ .lzo = .{} }, + .xz => .{ .xz = .{} }, + .lz4 => .{ .lz4 = .{} }, + .zstd => .{ .zstd = zstd.init(alloc) }, + }; + } pub fn deinit(self: *Decomp) void { switch (self.*) { .gzip => self.gzip.deinit(), - .lzma => self.lzma.deinit(), - .xz => self.xz.deinit(), .zstd => self.zstd.deinit(), - else => unreachable, + else => {}, } } - pub fn decompressor(self: *Decomp) *Decompressor { + pub fn decompressor(self: *Decomp) *const Decompressor { return switch (self.*) { .gzip => &self.gzip.interface, - .lzma => &self.lzma.interface, - .xz => &self.xz.interface, + .lzma => &lzma.stateless_decompressor, + .lzo => &lzo.stateless_decompressor, + .xz => &xz.stateless_decompressor, + .lz4 => &lz4.stateless_decompressor, .zstd => &self.zstd.interface, - else => unreachable, }; } }; diff --git a/src/decomp/c_lz4.zig b/src/decomp/c_lz4.zig new file mode 100644 index 0000000..09e95f2 --- /dev/null +++ b/src/decomp/c_lz4.zig @@ -0,0 +1,25 @@ +const std = @import("std"); + +const c = @import("c"); + +const Decompressor = @import("../util/decompressor.zig"); +const Error = Decompressor.Error; + +pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; + +fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + const res = c.LZ4_decompress_fast(in.ptr, out.ptr, out.len); + if (res < 0) return Error.ReadFailed; + return @abs(res); +} + +// lzma_allocator + +fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { + var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); + return alloc.rawAlloc(size, .@"1", 0); +} +fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { + var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); + alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); +} diff --git a/src/decomp/c_lzma.zig b/src/decomp/c_lzma.zig new file mode 100644 index 0000000..6e890b2 --- /dev/null +++ b/src/decomp/c_lzma.zig @@ -0,0 +1,48 @@ +const std = @import("std"); +const Io = std.Io; +const Reader = std.Io.Reader; +const zstd = std.compress.zstd; +const Node = std.SinglyLinkedList.Node; + +const c = @import("c"); + +const Decompressor = @import("../util/decompressor.zig"); +const Error = Decompressor.Error; + +const Queue = std.Io.Queue(c.lzma_stream); + +const Self = @This(); + +pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; + +fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + var stream: c.lzma_stream = .{ + .allocator = &.{ + .alloc = lzmaAlloc, + .free = lzmaFree, + .@"opaque" = @constCast(&alloc), + }, + .next_in = in.ptr, + .avail_in = in.len, + .next_out = out.ptr, + .avail_out = out.len, + }; + + var res = c.lzma_alone_decoder(&stream, stream.avail_out * 2); + if (res != c.LZMA_OK) return Error.ReadFailed; + while (res == c.LZMA_OK) + res = c.lzma_code(&stream, c.LZMA_RUN); + if (res != c.LZMA_FINISH) return Error.ReadFailed; + return stream.total_out; +} + +// lzma_allocator + +fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { + var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); + return alloc.rawAlloc(size, .@"1", 0); +} +fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { + var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); + alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); +} diff --git a/src/decomp/c_lzo.zig b/src/decomp/c_lzo.zig new file mode 100644 index 0000000..851519f --- /dev/null +++ b/src/decomp/c_lzo.zig @@ -0,0 +1,35 @@ +const std = @import("std"); +const Io = std.Io; +const Reader = std.Io.Reader; +const zstd = std.compress.zstd; +const Node = std.SinglyLinkedList.Node; + +const c = @import("c"); + +const Decompressor = @import("../util/decompressor.zig"); +const Error = Decompressor.Error; + +const Queue = std.Io.Queue(c.lzma_stream); + +const Self = @This(); + +pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; + +fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + _ = c.lzo_init(); + var out_len = out.len; + const res = c.lzo1x_decompress_safe(in.ptr, in.len, out.ptr, &out_len, null); + if (res != c.LZO_E_OK) return Error.ReadFailed; + return out_len; +} + +// lzma_allocator + +fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { + var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); + return alloc.rawAlloc(size, .@"1", 0); +} +fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { + var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); + alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); +} diff --git a/src/decomp/c_xz.zig b/src/decomp/c_xz.zig new file mode 100644 index 0000000..6e890b2 --- /dev/null +++ b/src/decomp/c_xz.zig @@ -0,0 +1,48 @@ +const std = @import("std"); +const Io = std.Io; +const Reader = std.Io.Reader; +const zstd = std.compress.zstd; +const Node = std.SinglyLinkedList.Node; + +const c = @import("c"); + +const Decompressor = @import("../util/decompressor.zig"); +const Error = Decompressor.Error; + +const Queue = std.Io.Queue(c.lzma_stream); + +const Self = @This(); + +pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; + +fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + var stream: c.lzma_stream = .{ + .allocator = &.{ + .alloc = lzmaAlloc, + .free = lzmaFree, + .@"opaque" = @constCast(&alloc), + }, + .next_in = in.ptr, + .avail_in = in.len, + .next_out = out.ptr, + .avail_out = out.len, + }; + + var res = c.lzma_alone_decoder(&stream, stream.avail_out * 2); + if (res != c.LZMA_OK) return Error.ReadFailed; + while (res == c.LZMA_OK) + res = c.lzma_code(&stream, c.LZMA_RUN); + if (res != c.LZMA_FINISH) return Error.ReadFailed; + return stream.total_out; +} + +// lzma_allocator + +fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { + var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); + return alloc.rawAlloc(size, .@"1", 0); +} +fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { + var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); + alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); +} diff --git a/src/decomp/c_zlib.zig b/src/decomp/c_zlib.zig new file mode 100644 index 0000000..4dd6833 --- /dev/null +++ b/src/decomp/c_zlib.zig @@ -0,0 +1,98 @@ +const std = @import("std"); +const Io = std.Io; +const Reader = std.Io.Reader; +const zstd = std.compress.zstd; +const Node = std.SinglyLinkedList.Node; + +const c = @import("c"); + +const Decompressor = @import("../util/decompressor.zig"); +const Error = Decompressor.Error; + +const Queue = std.Io.Queue(c.zng_stream); + +const Self = @This(); + +interface: Decompressor = .{ .decomp_fn = decomp }, + +io: Io, + +ctx: []c.zng_stream, +ctx_queue: Queue, + +pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self { + const buf = try alloc.alloc(c.zng_stream, 20); // TODO: Choose a better number instead of a random one. + var queue: Queue = .init(buf); + for (0..20) |_| + try queue.putOne(io, .{ + .zalloc = zalloc, + .zfree = zfree, + }); + + return .{ + .alloc = alloc, + .io = io, + + .block_size = block_size, + .ctx = buf, + .ctx_queue = queue, + }; +} +pub fn deinit(self: *Self, alloc: std.mem.Allocator) void { + self.ctx_queue.close(self.io); + alloc.free(self.ctx); +} + +fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + if (d == null) { + return statelessDecomp(d, alloc, in, out); + } + var self: *Self = @fieldParentPtr("interface", @constCast(d.?)); + + const stream = self.ctx_queue.getOne(self.io) catch return Error.ReadFailed; + defer self.ctx_queue.putOne(self.io, stream) catch {}; + + stream.@"opaque" = @constCast(&alloc); + stream.next_in = in.ptr; + stream.avail_in = @truncate(in.len); + stream.next_out = out.ptr; + stream.avail_out = @truncate(out.len); + + try zlibDecomp(&stream, in, out); + + return stream.total_out; +} + +inline fn zlibDecomp(stream: *c.zng_stream) !void { + _ = c.zng_inflateReset(stream); + + const res = c.zng_inflate(stream, c.Z_FULL_FLUSH); + if (res != c.Z_OK) return Error.ReadFailed; +} + +// Stateless + +pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; + +fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + var stream: c.zng_stream = .{ + .@"opaque" = @constCast(&alloc), + .next_in = in.ptr, + .avail_in = @truncate(in.len), + .next_out = out.ptr, + .avail_out = @truncate(out.len), + }; + try zlibDecomp(&stream); + return stream.total_out; +} + +// zalloc + +fn zalloc(ptr: ?*anyopaque, size: c_uint, len: c_uint) callconv(.c) ?*anyopaque { + var alloc: *std.mem.Allocator = @ptrCast(ptr); + return alloc.rawAlloc(size * len, .@"1", 0); +} +fn zfree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { + var alloc: *std.mem.Allocator = @ptrCast(ptr); + alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); +} diff --git a/src/decomp/c_zstd.zig b/src/decomp/c_zstd.zig new file mode 100644 index 0000000..e9c60f1 --- /dev/null +++ b/src/decomp/c_zstd.zig @@ -0,0 +1,71 @@ +const std = @import("std"); +const Io = std.Io; +const Reader = std.Io.Reader; +const zstd = std.compress.zstd; +const Node = std.SinglyLinkedList.Node; + +const c = @import("c"); + +const Decompressor = @import("../util/decompressor.zig"); +const Error = Decompressor.Error; + +const Queue = std.Io.Queue([]u8); + +const Self = @This(); + +interface: Decompressor = .{ .decomp_fn = decomp }, + +io: Io, + +ctx: []?*c.ZSTD_DCtx, +ctx_queue: Queue, + +pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self { + const buf = try alloc.alloc(?*c.ZSTD_DCtx, 20); // TODO: Choose a better number instead of a random one. + var queue: Queue = .init(buf); + for (0..20) |_| + try queue.putOne(io, c.ZSTD_createDCtx()); + + return .{ + .alloc = alloc, + .io = io, + + .block_size = block_size, + .ctx = buf, + .ctx_queue = queue, + }; +} +pub fn deinit(self: *Self, alloc: std.mem.Allocator) void { + self.ctx_queue.close(self.io); + for (self.ctx) |ctx| + c.ZSTD_freeDCtx(ctx); + alloc.free(self.ctx); +} + +fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + if (d == null) { + return statelessDecomp(d, alloc, in, out); + } + var self: *Self = @fieldParentPtr("interface", @constCast(d.?)); + + const ctx = self.ctx_queue.getOne(self.io) catch return Error.ReadFailed; + defer self.ctx_queue.putOne(self.io, ctx) catch {}; + + _ = c.ZSTD_DCtx_reset(ctx, c.ZSTD_reset_session_only); + + const res = c.ZSTD_decompressDCtx(ctx, out.ptr, out.len, in.ptr, in.len); + if (c.ZSTD_isError(res) != 0) + return Error.ReadFailed; + return res; +} + +// Stateless + +pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; + +fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize { + const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len); + if (c.ZSTD_isError(res) != 0) + return Error.ReadFailed; + return res; +} diff --git a/src/decomp/lzma.zig b/src/decomp/zig_lzma.zig similarity index 52% rename from src/decomp/lzma.zig rename to src/decomp/zig_lzma.zig index fa8a74f..622eac4 100644 --- a/src/decomp/lzma.zig +++ b/src/decomp/zig_lzma.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const Io = std.Io; const Reader = std.Io.Reader; const lzma = std.compress.lzma; const Node = std.SinglyLinkedList.Node; @@ -6,6 +7,8 @@ const Node = std.SinglyLinkedList.Node; const Decompressor = @import("../util/decompressor.zig"); const Error = Decompressor.Error; +const Queue = Io.Queue([]u8); + const Self = @This(); const Buffer = struct { @@ -16,57 +19,49 @@ const Buffer = struct { interface: Decompressor = .{ .decomp_fn = decomp }, alloc: std.mem.Allocator, +io: Io, block_size: u32, -buffers: std.ArrayList(Buffer), -buffer_queue: std.SinglyLinkedList = .{}, +buf: [][]u8, +buf_queue: Queue, + +pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self { + const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one. + var queue: Queue = .init(buf); + for (0..20) |_| + try queue.putOne(io, try alloc.alloc(u8, block_size)); -pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self { return .{ .alloc = alloc, + .io = io, .block_size = block_size, - .buffers = try .initCapacity(alloc, 5), + .buf = buf, + .buf_queue = queue, }; } pub fn deinit(self: *Self) void { - for (self.buffers.items) |buf| - self.alloc.free(buf.buf); - self.buffers.deinit(self.alloc); + self.buf_queue.close(self.io); + for (self.buf) |buf| + self.alloc.free(buf); + self.alloc.free(self.buf); } fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { if (d == null) { - var buf = try alloc.alloc(u8, in.len * 2); - defer alloc.free(buf); - return lzmaDecomp(alloc, &buf, in, out) catch |err| return switch (err) { - error.OutOfMemory => Error.OutOfMemory, - else => Error.ReadFailed, - }; + return statelessDecomp(d, alloc, in, out); } var self: *Self = @fieldParentPtr("interface", @constCast(d.?)); - const buf_node = self.buffer_queue.popFirst(); - var buf: *Buffer = undefined; - if (buf_node == null) { - const new_buf = try self.buffers.addOne(self.alloc); - new_buf.* = .{ .node = .{}, .buf = try self.alloc.alloc(u8, self.block_size) }; - buf = new_buf; - } else { - buf = @fieldParentPtr("node", buf_node.?); - } - defer self.buffer_queue.prepend(&buf.node); - return lzmaDecomp(self.alloc, &buf.buf, in, out) catch |err| { - // self.err = err; - return switch (err) { - error.OutOfMemory => Error.OutOfMemory, - else => Error.ReadFailed, - }; - }; + + const buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed; + defer self.buf_queue.putOne(self.io, buf) catch {}; + + return lzmaDecomp(self.alloc, &buf.buf, in, out) catch return Error.ReadFailed; } inline fn lzmaDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize { var rdr: Reader = .fixed(in); - var d = try lzma.Decompress.initOptions(&rdr, alloc, buffer.*, .{ .allow_incomplete = true }, 3 * 1024 * 1024); + var d = try lzma.Decompress.initOptions(&rdr, alloc, buffer.*, .{}); defer { buffer.* = d.takeBuffer(); d.deinit(); diff --git a/src/decomp/xz.zig b/src/decomp/zig_xz.zig similarity index 58% rename from src/decomp/xz.zig rename to src/decomp/zig_xz.zig index d13ff3e..2eae4ab 100644 --- a/src/decomp/xz.zig +++ b/src/decomp/zig_xz.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const Io = std.Io; const Reader = std.Io.Reader; const xz = std.compress.xz; const Node = std.SinglyLinkedList.Node; @@ -6,6 +7,8 @@ const Node = std.SinglyLinkedList.Node; const Decompressor = @import("../util/decompressor.zig"); const Error = Decompressor.Error; +const Queue = Io.Queue([]u8); + const Self = @This(); const Buffer = struct { @@ -16,46 +19,44 @@ const Buffer = struct { interface: Decompressor = .{ .decomp_fn = decomp }, alloc: std.mem.Allocator, +io: Io, block_size: u32, -buffers: std.ArrayList(Buffer), -buffer_queue: std.SinglyLinkedList = .{}, +buf: [][]u8, +buf_queue: Queue, + +pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self { + const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one. + var queue: Queue = .init(buf); + for (0..20) |_| + try queue.putOne(io, try alloc.alloc(u8, block_size)); -pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self { return .{ .alloc = alloc, + .io = io, .block_size = block_size, - .buffers = try .initCapacity(alloc, 5), + .buf = buf, + .buf_queue = queue, }; } pub fn deinit(self: *Self) void { - for (self.buffers.items) |buf| - self.alloc.free(buf.buf); - self.buffers.deinit(self.alloc); + self.buf_queue.close(self.io); + for (self.buf) |buf| + self.alloc.free(buf); + self.alloc.free(self.buf); } fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { if (d == null) { - var buf = try alloc.alloc(u8, in.len * 2); - defer alloc.free(buf); - return xzDecomp(alloc, &buf, in, out) catch return Error.ReadFailed; + return statelessDecomp(d, alloc, in, out); } var self: *Self = @fieldParentPtr("interface", @constCast(d.?)); - const buf_node = self.buffer_queue.popFirst(); - var buf: *Buffer = undefined; - if (buf_node == null) { - const new_buf = try self.buffers.addOne(self.alloc); - new_buf.* = .{ .node = .{}, .buf = try self.alloc.alloc(u8, self.block_size) }; - buf = new_buf; - } else { - buf = @fieldParentPtr("node", buf_node.?); - } - defer self.buffer_queue.prepend(&buf.node); - return xzDecomp(self.alloc, &buf.buf, in, out) catch { - // self.err = err; - return Error.ReadFailed; - }; + + const buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed; + defer self.buf_queue.putOne(self.io, buf) catch {}; + + return xzDecomp(self.alloc, &buf.buf, in, out) catch return Error.ReadFailed; } inline fn xzDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8) !usize { diff --git a/src/decomp/zlib.zig b/src/decomp/zig_zlib.zig similarity index 57% rename from src/decomp/zlib.zig rename to src/decomp/zig_zlib.zig index 8f11a7b..2f70c89 100644 --- a/src/decomp/zlib.zig +++ b/src/decomp/zig_zlib.zig @@ -1,11 +1,14 @@ const std = @import("std"); -const Reader = std.Io.Reader; +const Io = std.Io; const flate = std.compress.flate; const Node = std.SinglyLinkedList.Node; +const Reader = Io.Reader; const Decompressor = @import("../util/decompressor.zig"); const Error = Decompressor.Error; +const Queue = Io.Queue([]u8); + const Self = @This(); const Buffer = struct { @@ -16,42 +19,43 @@ const Buffer = struct { interface: Decompressor = .{ .decomp_fn = decomp }, alloc: std.mem.Allocator, +io: Io, block_size: u32, -buffers: std.ArrayList(Buffer), -buffer_queue: std.SinglyLinkedList = .{}, +buf: [][]u8, +buf_queue: Queue, + +pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self { + const buf = try alloc.alloc([]u8, 20); // TODO: Choose a better number instead of a random one. + var queue: Queue = .init(buf); + for (0..20) |_| + try queue.putOne(io, try alloc.alloc(u8, block_size)); -pub fn init(alloc: std.mem.Allocator, block_size: u32) !Self { return .{ .alloc = alloc, + .io = io, .block_size = block_size, - .buffers = try .initCapacity(alloc, 5), + .buf = buf, + .buf_queue = queue, }; } pub fn deinit(self: *Self) void { - for (self.buffers.items) |buf| - self.alloc.free(buf.buf); - self.buffers.deinit(self.alloc); + self.buf_queue.close(self.io); + for (self.buf) |buf| + self.alloc.free(buf); + self.alloc.free(self.buf); } fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { if (d == null) { - const buf = try alloc.alloc(u8, in.len * 2); - defer alloc.free(buf); - return zlibDecomp(buf, in, out); + return statelessDecomp(d, alloc, in, out); } var self: *Self = @fieldParentPtr("interface", @constCast(d.?)); - const buf_node = self.buffer_queue.popFirst(); - var buf: *Buffer = undefined; - if (buf_node == null) { - const new_buf = try self.buffers.addOne(self.alloc); - new_buf.* = .{ .node = .{}, .buf = try self.alloc.alloc(u8, self.block_size) }; - buf = new_buf; - } else { - buf = @fieldParentPtr("node", buf_node.?); - } - defer self.buffer_queue.prepend(&buf.node); + + const buf = self.buf_queue.getOne(self.io) catch return Error.ReadFailed; + defer self.buf_queue.putOne(self.io, buf) catch {}; + return zlibDecomp(buf.buf, in, out); } @@ -67,7 +71,7 @@ inline fn zlibDecomp(buffer: []u8, in: []u8, out: []u8) !usize { pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { - const buf = try alloc.alloc(u8, in.len * 2); + const buf = try alloc.alloc(u8, out.len); defer alloc.free(buf); return zlibDecomp(buf, in, out); } diff --git a/src/decomp/zstd.zig b/src/decomp/zig_zstd.zig similarity index 83% rename from src/decomp/zstd.zig rename to src/decomp/zig_zstd.zig index 5bd6d9a..731a4cf 100644 --- a/src/decomp/zstd.zig +++ b/src/decomp/zig_zstd.zig @@ -2,9 +2,6 @@ const std = @import("std"); const Io = std.Io; const Reader = std.Io.Reader; const zstd = std.compress.zstd; -const Node = std.SinglyLinkedList.Node; - -const c = @import("c"); const Decompressor = @import("../util/decompressor.zig"); const Error = Decompressor.Error; @@ -70,12 +67,7 @@ inline fn zstdDecomp(buffer: []u8, in: []u8, out: []u8) !usize { pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { - _ = alloc; - const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len); - if (c.ZSTD_isError(res) == 1) - return Error.ReadFailed; - return res; - // const buf = try alloc.alloc(u8, out.len + zstd.block_size_max); - // defer alloc.free(buf); - // return zstdDecomp(buf, in, out); + const buf = try alloc.alloc(u8, out.len + zstd.block_size_max); + defer alloc.free(buf); + return zstdDecomp(buf, in, out); } diff --git a/src/inode.zig b/src/inode.zig index aed5be4..b4b417c 100644 --- a/src/inode.zig +++ b/src/inode.zig @@ -213,21 +213,25 @@ const ExtractError = error{ MknodFailed, CannotSetXattr } || DataExtractor.Error const PathRet = struct { path: []const u8, inode: Inode, - xattr_idx: ?u32 = null, + origin: bool, - fn setMetadata(path_ret: PathRet, alloc: std.mem.Allocator, io: Io, id_table: *CachedTable(u16), xattr_table: ?*XattrTable, options: ExtractionOptions) !void { - var fil = Io.Dir.cwd().openFile(io, path_ret.path, .{}) catch |err| { - std.debug.print("{s}: {}\n", .{ path_ret.path, err }); - return err; - }; + fn deinit(self: PathRet, alloc: std.mem.Allocator) void { + if (self.origin) return; + alloc.free(self.path); + self.inode.deinit(alloc); + } + fn setMetadata(self: PathRet, alloc: std.mem.Allocator, io: Io, id_table: *CachedTable(u16), xattr_table: ?*XattrTable, options: ExtractionOptions) !void { + var fil = try Io.Dir.cwd().openFile(io, self.path, .{}); defer fil.close(io); + const inode = self.inode; + if (!options.ignore_permissions) { - try fil.setPermissions(io, @enumFromInt(path_ret.inode.hdr.permissions)); - try fil.setOwner(io, try id_table.get(io, path_ret.inode.hdr.uid_idx), try id_table.get(io, path_ret.inode.hdr.gid_idx)); + try fil.setPermissions(io, @enumFromInt(inode.hdr.permissions)); + try fil.setOwner(io, try id_table.get(io, inode.hdr.uid_idx), try id_table.get(io, inode.hdr.gid_idx)); } if (xattr_table != null) { - const idx = path_ret.inode.xattrIndex() catch return; + const idx = inode.xattrIndex() catch return; const xattrs = try xattr_table.?.get(alloc, io, idx); defer { @@ -236,7 +240,7 @@ const PathRet = struct { alloc.free(xattrs); } - const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path_ret.path}, 0); + const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{self.path}, 0); defer alloc.free(sentinel_path); for (xattrs) |x| { const xattr_ret = std.os.linux.fsetxattr(fil.handle, x.key, x.value.ptr, x.value.len, 0); @@ -271,19 +275,17 @@ pub fn extract( ) !void { const path = std.mem.trimEnd(u8, filepath, "/"); - const decomp = try @import("decomp.zig").StatelessDecomp(super.compression); + var decomp_base: Decomp = .init(super.compression, alloc); + const decomp = decomp_base.decompressor(); var frag_mgr: FragManager = try .init(alloc, fil, decomp, super.frag_start, super.frag_count, super.block_size); defer frag_mgr.deinit(io); - var arena: std.heap.ArenaAllocator = .init(alloc); - defer arena.deinit(); - var sel_buf: [10]ExtractReturnUnion = undefined; var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf); defer sel.cancelDiscard(); - sel.async(.path_ret, extractReal, .{ self, alloc, io, fil, super, decomp, &arena, &sel, &frag_mgr, path }); + sel.async(.path_ret, extractReal, .{ self, alloc, io, fil, super, decomp, &sel, &frag_mgr, path, true }); var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count); defer id_table.deinit(io); @@ -303,19 +305,42 @@ pub fn extract( const ret = try sel.await(); const path_ret = try ret.path_ret; - if (options.ignore_permissions and xattr_table == null) continue; + if (options.ignore_permissions and xattr_table == null) { + path_ret.deinit(alloc); + continue; + } if (path_ret.inode.hdr.inode_type == .dir or path_ret.inode.hdr.inode_type == .ext_dir) { try dir_queue.push(alloc, path_ret); continue; } + defer path_ret.deinit(alloc); + try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options); + } + + while (sel.cancel()) |ret| { + const path_ret = try ret.path_ret; + + if (options.ignore_permissions and xattr_table == null) { + path_ret.deinit(alloc); + continue; + } + + if (path_ret.inode.hdr.inode_type == .dir or path_ret.inode.hdr.inode_type == .ext_dir) { + try dir_queue.push(alloc, path_ret); + continue; + } + + defer path_ret.deinit(alloc); try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options); } var iter = dir_queue.iterator(); - while (iter.next()) |path_ret| + while (iter.next()) |path_ret| { + defer path_ret.deinit(alloc); try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options); + } } pub fn extractReal( self: Inode, @@ -324,11 +349,17 @@ pub fn extractReal( fil: OffsetFile, super: Archive.Superblock, decomp: *const Decompressor, - inode_arena: *std.heap.ArenaAllocator, sel: *Io.Select(ExtractReturnUnion), frag_mgr: *FragManager, path: []const u8, + origin: bool, ) ExtractError!PathRet { + errdefer { + if (!origin) { + self.deinit(alloc); + alloc.free(path); + } + } switch (self.hdr.inode_type) { .dir, .ext_dir => { try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777)); @@ -343,18 +374,18 @@ pub fn extractReal( alloc.free(entries); } - const inode_alloc = inode_arena.allocator(); - for (entries) |e| { - const new_path = try std.mem.concat(inode_alloc, u8, &[_][]const u8{ path, "/", e.name }); + const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", e.name }); + errdefer alloc.free(new_path); var rdr = fil.readerAt(super.inode_start + e.block_start); var meta: MetadataReader = .init(alloc, &rdr, decomp); try meta.interface.discardAll(e.block_offset); - const new_inode = try read(inode_alloc, &meta.interface, super.block_size); + const new_inode = try read(alloc, &meta.interface, super.block_size); + errdefer new_inode.deinit(alloc); - sel.async(.path_ret, extractReal, .{ new_inode, alloc, io, fil, super, decomp, inode_arena, sel, frag_mgr, new_path }); + sel.async(.path_ret, extractReal, .{ new_inode, alloc, io, fil, super, decomp, sel, frag_mgr, new_path, false }); } }, .file, .ext_file => { @@ -420,5 +451,6 @@ pub fn extractReal( return .{ .path = path, .inode = self, + .origin = origin, }; } diff --git a/src/test.zig b/src/test.zig index 0f5032b..4e8b379 100644 --- a/src/test.zig +++ b/src/test.zig @@ -15,6 +15,7 @@ test "Basics" { var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); defer fil.close(io); var sfs: Archive = try .init(io, fil, 0); + defer sfs.deinit(io); try std.testing.expectEqualDeep(sfs.super, LinuxPATestCorrectSuperblock); const root_file = try sfs.root(alloc, io); defer root_file.deinit(); @@ -30,6 +31,7 @@ test "ExtractSingleFile" { var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); defer fil.close(io); var sfs: Archive = try .init(io, fil, 0); + defer sfs.deinit(io); var test_fil = try sfs.open(alloc, io, TestFile); defer test_fil.deinit(); try test_fil.extract(alloc, io, TestFileExtractLocation, try .Default()); @@ -45,6 +47,7 @@ test "ExtractCompleteArchive" { var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); defer fil.close(io); var sfs: Archive = try .init(io, fil, 0); + defer sfs.deinit(io); try sfs.extract(alloc, io, TestFullExtractLocation, try .Default()); }