Belac/zig-squashfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed missed merge text (oops)

Browse Source 
Changed all *const Decompressor to *Decompressor
Changed all decompressors to only stateless (my queues are borked)
This commit is contained in:
Caleb Gardner
2026-05-24 15:26:37 -05:00
parent a9e50a0ff5
commit 1a8838b544
27 changed files with 120 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+7 -7
View File
@@ -44,17 +44,17 @@ This is some basic observation's I've made about this library's performance when
Currently, my only performance checks are checking execution time, nothing deeper.
* Currently, using my test archive, performance aproximately matches `unsquashfs` when multi-threaded, but significantly slower when single-threaded.
* Using Zig decompression libraries *significantly* increases decompression time by 5x. Under ideal circumstances.
* Using Zig decompression libraries *significantly* increases decompression time.
* Performance improvements/regressions will be common. I'm still learning Zig.
Example Times:
* *unsquashfs, multi-threaded*: .15s
* *unsquashfs, single-threaded*: .16s
* *C-libs, single-threaded*: .36s
* *C-libs, multi-threaded*: .14s
* *Zig-libs, single-threaded*: CURRENTLY UNTESTED
* *Zig-libs, multi-threaded*: .76s
* *unsquashfs, multi-threaded*: .11s
* *unsquashfs, single-threaded*: .13s
* *C-libs, multi-threaded*: .10s
* *C-libs, single-threaded*: ..28s
* *Zig-libs, single-threaded*: .74s
* *Zig-libs, multi-threaded*: 2.70s
## Build considerations
build.zig
-31
View File
@@ -3,10 +3,7 @@ const std = @import("std");
pub fn build(b: *std.Build) !void {
const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
<<<<<<< HEAD
=======
const dynamic = b.option(bool, "dynamic", "Dynamicly link C decompression libraries") orelse false;
>>>>>>> dfbfbda (Build is working again (on Zig master branch))
var debug = b.option(bool, "debug", "Enable options to make debugging easier.");
const version_string_option = b.option([]const u8, "version", "Version of the library/binary");
@@ -22,12 +19,6 @@ pub fn build(b: *std.Build) !void {
if (optimize == .Debug)
debug = true;
const c = b.addTranslateC(.{
.optimize = optimize,
.target = target,
.root_source_file = b.path("src/c.h"),
});
const lib = b.addLibrary(.{
.name = "squashfs",
.root_module = b.createModule(.{
@@ -35,15 +26,8 @@ pub fn build(b: *std.Build) !void {
.target = target,
.valgrind = debug,
.root_source_file = b.path("src/root.zig"),
<<<<<<< HEAD
// .link_libc = true,
.imports = &.{
.{ .name = "options", .module = zig_squashfs_options.createModule() },
.{ .name = "c", .module = c.createModule() },
=======
.imports = &.{
.{ .name = "options", .module = zig_squashfs_options.createModule() },
>>>>>>> dfbfbda (Build is working again (on Zig master branch))
},
}),
.use_llvm = debug,
@@ -51,20 +35,6 @@ pub fn build(b: *std.Build) !void {
const deps = try dependencies(b, optimize, target, use_zig_decomp, allow_lzo, dynamic);
defer b.allocator.free(deps);
<<<<<<< HEAD
const zng = b.dependency("zlib_ng", .{ .optimize = optimize, .target = target });
lib.root_module.linkLibrary(zng.artifact("zng"));
const xz = b.dependency("xz", .{ .optimize = optimize, .target = target });
lib.root_module.linkLibrary(xz.artifact("lzma"));
const minilzo = b.dependency("minilzo", .{ .optimize = optimize, .target = target });
lib.root_module.linkLibrary(minilzo.artifact("minilzo"));
const lz4 = b.dependency("lz4", .{ .optimize = optimize, .target = target });
lib.root_module.linkLibrary(lz4.artifact("lz4"));
=======
for (deps) |d|
lib.root_module.linkLibrary(d);
@@ -80,7 +50,6 @@ pub fn build(b: *std.Build) !void {
if (dynamic)
dynamicLinkLibraries(c, allow_lzo);
}
>>>>>>> dfbfbda (Build is working again (on Zig master branch))
var version = version_string_option orelse "0.0.0-testing";
if (version[0] == 'v') version = version[1..];
src/archive.zig
+10 -10
View File
@@ -16,7 +16,7 @@ const Archive = @This();
file: OffsetFile,
super: Superblock,
stateless_decomp: *const Decompressor,
stateless_decomp: Decompressor,
pub fn init(io: Io, file: std.Io.File, offset: u64) !Archive {
var rdr = file.reader(io, &[0]u8{});
@@ -37,20 +37,20 @@ pub fn deinit(self: *Archive, io: Io) void {
}
/// The root folder of the Archive. Used to open other Files.
pub fn root(self: Archive, alloc: std.mem.Allocator) !File {
pub fn root(self: *Archive, alloc: std.mem.Allocator) !File {
const root_inode = try Utils.inodeFromRef(
alloc,
self.file,
self.stateless_decomp,
&self.stateless_decomp,
self.super.inode_start,
self.super.block_size,
self.super.root_ref,
);
return .init(alloc, self, root_inode, "");
return .init(alloc, self.*, root_inode, "");
}
/// Opens a File within the archive.
pub fn open(self: Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
const root_file = try self.root(alloc);
pub fn open(self: *Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
var root_file = try self.root(alloc);
const path = std.mem.trim(u8, filepath, "/");
if (Utils.pathIsSelf(path))
return root_file;
@@ -60,7 +60,7 @@ pub fn open(self: Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u
/// Returns the inode with the given inode number.
/// Requires that the archive is exportable (has an export lookup table).
pub fn inode(self: Archive, alloc: std.mem.Allocator, io: Io, num: u32) !Inode {
pub fn inode(self: *Archive, alloc: std.mem.Allocator, io: Io, num: u32) !Inode {
if (!self.super.flags.exportable)
return error.NotExportable;
const ref = try LookupTable.lookupValue(
@@ -83,7 +83,7 @@ pub fn inode(self: Archive, alloc: std.mem.Allocator, io: Io, num: u32) !Inode {
);
}
/// Returns a value at the given index from the Archive's id (uid/gid) table.
pub fn idTable(self: Archive, alloc: std.mem.Allocator, io: Io, idx: u32) !u16 {
pub fn idTable(self: *Archive, alloc: std.mem.Allocator, io: Io, idx: u32) !u16 {
return LookupTable.lookupValue(
u16,
alloc,
@@ -96,11 +96,11 @@ pub fn idTable(self: Archive, alloc: std.mem.Allocator, io: Io, idx: u32) !u16 {
}
/// Extract the entire archive contents to the given directory.
pub fn extract(self: Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void {
pub fn extract(self: *Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void {
const root_inode = try Utils.inodeFromRef(
alloc,
self.file,
self.stateless_decomp,
&self.stateless_decomp,
self.super.inode_start,
self.super.block_size,
self.super.root_ref,
src/bin/unsquashfs.zig
+4 -1
View File
@@ -41,7 +41,10 @@ var force: bool = false;
pub fn main(init: std.process.Init) !void {
const alloc = init.gpa;
const io = init.io;
// const io = init.io;
var evented: Io.Evented = undefined;
try evented.init(alloc, .{});
const io = evented.io();
var stdout = std.Io.File.stdout();
defer stdout.close(io);
src/c.h
+3 -1
View File
@@ -1,5 +1,7 @@
#include <zstd.h>
#include <zlib-ng.h>
// #include <lzma.h>
#include <lzma.h>
#ifdef ALLOW_LZO
#include <lzo/minilzo.h>
#endif
#include <lz4.h>
src/decomp.zig
+13 -30
View File
@@ -3,18 +3,12 @@ const Io = std.Io;
const options = @import("options");
const lzma = @import("decomp/zig_lzma.zig");
const xz = @import("decomp/zig_xz.zig");
const Decompressor = @import("util/decompressor.zig");
const zlib = if (options.use_zig_decomp) @import("decomp/zig_zlib.zig") else @import("decomp/c_zlib.zig");
<<<<<<< HEAD
const lzo = if (options.use_zig_decomp or !options.allow_lzo) void else @import("decomp/c_lzo.zig");
=======
const lzma = if (options.use_zig_decomp) @import("decomp/zig_lzma.zig") else @import("decomp/c_lzma.zig");
const lzo = if (options.use_zig_decomp or options.allow_lzo) void else @import("decomp/c_lzo.zig");
const lzo = if (options.use_zig_decomp or !options.allow_lzo) void else @import("decomp/c_lzo.zig");
const xz = if (options.use_zig_decomp) @import("decomp/zig_xz.zig") else @import("decomp/c_xz.zig");
>>>>>>> dfbfbda (Build is working again (on Zig master branch))
const lz4 = if (options.use_zig_decomp) void else @import("decomp/c_lz4.zig");
const zstd = if (options.use_zig_decomp) @import("decomp/zig_zstd.zig") else @import("decomp/c_zstd.zig");
@@ -27,20 +21,20 @@ pub const Enum = enum(u16) {
zstd,
};
pub fn StatelessDecomp(val: Enum) !*const Decompressor {
pub fn StatelessDecomp(val: Enum) !Decompressor {
return switch (val) {
.gzip => &zlib.stateless_decompressor,
.lzma => &lzma.stateless_decompressor,
.gzip => zlib.stateless_decompressor,
.lzma => lzma.stateless_decompressor,
.lzo => if (options.use_zig_decomp or !options.allow_lzo)
error.LzoUnsupported
else
&lzo.stateless_decompressor,
.xz => &xz.stateless_decompressor,
lzo.stateless_decompressor,
.xz => xz.stateless_decompressor,
.lz4 => if (options.use_zig_decomp)
error.Lz4Unsupported
else
&lz4.stateless_decompressor,
.zstd => &zstd.stateless_decompressor,
lz4.stateless_decompressor,
.zstd => zstd.stateless_decompressor,
};
}
@@ -52,16 +46,6 @@ pub const Decomp = union(enum) {
lz4: lz4,
zstd: zstd,
<<<<<<< HEAD
pub fn init(val: Enum, alloc: std.mem.Allocator, io: std.Io, block_size: u32) !Decomp {
return switch (val) {
.gzip => .{ .gzip = zlib.init(alloc, io, block_size) },
.lzma => .{ .lzma = .{} },
.lzo => .{ .lzo = .{} },
.xz => .{ .xz = .{} },
.lz4 => .{ .lz4 = .{} },
.zstd => .{ .zstd = zstd.init(alloc, io, block_size) },
=======
pub fn init(val: Enum, alloc: std.mem.Allocator, io: Io, block_size: u32) !Decomp {
return switch (val) {
.gzip => .{ .gzip = if (options.use_zig_decomp) try zlib.init(alloc, io, block_size) else try zlib.init(alloc, io) },
@@ -70,7 +54,6 @@ pub const Decomp = union(enum) {
.xz => .{ .xz = if (options.use_zig_decomp) try xz.init(alloc, io, block_size) else .{} },
.lz4 => if (options.use_zig_decomp) error.Lz4Unsupported else .{ .lz4 = .{} },
.zstd => .{ .zstd = if (options.use_zig_decomp) try zstd.init(alloc, io, block_size) else try zstd.init(alloc, io) },
>>>>>>> dfbfbda (Build is working again (on Zig master branch))
};
}
pub fn deinit(self: *Decomp, alloc: std.mem.Allocator) void {
@@ -91,13 +74,13 @@ pub const Decomp = union(enum) {
}
}
pub fn decompressor(self: *Decomp) *const Decompressor {
pub fn decompressor(self: *Decomp) *Decompressor {
return switch (self.*) {
.gzip => &self.gzip.interface,
.lzma => &lzma.stateless_decompressor,
.lzo => if (options.use_zig_decomp or !options.allow_lzo) unreachable else &lzo.stateless_decompressor,
.xz => &xz.stateless_decompressor,
.lz4 => if (options.use_zig_decomp) unreachable else &lz4.stateless_decompressor,
.lzma => &self.lzma.interface,
.lzo => if (options.use_zig_decomp or !options.allow_lzo) unreachable else &self.lzo.interface,
.xz => &self.xz.interface,
.lz4 => if (options.use_zig_decomp) unreachable else &self.lz4.interface,
.zstd => &self.zstd.interface,
};
}
src/decomp/c_lz4.zig
+3 -5
View File
@@ -7,13 +7,11 @@ const Error = Decompressor.Error;
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
<<<<<<< HEAD
const res = c.LZ4_decompress_fast(in.ptr, out.ptr, @truncate(out.len));
=======
interface: Decompressor = .{ .decomp_fn = statelessDecomp },
fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
const out_len: c_int = @bitCast(@as(u32, @truncate(out.len)));
const res = c.LZ4_decompress_fast(in.ptr, out.ptr, out_len);
>>>>>>> dfbfbda (Build is working again (on Zig master branch))
if (res < 0) return Error.ReadFailed;
return @abs(res);
}
src/decomp/c_lzma.zig
+3 -1
View File
@@ -15,7 +15,9 @@ const Self = @This();
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
interface: Decompressor = .{ .decomp_fn = statelessDecomp },
fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var stream: c.lzma_stream = .{
.next_in = in.ptr,
.avail_in = in.len,
src/decomp/c_lzo.zig
+3 -1
View File
@@ -15,7 +15,9 @@ const Self = @This();
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
interface: Decompressor = .{ .decomp_fn = statelessDecomp },
fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
_ = c.lzo_init();
var out_len = out.len;
const res = c.lzo1x_decompress_safe(in.ptr, in.len, out.ptr, &out_len, null);
src/decomp/c_xz.zig
+3 -1
View File
@@ -15,7 +15,9 @@ const Self = @This();
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
interface: Decompressor = .{ .decomp_fn = statelessDecomp },
fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var stream: c.lzma_stream = .{
.next_in = in.ptr,
.avail_in = in.len,
src/decomp/c_zlib.zig
+2 -19
View File
@@ -38,7 +38,7 @@ pub fn deinit(self: *Self, alloc: std.mem.Allocator) void {
alloc.free(self.ctx);
}
fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
if (d == null) {
return statelessDecomp(d, alloc, in, out);
}
@@ -68,7 +68,7 @@ inline fn zlibDecomp(stream: *c.zng_stream) !void {
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var stream: c.zng_stream = .{
.next_in = in.ptr,
.avail_in = @truncate(in.len),
@@ -78,20 +78,3 @@ fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out:
try zlibDecomp(&stream);
return stream.total_out;
}
// zalloc
fn zalloc(ptr: ?*anyopaque, size: c_uint, len: c_uint) callconv(.c) ?*anyopaque {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
return alloc.rawAlloc(size * len, .@"1", 0);
}
fn zfree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
<<<<<<< HEAD
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0);
=======
if (mem_ptr == null) return;
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr));
alloc.free(@as([*]u8, @ptrCast(mem_ptr.?)));
>>>>>>> dfbfbda (Build is working again (on Zig master branch))
}
src/decomp/c_zstd.zig
+12 -14
View File
@@ -40,30 +40,28 @@ pub fn deinit(self: *Self, alloc: std.mem.Allocator) void {
alloc.free(self.ctx);
}
fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
// TODO: Fix
//
// if (d == null) {
fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
if (d == null) {
return statelessDecomp(d, alloc, in, out);
// }
// var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
}
var self: *Self = @fieldParentPtr("interface", @constCast(d.?));
// const ctx = self.ctx_queue.getOne(self.io) catch return Error.ReadFailed;
// defer self.ctx_queue.putOne(self.io, ctx) catch {};
const ctx = self.ctx_queue.getOne(self.io) catch return Error.ReadFailed;
defer self.ctx_queue.putOne(self.io, ctx) catch {};
// _ = c.ZSTD_DCtx_reset(ctx, c.ZSTD_reset_session_only);
_ = c.ZSTD_DCtx_reset(ctx, c.ZSTD_reset_session_only);
// const res = c.ZSTD_decompressDCtx(ctx, out.ptr, out.len, in.ptr, in.len);
// if (c.ZSTD_isError(res) != 0)
// return Error.ReadFailed;
// return res;
const res = c.ZSTD_decompressDCtx(ctx, out.ptr, out.len, in.ptr, in.len);
if (c.ZSTD_isError(res) != 0)
return Error.ReadFailed;
return res;
}
// Stateless
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn statelessDecomp(_: ?*Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
if (c.ZSTD_isError(res) != 0)
return Error.ReadFailed;
src/decomp/zig_lzma.zig
+2 -2
View File
@@ -47,7 +47,7 @@ pub fn deinit(self: *Self) void {
self.alloc.free(self.buf);
}
fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
if (d == null) {
return statelessDecomp(d, alloc, in, out);
}
@@ -74,7 +74,7 @@ inline fn lzmaDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var buf = try alloc.alloc(u8, in.len);
defer alloc.free(buf);
return lzmaDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
src/decomp/zig_xz.zig
+2 -2
View File
@@ -47,7 +47,7 @@ pub fn deinit(self: *Self) void {
self.alloc.free(self.buf);
}
fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
if (d == null) {
return statelessDecomp(d, alloc, in, out);
}
@@ -74,7 +74,7 @@ inline fn xzDecomp(alloc: std.mem.Allocator, buffer: *[]u8, in: []u8, out: []u8)
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var buf = try alloc.alloc(u8, in.len);
defer alloc.free(buf);
return xzDecomp(alloc, &buf, in, out) catch return Error.ReadFailed;
src/decomp/zig_zlib.zig
+2 -2
View File
@@ -47,7 +47,7 @@ pub fn deinit(self: *Self) void {
self.alloc.free(self.buf);
}
fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
if (d == null) {
return statelessDecomp(d, alloc, in, out);
}
@@ -70,7 +70,7 @@ inline fn zlibDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
const buf = try alloc.alloc(u8, out.len);
defer alloc.free(buf);
return zlibDecomp(buf, in, out);
src/decomp/zig_zstd.zig
+2 -2
View File
@@ -41,7 +41,7 @@ pub fn deinit(self: *Self) void {
self.alloc.free(self.buf);
}
fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn decomp(d: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
if (d == null) {
const buf = try alloc.alloc(u8, in.len * 2);
defer alloc.free(buf);
@@ -66,7 +66,7 @@ inline fn zstdDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
fn statelessDecomp(_: ?*Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
const buf = try alloc.alloc(u8, out.len + zstd.block_size_max);
defer alloc.free(buf);
return zstdDecomp(buf, in, out);
src/file.zig
+6 -7
View File
@@ -10,7 +10,6 @@ const Inode = @import("inode.zig");
const DataExtractor = @import("util/data_extractor.zig");
const Decompressor = @import("util/decompressor.zig");
const MetadataReader = @import("util/metadata.zig");
const SharedCache = @import("util/shared_cache.zig");
const File = @This();
@@ -35,25 +34,25 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, in: Inode, name: []const
.name = new_name,
};
}
pub fn fromDirEntry(alloc: std.mem.Allocator, archive: Archive, ent: DirEntry) !File {
pub fn fromDirEntry(alloc: std.mem.Allocator, archive: *Archive, ent: DirEntry) !File {
var rdr = archive.file.readerAt(archive.super.inode_start + ent.block_start);
var meta: MetadataReader = .init(alloc, &rdr, archive.stateless_decomp);
var meta: MetadataReader = .init(alloc, &rdr, &archive.stateless_decomp);
try meta.interface.discardAll(ent.block_offset);
var in: Inode = try .read(alloc, &meta.interface, archive.super.block_size);
errdefer in.deinit(alloc);
return .init(alloc, archive, in, ent.name);
return .init(alloc, archive.*, in, ent.name);
}
pub fn deinit(self: File) void {
self.alloc.free(self.name);
self.inode.deinit(self.alloc);
}
pub fn open(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
pub fn open(self: *File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File {
const entries = try self.inode.readDirectory(
alloc,
self.archive.file,
self.archive.stateless_decomp,
&self.archive.stateless_decomp,
self.archive.super.dir_start,
);
defer {
@@ -76,7 +75,7 @@ pub fn open(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8)
}
} else return Error.FileNotFound;
const first_elem_file = try fromDirEntry(alloc, self.archive, search_slice[idx]);
var first_elem_file = try fromDirEntry(alloc, &self.archive, search_slice[idx]);
if (first_element.len == path.len)
return first_elem_file;
defer first_elem_file.deinit();
src/frag.zig
+2 -2
View File
@@ -17,7 +17,7 @@ pub const FragEntry = extern struct {
alloc: std.mem.Allocator,
fil: OffsetFile,
decomp: *const Decompressor,
decomp: *Decompressor,
block_size: u32,
entries: []FragEntry,
@@ -25,7 +25,7 @@ entries: []FragEntry,
frag_cache: std.array_hash_map.Auto(u32, []u8),
cache_mut: std.Io.RwLock = .init,
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, frag_start: u64, frag_num: u32, block_size: u32) !FragManager {
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, frag_start: u64, frag_num: u32, block_size: u32) !FragManager {
const first_offset: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[frag_start .. frag_start + 8]), .little);
var rdr = fil.readerAt(first_offset);
src/inode.zig
+25 -27
View File
@@ -20,7 +20,6 @@ const DataReader = @import("util/data_reader.zig");
const Decompressor = @import("util/decompressor.zig");
const MetadataReader = @import("util/metadata.zig");
const OffsetFile = @import("util/offset_file.zig");
const SharedCache = @import("util/shared_cache.zig");
const XattrTable = @import("xattr_table.zig");
const Inode = @This();
@@ -64,14 +63,14 @@ pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
// Utility Functions
/// Read the directory entries
pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64) ![]DirEntry {
pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, dir_offset: u64) ![]DirEntry {
return switch (self.data) {
.dir => |d| readDirFromData(alloc, fil, decomp, dir_offset, d),
.ext_dir => |d| readDirFromData(alloc, fil, decomp, dir_offset, d),
else => Error.NotDirectory,
};
}
fn readDirFromData(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64, d: anytype) ![]DirEntry {
fn readDirFromData(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, dir_offset: u64, d: anytype) ![]DirEntry {
var rdr = fil.readerAt(dir_offset + d.block_start);
var meta: MetadataReader = .init(alloc, &rdr, decomp);
try meta.interface.discardAll(d.block_offset);
@@ -79,34 +78,34 @@ fn readDirFromData(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Dec
return DirEntry.readDirectory(alloc, &meta.interface, d.size);
}
/// Get a reader for a regular file's data.
pub fn dataReader(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32) !DataReader {
pub fn dataReader(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8) !DataReader {
return switch (self.data) {
.file => |f| getReaderFromData(alloc, io, fil, cache, decomp, block_size, f),
.ext_file => |f| getReaderFromData(alloc, io, fil, cache, decomp, block_size, f),
.file => |f| getReaderFromData(alloc, io, fil, decomp, block_size, frag_block, f),
.ext_file => |f| getReaderFromData(alloc, io, fil, decomp, block_size, frag_block, f),
else => Error.NotRegularFile,
};
}
fn getReaderFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32, d: anytype) !DataReader {
const ext: DataReader = .init(alloc, io, fil, cache, decomp, block_size, d.size, d.block_start, d.blocks);
fn getReaderFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8, d: anytype) !DataReader {
const ext: DataReader = .init(alloc, io, fil, decomp, block_size, d.size, d.block_start, d.blocks);
if (d.frag_block_offset == 0xFFFFFFFF) {
// TODO:
return error.TODO;
if (frag_block == null) return error.FragBlockNotProvided;
ext.addFrag(d.frag_block_offset, frag_block.?);
}
return ext;
}
/// Get an extractor for a regular file's data.
pub fn dataExtractor(self: Inode, fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32) !DataExtractor {
pub fn dataExtractor(self: Inode, fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8) !DataExtractor {
return switch (self.data) {
.file => |f| getExtractorFromData(fil, cache, decomp, block_size, f),
.ext_file => |f| getExtractorFromData(fil, cache, decomp, block_size, f),
.file => |f| getExtractorFromData(fil, decomp, block_size, frag_block, f),
.ext_file => |f| getExtractorFromData(fil, decomp, block_size, frag_block, f),
else => Error.NotRegularFile,
};
}
fn getExtractorFromData(fil: OffsetFile, cache: *SharedCache, decomp: *const Decompressor, block_size: u32, d: anytype) !DataExtractor {
const ext: DataExtractor = .init(fil, cache, decomp, block_size, d.size, d.block_start, d.blocks);
fn getExtractorFromData(fil: OffsetFile, decomp: *Decompressor, block_size: u32, frag_block: ?[]u8, d: anytype) !DataExtractor {
const ext: DataExtractor = .init(fil, decomp, block_size, d.size, d.block_start, d.blocks);
if (d.frag_block_offset == 0xFFFFFFFF) {
// TODO:
return error.TODO;
if (frag_block == null) return error.FragBlockNotProvided;
ext.addFrag(d.frag_block_offset, frag_block.?);
}
return ext;
}
@@ -119,11 +118,11 @@ pub fn symlinkTarget(self: Inode) ![]const u8 {
};
}
/// Get inode's gid
pub fn gid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, id_table_start: u64) !u16 {
pub fn gid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, id_table_start: u64) !u16 {
return LookupTable.lookupValue(u16, alloc, io, decomp, fil, id_table_start, self.hdr.gid_idx);
}
/// Get inode's uid
pub fn uid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, id_table_start: u64) !u16 {
pub fn uid(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, id_table_start: u64) !u16 {
return LookupTable.lookupValue(u16, alloc, io, decomp, fil, id_table_start, self.hdr.uid_idx);
}
/// Get the inode's xattr values as an index into the Archive's xattr table.
@@ -143,7 +142,7 @@ pub fn xattrIndex(self: Inode) !u32 {
return idx;
}
// Get an inode's xattr values. If the inode does not have xattr values (including if the inode is not an extended type), an empty slice is returned.
pub fn xattrValues(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, xattr_table_start: u64) ![]XattrTable.XattrOwned {
pub fn xattrValues(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, xattr_table_start: u64) ![]XattrTable.XattrOwned {
const idx = self.xattrIndex() catch &[0]XattrTable.XattrOwned{};
return XattrTable.statelessLookup(alloc, io, decomp, fil, xattr_table_start, idx);
}
@@ -275,9 +274,8 @@ pub fn extract(
) !void {
const path = std.mem.trimEnd(u8, filepath, "/");
var decomp_base: Decomp = try .init(super.compression, alloc, io, super.block_size);
decomp_base.deinit(alloc);
const decomp = decomp_base.decompressor();
var decomp_base: Decompressor = try @import("decomp.zig").StatelessDecomp(super.compression); // TODO: Replace with actual Decomp value to share states & caches for efficiency.
const decomp = &decomp_base;
var frag_mgr: FragManager = try .init(alloc, fil, decomp, super.frag_start, super.frag_count, super.block_size);
defer frag_mgr.deinit(io);
@@ -301,7 +299,7 @@ fn extractRealAsync(
io: Io,
fil: OffsetFile,
super: Archive.Superblock,
decomp: *const Decompressor,
decomp: *Decompressor,
sel: *Io.Select(ExtractReturnUnion),
frag_mgr: *FragManager,
path: []const u8,
@@ -407,7 +405,7 @@ fn extractRealAsync(
.origin = origin,
};
}
fn finishLoop(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, super: Archive.Superblock, options: ExtractionOptions, sel: *Io.Select(ExtractReturnUnion)) !void {
fn finishLoop(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, super: Archive.Superblock, options: ExtractionOptions, sel: *Io.Select(ExtractReturnUnion)) !void {
var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count);
defer id_table.deinit(io);
@@ -473,7 +471,7 @@ fn extractSinglethreaded(
super: Archive.Superblock,
path: []const u8,
options: ExtractionOptions,
decomp: *const Decompressor,
decomp: *Decompressor,
frag: *FragManager,
) !void {
var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count);
@@ -504,7 +502,7 @@ fn extractReal(
io: Io,
fil: OffsetFile,
super: Archive.Superblock,
decomp: *const Decompressor,
decomp: *Decompressor,
frag_mgr: *FragManager,
id_table: *CachedTable(u16),
xattr_table: ?*XattrTable,
src/lookup_table.zig
+3 -3
View File
@@ -5,7 +5,7 @@ const Decompressor = @import("util/decompressor.zig");
const MetadataReader = @import("util/metadata.zig");
const OffsetFile = @import("util/offset_file.zig");
pub fn lookupValue(comptime T: anytype, alloc: std.mem.Allocator, decomp: *const Decompressor, file: OffsetFile, table_start: u64, idx: u32) !T {
pub fn lookupValue(comptime T: anytype, alloc: std.mem.Allocator, decomp: *Decompressor, file: OffsetFile, table_start: u64, idx: u32) !T {
const T_PER_BLOCK: u16 = 8192 / @sizeOf(T);
const block = idx / T_PER_BLOCK;
@@ -33,7 +33,7 @@ pub fn CachedTable(comptime T: anytype) type {
alloc: std.mem.Allocator,
fil: OffsetFile,
decomp: *const Decompressor,
decomp: *Decompressor,
table_start: u64,
total_num: u32,
@@ -42,7 +42,7 @@ pub fn CachedTable(comptime T: anytype) type {
mut: Io.RwLock = .init,
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, offset: u64, total_num: u32) Table {
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, offset: u64, total_num: u32) Table {
return .{
.alloc = alloc,
.fil = fil,
src/util/data_extractor.zig
+2 -2
View File
@@ -15,7 +15,7 @@ pub const Error = Decompressor.Error || Io.File.MemoryMap.CreateError || Io.File
const DataExtractor = @This();
fil: OffsetFile,
decomp: *const Decompressor,
decomp: *Decompressor,
block_size: u32,
file_size: u64,
@@ -27,7 +27,7 @@ frag_block: ?[]u8 = null,
err: ?Error = null,
pub fn init(fil: OffsetFile, decomp: *const Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) DataExtractor {
pub fn init(fil: OffsetFile, decomp: *Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) DataExtractor {
return .{
.fil = fil,
.decomp = decomp,
src/util/data_reader.zig
+2 -2
View File
@@ -18,7 +18,7 @@ alloc: std.mem.Allocator,
fil: OffsetFile,
io: Io,
decomp: *const Decompressor,
decomp: *Decompressor,
block_size: u32,
file_size: u64,
@@ -33,7 +33,7 @@ sparse_block: bool = false,
interface: Io.Reader,
pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) !DataReader {
pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) !DataReader {
return .{
.alloc = alloc,
src/util/decompressor.zig
+2 -2
View File
@@ -8,8 +8,8 @@ pub const Error = std.Io.Reader.StreamError || std.mem.Allocator.Error;
/// The actual decompression function.
/// If the given decompressor is null, then the decompression should be done "stateless" without lasting allocations.
decomp_fn: *const fn (?*const Decompressor, std.mem.Allocator, in: []u8, out: []u8) Error!usize,
decomp_fn: *const fn (?*Decompressor, std.mem.Allocator, in: []u8, out: []u8) Error!usize,
pub fn Decompress(self: *const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
pub fn Decompress(self: *Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
return self.decomp_fn(self, alloc, in, out);
}
src/util/metadata.zig
+2 -2
View File
@@ -15,7 +15,7 @@ const This = @This();
alloc: std.mem.Allocator,
rdr: *Reader,
decomp: *const Decompressor,
decomp: *Decompressor,
cur_block_start: u32 = 0,
next_start_start: u32 = 0,
@@ -34,7 +34,7 @@ interface: Reader = .{
},
},
pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: *const Decompressor) This {
pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: *Decompressor) This {
return .{
.alloc = alloc,
.rdr = rdr,
src/util/misc.zig
+1 -1
View File
@@ -16,7 +16,7 @@ pub fn pathIsSelf(path: []const u8) bool {
return path[0] == '.';
}
/// Creates an Inode from an Inode.Ref.
pub fn inodeFromRef(alloc: std.mem.Allocator, file: OffsetFile, decomp: *const Decompressor, inode_start: u64, block_size: u32, ref: Inode.Ref) !Inode {
pub fn inodeFromRef(alloc: std.mem.Allocator, file: OffsetFile, decomp: *Decompressor, inode_start: u64, block_size: u32, ref: Inode.Ref) !Inode {
var rdr = file.readerAt(inode_start + ref.block_start);
var meta: MetadataReader = .init(alloc, &rdr, decomp);
try meta.interface.discardAll(ref.block_offset);
src/util/shared_cache.zig
-52
View File
@@ -1,52 +0,0 @@
const std = @import("std");
const Io = std.Io;
const Node = std.SinglyLinkedList.Node;
const SharedCache = @This();
pub const CACHE_SIZE = 1024 * 1024;
pub const BufferNode = struct {
node: Node,
cache: [CACHE_SIZE]u8,
};
alloc: std.mem.Allocator,
caches: std.ArrayList(BufferNode),
cache_queue: std.SinglyLinkedList,
queue_mut: Io.Mutex,
pub fn init(alloc: std.mem.Allocator, init_cache_size: u32) !SharedCache {
const caches: std.ArrayList(BufferNode) = try .initCapacity(alloc, init_cache_size);
var queue: std.SinglyLinkedList = .{};
for (caches.items) |item|
queue.prepend(&item.node);
return .{
.alloc = alloc,
.caches = caches,
.cache_queue = queue,
};
}
pub fn deinit(self: *SharedCache) void {
self.caches.deinit(self.alloc);
}
pub fn getCache(self: *SharedCache, io: Io) !*BufferNode {
self.queue_mut.lock(io);
const nxt = self.cache_queue.popFirst();
self.queue_mut.unlock(io);
if (nxt == null) {
const new = try self.caches.addOne(self.alloc);
new.* = .{
.node = .{},
.cache = undefined,
};
return new;
}
return @fieldParentPtr("node", nxt.?);
}
pub fn returnCache(self: *SharedCache, buf: *BufferNode) void {
self.cache_queue.prepend(buf);
}
src/xattr_table.zig
+3 -3
View File
@@ -12,7 +12,7 @@ const XattrCachedTable = @This();
alloc: std.mem.Allocator,
fil: OffsetFile,
decomp: *const Decompressor,
decomp: *Decompressor,
kv_start: u64,
@@ -20,7 +20,7 @@ table: LookupTable.CachedTable(TableValue),
value_cache: std.AutoHashMap(InodeRef, []const u8),
value_mut: Io.RwLock = .init,
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, xattr_start: u64) !XattrCachedTable {
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, xattr_start: u64) !XattrCachedTable {
const start: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[xattr_start .. xattr_start + 8]), .little);
const num: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[xattr_start + 8 .. xattr_start + 16]), .little);
@@ -212,7 +212,7 @@ const XattrPrefix = packed struct(u16) {
// Stateless
pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *const Decompressor, fil: OffsetFile, table_start: u64, idx: u16) ![]XattrOwned {
pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *Decompressor, fil: OffsetFile, table_start: u64, idx: u16) ![]XattrOwned {
const kv_start: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[table_start .. table_start + 8]), .little);
const lookup = try LookupTable.lookupValue(TableValue, alloc, io, decomp, fil, table_start + 16, idx);