25 Commits

Author SHA1 Message Date
Caleb J. Gardner 4b2b7021c7 Moved & organized decompression
Fully implemented Decompressor vtable
2026-04-02 06:27:34 -05:00
Caleb J. Gardner a1b9828578 Finished (?) decompression restructuring 2026-03-26 06:40:17 -05:00
Caleb J. Gardner 8e4661c4c6 Moving decompression to a vtable interface 2026-03-22 06:35:25 -05:00
Caleb J. Gardner 54aaf30ea5 Working on re-doing decompression 2026-03-21 02:13:36 -05:00
Caleb J. Gardner df22cf6529 Started work on stateful decompression 2026-03-20 01:55:00 -05:00
Caleb J. Gardner 6b5c830234 Actually fix build 2026-03-18 06:17:07 -05:00
Caleb J. Gardner 116234cf9c Fixed build 2026-03-18 06:15:52 -05:00
Caleb J. Gardner 4601e8f323 Updated build 2026-03-18 06:13:19 -05:00
Caleb J. Gardner 50cae8b63d Use zig packed versions of zlib-ng, lz4, and zstd.
Changed use_c_libs to use_zig_decomp so c libraries are now default
2026-03-18 05:24:58 -05:00
Caleb J. Gardner 8b8c9a772f Added -Ddebug build option
Re-added errors from settings xattr values
2026-03-09 00:16:19 -05:00
Caleb Gardner f563648b20 Merge pull request #4 from CalebQ42/inode_finish
Re-do much of extraction
2026-03-08 22:36:37 -05:00
Caleb Gardner 8d4f3d72f8 Merge branch 'main' into inode_finish 2026-03-08 22:34:50 -05:00
Caleb J. Gardner 0b6129f1ae Switch to zlib-ng 2026-03-08 22:15:28 -05:00
Caleb J. Gardner 7308faad36 Moved kv free's in setMetadata 2026-03-05 12:54:25 -06:00
Caleb J. Gardner c9499251f8 Moved lookup tables into separate struct to fix some race conditions
Fixed lingering issues due to zero work size InodeFinish
Fixed xattrs not applying due to the keys sometimes not being
null-terminated.
Updated performance numbers
2026-03-05 12:20:30 -06:00
Caleb J. Gardner d470ca98e3 Added --force to unsquashfs
Fixing race condition bugs (yay)
2026-03-05 07:04:24 -06:00
Caleb J. Gardner a606f5e11a Move extract logic to util/extract.zig to make it easier to read. 2026-03-05 03:03:37 -06:00
Caleb J. Gardner a4e23a840d Updated performance values in README.
Added ability to ignore xattrs & permissions.
Ignore setting xattr errors due to an unknown issues.
2026-03-05 03:03:34 -06:00
Caleb J. Gardner 3a10572953 Updated performance values in README.
Added ability to ignore xattrs & permissions.
Ignore setting xattr errors due to an unknown issues.
2026-03-04 13:28:29 -06:00
Caleb J. Gardner edfe919c1b Expirementation with a new way to finish threads. Currently not working. 2026-03-04 06:39:44 -06:00
Caleb J. Gardner 4515610082 Added xattr support (currently untested)
Use stack fallback allocator on extraction for better performance.
2026-03-04 03:24:30 -06:00
Caleb J. Gardner beca6a2ae6 Small tweaks 2026-03-01 02:27:54 -06:00
Caleb J. Gardner 760e11df0b Fixed github workflow 2026-02-28 22:38:47 -06:00
Caleb J. Gardner 5f629df47c alloc-ified many functions.
Updated README
2026-02-28 22:33:57 -06:00
Caleb J. Gardner b0160e005b Some fixes 2026-02-17 05:54:32 -06:00
32 changed files with 1766 additions and 928 deletions
+8 -8
View File
@@ -13,19 +13,19 @@ jobs:
uses: actions/checkout@v6
- uses: mlugg/setup-zig@v2
- name: Install deps
run: sudo apt update && sudo apt install -y zlib1g-dev libzstd-dev liblzma-dev liblz4-dev liblzo2-dev
run: sudo apt update && sudo apt install -y liblzma-dev liblzo2-dev
- name: Build normal version
run: zig build -Drelease=true -Dversion=${{ github.ref_name }}
- name: Move normal build out
run: mv zig-out/bin/unsquashfs ./
run: zig build --release=fast -Duse_zig_decomp=true -Dversion=${{ github.ref_name }}
- name: Move zig build out
run: mv zig-out/bin/unsquashfs ./unsquashfs-x86_64-zig-libs
- name: Rebuild with C libraries
run: zig build -Drelease=true -Duse_c_libs=true -Dversion="${{ github.ref_name }}"
run: zig build --release=fast -Dversion="${{ github.ref_name }}"
- name: Move C build out
run: mv zig-out/bin/unsquashfs ./unsquashfs-c-libs
run: mv zig-out/bin/unsquashfs ./unsquashfs-x86_64-c-libs
- name: Release
uses: softprops/action-gh-release@v2
with:
prerelease: true
files: |
unsquashfs
unsquashfs-c-libs
unsquashfs-x86_64-zig-libs
unsquashfs-x86_64-c-libs
+25
View File
@@ -0,0 +1,25 @@
// Project-local debug tasks
//
// For more documentation on how to configure debug tasks,
// see: https://zed.dev/docs/debugger
[
{
"label": "Build & Run",
"adapter": "CodeLLDB",
"request": "launch",
"build": {
"command": "zig",
"args": ["build", "-Duse_c_libs=true", "-Ddebug=true"],
},
"program": "zig-out/bin/unsquashfs",
"args": [
"--force",
"-d",
"testing/TestExtractUnsquashfs",
"testing/LinuxPATest.sfs",
],
},
]
+29 -7
View File
@@ -6,19 +6,23 @@ A library and application to decompress or view squashfs archives.
## Current State
Overall works, but currently is missing some features (see below). Extraction is a bit slow compared to the normal `unsquashfs` (from my _very_ basic testing it's about ~3x slower). Only properly work on Linux, any other OSes probably won't work fully and are untested.
Overall works, but currently is missing some features ([see below](#capabilities)) and has significantly slow performance compared to `unsquashfs` ([see below](#performance)).
## Build options
> `-Duse_c_libs`
> `-Duse_c_libs=true`
Instead of using Zig's standard library for decompression, use the system's C libraries. Has the benefit of being much faster and enabling LZO and LZ4 decompression.
> `-Dallow_lzo`
> `-Dallow_lzo=true`
Enable compiling with LZO decompression support. The LZO library currently has some issues with Zig when imported so it's easier to just disable it by default. Only has an effect when using `-Duse_c_libs=true`.
> `-Dversion`
> `-Ddebug=true`
Sets various build options that make debugging easier. Specifically, debug optimization is forced, valgrind support is enabled, error tracing is enabled, stipping is disabled, and copmilation uses LLVM (this is due to some linking issues when on Debug optimization and is required for debugging tools such as `lldb`. In the future this may be removed from the debug flag).
> `-Dversion=0.0.0`
Sets the version of `unsquashfs` shown when `--version` is passed.
@@ -26,11 +30,29 @@ Sets the version of `unsquashfs` shown when `--version` is passed.
Most features are present except for the following:
* mod_time is not set on extraction
* xattrs are not applied on extraction
* When using Zig decompression libraries then lzo and lz4 compression types are unavailable. I don't _currently_ plan on spending the time to find and validate a library since neither is popular.
* When using C decompression libraries, lzo is not supported by default due to [some issues](#build-considerations). If it's needed it's trivial to fix, but it's easiest to just leave it disabled.
## Building considerations
## Performance
This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive (which is fairly small and uses zstd compression) and with `--release=fast`.
Currently, my only performance checks are checking execution time, nothing deeper.
* Under ideal circumstances, my library is ~70% slower (.12s vs .20s).
* Using Zig decompression libraries *significantly* increases decompression time by ~600%. Under ideal circumstances.
* Performance improvements/regressions will be common. I'm still learning Zig.
Example Times:
* *unsquashfs, multi-threaded*: .12s
* *unsquashfs, single-threaded*: .13s
* *C-libs, single-threaded*: .45s
* *C-libs, multi-threaded*: .20s
* *Zig-libs, single-threaded*: 5.78s
* *Zig-libs, multi-threaded*: 1.08s
## Build considerations
Compilation without `use_c_libs` works completely fine, but Zig has issues with some symbols from the lzo library that needs to be manually fixed. In particular you need to fix the definitions for `lzo_bytep` and `lzo_voidp` to be `*u8` and `?*anyopaque` respectively. Due to this, you have to manually enable LZO decompression using `-Dallow_lzo=true` when building.
+62 -28
View File
@@ -1,30 +1,53 @@
const std = @import("std");
pub fn build(b: *std.Build) !void {
const use_c_libs_option = b.option(bool, "use_c_libs", "Use C versions of decompression libraries instead of the Zig standard library ones");
const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support");
const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
// const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
const debug = b.option(bool, "debug", "Enable options to make debugging easier.") orelse false;
const version_string_option = b.option([]const u8, "version", "Version of the library/binary");
const zig_squashfs_options = b.addOptions();
zig_squashfs_options.addOption(bool, "use_c_libs", use_c_libs_option orelse false);
zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo orelse false);
zig_squashfs_options.addOption(bool, "use_zig_decomp", use_zig_decomp);
// zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo);
const target = b.standardTargetOptions(.{});
const optimize = b.standardOptimizeOption(.{ .preferred_optimize_mode = .ReleaseFast });
const optimize = b.standardOptimizeOption(.{});
const mod = b.addModule("zig_squashfs", .{
.root_source_file = b.path("src/root.zig"),
.target = target,
.optimize = optimize,
.link_libc = if (use_c_libs_option == true) true else false,
.optimize = if (debug == true) .Debug else optimize,
.link_libc = !use_zig_decomp,
.valgrind = debug,
.error_tracing = debug,
.strip = if (debug == true) false else null,
});
mod.addOptions("config", zig_squashfs_options);
if (use_c_libs_option == true) {
mod.linkSystemLibrary("zlib", .{});
mod.linkSystemLibrary("lzma", .{});
if (allow_lzo == true)
mod.linkSystemLibrary("minilzo", .{});
mod.linkSystemLibrary("lz4", .{});
mod.linkSystemLibrary("zstd", .{});
if (!use_zig_decomp) {
var zlib_ng = b.dependency("zlib_ng", .{
.target = target,
.optimize = optimize,
});
mod.linkLibrary(zlib_ng.artifact("zng"));
mod.linkSystemLibrary("lzma", .{ .preferred_link_mode = .static });
var minilzo = b.dependency("minilzo", .{
.target = target,
.optimize = optimize,
});
mod.linkLibrary(minilzo.artifact("minilzo"));
var lz4 = b.dependency("lz4", .{
.target = target,
.optimize = optimize,
});
mod.linkLibrary(lz4.artifact("lz4"));
var zstd = b.dependency("zstd", .{
.target = target,
.optimize = optimize,
});
mod.linkLibrary(zstd.artifact("zstd"));
}
var version = version_string_option orelse "0.0.0-testing";
@@ -39,41 +62,52 @@ pub fn build(b: *std.Build) !void {
var exe_mod = b.createModule(.{
.root_source_file = b.path("src/bin/unsquashfs.zig"),
.target = target,
.optimize = optimize,
.link_libc = if (use_c_libs_option == true) true else false,
.optimize = if (debug == true) .Debug else optimize,
.link_libc = !use_zig_decomp,
.imports = &.{
.{ .name = "zig_squashfs", .module = mod },
},
.valgrind = debug,
.error_tracing = debug,
.strip = if (debug == true) false else null,
});
exe_mod.addOptions("config", unsquashfs_options);
const exe = b.addExecutable(.{
.name = "unsquashfs",
.root_module = exe_mod,
.use_llvm = debug,
});
const lib = b.addLibrary(.{
.name = "squashfs",
.root_module = mod,
.use_llvm = debug,
});
b.installArtifact(lib);
b.installArtifact(exe);
const run_step = b.step("run", "Run the app");
const run_cmd = b.addRunArtifact(exe);
run_step.dependOn(&run_cmd.step);
run_cmd.step.dependOn(b.getInstallStep());
if (b.args) |args| {
run_cmd.addArgs(args);
}
const mod_tests = b.addTest(.{
.root_module = mod,
.test_runner = .{
.mode = .simple,
.path = b.path("src/test.zig"),
},
});
const run_mod_tests = b.addRunArtifact(mod_tests);
const exe_tests = b.addTest(.{
.root_module = exe.root_module,
});
const run_exe_tests = b.addRunArtifact(exe_tests);
const test_step = b.step("test", "Run tests");
test_step.dependOn(&run_mod_tests.step);
test_step.dependOn(&run_exe_tests.step);
// zls build check steps
const lib_check = b.addLibrary(.{
.name = "squashfs",
.root_module = mod,
});
const exe_check = b.addExecutable(.{
.name = "unsquashfs",
.root_module = exe_mod,
});
const check = b.step("check", "Check if unsquashfs compiles");
check.dependOn(&lib_check.step);
check.dependOn(&exe_check.step);
}
+17 -35
View File
@@ -1,43 +1,25 @@
.{
.name = .squashfs,
.version = "0.0.1",
.version = "0.0.6",
.fingerprint = 0x37ba29474b87f145, // Changing this has security and trust implications.
.minimum_zig_version = "0.15.2",
// This field is optional.
// Each dependency must either provide a `url` and `hash`, or a `path`.
// `zig build --fetch` can be used to fetch all dependencies of a package, recursively.
// Once all dependencies are fetched, `zig build` no longer requires
// internet connectivity.
.dependencies = .{
// See `zig fetch --save <url>` for a command-line interface for adding dependencies.
//.example = .{
// // When updating this field to a new URL, be sure to delete the corresponding
// // `hash`, otherwise you are communicating that you expect to find the old hash at
// // the new URL. If the contents of a URL change this will result in a hash mismatch
// // which will prevent zig from using it.
// .url = "https://example.com/foo.tar.gz",
//
// // This is computed from the file contents of the directory of files that is
// // obtained after fetching `url` and applying the inclusion rules given by
// // `paths`.
// //
// // This field is the source of truth; packages do not come from a `url`; they
// // come from a `hash`. `url` is just one of many possible mirrors for how to
// // obtain a package matching this `hash`.
// //
// // Uses the [multihash](https://multiformats.io/multihash/) format.
// .hash = "...",
//
// // When this is provided, the package is found in a directory relative to the
// // build root. In this case the package's hash is irrelevant and therefore not
// // computed. This field and `url` are mutually exclusive.
// .path = "foo",
//
// // When this is set to `true`, a package is declared to be lazily
// // fetched. This makes the dependency only get fetched if it is
// // actually used.
// .lazy = false,
//},
.zlib_ng = .{
.url = "git+https://github.com/CalebQ42/zig-zlib-ng#5f2f02dfb28acca2517dacbbd09e9b987f57b133",
.hash = "zlib_ng-2.3.3-pre1-2HYS4ClFAABW8KlHMyBHtlNKE3V7kCS8wqfxawG7xeaa",
},
.zstd = .{
.url = "git+https://github.com/allyourcodebase/zstd.git?ref=1.5.7-1#e1a501be57f42c541e8a5597e4b59a074dfd09a3",
.hash = "zstd-1.5.7-1-KEItkAMwAAD6OKY3m0OOmXG7aL-aLUfrDqbP5J5oYapU",
},
.lz4 = .{
.url = "git+https://github.com/allyourcodebase/lz4.git?ref=1.10.0-6#41f52ab227caf9d48cf88c89a4d2946caa12b102",
.hash = "lz4-1.10.0-6-ewyzw-4NAAAWDpY4xpiqr4LQhZQAC0x_rGnW2iPh6jk2",
},
.minilzo = .{
.url = "git+https://github.com/CalebQ42/zig-minilzo.git#7cbae997b91a44d74b7cd6c073584dc9562a6c90",
.hash = "minilzo-2.10.0-Ij7BO8wLAADeWI4Pe4jp8XTDsDaquZR14oZ7_9yKKDWP",
},
},
.paths = .{
"build.zig",
+50 -113
View File
@@ -5,164 +5,101 @@ const std = @import("std");
const File = std.fs.File;
const builtin = @import("builtin");
const Decomp = @import("decomp.zig");
const config = @import("config");
const cDecomp = @import("decomp/misc_c.zig");
const Decomp = @import("decomp/zig_decomp.zig");
const ExtractionOptions = @import("options.zig");
const Inode = @import("inode.zig");
const InodeRef = Inode.Ref;
const BlockSize = @import("inode_data/file.zig").BlockSize;
const SfsFile = @import("file.zig");
const Superblock = @import("super.zig").Superblock;
const Table = @import("table.zig").Table;
const Tables = @import("tables.zig");
const MetadataReader = @import("util/metadata.zig");
const OffsetFile = @import("util/offset_file.zig");
const config = if (builtin.is_test) .{
.use_c_libs = true,
.allow_lzo = false,
} else @import("config");
/// Information about a fragment section. Multiple fragments are contained in the block described by a single FragEntry.
/// The offset into the block and fragment size is stored in the file's inode.
pub const FragEntry = packed struct {
start: u64,
size: BlockSize,
_: u32,
};
const XattrTable = @import("xattr.zig");
const Archive = @This();
// 4 Gigs
const DEFAULT_MEM_SIZE = 4 * 1024 * 1024 * 1024;
parent_alloc: std.mem.Allocator,
alloc: std.heap.ThreadSafeAllocator,
// alloc: std.heap.FixedBufferAllocator,
// fixed_buf: []u8,
thread_count: usize,
alloc: std.mem.Allocator,
fil: OffsetFile,
super: Superblock,
setup: bool = false,
tables: ?Tables = null,
decomp: Decomp.DecompFn,
frag_table: Table(FragEntry) = undefined,
id_table: Table(u16) = undefined,
export_table: Table(InodeRef) = undefined,
decomp: @import("decomp/types.zig").Decomp,
/// Default settings using std.Thread.getCpuCount() threads and the minimum of 4gb or half of system memory for memory usage.
pub fn init(alloc: std.mem.Allocator, fil: File) !Archive {
return initAdvanced(
alloc,
fil,
0,
try std.Thread.getCpuCount(),
);
}
/// Create the Archive dictating the amount of threads used for extraction.
/// If you're planning on only interacting with a small number of files, it should be fine to use few (or one) threads.
pub fn initAdvanced(alloc: std.mem.Allocator, fil: File, offset: u64, threads: usize) !Archive {
pub fn init(alloc: std.mem.Allocator, fil: File, offset: u64) !Archive {
var super: Superblock = undefined;
const red = try fil.pread(@ptrCast(&super), offset);
std.debug.assert(red == @sizeOf(Superblock));
try super.validate();
// const fixed_buf = try alloc.alloc(u8, mem);
const off_fil: OffsetFile = .init(fil, offset);
return .{
.parent_alloc = alloc,
.alloc = .{ .child_allocator = alloc },
// .fixed_buf = fixed_buf,
.thread_count = if (threads > 0) threads else try std.Thread.getCpuCount(),
.fil = .init(fil, offset),
.decomp = switch (super.compression) {
.gzip => Decomp.gzipDecompress,
.lzma => Decomp.lzmaDecompress,
.xz => Decomp.xzDecompress,
.zstd => Decomp.zstdDecompress,
.lz4 => if (config.use_c_libs) Decomp.cLz4 else return error.Lz4Unsupported,
.lzo => if (config.use_c_libs and config.allow_lzo) Decomp.lzoDecompress else return error.LzoUnsupported,
.alloc = alloc,
.fil = off_fil,
.decomp = if (config.use_zig_decomp)
switch (super.compression) {
.lz4 => return error.Lz4Unsupported,
.lzo => return error.LzoUnsupported,
.gzip => .{ .gzip = .{} },
.lzma => .{ .lzma = .{} },
.xz => .{ .xz = .{} },
.zstd => .{ .zstd = .{} },
}
else switch (super.compression) {
.gzip => .{ .gzip = .init(alloc) },
.zstd => .{ .zstd = .init(alloc) },
.xz => .{ .xz = .init(alloc) },
.lzma => .{ .lzma = .init(alloc) },
.lzo => .{ .lzo = .{} },
.lz4 => .{ .lz4 = .{} },
},
.super = super,
};
}
pub fn deinit(self: *Archive) void {
// self.parent_alloc.free(self.fixed_buf);
if (self.setup) {
self.frag_table.deinit();
self.export_table.deinit();
self.id_table.deinit();
}
if (self.tables != null)
self.tables.?.deinit();
self.decomp.deinit();
}
pub fn allocator(self: *Archive) std.mem.Allocator {
return self.alloc.allocator();
}
fn setupValues(self: *Archive) !void {
const alloc = self.allocator();
self.frag_table = try .init(alloc, self.fil, self.decomp, self.super.frag_start, self.super.frag_count);
self.id_table = try .init(alloc, self.fil, self.decomp, self.super.id_start, self.super.id_count);
self.export_table = try .init(alloc, self.fil, self.decomp, self.super.export_start, self.super.inode_count);
self.setup = true;
}
pub fn id(self: *Archive, idx: u32) !u16 {
if (!self.setup) try self.setupValues();
return self.id_table.get(idx);
}
pub fn frag(self: *Archive, idx: u32) !FragEntry {
if (!self.setup) try self.setupValues();
return self.frag_table.get(idx);
}
pub fn inode(self: *Archive, num: u32) !Inode {
if (!self.setup) try self.setupValues();
pub fn inode(self: *Archive, alloc: std.mem.Allocator, num: u32) !Inode {
if (self.tables == null)
self.tables = try .init(alloc, self);
const ref = try self.export_table.get(num - 1);
var rdr = try self.fil.readerAt(ref.block_start + self.super.inode_start, &[0]u8{});
var meta: MetadataReader = .init(self.allocator(), &rdr.interface, &self.decomp);
var meta: MetadataReader = .init(alloc, &rdr.interface, &self.decomp);
try meta.interface.discardAll(ref.block_offset);
return try .read(self.allocator(), &meta.interface, self.super.block_size);
return try .read(alloc, &meta.interface, self.super.block_size);
}
pub fn root(self: *Archive) !SfsFile {
if (!self.setup) try self.setupValues();
pub fn root(self: *Archive, alloc: std.mem.Allocator) !SfsFile {
if (self.tables == null)
self.tables = try .init(alloc, self);
var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{});
var meta: MetadataReader = .init(self.allocator(), &rdr.interface, self.decomp);
var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
try meta.interface.discardAll(self.super.root_ref.block_offset);
const in: Inode = try .read(self.allocator(), &meta.interface, self.super.block_size);
const in: Inode = try .read(alloc, &meta.interface, self.super.block_size);
return .init(self, in, "");
}
pub fn open(self: *Archive, path: []const u8) !SfsFile {
if (!self.setup) try self.setupValues();
var root_fil = try self.root();
pub fn open(self: *Archive, alloc: std.mem.Allocator, path: []const u8) !SfsFile {
if (self.tables == null)
self.tables = try .init(alloc, self);
var root_fil = try self.root(alloc);
defer if (!SfsFile.pathIsSelf(path)) root_fil.deinit();
return root_fil.open(path);
}
pub fn extract(self: *Archive, path: []const u8, options: ExtractionOptions) !void {
if (!self.setup) try self.setupValues();
var alloc = self.allocator();
var ext_path: []u8 = undefined;
if (std.fs.cwd().statFile(path)) |stat| {
if (stat.kind == .directory) {
ext_path = @constCast(path);
} else return error.ExtractionPathExists;
} else |err| {
if (err == error.FileNotFound) {
ext_path = @constCast(path);
} else {
std.log.err("Error stat-ing extraction path {s}: {}\n", .{ path, err });
return err;
}
}
defer if (ext_path.len > path.len) alloc.free(ext_path);
pub fn extract(self: Archive, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void {
var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{});
var meta: MetadataReader = .init(self.allocator(), &rdr.interface, self.decomp);
var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
try meta.interface.discardAll(self.super.root_ref.block_offset);
const in: Inode = try .read(self.allocator(), &meta.interface, self.super.block_size);
try in.extractToThreaded(self, ext_path, options, self.thread_count);
const in: Inode = try .read(self.alloc, &meta.interface, self.super.block_size);
try in.extractTo(alloc, self, path, options);
}
+27 -2
View File
@@ -14,10 +14,14 @@ const help_mgs =
\\ -d <location> Extract to the given location instead of "squashfs-root"
\\
\\ -o <offset> Start reading the archive at the given offset.
\\ -dx Don't set xattr values
\\ -dp Don't set permissions (includes setting uid & gid owner)
\\
\\ -p <threads> Specify how many threads to use. If no present or zero, the system's logical cores count is used.
\\ -v Verbose
\\
\\ --force Force extraction. If the destination already exists, it will be deleted.
\\
\\ --help Display this messages
\\ --version Display the version
\\
@@ -30,6 +34,9 @@ var extLoc: []const u8 = "squashfs-root";
var offset: u64 = 0;
var threads: u32 = 0;
var verbose: bool = false;
var ignore_xattrs: bool = false;
var ignore_permissions: bool = false;
var force: bool = false;
pub fn main() !void {
const alloc = std.heap.smp_allocator;
@@ -44,9 +51,18 @@ pub fn main() !void {
}
var fil: std.fs.File = try std.fs.cwd().openFile(archive, .{}); //TODO: Handle error gracefully.
defer fil.close();
var arc: squashfs.Archive = try .initAdvanced(alloc, fil, offset, threads); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
var arc: squashfs.Archive = try .init(alloc, fil, offset); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
defer arc.deinit();
try arc.extract(extLoc, if (verbose) .VerboseDefault(&out.interface) else .Default); //TODO: Handle error gracefully.
const options: squashfs.ExtractionOptions = .{
.threads = if (threads == 0) try std.Thread.getCpuCount() else threads,
.verbose = verbose,
.verbose_writer = if (verbose) &out.interface else null,
.ignore_xattr = ignore_xattrs,
.ignore_permissions = ignore_permissions,
};
if (force)
try std.fs.cwd().deleteTree(extLoc);
try arc.extract(alloc, extLoc, options); //TODO: Handle error gracefully.
}
fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
@@ -87,6 +103,15 @@ fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
} else if (std.mem.eql(u8, arg, "-v")) {
verbose = true;
continue;
} else if (std.mem.eql(u8, arg, "-dx")) {
ignore_xattrs = true;
continue;
} else if (std.mem.eql(u8, arg, "-dp")) {
ignore_permissions = true;
continue;
} else if (std.mem.eql(u8, arg, "--force")) {
force = true;
continue;
} else if (std.mem.eql(u8, arg, "--version")) {
try out.print("zig-unsquashfs v", .{});
try config.version.format(out);
+8
View File
@@ -0,0 +1,8 @@
pub const c = @cImport({
@cInclude("zlib-ng.h");
@cInclude("lzma.h");
@cInclude("lz4.h");
@cInclude("zstd.h");
@cInclude("zstd_errors.h");
@cInclude("lzo/minilzo.h");
});
+16 -251
View File
@@ -1,260 +1,25 @@
//! Implementations for decompression.
//! TODO: change to vtable interface to allow for shared decompressors for better performance/resource usage.
const std = @import("std");
const Reader = std.Io.Reader;
const builtin = @import("builtin");
const config = if (builtin.is_test) .{
.use_c_libs = builtin.link_libc == true,
.allow_lzo = false, // Change once LZO compilation is fixed
} else @import("config");
const Decompressor = @This();
const c = @cImport({
@cInclude("zlib.h");
@cInclude("lzma.h");
@cInclude("lz4.h");
@cInclude("zstd.h");
@cInclude("zstd_errors.h");
if (config.allow_lzo)
@cInclude("lzo/minilzo.h");
});
pub const CompressionType = enum(u16) {
gzip = 1,
lzma,
lzo,
xz,
lz4,
zstd,
pub const Error = error{
OutOfMemory,
BadInput,
OutputTooSmall,
ReadFailed,
WriteFailed,
EndOfStream,
};
pub const DecompFn = *const fn (alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize; // TODO: replace anyerror to definitive error types.
vtable: *const struct {
decompress: *const fn (*Decompressor, []u8, []u8) Error!usize = DefaultDecompress,
stateless: *const fn (std.mem.Allocator, []u8, []u8) Error!usize,
},
pub const gzipDecompress = if (config.use_c_libs) cGzip else zigGzip;
fn zigGzip(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
var rdr: Reader = .fixed(in);
const buf = try alloc.alloc(u8, out.len);
defer alloc.free(buf);
var decomp = std.compress.flate.Decompress.init(&rdr, .zlib, buf);
return decomp.reader.readSliceShort(out);
}
fn cGzip(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
_ = alloc;
var out_len: usize = out.len;
const res = c.uncompress(out.ptr, &out_len, in.ptr, in.len);
return switch (res) {
c.Z_OK => out_len,
c.Z_MEM_ERROR => error.NotEnoughMemory,
c.Z_BUF_ERROR => error.OutBufferTooSmall,
c.Z_DATA_ERROR => error.BadData,
else => error.UnknownResult,
};
pub fn decompress(self: *Decompressor, in: []u8, out: []u8) Error!usize {
return self.vtable.decompress(self, in, out);
}
pub const lzmaDecompress = if (config.use_c_libs) cLzma else zigLzma;
fn zigLzma(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
var rdr: Reader = .fixed(in);
var decomp = try std.compress.lzma.decompress(alloc, rdr.adaptToOldInterface());
return decomp.read(out);
fn DefaultDecompress(self: *Decompressor, in: []u8, out: []u8) Error!usize {
return self.vtable.stateless(std.heap.smp_allocator, in, out);
}
fn cLzma(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
_ = alloc;
var stream: c.lzma_stream = .{
.next_in = in.ptr,
.avail_in = in.len,
.next_out = out.ptr,
.avail_out = out.len,
};
var res = c.lzma_alone_decoder(&stream, in.len * 2);
switch (res) {
c.LZMA_OK => {},
c.LZMA_MEM_ERROR => return error.LzmaMemoryError,
c.LZMA_PROG_ERROR => return error.LzmaProgramError,
else => return error.UnknownResult,
}
defer c.lzma_end(&stream);
while (res == c.LZMA_OK)
res = c.lzma_code(&stream, c.LZMA_RUN);
return switch (res) {
c.LZMA_STREAM_END => stream.total_out,
c.LZMA_MEM_ERROR => error.LzmaMemoryError,
c.LZMA_MEMLIMIT_ERROR => error.LzmaMemoryLimit,
c.LZMA_FORMAT_ERROR => error.LzmaBadFormat,
c.LZMA_DATA_ERROR => error.LzmaDataCorrupt,
c.LZMA_BUF_ERROR => error.LzmaCannotProgress,
c.LZMA_PROG_ERROR => error.LzmaProgramError,
else => error.UnknownResult,
};
}
// pub const lzoDecompress = if (config.use_c_libs) cLzo else zigLzo;
// fn zigLzo(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
// _ = alloc;
// _ = in;
// _ = out;
// return error.LzoUnsupported;
// }
pub fn cLzo(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
_ = alloc;
var res = c.lzo_init();
if (res != 0) return error.LzoInitFailed;
var out_len: usize = out.len;
res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &out_len, null);
return switch (res) {
c.LZO_E_OK => out_len,
c.LZO_E_ERROR => error.LzoError,
c.LZO_E_OUT_OF_MEMORY => error.LzoOutOfMemory,
c.LZO_E_NOT_COMPRESSIBLE => error.LzoNotCompressible,
c.LZO_E_INPUT_OVERRUN => error.LzoInputOverrun,
c.LZO_E_OUTPUT_OVERRUN => error.LzoOutputOverrun,
c.LZO_E_LOOKBEHIND_OVERRUN => error.LzoLookbehindOverrun,
c.LZO_E_EOF_NOT_FOUND => error.LzoEofNotFound,
c.LZO_E_INPUT_NOT_CONSUMED => error.LzoInputNotConsumed,
c.LZO_E_NOT_YET_IMPLEMENTED => error.LzoNotYetImplemented,
c.LZO_E_INVALID_ARGUMENT => error.LzoInvalidArgument,
c.LZO_E_INVALID_ALIGNMENT => error.LzoInvalidAlignment,
c.LZO_E_OUTPUT_NOT_CONSUMED => error.LzoOutputNotConsumed,
c.LZO_E_INTERNAL_ERROR => error.LzoInternalError,
else => error.UnknownResult,
};
}
pub const xzDecompress = if (config.use_c_libs) cXz else zigXz;
fn zigXz(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
var rdr: Reader = .fixed(in);
var decomp = try std.compress.xz.decompress(alloc, rdr.adaptToOldInterface());
return decomp.read(out);
}
fn cXz(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
_ = alloc;
var stream: c.lzma_stream = .{
.next_in = in.ptr,
.avail_in = in.len,
.next_out = out.ptr,
.avail_out = out.len,
};
var res = c.lzma_stream_decoder(&stream, in.len * 2, 0);
switch (res) {
c.LZMA_OK => {},
c.LZMA_MEM_ERROR => return error.LzmaMemoryError,
c.LZMA_PROG_ERROR => return error.LzmaProgramError,
else => return error.UnknownResult,
}
defer c.lzma_end(&stream);
while (res == c.LZMA_OK)
res = c.lzma_code(&stream, c.LZMA_RUN);
return switch (res) {
c.LZMA_STREAM_END => stream.total_out,
c.LZMA_MEM_ERROR => error.LzmaMemoryError,
c.LZMA_MEMLIMIT_ERROR => error.LzmaMemoryLimit,
c.LZMA_FORMAT_ERROR => error.LzmaBadFormat,
c.LZMA_DATA_ERROR => error.LzmaDataCorrupt,
c.LZMA_BUF_ERROR => error.LzmaCannotProgress,
c.LZMA_PROG_ERROR => error.LzmaProgramError,
else => error.UnknownResult,
};
}
// pub const lz4Decompress = if (config.use_c_libs) cLz4 else zigLz4;
// fn zigLz4(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
// _ = alloc;
// _ = in;
// _ = out;
// return error.Lz4Unsupported;
// }
pub fn cLz4(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
_ = alloc;
const res = c.LZ4_decompress_safe(in.ptr, out.ptr, @intCast(in.len), @intCast(out.len));
if (res > 0) return @abs(res); // TODO: Find out what error values it can return.
return error.Lz4DecompressFailed;
}
pub const zstdDecompress = if (config.use_c_libs) cZstd else zigZstd;
pub fn zigZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
var rdr: Reader = .fixed(in);
const buf = try alloc.alloc(u8, 1024 * 1024);
defer alloc.free(buf);
var decomp = std.compress.zstd.Decompress.init(&rdr, buf, .{});
return decomp.reader.readSliceShort(out) catch |err| {
return decomp.err orelse err;
};
}
fn cZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) anyerror!usize {
_ = alloc;
const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
if (c.ZSTD_isError(res) == 0) return res;
return switch (c.ZSTD_getErrorCode(res)) {
c.ZSTD_error_prefix_unknown => cZstdError.PrefixUnknown,
c.ZSTD_error_version_unsupported => cZstdError.VersionUnsupported,
c.ZSTD_error_frameParameter_unsupported => cZstdError.FrameParameterUnsupported,
c.ZSTD_error_frameParameter_windowTooLarge => cZstdError.FrameParameterWindowTooLarge,
c.ZSTD_error_corruption_detected => cZstdError.CorruptionDetected,
c.ZSTD_error_checksum_wrong => cZstdError.ChecksumWrong,
c.ZSTD_error_literals_headerWrong => cZstdError.LiteralsHeaderWrong,
c.ZSTD_error_dictionary_corrupted => cZstdError.DictionaryCorrupted,
c.ZSTD_error_dictionary_wrong => cZstdError.DictionaryWrong,
c.ZSTD_error_dictionaryCreation_failed => cZstdError.DictionaryCreationFailed,
c.ZSTD_error_parameter_unsupported => cZstdError.ParameterUnsupported,
c.ZSTD_error_parameter_combination_unsupported => cZstdError.ParameterCombinationUnsupported,
c.ZSTD_error_parameter_outOfBound => cZstdError.ParameterOutOfBound,
c.ZSTD_error_tableLog_tooLarge => cZstdError.TableLogTooLarge,
c.ZSTD_error_maxSymbolValue_tooLarge => cZstdError.MaxSymbolValueTooLarge,
c.ZSTD_error_maxSymbolValue_tooSmall => cZstdError.MaxSymbolValueTooSmall,
c.ZSTD_error_stabilityCondition_notRespected => cZstdError.StabilityConditionNotRespected,
c.ZSTD_error_stage_wrong => cZstdError.StageWrong,
c.ZSTD_error_init_missing => cZstdError.InitMissing,
c.ZSTD_error_memory_allocation => cZstdError.MemoryAllocation,
c.ZSTD_error_workSpace_tooSmall => cZstdError.WorkSpaceTooSmall,
c.ZSTD_error_dstSize_tooSmall => cZstdError.DstSizeTooSmall,
c.ZSTD_error_srcSize_wrong => cZstdError.SrcSizeWrong,
c.ZSTD_error_dstBuffer_null => cZstdError.DstBufferNull,
c.ZSTD_error_noForwardProgress_destFull => cZstdError.NoForwardProgressDestFull,
c.ZSTD_error_noForwardProgress_inputEmpty => cZstdError.NoForwardProgressInputEmpty,
else => cZstdError.Generic,
};
}
pub const cZstdError = error{
Generic,
PrefixUnknown,
VersionUnsupported,
FrameParameterUnsupported,
FrameParameterWindowTooLarge,
CorruptionDetected,
ChecksumWrong,
LiteralsHeaderWrong,
DictionaryCorrupted,
DictionaryWrong,
DictionaryCreationFailed,
ParameterUnsupported,
ParameterCombinationUnsupported,
ParameterOutOfBound,
TableLogTooLarge,
MaxSymbolValueTooLarge,
MaxSymbolValueTooSmall,
CannotProduceUncompressedBlock,
StabilityConditionNotRespected,
StageWrong,
InitMissing,
MemoryAllocation,
WorkSpaceTooSmall,
DstSizeTooSmall,
SrcSizeWrong,
DstBufferNull,
NoForwardProgressDestFull,
NoForwardProgressInputEmpty,
FrameIndexTooLarge,
SeekableIo,
DstBufferWrong,
SrcBufferWrong,
SequenceProducerFailed,
ExternalSequencesInvalid,
MaxCode,
};
+14
View File
@@ -0,0 +1,14 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
const res = c.LZ4_decompress_fast(in.ptr, out.ptr, @intCast(out.len));
if (res < 0) return Decompressor.Error.ReadFailed;
return @abs(res);
}
+127
View File
@@ -0,0 +1,127 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
alloc: std.mem.Allocator,
streams: std.AutoHashMap(std.Thread.Id, c.lzma_stream),
interface: Decompressor,
pub fn init(alloc: std.mem.Allocator) Self {
return .{
.alloc = alloc,
.streams = .init(alloc),
.interface = .{
.vtable = &.{
.decompress = decompress,
.stateless = stateless,
},
},
};
}
pub fn deinit(self: *Self) void {
self.streams.deinit();
}
fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
var self: *Self = @fieldParentPtr("interface", decomp);
var strm = try self.getOrCreate();
strm.next_in = in.ptr;
strm.avail_in = in.len;
strm.next_out = out.ptr;
strm.avail_out = out.len;
var res = c.lzma_alone_decoder(strm, out.len * 2);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
while (res == c.LZMA_OK)
res = c.lzma_code(strm, c.LZMA_RUN);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
return strm.total_out;
}
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var strm: c.lzma_stream = .{
.allocator = &.{
.alloc = lzmaAlloc,
.free = lzmaFree,
.@"opaque" = @ptrCast(@constCast(&alloc)),
},
.next_in = in.ptr,
.avail_in = in.len,
.next_out = out.ptr,
.avail_out = out.len,
};
var res = c.lzma_alone_decoder(&strm, out.len * 2);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
while (res == c.LZMA_OK)
res = c.lzma_code(&strm, c.LZMA_RUN);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
return strm.total_out;
}
inline fn getOrCreate(self: *Self) !*c.lzma_stream {
const res = try self.streams.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) return res.value_ptr;
res.value_ptr.* = .{ .allocator = &.{
.alloc = lzmaAlloc,
.free = lzmaFree,
.@"opaque" = @ptrCast(&self.alloc),
} };
return res.value_ptr;
}
inline fn decodeResult(res: usize) Error!void {
return switch (res) {
c.LZMA_OK, c.LZMA_STREAM_END => {},
c.LZMA_NO_CHECK => Error.NoCheck,
c.LZMA_UNSUPPORTED_CHECK => Error.UnsupportedCheck,
c.LZMA_GET_CHECK => Error.GetCheck,
c.LZMA_MEM_ERROR, c.LZMA_MEMLIMIT_ERROR => Error.OutOfMemory,
c.LZMA_FORMAT_ERROR => Error.Format,
c.LZMA_OPTIONS_ERROR => Error.Options,
c.LZMA_DATA_ERROR => Error.Data,
c.LZMA_BUF_ERROR => Error.Buffer,
c.LZMA_PROG_ERROR => Error.Program,
c.LZMA_SEEK_NEEDED => Error.SeekNeeded,
else => Error.Unknown,
};
}
inline fn lzmaErrToDecompErr(err: Error) Decompressor.Error {
return switch (err) {
Error.OutOfMemory => Decompressor.Error.OutOfMemory,
Error.NoCheck => Decompressor.Error.ReadFailed,
Error.UnsupportedCheck => Decompressor.Error.ReadFailed,
Error.GetCheck => Decompressor.Error.ReadFailed,
Error.Format => Decompressor.Error.ReadFailed,
Error.Options => Decompressor.Error.ReadFailed,
Error.Data => Decompressor.Error.ReadFailed,
Error.Buffer => Decompressor.Error.WriteFailed,
Error.Program => Decompressor.Error.ReadFailed,
Error.SeekNeeded => Decompressor.Error.ReadFailed,
else => Decompressor.Error.ReadFailed,
};
}
fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
return alloc.rawAlloc(size, .@"1", 0);
}
fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0);
}
const Error = error{
OutOfMemory,
NoCheck,
UnsupportedCheck,
GetCheck,
Format,
Options,
Data,
Buffer,
Program,
SeekNeeded,
Unknown,
};
+67
View File
@@ -0,0 +1,67 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var out_len: usize = out.len;
const res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &out_len, null);
decodeError(res) catch |err| return lzoErrToDecompErr(err);
return out_len;
}
inline fn decodeError(res: c_int) Error!void {
return switch (res) {
c.LZO_E_OK => {},
c.LZO_E_EOF_NOT_FOUND => Error.EofNotFound,
c.LZO_E_INPUT_NOT_CONSUMED => Error.InputNotConsumed,
c.LZO_E_INPUT_OVERRUN => Error.InputOverrun,
c.LZO_E_INTERNAL_ERROR => Error.InternalError,
c.LZO_E_INVALID_ALIGNMENT => Error.InvalidAlignment,
c.LZO_E_INVALID_ARGUMENT => Error.InvalidArgument,
c.LZO_E_LOOKBEHIND_OVERRUN => Error.LookbehindOverrun,
c.LZO_E_NOT_COMPRESSIBLE => Error.NotCompressible,
c.LZO_E_NOT_YET_IMPLEMENTED => Error.NotYetImplemented,
c.LZO_E_OUTPUT_NOT_CONSUMED => Error.OutputNotConsumed,
c.LZO_E_OUTPUT_OVERRUN => Error.OutputOverrun,
c.LZO_E_OUT_OF_MEMORY => Error.OutOfMemory,
else => Error.Unknown,
};
}
inline fn lzoErrToDecompErr(err: Error) Decompressor.Error {
return switch (err) {
Error.EofNotFound => Decompressor.Error.ReadFailed,
Error.InputNotConsumed => Decompressor.Error.ReadFailed,
Error.InputOverrun => Decompressor.Error.ReadFailed,
Error.InternalError => Decompressor.Error.ReadFailed,
Error.InvalidAlignment => Decompressor.Error.ReadFailed,
Error.InvalidArgument => Decompressor.Error.ReadFailed,
Error.LookbehindOverrun => Decompressor.Error.ReadFailed,
Error.NotCompressible => Decompressor.Error.ReadFailed,
Error.NotYetImplemented => Decompressor.Error.ReadFailed,
Error.OutputNotConsumed => Decompressor.Error.WriteFailed,
Error.OutputOverrun => Decompressor.Error.WriteFailed,
Error.OutOfMemory => Decompressor.Error.OutOfMemory,
else => Decompressor.Error.ReadFailed,
};
}
const Error = error{
EofNotFound,
InputNotConsumed,
InputOverrun,
InternalError,
InvalidAlignment,
InvalidArgument,
LookbehindOverrun,
NotCompressible,
NotYetImplemented,
OutputNotConsumed,
OutputOverrun,
OutOfMemory,
Unknown,
};
+127
View File
@@ -0,0 +1,127 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
alloc: std.mem.Allocator,
streams: std.AutoHashMap(std.Thread.Id, c.lzma_stream),
interface: Decompressor,
pub fn init(alloc: std.mem.Allocator) Self {
return .{
.alloc = alloc,
.streams = .init(alloc),
.interface = .{
.vtable = &.{
.decompress = decompress,
.stateless = stateless,
},
},
};
}
pub fn deinit(self: *Self) void {
self.streams.deinit();
}
fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
var self: *Self = @fieldParentPtr("interface", decomp);
var strm = try self.getOrCreate();
strm.next_in = in.ptr;
strm.avail_in = in.len;
strm.next_out = out.ptr;
strm.avail_out = out.len;
var res = c.lzma_stream_decoder(strm, out.len * 2, 0);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
while (res == c.LZMA_OK)
res = c.lzma_code(strm, c.LZMA_RUN);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
return strm.total_out;
}
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var strm: c.lzma_stream = .{
.allocator = &.{
.alloc = lzmaAlloc,
.free = lzmaFree,
.@"opaque" = @ptrCast(@constCast(&alloc)),
},
.next_in = in.ptr,
.avail_in = in.len,
.next_out = out.ptr,
.avail_out = out.len,
};
var res = c.lzma_stream_decoder(&strm, out.len * 2, 0);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
while (res == c.LZMA_OK)
res = c.lzma_code(&strm, c.LZMA_RUN);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
return strm.total_out;
}
inline fn getOrCreate(self: *Self) !*c.lzma_stream {
const res = try self.streams.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) return res.value_ptr;
res.value_ptr.* = .{ .allocator = &.{
.alloc = lzmaAlloc,
.free = lzmaFree,
.@"opaque" = @ptrCast(&self.alloc),
} };
return res.value_ptr;
}
inline fn decodeResult(res: usize) Error!void {
return switch (res) {
c.LZMA_OK, c.LZMA_STREAM_END => {},
c.LZMA_NO_CHECK => Error.NoCheck,
c.LZMA_UNSUPPORTED_CHECK => Error.UnsupportedCheck,
c.LZMA_GET_CHECK => Error.GetCheck,
c.LZMA_MEM_ERROR, c.LZMA_MEMLIMIT_ERROR => Error.OutOfMemory,
c.LZMA_FORMAT_ERROR => Error.Format,
c.LZMA_OPTIONS_ERROR => Error.Options,
c.LZMA_DATA_ERROR => Error.Data,
c.LZMA_BUF_ERROR => Error.Buffer,
c.LZMA_PROG_ERROR => Error.Program,
c.LZMA_SEEK_NEEDED => Error.SeekNeeded,
else => Error.Unknown,
};
}
inline fn lzmaErrToDecompErr(err: Error) Decompressor.Error {
return switch (err) {
Error.OutOfMemory => Decompressor.Error.OutOfMemory,
Error.NoCheck => Decompressor.Error.ReadFailed,
Error.UnsupportedCheck => Decompressor.Error.ReadFailed,
Error.GetCheck => Decompressor.Error.ReadFailed,
Error.Format => Decompressor.Error.ReadFailed,
Error.Options => Decompressor.Error.ReadFailed,
Error.Data => Decompressor.Error.ReadFailed,
Error.Buffer => Decompressor.Error.WriteFailed,
Error.Program => Decompressor.Error.ReadFailed,
Error.SeekNeeded => Decompressor.Error.ReadFailed,
else => Decompressor.Error.ReadFailed,
};
}
fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
return alloc.rawAlloc(size, .@"1", 0);
}
fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0);
}
const Error = error{
OutOfMemory,
NoCheck,
UnsupportedCheck,
GetCheck,
Format,
Options,
Data,
Buffer,
Program,
SeekNeeded,
Unknown,
};
+112
View File
@@ -0,0 +1,112 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
alloc: std.mem.Allocator,
streams: std.AutoHashMap(std.Thread.Id, c.zng_stream),
interface: Decompressor,
pub fn init(alloc: std.mem.Allocator) Self {
return .{
.alloc = alloc,
.streams = .init(alloc),
.interface = .{
.vtable = &.{
.decompress = decompress,
.stateless = stateless,
},
},
};
}
pub fn deinit(self: *Self) void {
self.streams.deinit();
}
fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
const self: *Self = @fieldParentPtr("interface", decomp);
var strm = try self.getOrCreate();
strm.next_in = in.ptr;
strm.avail_in = @truncate(in.len);
strm.next_out = out.ptr;
strm.total_out = out.len;
var res = c.zng_inflateReset(strm);
decodeError(res) catch |err| return zlibErrToDecompErr(err);
res = c.zng_inflate(strm, c.Z_FULL_FLUSH);
decodeError(res) catch |err| return zlibErrToDecompErr(err);
return strm.total_out;
}
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var strm: c.zng_stream = .{
.zalloc = zalloc,
.zfree = zfree,
.@"opaque" = @constCast(&alloc),
.next_in = in.ptr,
.avail_in = @truncate(in.len),
.next_out = out.ptr,
.total_out = out.len,
};
var res = c.zng_inflateInit(&strm);
decodeError(res) catch |err| return zlibErrToDecompErr(err);
res = c.zng_inflate(&strm, c.Z_FULL_FLUSH);
decodeError(res) catch |err| return zlibErrToDecompErr(err);
return strm.total_out;
}
fn getOrCreate(self: *Self) !*c.zng_stream {
const res = try self.streams.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) return res.value_ptr;
res.value_ptr.* = .{
.zalloc = zalloc,
.zfree = zfree,
.@"opaque" = &self.alloc,
};
return res.value_ptr;
}
fn zalloc(ptr: ?*anyopaque, size: c_uint, len: c_uint) callconv(.c) ?*anyopaque {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
return alloc.rawAlloc(size * len, .@"1", 0);
}
fn zfree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0);
}
inline fn decodeError(res: i32) Error!void {
if (res >= 0) return;
return switch (res) {
c.Z_STREAM_ERROR => Error.Stream,
c.Z_DATA_ERROR => Error.Data,
c.Z_MEM_ERROR => Error.OutOfMemory,
c.Z_BUF_ERROR => Error.Buffer,
c.Z_VERSION_ERROR => Error.Version,
else => Error.Misc,
};
}
inline fn zlibErrToDecompErr(err: Error) Decompressor.Error {
return switch (err) {
Error.OutOfMemory => Decompressor.Error.OutOfMemory,
Error.Misc => Decompressor.Error.ReadFailed,
Error.Stream => Decompressor.Error.ReadFailed,
Error.Data => Decompressor.Error.ReadFailed,
Error.Buffer => Decompressor.Error.WriteFailed,
Error.Version => Decompressor.Error.ReadFailed,
};
}
const Error = error{
OutOfMemory,
Misc,
Stream,
Data,
Buffer,
Version,
};
+165
View File
@@ -0,0 +1,165 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
alloc: std.mem.Allocator,
ctx: std.AutoHashMap(std.Thread.Id, ?*c.ZSTD_DCtx),
interface: Decompressor,
pub fn init(alloc: std.mem.Allocator) Self {
return .{
.alloc = alloc,
.ctx = .init(alloc),
.interface = .{
.vtable = &.{
.decompress = decompress,
.stateless = stateless,
},
},
};
}
pub fn deinit(self: *Self) void {
self.ctx.deinit();
}
fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
var self: *Self = @fieldParentPtr("interface", decomp);
const ctx = try self.getOrCreate();
const res = c.ZSTD_decompressDCtx(ctx, out.ptr, out.len, in.ptr, in.len);
decodeError(res) catch |err| return zstdErrToDecompErr(err);
return res;
}
fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
decodeError(res) catch |err| return zstdErrToDecompErr(err);
return res;
}
inline fn getOrCreate(self: *Self) !?*c.ZSTD_DCtx {
const res = try self.ctx.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) return res.value_ptr.*;
res.value_ptr.* = c.ZSTD_createDCtx();
return res.value_ptr.*;
}
inline fn decodeError(res: usize) Error!void {
if (c.ZSTD_isError(res) == 0) return;
return switch (c.ZSTD_getErrorCode(res)) {
c.ZSTD_error_prefix_unknown => Error.PrefixUnknown,
c.ZSTD_error_version_unsupported => Error.VersionUnsupported,
c.ZSTD_error_frameParameter_unsupported => Error.FrameParameterUnsupported,
c.ZSTD_error_frameParameter_windowTooLarge => Error.FrameParameterWindowTooLarge,
c.ZSTD_error_corruption_detected => Error.CorruptionDetected,
c.ZSTD_error_checksum_wrong => Error.ChecksumWrong,
c.ZSTD_error_literals_headerWrong => Error.LiteralsHeaderWrong,
c.ZSTD_error_dictionary_corrupted => Error.DictionaryCorrupted,
c.ZSTD_error_dictionary_wrong => Error.DictionaryWrong,
c.ZSTD_error_dictionaryCreation_failed => Error.DictionaryCreationFailed,
c.ZSTD_error_parameter_unsupported => Error.ParameterUnsupported,
c.ZSTD_error_parameter_combination_unsupported => Error.ParameterCombinationUnsupported,
c.ZSTD_error_parameter_outOfBound => Error.ParameterOutOfBound,
c.ZSTD_error_tableLog_tooLarge => Error.TableLogTooLarge,
c.ZSTD_error_maxSymbolValue_tooLarge => Error.MaxSymbolValueTooLarge,
c.ZSTD_error_maxSymbolValue_tooSmall => Error.MaxSymbolValueTooSmall,
c.ZSTD_error_cannotProduce_uncompressedBlock => Error.CannotProduceUncompressedBlock,
c.ZSTD_error_stabilityCondition_notRespected => Error.StabilityConditionNotRespected,
c.ZSTD_error_stage_wrong => Error.StageWrong,
c.ZSTD_error_init_missing => Error.InitMissing,
c.ZSTD_error_memory_allocation => Error.MemoryAllocation,
c.ZSTD_error_workSpace_tooSmall => Error.WorkSpaceTooSmall,
c.ZSTD_error_dstSize_tooSmall => Error.DstSizeTooSmall,
c.ZSTD_error_srcSize_wrong => Error.SrcSizeWrong,
c.ZSTD_error_dstBuffer_null => Error.DstBufferNull,
c.ZSTD_error_noForwardProgress_destFull => Error.NoForwardProgressDestFull,
c.ZSTD_error_noForwardProgress_inputEmpty => Error.NoForwardProgressInputEmpty,
c.ZSTD_error_frameIndex_tooLarge => Error.FrameIndexTooLarge,
c.ZSTD_error_seekableIO => Error.SeekableIo,
c.ZSTD_error_dstBuffer_wrong => Error.DstBufferWrong,
c.ZSTD_error_srcBuffer_wrong => Error.SrcBufferWrong,
c.ZSTD_error_sequenceProducer_failed => Error.SequenceProducerFailed,
c.ZSTD_error_externalSequences_invalid => Error.ExternalSequencesInvalid,
else => Error.Generic,
};
}
inline fn zstdErrToDecompErr(err: Error) Decompressor.Error {
return switch (err) {
Error.OutOfMemory => Decompressor.Error.OutOfMemory,
Error.Generic => Decompressor.Error.ReadFailed,
Error.PrefixUnknown => Decompressor.Error.ReadFailed,
Error.VersionUnsupported => Decompressor.Error.ReadFailed,
Error.FrameParameterUnsupported => Decompressor.Error.ReadFailed,
Error.FrameParameterWindowTooLarge => Decompressor.Error.ReadFailed,
Error.CorruptionDetected => Decompressor.Error.ReadFailed,
Error.ChecksumWrong => Decompressor.Error.ReadFailed,
Error.LiteralsHeaderWrong => Decompressor.Error.ReadFailed,
Error.DictionaryCorrupted => Decompressor.Error.ReadFailed,
Error.DictionaryWrong => Decompressor.Error.ReadFailed,
Error.DictionaryCreationFailed => Decompressor.Error.ReadFailed,
Error.ParameterUnsupported => Decompressor.Error.ReadFailed,
Error.ParameterCombinationUnsupported => Decompressor.Error.ReadFailed,
Error.ParameterOutOfBound => Decompressor.Error.ReadFailed,
Error.TableLogTooLarge => Decompressor.Error.ReadFailed,
Error.MaxSymbolValueTooLarge => Decompressor.Error.ReadFailed,
Error.MaxSymbolValueTooSmall => Decompressor.Error.ReadFailed,
Error.CannotProduceUncompressedBlock => Decompressor.Error.ReadFailed,
Error.StabilityConditionNotRespected => Decompressor.Error.ReadFailed,
Error.StageWrong => Decompressor.Error.ReadFailed,
Error.InitMissing => Decompressor.Error.ReadFailed,
Error.MemoryAllocation => Decompressor.Error.OutOfMemory,
Error.WorkSpaceTooSmall => Decompressor.Error.WriteFailed,
Error.DstSizeTooSmall => Decompressor.Error.WriteFailed,
Error.SrcSizeWrong => Decompressor.Error.ReadFailed,
Error.DstBufferNull => Decompressor.Error.WriteFailed,
Error.NoForwardProgressDestFull => Decompressor.Error.WriteFailed,
Error.NoForwardProgressInputEmpty => Decompressor.Error.ReadFailed,
Error.FrameIndexTooLarge => Decompressor.Error.ReadFailed,
Error.SeekableIo => Decompressor.Error.ReadFailed,
Error.DstBufferWrong => Decompressor.Error.WriteFailed,
Error.SrcBufferWrong => Decompressor.Error.ReadFailed,
Error.SequenceProducerFailed => Decompressor.Error.ReadFailed,
Error.ExternalSequencesInvalid => Decompressor.Error.ReadFailed,
};
}
const Error = error{
OutOfMemory,
Generic,
PrefixUnknown,
VersionUnsupported,
FrameParameterUnsupported,
FrameParameterWindowTooLarge,
CorruptionDetected,
ChecksumWrong,
LiteralsHeaderWrong,
DictionaryCorrupted,
DictionaryWrong,
DictionaryCreationFailed,
ParameterUnsupported,
ParameterCombinationUnsupported,
ParameterOutOfBound,
TableLogTooLarge,
MaxSymbolValueTooLarge,
MaxSymbolValueTooSmall,
CannotProduceUncompressedBlock,
StabilityConditionNotRespected,
StageWrong,
InitMissing,
MemoryAllocation,
WorkSpaceTooSmall,
DstSizeTooSmall,
SrcSizeWrong,
DstBufferNull,
NoForwardProgressDestFull,
NoForwardProgressInputEmpty,
FrameIndexTooLarge,
SeekableIo,
DstBufferWrong,
SrcBufferWrong,
SequenceProducerFailed,
ExternalSequencesInvalid,
};
+43
View File
@@ -0,0 +1,43 @@
const config = @import("config");
const Decompressor = @import("../decomp.zig");
const cLz4 = @import("c/lz4.zig");
const cLzma = @import("c/lzma.zig");
const cLzo = @import("c/lzo.zig");
const cXz = @import("c/xz.zig");
const cZlib = @import("c/zlib.zig");
const cZstd = @import("c/zstd.zig");
const zigLzma = @import("zig/lzma.zig");
const zigXz = @import("zig/xz.zig");
const zigZlib = @import("zig/zstd.zig");
const zigZstd = @import("zig/zstd.zig");
pub const Decomp = union(enum) {
gzip: if (config.use_zig_decomp) zigZlib else cZlib,
lzma: if (config.use_zig_decomp) zigLzma else cLzma,
lzo: if (config.use_zig_decomp) void else cLzo,
xz: if (config.use_zig_decomp) zigXz else cXz,
lz4: if (config.use_zig_decomp) void else cLz4,
zstd: if (config.use_zig_decomp) zigZstd else cZstd,
pub fn deinit(self: *Decomp) void {
switch (self) {
.gzip => self.gzip.deinit(),
.lzma => self.lzma.deinit(),
.xz => self.xz.deinit(),
.zstd => self.zstd.deinit(),
else => {},
}
}
pub fn decompressor(self: *Decomp) *Decompressor {
return switch (self) {
.gzip => &self.gzip.interface,
.lzma => &self.lzma.interface,
.lzo => &self.lzo.interface,
.xz => &self.xz.interface,
.lz4 => &self.lz4.interface,
.zstd => &self.zstd.interface,
};
}
};
+19
View File
@@ -0,0 +1,19 @@
const std = @import("std");
const lzma = std.compress.lzma;
const Reader = std.Io.Reader;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var rdr: Reader = .fixed(in);
var decomp = try lzma.decompress(alloc, rdr.adaptToOldInterface());
defer decomp.deinit();
return decomp.read(out) catch |err| switch (err) {
error.CorruptInput, error.EndOfStream, error.Overflow => return Decompressor.Error.ReadFailed,
else => return err,
};
}
+25
View File
@@ -0,0 +1,25 @@
const std = @import("std");
const xz = std.compress.xz;
const Reader = std.Io.Reader;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var rdr: Reader = .fixed(in);
var decomp = try xz.decompress(alloc, rdr.adaptToOldInterface());
defer decomp.deinit();
return decomp.read(out) catch |err| switch (err) {
error.CorruptInput,
error.EndOfStream,
error.EndOfStreamWithNoError,
error.WrongChecksum,
error.Unsupported,
error.Overflow,
=> Decompressor.Error.ReadFailed,
else => return err,
};
}
+18
View File
@@ -0,0 +1,18 @@
const std = @import("std");
const Reader = std.Io.Reader;
const flate = std.compress.flate;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
const buf = try alloc.alloc(u8, out.len);
defer alloc.free(buf);
var rdr: Reader = .fixed(in);
var decomp = flate.Decompress.init(&rdr, .zlib, buf);
return decomp.reader.readSliceShort(out);
}
+18
View File
@@ -0,0 +1,18 @@
const std = @import("std");
const Reader = std.Io.Reader;
const zstd = std.compress.zstd;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
const buf = try alloc.alloc(u8, out.len * 2);
defer alloc.free(buf);
var rdr: Reader = .fixed(in);
var decomp = zstd.Decompress.init(&rdr, buf, .{ .window_len = @min(out.len, zstd.default_window_len) });
return decomp.reader.readSliceShort(out);
}
+4 -32
View File
@@ -96,7 +96,7 @@ pub fn iterate(self: SfsFile) !Iterator {
}
/// Open a sub-file/folder within a directory at the given path.
/// If path is "", ".", "/", or "./", this File is returned.
pub fn open(self: SfsFile, path: []const u8) !SfsFile {
pub fn open(self: SfsFile, alloc: std.mem.Allocator, path: []const u8) !SfsFile {
if (!self.isDir()) return FileError.NotDirectory;
if (pathIsSelf(path)) return self;
@@ -109,7 +109,6 @@ pub fn open(self: SfsFile, path: []const u8) !SfsFile {
if (std.mem.eql(u8, first_element, ".")) return self.open(path[idx + 1 ..]);
const entries = try self.getEntries();
defer {
var alloc = self.archive.allocator();
for (entries) |e| {
e.deinit(alloc);
}
@@ -127,7 +126,7 @@ pub fn open(self: SfsFile, path: []const u8) !SfsFile {
return fil;
}
defer fil.deinit();
return fil.open(path[idx + 1 ..]);
return fil.open(alloc, path[idx + 1 ..]);
},
.lt => cur_slice = cur_slice[0..split],
.gt => cur_slice = cur_slice[split + 1 ..],
@@ -170,35 +169,8 @@ pub fn devNum(self: SfsFile) !u32 {
/// Extract the given File to the path. If File is a regular file, the path must be a directory or not exist.
/// If the gievn path is a folder, the File's contents will be extracted within.
pub fn extract(self: *SfsFile, path: []const u8, options: ExtractionOptions) !void {
var alloc = self.archive.allocator();
var ext_path: []u8 = undefined;
if (std.fs.cwd().statFile(path)) |stat| {
if (stat.kind == .directory) {
if (!self.isDir()) {
const has_end_sep = path[path.len - 1] == '/';
const alloc_size = if (has_end_sep)
path.len + self.name.len
else
path.len + self.name.len + 1;
ext_path = try alloc.alloc(u8, alloc_size);
@memcpy(ext_path[0..path.len], path);
@memcpy(ext_path[ext_path.len - self.name.len ..], self.name);
if (!has_end_sep) ext_path[path.len] = '/';
} else {
ext_path = @constCast(path);
}
} else return FileError.ExtractionPathExists;
} else |err| {
if (err == error.FileNotFound) {
ext_path = @constCast(path);
} else {
std.log.err("Error stat-ing extraction path {s}: {}\n", .{ path, err });
return err;
}
}
defer if (ext_path.len > path.len) alloc.free(ext_path);
return self.inode.extractToThreaded(self.archive, path, options, self.archive.thread_count);
pub fn extract(self: *SfsFile, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void {
return self.inode.extractToThreaded(alloc, self.archive, path, options);
}
/// Utility function.
+43 -392
View File
@@ -12,8 +12,12 @@ const ExtractionOptions = @import("options.zig");
const dir = @import("inode_data/dir.zig");
const file = @import("inode_data/file.zig");
const misc = @import("inode_data/misc.zig");
const Tables = @import("tables.zig");
const DataReader = @import("util/data.zig");
const ThreadedDataReader = @import("util/data_threaded.zig");
const InodeExtract = @import("util/extract.zig");
const InodeFinish = @import("util/inode_finish.zig");
const FinishUnion = InodeFinish.FinishUnion;
const MetadataReader = @import("util/metadata.zig");
pub const Ref = packed struct {
@@ -93,7 +97,7 @@ pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode {
},
};
}
pub fn readFromEntry(alloc: std.mem.Allocator, archive: *Archive, entry: DirEntry) !Inode {
pub fn readFromEntry(alloc: std.mem.Allocator, archive: Archive, entry: DirEntry) !Inode {
var rdr = try archive.fil.readerAt(archive.super.inode_start + entry.block_start, &[0]u8{});
var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp);
try meta.interface.discardAll(entry.block_offset);
@@ -111,31 +115,17 @@ pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
}
/// Get the data reader for a file inode.
pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: *Archive) !DataReader {
pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive, tables: *Tables) !DataReader {
return switch (self.hdr.inode_type) {
.file => readerFromData(alloc, archive, self.data.file),
.ext_file => readerFromData(alloc, archive, self.data.ext_file),
.file => readerFromData(alloc, archive, tables, self.data.file),
.ext_file => readerFromData(alloc, archive, tables, self.data.ext_file),
else => error.NotRegularFile,
};
}
fn readerFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !DataReader {
var out: DataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size);
fn readerFromData(alloc: std.mem.Allocator, archive: Archive, tables: *Tables, data: anytype) !DataReader {
var out: DataReader = .init(alloc, archive, data.block_sizes, data.block_start, data.size);
if (data.frag_idx != 0xFFFFFFFF)
out.addFragment(try archive.frag(data.frag_idx), data.frag_block_offset);
return out;
}
/// Get a threaded data reader for a file inode.
pub fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: *Archive) !ThreadedDataReader {
return switch (self.hdr.inode_type) {
.file => threadedReaderFromData(alloc, archive, self.data.file),
.ext_file => threadedReaderFromData(alloc, archive, self.data.ext_file),
else => error.NotRegularFile,
};
}
fn threadedReaderFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !ThreadedDataReader {
var out: ThreadedDataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size);
if (data.frag_idx != 0xFFFFFFFF)
out.addFragment(try archive.frag(data.frag_idx), data.frag_block_offset);
out.addFragment(try tables.frag_table.get(data.frag_idx), data.frag_block_offset);
return out;
}
@@ -154,385 +144,46 @@ fn entriesFromData(alloc: std.mem.Allocator, archive: Archive, data: anytype) ![
return DirEntry.readDir(alloc, &meta.interface, data.size);
}
/// Extract the inode to the given path. Single threaded.
pub fn extractTo(self: Inode, archive: *Archive, path: []const u8, options: ExtractionOptions) !void {
switch (self.hdr.inode_type) {
.dir, .ext_dir => {
// Removing any trailing separators since that's the easiest path forward.
if (path[path.len - 1] == '/') return self.extractTo(archive, path[0 .. path.len - 1], options);
std.fs.cwd().makeDir(path) catch |err| {
if (err != std.fs.Dir.MakeError.PathAlreadyExists) return err;
/// Returns the xattr index for the given inode. If the inode isn't an extended variant or doesn't have any, the u32 max is returned (0xFFFFFFFF).
pub fn xattrIdx(self: Inode) u32 {
return switch (self.data) {
.ext_dir => |d| d.xattr_id,
.ext_file => |f| f.xattr_idx,
.ext_symlink => |s| s.xattr_idx,
.ext_block_dev, .ext_char_dev => |d| d.xattr_idx,
.ext_fifo, .ext_socket => |i| i.xattr_idx,
else => 0xFFFFFFFF,
};
var alloc = archive.allocator();
const entries = try self.dirEntries(alloc, archive.*);
defer {
for (entries) |entry| entry.deinit(alloc);
alloc.free(entries);
}
for (entries) |entry| {
var new_path = try alloc.alloc(u8, path.len + 1 + entry.name.len);
@memcpy(new_path[0..path.len], path);
@memcpy(new_path[path.len + 1 ..], entry.name);
new_path[path.len] = '/';
defer alloc.free(new_path);
var inode: Inode = try readFromEntry(alloc, archive, entry);
defer inode.deinit(alloc);
try inode.extractTo(archive, new_path, options);
}
},
.file, .ext_file => try self.extractRegFile(archive.allocator(), archive, path, options),
.symlink, .ext_symlink => try self.extractSymlink(path),
else => try self.extractDevice(archive, path, options),
}
}
const Parent = struct {
alloc: std.mem.Allocator,
path: []const u8,
uid: u16,
gid: u16,
perm: u16,
mod_time: u32,
ignore_permissions: bool,
ignore_xattr: bool,
wg: WaitGroup,
mut: Mutex = .{},
fn create(alloc: std.mem.Allocator, hdr: Header, archive: *Archive, path: []const u8, options: ExtractionOptions, dir_size: usize) !*Parent {
const out = try alloc.create(Parent);
errdefer alloc.destroy(out);
out.* = .{
.alloc = alloc,
.path = path,
.uid = try archive.id(hdr.uid_idx),
.gid = try archive.id(hdr.gid_idx),
.perm = hdr.permissions,
.mod_time = hdr.mod_time,
.ignore_permissions = options.ignore_permissions,
.ignore_xattr = options.ignore_xattr,
.wg = .{ .state = .init(dir_size) },
};
return out;
}
fn finish(p: *Parent) !void {
p.mut.lock();
{
defer p.mut.unlock();
p.wg.finish();
if (!p.wg.isDone()) return;
}
defer p.alloc.destroy(p);
var fil = try std.fs.cwd().openFile(p.path, .{});
defer fil.close();
const time = p.mod_time * 1000000000;
try fil.updateTimes(time, time);
if (p.ignore_permissions) {
try fil.chmod(p.perm);
try fil.chown(p.uid, p.gid);
}
}
};
/// Extract the inode to the given path. Multi-threaded.
/// Functions identically to extractTo on all but regular files and directories.
///
/// If threads <= 1, then this just calls extractTo.
pub fn extractToThreaded(self: Inode, archive: *Archive, path: []const u8, options: ExtractionOptions, threads: usize) !void {
if (threads <= 1) return self.extractTo(archive, path, options);
switch (self.hdr.inode_type) {
.dir, .ext_dir => {
// Removing any trailing separators since that's the easiest path forward.
if (path[path.len - 1] == '/') return self.extractToThreaded(archive, path[0 .. path.len - 1], options, threads);
// Fixed Allocator
// const mem_buf = archive.allocator().alloc(u8, 2 * 1024 * 1024 * 1024);
// defer archive.allocator().free(mem_buf);
// var fixed_alloc: std.heap.FixedBufferAllocator = .init(mem_buf);
// const alloc = fixed_alloc.threadSafeAllocator();
// Arena Allocator
var arena_alloc: std.heap.ArenaAllocator = .init(archive.allocator());
defer arena_alloc.deinit();
var thread_alloc: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena_alloc.allocator() };
const alloc = thread_alloc.allocator();
var wg: WaitGroup = .{};
// defer if(!options.ignore_permissions) perms.?.deinit(alloc); We don't need to do this due to ArenaAllocator
var pool: Pool = undefined;
try pool.init(.{ .allocator = alloc, .n_jobs = threads - 1 });
defer pool.deinit();
var out_err: ?anyerror = null;
wg.start();
self.extractThread(alloc, archive, path, options, &wg, &pool, &out_err, null);
pool.waitAndWork(&wg);
if (out_err != null) return out_err.?;
var fil = try std.fs.cwd().openFile(path, .{});
defer fil.close();
const time = @as(i128, self.hdr.mod_time) * 1000000000;
try fil.updateTimes(time, time);
if (options.ignore_permissions) {
try fil.chmod(self.hdr.permissions);
try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx));
}
},
.file, .ext_file => {
const alloc = archive.allocator();
var pool: Pool = undefined;
try pool.init(.{ .allocator = alloc, .n_jobs = threads });
defer pool.deinit();
try self.extractRegFileThreaded(alloc, archive, path, options, &pool);
var fil = try std.fs.cwd().openFile(path, .{});
defer fil.close();
/// Applies the Inode's metadata to the given File.
/// Mod time is always set, but permissions and xattrs are set based on the given ExtractionOptions.
pub fn setMetadata(self: Inode, alloc: std.mem.Allocator, tables: *Tables, fil: std.fs.File, options: ExtractionOptions) !void {
const time = @as(i128, self.hdr.mod_time) * 1000000000;
try fil.updateTimes(time, time);
if (!options.ignore_permissions) {
try fil.chmod(self.hdr.permissions);
try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx));
}
},
.symlink, .ext_symlink => try self.extractSymlink(path),
else => try self.extractDevice(archive, path, options),
}
}
fn extractThreadEntry(
entry: DirEntry,
alloc: std.mem.Allocator,
archive: *Archive,
path: []const u8,
options: ExtractionOptions,
wg: *WaitGroup,
pool: *Pool,
out_err: *?anyerror,
parent: ?*Parent,
) void {
var new_path = alloc.alloc(u8, path.len + entry.name.len + 1) catch |err| {
wg.finish();
out_err.* = err;
return;
};
@memcpy(new_path[0..path.len], path);
@memcpy(new_path[path.len + 1 ..], entry.name);
new_path[path.len] = '/';
var inode = readFromEntry(alloc, archive, entry) catch |err| {
out_err.* = err;
wg.finish();
return;
};
inode.extractThread(alloc, archive, new_path, options, wg, pool, out_err, parent);
}
/// Extract threadedly the inode to the path.
fn extractThread(
self: Inode,
alloc: std.mem.Allocator,
archive: *Archive,
path: []const u8,
options: ExtractionOptions,
wg: *WaitGroup,
pool: *Pool,
out_err: *?anyerror,
parent: ?*Parent,
) void {
if (options.verbose)
options.verbose_writer.?.print("Extracting inode #{} to {s}\n", .{ self.hdr.num, path }) catch {};
defer {
if (parent != null) parent.?.finish() catch |err| {
if (options.verbose)
options.verbose_writer.?.print("Error setting folder permission to {s}: {}\n", .{ path, err }) catch {};
out_err.* = err;
};
wg.finish();
}
if (out_err.* != null) return;
switch (self.hdr.inode_type) {
.dir, .ext_dir => {
_ = std.fs.cwd().makePathStatus(path) catch |err| {
if (options.verbose)
options.verbose_writer.?.print("Error creating {s}: {}\n", .{ path, err }) catch {};
out_err.* = err;
return;
};
const entries = self.dirEntries(alloc, archive.*) catch |err| {
if (options.verbose)
options.verbose_writer.?.print("Error getting directory entries for inode #{} (extracting to {s}): {}\n", .{ self.hdr.num, path, err }) catch {};
out_err.* = err;
return;
};
const p = Parent.create(alloc, self.hdr, archive, path, options, entries.len) catch |err| {
out_err.* = err;
return;
};
wg.startMany(entries.len);
// defer files.deinit(alloc); We don't need to do this due to ArenaAllocator
for (entries) |entry| {
if (entry.inode_type == .dir) {
extractThreadEntry(entry, alloc, archive, path, options, wg, pool, out_err, p);
continue;
}
pool.spawn(
extractThreadEntry,
.{
entry,
alloc,
archive,
path,
options,
wg,
pool,
out_err,
p,
},
) catch |err| {
wg.finish();
if (options.verbose)
options.verbose_writer.?.print("Error starting extraction thread: {}\n", .{err}) catch {};
out_err.* = err;
continue;
};
}
},
.file, .ext_file => {
self.extractRegFileThreaded(alloc, archive, path, options, pool) catch |err| {
if (options.verbose)
options.verbose_writer.?.print("Error extracting file inode #{} to {s}: {}\n", .{ self.hdr.num, path, err }) catch {};
out_err.* = err;
};
},
.symlink, .ext_symlink => {
self.extractSymlink(path) catch |err| {
if (options.verbose)
options.verbose_writer.?.print("Error extracting symlink inode #{} to {s}: {}\n", .{ self.hdr.num, path, err }) catch {};
out_err.* = err;
};
},
else => {
self.extractDevice(archive, path, options) catch |err| {
if (options.verbose)
options.verbose_writer.?.print("Error extracting device/IPC inode #{} to {s}: {}\n", .{ self.hdr.num, path, err }) catch {};
out_err.* = err;
};
},
}
}
/// Creates and writes the inode file contents to the given path.
/// Optionally set owner & permissions.
///
/// Assumes the inode is a file or ext_file type.
fn extractRegFile(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void {
var fil = try std.fs.cwd().createFile(path, .{ .exclusive = true });
defer fil.close();
var wrt = fil.writer(&[0]u8{});
var dat_rdr = try self.dataReader(alloc, archive);
defer dat_rdr.deinit();
_ = try dat_rdr.interface.streamRemaining(&wrt.interface);
try wrt.interface.flush();
const time = @as(i128, self.hdr.mod_time) * 1000000000;
try fil.updateTimes(time, time);
if (!options.ignore_permissions) {
try fil.chmod(self.hdr.permissions);
try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx));
}
}
/// Extract the inode file contents to the given path threadedly.
/// pool is used to spawn threads.
///
/// Assumes the inode is a file or ext_file type.
fn extractRegFileThreaded(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions, pool: *Pool) !void {
var fil = try std.fs.cwd().createFile(path, .{});
defer fil.close();
var data = try self.threadedDataReader(alloc, archive);
try data.extractThreaded(fil, pool);
const time = @as(i128, self.hdr.mod_time) * 1000000000;
try fil.updateTimes(time, time);
if (!options.ignore_permissions) {
try fil.chmod(self.hdr.permissions);
try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx));
}
}
/// Creates the symlink described by the inode.
///
/// Assumes the inode is a symlink or ext_symlink type.
fn extractSymlink(self: Inode, path: []const u8) !void {
const target = switch (self.data) {
.symlink => |s| s.target,
.ext_symlink => |s| s.target,
else => unreachable,
};
try std.fs.cwd().symLink(target, path, .{});
}
/// Creates the device described by the inode.
///
/// Optionally set owner & permissions.
/// Assumes the inode is a char_dev, block_dev, fifo, socket, or their extended counterparts.
fn extractDevice(self: Inode, archive: *Archive, path: []const u8, options: ExtractionOptions) !void {
var mode: u32 = undefined;
var dev: u32 = 0;
switch (self.data) {
.char_dev => |d| {
mode = std.posix.S.IFCHR;
dev = d.dev;
},
.ext_char_dev => |d| {
mode = std.posix.S.IFCHR;
dev = d.dev;
},
.block_dev => |d| {
mode = std.posix.S.IFBLK;
dev = d.dev;
},
.ext_block_dev => |d| {
mode = std.posix.S.IFBLK;
dev = d.dev;
},
.fifo, .ext_fifo => mode = std.posix.S.IFIFO,
.socket, .ext_socket => mode = std.posix.S.IFSOCK,
else => unreachable,
}
const res: std.os.linux.E = @enumFromInt(std.os.linux.mknod(@ptrCast(path), mode, dev));
switch (res) {
.SUCCESS => {},
.ACCES => return std.fs.Dir.MakeError.AccessDenied,
.DQUOT => return std.fs.Dir.MakeError.DiskQuota,
.EXIST => return std.fs.Dir.MakeError.PathAlreadyExists,
.FAULT, .NOENT => return std.fs.Dir.MakeError.BadPathName,
.LOOP => return std.fs.Dir.MakeError.SymLinkLoop,
.NAMETOOLONG => return std.fs.Dir.MakeError.NameTooLong,
.NOMEM => return std.fs.Dir.MakeError.SystemResources,
.NOSPC => return std.fs.Dir.MakeError.NoSpaceLeft,
.NOTDIR => return std.fs.Dir.MakeError.NotDir,
.PERM => return std.fs.Dir.MakeError.PermissionDenied,
.ROFS => return std.fs.Dir.MakeError.ReadOnlyFileSystem,
else => return blk: {
std.debug.print("unhandled mknod result: {}\n", .{res});
break :blk std.fs.Dir.MakeError.Unexpected;
},
}
var fil = try std.fs.cwd().openFile(path, .{});
defer fil.close();
const time = @as(i128, self.hdr.mod_time) * 1000000000;
try fil.updateTimes(time, time);
if (!options.ignore_permissions) {
try fil.chmod(self.hdr.permissions);
try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx));
try fil.chown(try tables.id_table.get(self.hdr.uid_idx), try tables.id_table.get(self.hdr.gid_idx));
}
if (!options.ignore_xattr) {
// TODO
const idx = self.xattrIdx();
if (idx == 0xFFFFFFFF) return;
const xattrs = try tables.xattr_table.get(alloc, idx);
defer alloc.free(xattrs);
for (xattrs) |kv| {
const res = std.os.linux.fsetxattr(fil.handle, kv.key, kv.value.ptr, kv.value.len, 0);
alloc.free(kv.key);
alloc.free(kv.value);
if (res != 0) {
if (options.verbose)
options.verbose_writer.?.print("fsetxattr has result of: {}\n", .{res}) catch {};
return error.SetXattr;
}
}
}
}
/// Extract the inode to the given path.
pub fn extractTo(self: Inode, alloc: std.mem.Allocator, archive: Archive, path: []const u8, options: ExtractionOptions) !void {
return InodeExtract.extractTo(alloc, self, archive, path, options);
}
+10 -2
View File
@@ -5,6 +5,8 @@ const Writer = std.Io.Writer;
const ExtractionOptions = @This();
/// The number of threads used for extraction. 0 implies single threaded.
threads: usize = 1,
/// Don't set the file's owner & permissions after extraction
ignore_permissions: bool = false,
/// Don't set xattr values. Currently xattrs are never set anyway.
@@ -16,10 +18,16 @@ verbose: bool = false,
/// Where to print verbose log.
verbose_writer: ?*Writer = null,
pub const Default: ExtractionOptions = .{};
pub fn VerboseDefault(wrt: *Writer) ExtractionOptions {
pub const SingleThreadedDefault: ExtractionOptions = .{};
pub fn Default() !ExtractionOptions {
return .{
.threads = try std.Thread.getCpuCount(),
};
}
pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions {
return .{
.verbose = true,
.verbose_writer = wrt,
.threads = try std.Thread.getCpuCount(),
};
}
+1
View File
@@ -1 +1,2 @@
pub const Archive = @import("archive.zig");
pub const ExtractionOptions = @import("options.zig");
+8 -2
View File
@@ -1,7 +1,6 @@
const std = @import("std");
const math = std.math;
const CompressionType = @import("decomp.zig").CompressionType;
const InodeRef = @import("inode.zig").Ref;
const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little);
@@ -20,7 +19,14 @@ pub const Superblock = packed struct {
mod_time: u32,
block_size: u32,
frag_count: u32,
compression: CompressionType,
compression: enum(u16) {
gzip = 1,
lzma,
lzo,
xz,
lz4,
zstd,
},
block_log: u16,
flags: packed struct {
inode_uncompressed: bool,
+37 -6
View File
@@ -1,14 +1,45 @@
const std = @import("std");
const Mutex = std.Thread.Mutex;
const DecompFn = @import("decomp.zig").DecompFn;
const Archive = @import("archive.zig");
const BlockSize = @import("inode_data/file.zig").BlockSize;
const InodeRef = @import("inode.zig").Ref;
const Superblock = @import("super.zig").Superblock;
const MetadataReader = @import("util/metadata.zig");
const OffsetFile = @import("util/offset_file.zig");
const XattrTable = @import("xattr.zig");
const Decompressor = @import("decomp.zig");
const TableError = error{
InvalidIndex,
/// Information about a fragment section. Multiple fragments are contained in the block described by a single FragEntry.
/// The offset into the block and fragment size is stored in the file's inode.
pub const FragEntry = packed struct {
start: u64,
size: BlockSize,
_: u32,
};
const Tables = @This();
frag_table: Table(FragEntry),
id_table: Table(u16),
export_table: Table(InodeRef),
xattr_table: XattrTable,
pub fn init(alloc: std.mem.Allocator, archive: Archive) !Tables {
return .{
.frag_table = try .init(alloc, archive.fil, archive.decomp, archive.super.frag_start, archive.super.frag_count),
.id_table = try .init(alloc, archive.fil, archive.decomp, archive.super.id_start, archive.super.id_count),
.export_table = try .init(alloc, archive.fil, archive.decomp, archive.super.export_start, archive.super.inode_count),
.xattr_table = try .init(alloc, archive.fil, archive.decomp, archive.super.xattr_start),
};
}
pub fn deinit(self: *Tables) void {
self.frag_table.deinit();
self.id_table.deinit();
self.export_table.deinit();
self.xattr_table.deinit();
}
/// A two-layer metadata table.
pub fn Table(T: anytype) type {
return struct {
@@ -18,7 +49,7 @@ pub fn Table(T: anytype) type {
alloc: std.mem.Allocator,
fil: OffsetFile,
decomp: DecompFn,
decomp: *Decompressor,
tab_start: u64,
tab: std.AutoHashMap(u32, []T),
@@ -26,7 +57,7 @@ pub fn Table(T: anytype) type {
mut: Mutex = .{},
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: DecompFn, tab_start: u64, values: u32) !Self {
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, tab_start: u64, values: u32) !Self {
return .{
.alloc = alloc,
.fil = fil,
@@ -47,7 +78,7 @@ pub fn Table(T: anytype) type {
}
pub fn get(self: *Self, idx: u32) !T {
if (idx >= self.values) return TableError.InvalidIndex;
if (idx >= self.values) return error.InvalidIndex;
const block_num = idx / VALS_PER_BLOCK;
const idx_offset = idx - (block_num * VALS_PER_BLOCK);
if (self.tab.contains(block_num)) {
+1 -1
View File
@@ -6,9 +6,9 @@ const Writer = std.Io.Writer;
const Limit = std.Io.Limit;
const Archive = @import("../archive.zig");
const FragEntry = Archive.FragEntry;
const DecompFn = @import("../decomp.zig").DecompFn;
const BlockSize = @import("../inode_data/file.zig").BlockSize;
const FragEntry = @import("../tables.zig").FragEntry;
const OffsetFile = @import("offset_file.zig");
const DataReader = @This();
+73 -49
View File
@@ -8,9 +8,10 @@ const WaitGroup = std.Thread.WaitGroup;
const Pool = std.Thread.Pool;
const Archive = @import("../archive.zig");
const FragEntry = Archive.FragEntry;
const DecompFn = @import("../decomp.zig").DecompFn;
const BlockSize = @import("../inode_data/file.zig").BlockSize;
const FragEntry = @import("../tables.zig").FragEntry;
const InodeFinish = @import("inode_finish.zig");
const OffsetFile = @import("offset_file.zig");
const ThreadedDataReader = @This();
@@ -25,6 +26,7 @@ blocks: []BlockSize,
frag: ?FragEntry = null, // TODO: do something better?
frag_offset: u32 = 0,
size: u64,
num_blocks: usize,
start_offset: u64,
@@ -34,8 +36,12 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, sta
.fil = archive.fil,
.decomp = archive.decomp,
.block_size = archive.super.block_size,
.blocks = blocks,
.size = size,
.num_blocks = blocks.len,
.start_offset = start,
};
}
@@ -43,141 +49,159 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, sta
pub fn addFragment(self: *ThreadedDataReader, entry: FragEntry, frag_offset: u32) void {
self.frag = entry;
self.frag_offset = frag_offset;
}
fn numBlocks(self: ThreadedDataReader) usize {
var res = self.blocks.len;
if (self.frag != null) res += 1;
return res;
self.num_blocks = self.blocks.len + 1;
}
/// Extract the data to the file threadedly, using pool to spawn threads.
/// If multiple errors occur, thread spawning errors will have, then the last decompression error that occurs;
///
/// The function must be called from an unused DataReader. The DataReader is still usable afterwards.
/// If only extractThreaded is used, there is no need to call deinit() afterwards.
///
/// The file will always be written to starting at 0.
pub fn extractThreaded(self: ThreadedDataReader, file: std.fs.File, pool: *Pool) !void {
var wg: WaitGroup = .{};
wg.startMany(self.numBlocks());
var out_err: ?anyerror = null;
/// If errors occur, they are set to finish.out_err.
pub fn extractThreaded(self: ThreadedDataReader, file: std.fs.File, pool: *Pool, finish: *InodeFinish) void {
var cur_write_offset: u64 = 0;
var cur_read_offset: u64 = self.start_offset;
for (0..self.blocks.len) |i| {
const cur_block_size = if (i == self.numBlocks() - 1) self.size % self.block_size else self.block_size;
try pool.spawn(workThreadBlocks, .{ self, file, cur_write_offset, cur_read_offset, self.blocks[i], cur_block_size, &wg, &out_err });
const cur_block_size = if (i == self.num_blocks - 1) self.size % self.block_size else self.block_size;
pool.spawn(workThreadBlocks, .{ self, file, cur_write_offset, cur_read_offset, self.blocks[i], cur_block_size, finish }) catch |res_err| {
finish.logError("Can't spawn pool task: {}", .{res_err});
finish.out_err.* = res_err;
finish.finish();
};
cur_write_offset += cur_block_size;
cur_read_offset += self.blocks[i].size;
}
if (self.frag != null) {
try pool.spawn(workThreadFragment, .{ self, file, cur_write_offset, &wg, &out_err });
}
pool.waitAndWork(&wg);
if (out_err != null) return out_err.?;
if (self.frag != null)
pool.spawn(workThreadFragment, .{ self, file, cur_write_offset, finish }) catch |res_err| {
finish.logError("Can't spawn pool task: {}", .{res_err});
finish.out_err.* = res_err;
finish.finish();
};
}
fn workThreadBlocks(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, read_offset: u64, block: BlockSize, cur_block_size: u64, wg: *WaitGroup, out_err: *?anyerror) void {
defer wg.finish();
fn workThreadBlocks(
self: ThreadedDataReader,
fil: std.fs.File,
write_offset: u64,
read_offset: u64,
block: BlockSize,
cur_block_size: u64,
finish: *InodeFinish,
) void {
defer finish.finish();
var wrt = fil.writer(&[0]u8{});
wrt.seekTo(write_offset) catch |err| {
out_err.* = err;
finish.logError("Error seeking file writer: {}", .{err});
finish.out_err.* = err;
return;
};
defer wrt.interface.flush() catch |err| {
out_err.* = err;
finish.logError("Error flushing file writer: {}", .{err});
finish.out_err.* = err;
};
if (block.size == 0) {
wrt.interface.splatByteAll(0, cur_block_size) catch |err| {
out_err.* = err;
finish.logError("Error writing zeroes: {}", .{err});
finish.out_err.* = err;
return;
};
return;
}
var rdr = self.fil.readerAt(read_offset, &[0]u8{}) catch |err| {
out_err.* = err;
finish.logError("Error creating file reader: {}", .{err});
finish.out_err.* = err;
return;
};
if (block.uncompressed) {
rdr.interface.streamExact(&wrt.interface, block.size) catch |err| {
out_err.* = err;
finish.logError("Error streaming data: {}", .{err});
finish.out_err.* = err;
return;
};
return;
}
// TODO: shared buffers
const read_buf = self.alloc.alloc(u8, block.size) catch |err| {
out_err.* = err;
finish.logError("Error creating reader buffer: {}", .{err});
finish.out_err.* = err;
return;
};
defer self.alloc.free(read_buf);
rdr.interface.readSliceAll(read_buf) catch |err| {
out_err.* = err;
finish.logError("Error reading data into reader buffer: {}", .{err});
finish.out_err.* = err;
return;
};
// TODO: shared buffers
const res_buf = self.alloc.alloc(u8, cur_block_size) catch |err| {
out_err.* = err;
finish.logError("Error creating result buffer: {}", .{err});
finish.out_err.* = err;
return;
};
defer self.alloc.free(res_buf);
_ = self.decomp(self.alloc, read_buf, res_buf) catch |err| {
out_err.* = err;
finish.logError("Error decompressing data block: {}", .{err});
finish.out_err.* = err;
return;
};
wrt.interface.writeAll(res_buf) catch |err| {
out_err.* = err;
finish.logError("Error writing to file: {}", .{err});
finish.out_err.* = err;
return;
};
}
fn workThreadFragment(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, wg: *WaitGroup, out_err: *?anyerror) void {
defer wg.finish();
fn workThreadFragment(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, finish: *InodeFinish) void {
defer finish.finish();
var wrt = fil.writer(&[0]u8{});
wrt.seekTo(write_offset) catch |err| {
out_err.* = err;
finish.logError("Error seeking file writer for file fragment: {}", .{err});
finish.out_err.* = err;
return;
};
defer wrt.interface.flush() catch |err| {
out_err.* = err;
finish.out_err.* = err;
};
var rdr = self.fil.readerAt(self.frag.?.start, &[0]u8{}) catch |err| {
out_err.* = err;
finish.logError("Error creating file reader for file fragment: {}", .{err});
finish.out_err.* = err;
return;
};
if (self.frag.?.size.uncompressed) {
rdr.interface.discardAll(self.frag_offset) catch |err| {
out_err.* = err;
finish.logError("Error discarding useless fragment data: {}", .{err});
finish.out_err.* = err;
return;
};
rdr.interface.streamExact(&wrt.interface, self.size % self.block_size) catch |err| {
out_err.* = err;
finish.logError("Error streaming fragment data: {}", .{err});
finish.out_err.* = err;
return;
};
return;
}
const tmp_buf = self.alloc.alloc(u8, self.frag.?.size.size) catch |err| {
out_err.* = err;
finish.logError("Error creating a temporary buffer for a file fragment: {}", .{err});
finish.out_err.* = err;
return;
};
defer self.alloc.free(tmp_buf);
rdr.interface.readSliceAll(tmp_buf) catch |err| {
out_err.* = err;
finish.logError("Error reading data into fragment buffer: {}", .{err});
finish.out_err.* = err;
return;
};
const needed_block = self.alloc.alloc(u8, self.block_size) catch |err| {
out_err.* = err;
finish.logError("Error allocating fragment decompression results: {}", .{err});
finish.out_err.* = err;
return;
};
defer self.alloc.free(needed_block);
_ = self.decomp(self.alloc, tmp_buf, needed_block) catch |err| {
out_err.* = err;
finish.logError("Error decompressing fragment: {}", .{err});
finish.out_err.* = err;
return;
};
wrt.interface.writeAll(needed_block[self.frag_offset .. self.frag_offset + (self.size % self.block_size)]) catch |err| {
out_err.* = err;
finish.logError("Error writing fragment: {}", .{err});
finish.out_err.* = err;
return;
};
}
+353
View File
@@ -0,0 +1,353 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const Pool = std.Thread.Pool;
const WaitGroup = std.Thread.WaitGroup;
const Archive = @import("../archive.zig");
const DirEntry = @import("../dir_entry.zig");
const Inode = @import("../inode.zig");
const ExtractionOptions = @import("../options.zig");
const Tables = @import("../tables.zig");
const InodeFinish = @import("inode_finish.zig");
const FinishUnion = InodeFinish.FinishUnion;
const ThreadedDataReader = @import("data_threaded.zig");
// 1 MB
const STACK_ALLOC_SIZE = 1024 * 1024;
pub fn extractTo(
allocator: Allocator,
inode: Inode,
archive: Archive,
path: []const u8,
options: ExtractionOptions,
) !void {
if (path[path.len - 1] == '/')
return extractTo(allocator, inode, archive, path[0 .. path.len - 2], options);
var stack_alloc = std.heap.stackFallback(STACK_ALLOC_SIZE, allocator);
var arena: std.heap.ArenaAllocator = .init(stack_alloc.get());
defer arena.deinit();
if (options.threads <= 1) {
const alloc = arena.allocator();
var tables: Tables = try .init(alloc, archive);
return extractSingleThread(arena.allocator(), inode, archive, &tables, path, options);
}
var thread_alloc = std.heap.ThreadSafeAllocator{ .child_allocator = arena.allocator() };
const alloc = thread_alloc.allocator();
var tables: Tables = try .init(alloc, archive);
var pool_alloc = std.heap.stackFallback(10 * 1024, alloc);
var pool: Pool = undefined;
try pool.init(.{ .allocator = pool_alloc.get(), .n_jobs = options.threads - 1 });
var wg: WaitGroup = .{};
var err: ?anyerror = null;
wg.start();
try pool.spawn(extractMultiThread, .{
alloc,
inode,
archive,
&tables,
path,
options,
&pool,
FinishUnion{ .wg = &wg },
&err,
});
pool.waitAndWork(&wg);
if (err != null) return err.?;
}
fn extractSingleThread(
alloc: Allocator,
inode: Inode,
archive: Archive,
tables: *Tables,
path: []const u8,
options: ExtractionOptions,
) !void {
switch (inode.hdr.inode_type) {
.dir, .ext_dir => {
_ = std.fs.cwd().makeDir(path) catch |err| switch (err) {
std.fs.Dir.MakeError.PathAlreadyExists => {},
else => return err,
};
// Currently we are ignoring any deinit or free calls since we know we are under an ArenaAllocator.
// Possibly in the future, do some simple math to see if it would be safe to ONLY deinit via Arena,
// otherwise be more conscientious about freeing memory.
// For now, this is good enough.
const entries = try inode.dirEntries(alloc, archive);
for (entries) |ent| {
const sub_inode: Inode = try .readFromEntry(alloc, archive, ent);
const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", ent.name });
try extractSingleThread(alloc, sub_inode, archive, tables, new_path, options);
}
const fil = try std.fs.cwd().openFile(path, .{});
defer fil.close();
try inode.setMetadata(alloc, tables, fil, options);
},
.file, .ext_file => {
var fil = try std.fs.cwd().createFile(path, .{ .exclusive = true });
defer fil.close();
var wrt = fil.writer(&[0]u8{});
var dat_rdr = try inode.dataReader(alloc, archive, tables);
defer dat_rdr.deinit();
_ = try dat_rdr.interface.streamRemaining(&wrt.interface);
try wrt.interface.flush();
try inode.setMetadata(alloc, tables, fil, options);
},
.symlink, .ext_symlink => return extractSymlink(inode, path),
else => return extractDeviceAndIPC(inode, alloc, tables, path, options),
}
}
fn extractMultiThread(
alloc: Allocator,
inode: Inode,
archive: Archive,
tables: *Tables,
path: []const u8,
options: ExtractionOptions,
pool: *Pool,
fin: FinishUnion,
err: *?anyerror,
) void {
if (err.* != null) {
fin.finish();
return;
}
switch (inode.hdr.inode_type) {
.dir, .ext_dir => {
_ = std.fs.cwd().makeDir(path) catch |res_err| switch (res_err) {
std.fs.Dir.MakeError.PathAlreadyExists => {},
else => {
err.* = res_err;
fin.finish();
return;
},
};
// Currently we are ignoring any deinit or free calls since we know we are under an ArenaAllocator.
// Possibly in the future, do some simple math to see if it would be safe to ONLY deinit via Arena,
// otherwise be more conscientious about freeing memory.
// For now, this is good enough.
const entries = inode.dirEntries(alloc, archive) catch |res_err| {
err.* = res_err;
fin.finish();
return;
};
if (entries.len == 0) {
fin.finish();
return;
}
var dir_fin = InodeFinish.create(
alloc,
inode,
path,
tables,
options,
fin,
err,
null,
entries.len,
) catch |res_err| {
err.* = res_err;
fin.finish();
return;
};
for (entries) |ent| {
if (ent.inode_type == .dir) {
extractEntry(
alloc,
ent,
archive,
tables,
path,
options,
pool,
.{ .fin = dir_fin },
err,
);
continue;
}
pool.spawn(
extractEntry,
.{ alloc, ent, archive, tables, path, options, pool, FinishUnion{ .fin = dir_fin }, err },
) catch |res_err| {
err.* = res_err;
dir_fin.finish();
return;
};
}
},
.file, .ext_file => {
const fil = std.fs.cwd().createFile(path, .{ .exclusive = true }) catch |res_err| {
if (options.verbose)
options.verbose_writer.?.print("Can't create file at {s}: {}\n", .{ path, res_err }) catch {};
err.* = res_err;
fin.finish();
return;
};
var data_rdr = threadedDataReader(inode, alloc, archive, tables) catch |res_err| {
if (options.verbose)
options.verbose_writer.?.print("Can't create data reader for inode #{} (extracting to {s}): {}\n", .{ inode.hdr.num, path, res_err }) catch {};
err.* = res_err;
fin.finish();
return;
};
if (data_rdr == null) {
inode.setMetadata(alloc, tables, fil, options) catch |res_err| {
if (options.verbose)
options.verbose_writer.?.print("Can't set metadata to {s}: {}\n", .{ path, res_err }) catch {};
err.* = res_err;
};
fin.finish();
return;
}
const file_fin = InodeFinish.create(
alloc,
inode,
path,
tables,
options,
fin,
err,
fil,
data_rdr.?.num_blocks,
) catch |res_err| {
if (options.verbose)
options.verbose_writer.?.print("Can't create callback for inode #{} (extracting to {s}): {}\n", .{ inode.hdr.num, path, res_err }) catch {};
err.* = res_err;
fin.finish();
return;
};
data_rdr.?.extractThreaded(fil, pool, file_fin);
},
.symlink, .ext_symlink => {
extractSymlink(inode, path) catch |res_err| {
err.* = res_err;
};
fin.finish();
},
else => {
extractDeviceAndIPC(inode, alloc, tables, path, options) catch |res_err| {
err.* = res_err;
};
fin.finish();
},
}
}
fn extractEntry(
alloc: Allocator,
ent: DirEntry,
archive: Archive,
tables: *Tables,
path: []const u8,
options: ExtractionOptions,
pool: *Pool,
fin: FinishUnion,
err: *?anyerror,
) void {
const new_path = std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", ent.name }) catch |res_err| {
err.* = res_err;
fin.finish();
return;
};
const inode = Inode.readFromEntry(alloc, archive, ent) catch |res_err| {
err.* = res_err;
fin.finish();
return;
};
extractMultiThread(alloc, inode, archive, tables, new_path, options, pool, fin, err);
}
/// Get a threaded data reader for a file inode.
fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive, tables: *Tables) !?ThreadedDataReader {
return switch (self.hdr.inode_type) {
.file => threadedReaderFromData(alloc, archive, tables, self.data.file),
.ext_file => threadedReaderFromData(alloc, archive, tables, self.data.ext_file),
else => error.NotRegularFile,
};
}
fn threadedReaderFromData(alloc: std.mem.Allocator, archive: Archive, tables: *Tables, data: anytype) !?ThreadedDataReader {
if (data.block_sizes.len == 0 and data.frag_idx == 0xFFFFFFFF) return null;
var out: ThreadedDataReader = .init(alloc, archive, data.block_sizes, data.block_start, data.size);
if (data.frag_idx != 0xFFFFFFFF)
out.addFragment(try tables.frag_table.get(data.frag_idx), data.frag_block_offset);
return out;
}
/// Creates the symlink described by the inode.
/// Sets metadata.
fn extractSymlink(self: Inode, path: []const u8) !void {
const target = switch (self.data) {
.symlink => |s| s.target,
.ext_symlink => |s| s.target,
else => unreachable,
};
try std.fs.cwd().symLink(target, path, .{});
}
/// Creates the device described by the inode.
/// Sets metadata.
fn extractDeviceAndIPC(self: Inode, alloc: std.mem.Allocator, tables: *Tables, path: []const u8, options: ExtractionOptions) !void {
var mode: u32 = undefined;
var dev: u32 = 0;
switch (self.data) {
.char_dev => |d| {
mode = std.posix.S.IFCHR;
dev = d.dev;
},
.ext_char_dev => |d| {
mode = std.posix.S.IFCHR;
dev = d.dev;
},
.block_dev => |d| {
mode = std.posix.S.IFBLK;
dev = d.dev;
},
.ext_block_dev => |d| {
mode = std.posix.S.IFBLK;
dev = d.dev;
},
.fifo, .ext_fifo => mode = std.posix.S.IFIFO,
.socket, .ext_socket => mode = std.posix.S.IFSOCK,
else => unreachable,
}
const res: std.os.linux.E = @enumFromInt(std.os.linux.mknod(@ptrCast(path), mode, dev));
switch (res) {
.SUCCESS => {},
.ACCES => return std.fs.Dir.MakeError.AccessDenied,
.DQUOT => return std.fs.Dir.MakeError.DiskQuota,
.EXIST => return std.fs.Dir.MakeError.PathAlreadyExists,
.FAULT, .NOENT => return std.fs.Dir.MakeError.BadPathName,
.LOOP => return std.fs.Dir.MakeError.SymLinkLoop,
.NAMETOOLONG => return std.fs.Dir.MakeError.NameTooLong,
.NOMEM => return std.fs.Dir.MakeError.SystemResources,
.NOSPC => return std.fs.Dir.MakeError.NoSpaceLeft,
.NOTDIR => return std.fs.Dir.MakeError.NotDir,
.PERM => return std.fs.Dir.MakeError.PermissionDenied,
.ROFS => return std.fs.Dir.MakeError.ReadOnlyFileSystem,
else => return blk: {
std.debug.print("unhandled mknod result: {}\n", .{res});
break :blk std.fs.Dir.MakeError.Unexpected;
},
}
var fil = try std.fs.cwd().openFile(path, .{});
defer fil.close();
try self.setMetadata(alloc, tables, fil, options);
}
+101
View File
@@ -0,0 +1,101 @@
const std = @import("std");
const WaitGroup = std.Thread.WaitGroup;
const Mutex = std.Thread.Mutex;
const Archive = @import("../archive.zig");
const Inode = @import("../inode.zig");
const ExtractionOptions = @import("../options.zig");
const Tables = @import("../tables.zig");
const InodeFinish = @This();
const FinishEnum = enum {
wg,
fin,
};
pub const FinishUnion = union(FinishEnum) {
wg: *WaitGroup,
fin: *InodeFinish,
pub fn finish(self: FinishUnion) void {
switch (self) {
.wg => |wg| wg.finish(),
.fin => |fin| fin.finish(),
}
}
};
alloc: std.mem.Allocator,
inode: Inode,
path: []const u8,
tables: *Tables,
options: ExtractionOptions,
parent_finish: FinishUnion,
fil: ?std.fs.File,
out_err: *?anyerror,
wg: WaitGroup = .{},
mut: Mutex = .{},
pub fn create(
alloc: std.mem.Allocator,
inode: Inode,
path: []const u8,
tables: *Tables,
options: ExtractionOptions,
parent_finish: FinishUnion,
out_err: *?anyerror,
fil: ?std.fs.File,
work_size: usize,
) !*InodeFinish {
if (work_size == 0)
return error.InvalidWorkSize;
const out = try alloc.create(InodeFinish);
errdefer alloc.destroy(out);
out.* = .{
.alloc = alloc,
.inode = inode,
.path = path,
.tables = tables,
.options = options,
.parent_finish = parent_finish,
.out_err = out_err,
.fil = fil,
};
out.wg.startMany(work_size);
return out;
}
pub fn logError(self: *InodeFinish, comptime fmt: []const u8, args: anytype) void {
if (self.options.verbose)
self.options.verbose_writer.?.print(fmt, args) catch {};
}
pub fn finish(self: *InodeFinish) void {
self.mut.lock();
{
defer self.mut.unlock();
self.wg.finish();
if (!self.wg.isDone()) return;
}
defer {
self.parent_finish.finish();
self.alloc.destroy(self);
}
if (self.fil == null)
self.fil = std.fs.cwd().openFile(self.path, .{}) catch |err| {
if (self.options.verbose)
self.options.verbose_writer.?.print("Error opening {s} to set metadata: {}\n", .{ self.path, err }) catch {};
self.out_err.* = err;
return;
};
defer self.fil.?.close();
self.inode.setMetadata(self.alloc, self.tables, self.fil.?, self.options) catch |err| {
if (self.options.verbose)
self.options.verbose_writer.?.print("Error setting metadata to {s}: {}\n", .{ self.path, err }) catch {};
self.out_err.* = err;
return;
};
}
+41
View File
@@ -0,0 +1,41 @@
const std = @import("std");
const OffsetFile = @import("offset_file.zig");
const MetadataCache = @This();
alloc: std.mem.Allocator,
buf: []u8,
fixed_alloc: std.heap.FixedBufferAllocator,
cache: std.AutoArrayHashMap(u64, [8192]u8),
mut: std.Thread.Mutex = .{},
cache_mut: std.AutoArrayHashMap(u64, std.Thread.Mutex),
fil: OffsetFile,
pub fn init(alloc: std.mem.Allocator, cache_size: u64) !MetadataCache {}
pub fn deinit(self: *MetadataCache) void {
self.mut.lock();
defer self.mut.unlock();
self.cache.deinit();
self.cache_mut.deinit();
self.alloc.free(self.buf);
}
pub fn getChunk(self: *MetadataCache, offset: u64) ![8192]u8 {
var res = self.cache.get(offset);
if (res != null) return res.?;
var offset_mut = blk: {
self.mut.lock();
defer self.mut.unlock();
const mut = try self.cache_mut.getOrPut(offset);
if (!mut.found_existing)
mut.value_ptr.* = .{};
break :blk mut.value_ptr;
};
offset_mut.lock();
defer offset_mut.unlock();
}
+117
View File
@@ -0,0 +1,117 @@
const std = @import("std");
const Decompressor = @import("decomp.zig");
const Table = @import("tables.zig").Table;
const MetadataReader = @import("util/metadata.zig");
const OffsetFile = @import("util/offset_file.zig");
const Ref = packed struct {
block_offset: u16,
block_start: u32,
_: u16,
};
const Entry = packed struct {
ref: Ref,
count: u32,
size: u32,
};
const KeyPrefix = enum(u8) {
user,
trusted,
security,
};
const KeyRaw = packed struct {
type: packed struct {
prefix: KeyPrefix,
out_of_line: bool,
_: u7,
},
name_size: u16,
};
pub const KeyValue = struct {
key: [:0]u8,
value: []u8,
};
const XattrTable = @This();
alloc: std.mem.Allocator,
fil: OffsetFile,
decomp: *Decompressor,
count: u32,
start: u64,
table: Table(Entry),
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, table_start: u64) !XattrTable {
var info = packed struct {
start: u64 = undefined,
count: u32 = undefined,
_: u32 = undefined,
}{};
var rdr = try fil.readerAt(table_start, &[0]u8{});
try rdr.interface.readSliceEndian(@TypeOf(info), @ptrCast(&info), .little);
return .{
.alloc = alloc,
.fil = fil,
.decomp = decomp,
.count = info.count,
.start = info.start,
.table = try .init(alloc, fil, decomp, table_start + 16, info.count),
};
}
pub fn deinit(self: *XattrTable) void {
self.table.deinit();
}
pub fn get(self: *XattrTable, alloc: std.mem.Allocator, idx: u32) ![]KeyValue {
const entry: Entry = try self.table.get(idx);
const out = try alloc.alloc(KeyValue, entry.count);
for (out) |*kv| {
var rdr = try self.fil.readerAt(self.start + entry.ref.block_start, &[0]u8{});
var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
try meta.interface.discardAll(entry.ref.block_offset);
var key_raw: KeyRaw = undefined;
try meta.interface.readSliceEndian(KeyRaw, @ptrCast(&key_raw), .little);
switch (key_raw.type.prefix) {
.user => {
kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 5 + 1));
@memcpy(kv.key[0..5], "user.");
try meta.interface.readSliceAll(kv.key[5 .. kv.key.len - 1]);
kv.key[kv.key.len - 1] = 0;
},
.security => {
kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 9 + 1));
@memcpy(kv.key[0..9], "security.");
try meta.interface.readSliceAll(kv.key[9 .. kv.key.len - 1]);
kv.key[kv.key.len - 1] = 0;
},
.trusted => {
kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 8 + 1));
@memcpy(kv.key[0..8], "trusted.");
try meta.interface.readSliceAll(kv.key[8 .. kv.key.len - 1]);
kv.key[kv.key.len - 1] = 0;
},
}
if (key_raw.type.out_of_line) {
try meta.interface.discardAll(4);
var ref: Ref = undefined;
try meta.interface.readSliceEndian(Ref, @ptrCast(&ref), .little);
rdr = try self.fil.readerAt(self.start + ref.block_start, &[0]u8{});
meta = .init(alloc, &rdr.interface, self.decomp);
try meta.interface.discardAll(ref.block_offset);
}
var value_size: u32 = undefined;
try meta.interface.readSliceEndian(u32, @ptrCast(&value_size), .little);
kv.value = try alloc.alloc(u8, value_size);
try meta.interface.readSliceAll(kv.value);
}
return out;
}