alloc-ified many functions.

Updated README
This commit is contained in:
Caleb J. Gardner
2026-02-28 22:33:57 -06:00
parent b0160e005b
commit 5f629df47c
8 changed files with 126 additions and 169 deletions
+26 -6
View File
@@ -6,19 +6,23 @@ A library and application to decompress or view squashfs archives.
## Current State ## Current State
Overall works, but currently is missing some features (see below). Extraction is a bit slow compared to the normal `unsquashfs` (from my _very_ basic testing it's about ~3x slower). Only properly work on Linux, any other OSes probably won't work fully and are untested. Overall works, but currently is missing some features ([see below](#capabilities)) and has significantly slow performance compared to `unsquashfs` ([see below](#performance)).
## Build options ## Build options
> `-Duse_c_libs` > `-Duse_c_libs=true`
Instead of using Zig's standard library for decompression, use the system's C libraries. Has the benefit of being much faster and enabling LZO and LZ4 decompression. Instead of using Zig's standard library for decompression, use the system's C libraries. Has the benefit of being much faster and enabling LZO and LZ4 decompression.
> `-Dallow_lzo` > `-Dallow_lzo=true`
Enable compiling with LZO decompression support. The LZO library currently has some issues with Zig when imported so it's easier to just disable it by default. Only has an effect when using `-Duse_c_libs=true`. Enable compiling with LZO decompression support. The LZO library currently has some issues with Zig when imported so it's easier to just disable it by default. Only has an effect when using `-Duse_c_libs=true`.
> `-Dversion` > `-Dvalgrind=true`
Just sets the valgrind build option.
> `-Dversion=0.0.0`
Sets the version of `unsquashfs` shown when `--version` is passed. Sets the version of `unsquashfs` shown when `--version` is passed.
@@ -26,11 +30,27 @@ Sets the version of `unsquashfs` shown when `--version` is passed.
Most features are present except for the following: Most features are present except for the following:
* mod_time is not set on extraction
* xattrs are not applied on extraction * xattrs are not applied on extraction
* When using Zig decompression libraries then lzo and lz4 compression types are unavailable. I don't _currently_ plan on spending the time to find and validate a library since neither is popular. * When using Zig decompression libraries then lzo and lz4 compression types are unavailable. I don't _currently_ plan on spending the time to find and validate a library since neither is popular.
* When using C decompression libraries, lzo is not supported by default due to [some issues](#build-considerations). If it's needed it's trivial to fix, but it's easiest to just leave it disabled.
## Building considerations ## Performance
This is some basic observation's I've made about this library's performance when compared to `unsquashfs`. Unless otherwise stated, most observations were made when extracting my test archive (which is fairly small and uses zstd compression) and with `--release=fast`.
* Under ideal circumstances, my library is ~70% slower (.11s vs .18s)
* Mutli-threading on small archives noticably increases extraction times (when using C libraries) (.18s vs .57s). This should theoretically reverse on larger archives with many inodes, but I haven't tested that yet.
* Using Zig libraries *significantly* increases decompression time by ~600% under ideal circumstances.
Times:
* *unsquashfs*: .11s
* *C-libs, single-threaded*: .18s
* *C-libs, multi-threaded*: .57s
* *Zig-libs, single-threaded*: 5.87s
* *Zig-libs, multi-threaded*: 1.10s
## Build considerations
Compilation without `use_c_libs` works completely fine, but Zig has issues with some symbols from the lzo library that needs to be manually fixed. In particular you need to fix the definitions for `lzo_bytep` and `lzo_voidp` to be `*u8` and `?*anyopaque` respectively. Due to this, you have to manually enable LZO decompression using `-Dallow_lzo=true` when building. Compilation without `use_c_libs` works completely fine, but Zig has issues with some symbols from the lzo library that needs to be manually fixed. In particular you need to fix the definitions for `lzo_bytep` and `lzo_voidp` to be `*u8` and `?*anyopaque` respectively. Due to this, you have to manually enable LZO decompression using `-Dallow_lzo=true` when building.
+19 -9
View File
@@ -3,6 +3,7 @@ const std = @import("std");
pub fn build(b: *std.Build) !void { pub fn build(b: *std.Build) !void {
const use_c_libs_option = b.option(bool, "use_c_libs", "Use C versions of decompression libraries instead of the Zig standard library ones"); const use_c_libs_option = b.option(bool, "use_c_libs", "Use C versions of decompression libraries instead of the Zig standard library ones");
const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support"); const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support");
const valgrind = b.option(bool, "valgrind", "Compile with valgrind integration");
const version_string_option = b.option([]const u8, "version", "Version of the library/binary"); const version_string_option = b.option([]const u8, "version", "Version of the library/binary");
const zig_squashfs_options = b.addOptions(); const zig_squashfs_options = b.addOptions();
@@ -15,7 +16,8 @@ pub fn build(b: *std.Build) !void {
.root_source_file = b.path("src/root.zig"), .root_source_file = b.path("src/root.zig"),
.target = target, .target = target,
.optimize = optimize, .optimize = optimize,
.link_libc = if (use_c_libs_option == true) true else false, .link_libc = use_c_libs_option,
.valgrind = valgrind,
}); });
mod.addOptions("config", zig_squashfs_options); mod.addOptions("config", zig_squashfs_options);
if (use_c_libs_option == true) { if (use_c_libs_option == true) {
@@ -40,10 +42,11 @@ pub fn build(b: *std.Build) !void {
.root_source_file = b.path("src/bin/unsquashfs.zig"), .root_source_file = b.path("src/bin/unsquashfs.zig"),
.target = target, .target = target,
.optimize = optimize, .optimize = optimize,
.link_libc = if (use_c_libs_option == true) true else false, .link_libc = use_c_libs_option,
.imports = &.{ .imports = &.{
.{ .name = "zig_squashfs", .module = mod }, .{ .name = "zig_squashfs", .module = mod },
}, },
.valgrind = valgrind,
}); });
exe_mod.addOptions("config", unsquashfs_options); exe_mod.addOptions("config", unsquashfs_options);
const exe = b.addExecutable(.{ const exe = b.addExecutable(.{
@@ -58,13 +61,7 @@ pub fn build(b: *std.Build) !void {
b.installArtifact(lib); b.installArtifact(lib);
b.installArtifact(exe); b.installArtifact(exe);
const run_step = b.step("run", "Run the app");
const run_cmd = b.addRunArtifact(exe);
run_step.dependOn(&run_cmd.step);
run_cmd.step.dependOn(b.getInstallStep());
if (b.args) |args| {
run_cmd.addArgs(args);
}
const mod_tests = b.addTest(.{ const mod_tests = b.addTest(.{
.root_module = mod, .root_module = mod,
}); });
@@ -76,4 +73,17 @@ pub fn build(b: *std.Build) !void {
const test_step = b.step("test", "Run tests"); const test_step = b.step("test", "Run tests");
test_step.dependOn(&run_mod_tests.step); test_step.dependOn(&run_mod_tests.step);
test_step.dependOn(&run_exe_tests.step); test_step.dependOn(&run_exe_tests.step);
// zls build check steps
const lib_check = b.addLibrary(.{
.name = "squashfs",
.root_module = mod,
});
const exe_check = b.addExecutable(.{
.name = "unsquashfs",
.root_module = exe_mod,
});
const check = b.step("check", "Check if unsquashfs compiles");
check.dependOn(&lib_check.step);
check.dependOn(&exe_check.step);
} }
+27 -88
View File
@@ -31,138 +31,77 @@ pub const FragEntry = packed struct {
const Archive = @This(); const Archive = @This();
// 4 Gigs alloc: std.mem.Allocator,
const DEFAULT_MEM_SIZE = 4 * 1024 * 1024 * 1024;
parent_alloc: std.mem.Allocator,
alloc: std.heap.ThreadSafeAllocator,
// alloc: std.heap.FixedBufferAllocator,
// fixed_buf: []u8,
thread_count: usize,
fil: OffsetFile, fil: OffsetFile,
decomp: Decomp.DecompFn,
super: Superblock, super: Superblock,
setup: bool = false,
decomp: Decomp.DecompFn,
frag_table: Table(FragEntry) = undefined, frag_table: Table(FragEntry) = undefined,
id_table: Table(u16) = undefined, id_table: Table(u16) = undefined,
export_table: Table(InodeRef) = undefined, export_table: Table(InodeRef) = undefined,
/// Default settings using std.Thread.getCpuCount() threads and the minimum of 4gb or half of system memory for memory usage. /// Default settings using std.Thread.getCpuCount() threads and the minimum of 4gb or half of system memory for memory usage.
pub fn init(alloc: std.mem.Allocator, fil: File) !Archive { pub fn init(alloc: std.mem.Allocator, fil: File, offset: u64) !Archive {
return initAdvanced(
alloc,
fil,
0,
try std.Thread.getCpuCount(),
);
}
/// Create the Archive dictating the amount of threads used for extraction.
/// If you're planning on only interacting with a small number of files, it should be fine to use few (or one) threads.
pub fn initAdvanced(alloc: std.mem.Allocator, fil: File, offset: u64, threads: usize) !Archive {
var super: Superblock = undefined; var super: Superblock = undefined;
const red = try fil.pread(@ptrCast(&super), offset); const red = try fil.pread(@ptrCast(&super), offset);
std.debug.assert(red == @sizeOf(Superblock)); std.debug.assert(red == @sizeOf(Superblock));
try super.validate(); try super.validate();
// const fixed_buf = try alloc.alloc(u8, mem); const off_fil: OffsetFile = .init(fil, offset);
return .{ const decomp: Decomp.DecompFn = switch (super.compression) {
.parent_alloc = alloc,
.alloc = .{ .child_allocator = alloc },
// .fixed_buf = fixed_buf,
.thread_count = if (threads > 0) threads else try std.Thread.getCpuCount(),
.fil = .init(fil, offset),
.decomp = switch (super.compression) {
.gzip => Decomp.gzipDecompress, .gzip => Decomp.gzipDecompress,
.lzma => Decomp.lzmaDecompress, .lzma => Decomp.lzmaDecompress,
.xz => Decomp.xzDecompress, .xz => Decomp.xzDecompress,
.zstd => Decomp.zstdDecompress, .zstd => Decomp.zstdDecompress,
.lz4 => if (config.use_c_libs) Decomp.cLz4 else return error.Lz4Unsupported, .lz4 => if (config.use_c_libs) Decomp.cLz4 else return error.Lz4Unsupported,
.lzo => if (config.use_c_libs and config.allow_lzo) Decomp.lzoDecompress else return error.LzoUnsupported, .lzo => if (config.use_c_libs and config.allow_lzo) Decomp.lzoDecompress else return error.LzoUnsupported,
}, };
return .{
.alloc = alloc,
.fil = off_fil,
.decomp = decomp,
.super = super, .super = super,
.frag_table = try .init(alloc, off_fil, decomp, super.frag_start, super.frag_count),
.id_table = try .init(alloc, off_fil, decomp, super.id_start, super.id_count),
.export_table = try .init(alloc, off_fil, decomp, super.export_start, super.inode_count),
}; };
} }
pub fn deinit(self: *Archive) void { pub fn deinit(self: *Archive) void {
// self.parent_alloc.free(self.fixed_buf);
if (self.setup) {
self.frag_table.deinit(); self.frag_table.deinit();
self.export_table.deinit(); self.export_table.deinit();
self.id_table.deinit(); self.id_table.deinit();
} }
}
pub fn allocator(self: *Archive) std.mem.Allocator { pub fn inode(self: *Archive, alloc: std.mem.Allocator, num: u32) !Inode {
return self.alloc.allocator();
}
fn setupValues(self: *Archive) !void {
const alloc = self.allocator();
self.frag_table = try .init(alloc, self.fil, self.decomp, self.super.frag_start, self.super.frag_count);
self.id_table = try .init(alloc, self.fil, self.decomp, self.super.id_start, self.super.id_count);
self.export_table = try .init(alloc, self.fil, self.decomp, self.super.export_start, self.super.inode_count);
self.setup = true;
}
pub fn id(self: *Archive, idx: u32) !u16 {
if (!self.setup) try self.setupValues();
return self.id_table.get(idx);
}
pub fn frag(self: *Archive, idx: u32) !FragEntry {
if (!self.setup) try self.setupValues();
return self.frag_table.get(idx);
}
pub fn inode(self: *Archive, num: u32) !Inode {
if (!self.setup) try self.setupValues();
const ref = try self.export_table.get(num - 1); const ref = try self.export_table.get(num - 1);
var rdr = try self.fil.readerAt(ref.block_start + self.super.inode_start, &[0]u8{}); var rdr = try self.fil.readerAt(ref.block_start + self.super.inode_start, &[0]u8{});
var meta: MetadataReader = .init(self.allocator(), &rdr.interface, &self.decomp); var meta: MetadataReader = .init(alloc, &rdr.interface, &self.decomp);
try meta.interface.discardAll(ref.block_offset); try meta.interface.discardAll(ref.block_offset);
return try .read(self.allocator(), &meta.interface, self.super.block_size); return try .read(alloc, &meta.interface, self.super.block_size);
} }
pub fn root(self: *Archive) !SfsFile { pub fn root(self: *Archive, alloc: std.mem.Allocator) !SfsFile {
if (!self.setup) try self.setupValues();
var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{}); var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{});
var meta: MetadataReader = .init(self.allocator(), &rdr.interface, self.decomp); var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
try meta.interface.discardAll(self.super.root_ref.block_offset); try meta.interface.discardAll(self.super.root_ref.block_offset);
const in: Inode = try .read(self.allocator(), &meta.interface, self.super.block_size); const in: Inode = try .read(alloc, &meta.interface, self.super.block_size);
return .init(self, in, ""); return .init(self, in, "");
} }
pub fn open(self: *Archive, path: []const u8) !SfsFile { pub fn open(self: *Archive, alloc: std.mem.Allocator, path: []const u8) !SfsFile {
if (!self.setup) try self.setupValues(); var root_fil = try self.root(alloc);
var root_fil = try self.root();
defer if (!SfsFile.pathIsSelf(path)) root_fil.deinit(); defer if (!SfsFile.pathIsSelf(path)) root_fil.deinit();
return root_fil.open(path); return root_fil.open(path);
} }
pub fn extract(self: *Archive, path: []const u8, options: ExtractionOptions) !void { pub fn extract(self: *Archive, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void {
if (!self.setup) try self.setupValues();
var alloc = self.allocator();
var ext_path: []u8 = undefined;
if (std.fs.cwd().statFile(path)) |stat| {
if (stat.kind == .directory) {
ext_path = @constCast(path);
} else return error.ExtractionPathExists;
} else |err| {
if (err == error.FileNotFound) {
ext_path = @constCast(path);
} else {
std.log.err("Error stat-ing extraction path {s}: {}\n", .{ path, err });
return err;
}
}
defer if (ext_path.len > path.len) alloc.free(ext_path);
var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{}); var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{});
var meta: MetadataReader = .init(self.allocator(), &rdr.interface, self.decomp); var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
try meta.interface.discardAll(self.super.root_ref.block_offset); try meta.interface.discardAll(self.super.root_ref.block_offset);
const in: Inode = try .read(self.allocator(), &meta.interface, self.super.block_size); const in: Inode = try .read(self.alloc, &meta.interface, self.super.block_size);
try in.extractToThreaded(self, ext_path, options, self.thread_count); try in.extractTo(alloc, self, path, options);
} }
+7 -2
View File
@@ -44,9 +44,14 @@ pub fn main() !void {
} }
var fil: std.fs.File = try std.fs.cwd().openFile(archive, .{}); //TODO: Handle error gracefully. var fil: std.fs.File = try std.fs.cwd().openFile(archive, .{}); //TODO: Handle error gracefully.
defer fil.close(); defer fil.close();
var arc: squashfs.Archive = try .initAdvanced(alloc, fil, offset, threads); //TODO: Update when memory size matters. //TODO: Handle error gracefully. var arc: squashfs.Archive = try .init(alloc, fil, offset); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
defer arc.deinit(); defer arc.deinit();
try arc.extract(extLoc, if (verbose) .VerboseDefault(&out.interface) else .Default); //TODO: Handle error gracefully. const options: squashfs.ExtractionOptions = .{
.threads = if (threads == 0) try std.Thread.getCpuCount() else threads,
.verbose = verbose,
.verbose_writer = if (verbose) &out.interface else null,
};
try arc.extract(alloc, extLoc, options); //TODO: Handle error gracefully.
} }
fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void { fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
+4 -32
View File
@@ -96,7 +96,7 @@ pub fn iterate(self: SfsFile) !Iterator {
} }
/// Open a sub-file/folder within a directory at the given path. /// Open a sub-file/folder within a directory at the given path.
/// If path is "", ".", "/", or "./", this File is returned. /// If path is "", ".", "/", or "./", this File is returned.
pub fn open(self: SfsFile, path: []const u8) !SfsFile { pub fn open(self: SfsFile, alloc: std.mem.Allocator, path: []const u8) !SfsFile {
if (!self.isDir()) return FileError.NotDirectory; if (!self.isDir()) return FileError.NotDirectory;
if (pathIsSelf(path)) return self; if (pathIsSelf(path)) return self;
@@ -109,7 +109,6 @@ pub fn open(self: SfsFile, path: []const u8) !SfsFile {
if (std.mem.eql(u8, first_element, ".")) return self.open(path[idx + 1 ..]); if (std.mem.eql(u8, first_element, ".")) return self.open(path[idx + 1 ..]);
const entries = try self.getEntries(); const entries = try self.getEntries();
defer { defer {
var alloc = self.archive.allocator();
for (entries) |e| { for (entries) |e| {
e.deinit(alloc); e.deinit(alloc);
} }
@@ -127,7 +126,7 @@ pub fn open(self: SfsFile, path: []const u8) !SfsFile {
return fil; return fil;
} }
defer fil.deinit(); defer fil.deinit();
return fil.open(path[idx + 1 ..]); return fil.open(alloc, path[idx + 1 ..]);
}, },
.lt => cur_slice = cur_slice[0..split], .lt => cur_slice = cur_slice[0..split],
.gt => cur_slice = cur_slice[split + 1 ..], .gt => cur_slice = cur_slice[split + 1 ..],
@@ -170,35 +169,8 @@ pub fn devNum(self: SfsFile) !u32 {
/// Extract the given File to the path. If File is a regular file, the path must be a directory or not exist. /// Extract the given File to the path. If File is a regular file, the path must be a directory or not exist.
/// If the gievn path is a folder, the File's contents will be extracted within. /// If the gievn path is a folder, the File's contents will be extracted within.
pub fn extract(self: *SfsFile, path: []const u8, options: ExtractionOptions) !void { pub fn extract(self: *SfsFile, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void {
var alloc = self.archive.allocator(); return self.inode.extractToThreaded(alloc, self.archive, path, options);
var ext_path: []u8 = undefined;
if (std.fs.cwd().statFile(path)) |stat| {
if (stat.kind == .directory) {
if (!self.isDir()) {
const has_end_sep = path[path.len - 1] == '/';
const alloc_size = if (has_end_sep)
path.len + self.name.len
else
path.len + self.name.len + 1;
ext_path = try alloc.alloc(u8, alloc_size);
@memcpy(ext_path[0..path.len], path);
@memcpy(ext_path[ext_path.len - self.name.len ..], self.name);
if (!has_end_sep) ext_path[path.len] = '/';
} else {
ext_path = @constCast(path);
}
} else return FileError.ExtractionPathExists;
} else |err| {
if (err == error.FileNotFound) {
ext_path = @constCast(path);
} else {
std.log.err("Error stat-ing extraction path {s}: {}\n", .{ path, err });
return err;
}
}
defer if (ext_path.len > path.len) alloc.free(ext_path);
return self.inode.extractToThreaded(self.archive, path, options, self.archive.thread_count);
} }
/// Utility function. /// Utility function.
+24 -22
View File
@@ -121,7 +121,7 @@ pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: *Archive) !Dat
fn readerFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !DataReader { fn readerFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !DataReader {
var out: DataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size); var out: DataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size);
if (data.frag_idx != 0xFFFFFFFF) if (data.frag_idx != 0xFFFFFFFF)
out.addFragment(try archive.frag(data.frag_idx), data.frag_block_offset); out.addFragment(try archive.frag_table.get(data.frag_idx), data.frag_block_offset);
return out; return out;
} }
/// Get a threaded data reader for a file inode. /// Get a threaded data reader for a file inode.
@@ -135,7 +135,7 @@ pub fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: *Archi
fn threadedReaderFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !ThreadedDataReader { fn threadedReaderFromData(alloc: std.mem.Allocator, archive: *Archive, data: anytype) !ThreadedDataReader {
var out: ThreadedDataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size); var out: ThreadedDataReader = .init(alloc, archive.*, data.block_sizes, data.block_start, data.size);
if (data.frag_idx != 0xFFFFFFFF) if (data.frag_idx != 0xFFFFFFFF)
out.addFragment(try archive.frag(data.frag_idx), data.frag_block_offset); out.addFragment(try archive.frag_table.get(data.frag_idx), data.frag_block_offset);
return out; return out;
} }
@@ -155,15 +155,15 @@ fn entriesFromData(alloc: std.mem.Allocator, archive: Archive, data: anytype) ![
} }
/// Extract the inode to the given path. Single threaded. /// Extract the inode to the given path. Single threaded.
pub fn extractTo(self: Inode, archive: *Archive, path: []const u8, options: ExtractionOptions) !void { pub fn extractTo(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void {
if (options.threads > 1) return self.extractToThreaded(alloc, archive, path, options);
switch (self.hdr.inode_type) { switch (self.hdr.inode_type) {
.dir, .ext_dir => { .dir, .ext_dir => {
// Removing any trailing separators since that's the easiest path forward. // Removing any trailing separators since that's the easiest path forward.
if (path[path.len - 1] == '/') return self.extractTo(archive, path[0 .. path.len - 1], options); if (path[path.len - 1] == '/') return self.extractTo(alloc, archive, path[0 .. path.len - 1], options);
std.fs.cwd().makeDir(path) catch |err| { std.fs.cwd().makeDir(path) catch |err| {
if (err != std.fs.Dir.MakeError.PathAlreadyExists) return err; if (err != std.fs.Dir.MakeError.PathAlreadyExists) return err;
}; };
var alloc = archive.allocator();
const entries = try self.dirEntries(alloc, archive.*); const entries = try self.dirEntries(alloc, archive.*);
defer { defer {
for (entries) |entry| entry.deinit(alloc); for (entries) |entry| entry.deinit(alloc);
@@ -178,10 +178,10 @@ pub fn extractTo(self: Inode, archive: *Archive, path: []const u8, options: Extr
var inode: Inode = try readFromEntry(alloc, archive, entry); var inode: Inode = try readFromEntry(alloc, archive, entry);
defer inode.deinit(alloc); defer inode.deinit(alloc);
try inode.extractTo(archive, new_path, options); try inode.extractTo(alloc, archive, new_path, options);
} }
}, },
.file, .ext_file => try self.extractRegFile(archive.allocator(), archive, path, options), .file, .ext_file => try self.extractRegFile(alloc, archive, path, options),
.symlink, .ext_symlink => try self.extractSymlink(path), .symlink, .ext_symlink => try self.extractSymlink(path),
else => try self.extractDevice(archive, path, options), else => try self.extractDevice(archive, path, options),
} }
@@ -209,8 +209,8 @@ const Parent = struct {
.alloc = alloc, .alloc = alloc,
.path = path, .path = path,
.uid = try archive.id(hdr.uid_idx), .uid = try archive.id_table.get(hdr.uid_idx),
.gid = try archive.id(hdr.gid_idx), .gid = try archive.id_table.get(hdr.gid_idx),
.perm = hdr.permissions, .perm = hdr.permissions,
.mod_time = hdr.mod_time, .mod_time = hdr.mod_time,
@@ -244,12 +244,11 @@ const Parent = struct {
/// Functions identically to extractTo on all but regular files and directories. /// Functions identically to extractTo on all but regular files and directories.
/// ///
/// If threads <= 1, then this just calls extractTo. /// If threads <= 1, then this just calls extractTo.
pub fn extractToThreaded(self: Inode, archive: *Archive, path: []const u8, options: ExtractionOptions, threads: usize) !void { fn extractToThreaded(self: Inode, allocator: std.mem.Allocator, archive: *Archive, path: []const u8, options: ExtractionOptions) !void {
if (threads <= 1) return self.extractTo(archive, path, options);
switch (self.hdr.inode_type) { switch (self.hdr.inode_type) {
.dir, .ext_dir => { .dir, .ext_dir => {
// Removing any trailing separators since that's the easiest path forward. // Removing any trailing separators since that's the easiest path forward.
if (path[path.len - 1] == '/') return self.extractToThreaded(archive, path[0 .. path.len - 1], options, threads); if (path[path.len - 1] == '/') return self.extractToThreaded(allocator, archive, path[0 .. path.len - 1], options);
// Fixed Allocator // Fixed Allocator
// const mem_buf = archive.allocator().alloc(u8, 2 * 1024 * 1024 * 1024); // const mem_buf = archive.allocator().alloc(u8, 2 * 1024 * 1024 * 1024);
@@ -258,7 +257,7 @@ pub fn extractToThreaded(self: Inode, archive: *Archive, path: []const u8, optio
// const alloc = fixed_alloc.threadSafeAllocator(); // const alloc = fixed_alloc.threadSafeAllocator();
// Arena Allocator // Arena Allocator
var arena_alloc: std.heap.ArenaAllocator = .init(archive.allocator()); var arena_alloc: std.heap.ArenaAllocator = .init(allocator);
defer arena_alloc.deinit(); defer arena_alloc.deinit();
var thread_alloc: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena_alloc.allocator() }; var thread_alloc: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena_alloc.allocator() };
const alloc = thread_alloc.allocator(); const alloc = thread_alloc.allocator();
@@ -266,7 +265,7 @@ pub fn extractToThreaded(self: Inode, archive: *Archive, path: []const u8, optio
var wg: WaitGroup = .{}; var wg: WaitGroup = .{};
// defer if(!options.ignore_permissions) perms.?.deinit(alloc); We don't need to do this due to ArenaAllocator // defer if(!options.ignore_permissions) perms.?.deinit(alloc); We don't need to do this due to ArenaAllocator
var pool: Pool = undefined; var pool: Pool = undefined;
try pool.init(.{ .allocator = alloc, .n_jobs = threads - 1 }); try pool.init(.{ .allocator = allocator, .n_jobs = options.threads - 1 });
defer pool.deinit(); defer pool.deinit();
var out_err: ?anyerror = null; var out_err: ?anyerror = null;
@@ -281,16 +280,19 @@ pub fn extractToThreaded(self: Inode, archive: *Archive, path: []const u8, optio
try fil.updateTimes(time, time); try fil.updateTimes(time, time);
if (options.ignore_permissions) { if (options.ignore_permissions) {
try fil.chmod(self.hdr.permissions); try fil.chmod(self.hdr.permissions);
try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx)); try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx));
} }
}, },
.file, .ext_file => { .file, .ext_file => {
const alloc = archive.allocator();
var pool: Pool = undefined; var pool: Pool = undefined;
try pool.init(.{ .allocator = alloc, .n_jobs = threads }); try pool.init(.{ .allocator = allocator, .n_jobs = options.threads - 1 });
defer pool.deinit(); defer pool.deinit();
var arena_alloc: std.heap.ArenaAllocator = .init(allocator);
defer arena_alloc.deinit();
var thread_alloc: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena_alloc.allocator() };
const alloc = thread_alloc.allocator();
try self.extractRegFileThreaded(alloc, archive, path, options, &pool); try self.extractRegFileThreaded(alloc, archive, path, options, &pool);
var fil = try std.fs.cwd().openFile(path, .{}); var fil = try std.fs.cwd().openFile(path, .{});
@@ -299,7 +301,7 @@ pub fn extractToThreaded(self: Inode, archive: *Archive, path: []const u8, optio
try fil.updateTimes(time, time); try fil.updateTimes(time, time);
if (!options.ignore_permissions) { if (!options.ignore_permissions) {
try fil.chmod(self.hdr.permissions); try fil.chmod(self.hdr.permissions);
try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx)); try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx));
} }
}, },
.symlink, .ext_symlink => try self.extractSymlink(path), .symlink, .ext_symlink => try self.extractSymlink(path),
@@ -445,7 +447,7 @@ fn extractRegFile(self: Inode, alloc: std.mem.Allocator, archive: *Archive, path
try fil.updateTimes(time, time); try fil.updateTimes(time, time);
if (!options.ignore_permissions) { if (!options.ignore_permissions) {
try fil.chmod(self.hdr.permissions); try fil.chmod(self.hdr.permissions);
try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx)); try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx));
} }
} }
/// Extract the inode file contents to the given path threadedly. /// Extract the inode file contents to the given path threadedly.
@@ -462,7 +464,7 @@ fn extractRegFileThreaded(self: Inode, alloc: std.mem.Allocator, archive: *Archi
try fil.updateTimes(time, time); try fil.updateTimes(time, time);
if (!options.ignore_permissions) { if (!options.ignore_permissions) {
try fil.chmod(self.hdr.permissions); try fil.chmod(self.hdr.permissions);
try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx)); try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx));
} }
} }
/// Creates the symlink described by the inode. /// Creates the symlink described by the inode.
@@ -529,7 +531,7 @@ fn extractDevice(self: Inode, archive: *Archive, path: []const u8, options: Extr
try fil.updateTimes(time, time); try fil.updateTimes(time, time);
if (!options.ignore_permissions) { if (!options.ignore_permissions) {
try fil.chmod(self.hdr.permissions); try fil.chmod(self.hdr.permissions);
try fil.chown(try archive.id(self.hdr.uid_idx), try archive.id(self.hdr.gid_idx)); try fil.chown(try archive.id_table.get(self.hdr.uid_idx), try archive.id_table.get(self.hdr.gid_idx));
} }
if (!options.ignore_xattr) { if (!options.ignore_xattr) {
// TODO // TODO
+10 -2
View File
@@ -5,6 +5,8 @@ const Writer = std.Io.Writer;
const ExtractionOptions = @This(); const ExtractionOptions = @This();
/// The number of threads used for extraction. 0 implies single threaded.
threads: usize = 1,
/// Don't set the file's owner & permissions after extraction /// Don't set the file's owner & permissions after extraction
ignore_permissions: bool = false, ignore_permissions: bool = false,
/// Don't set xattr values. Currently xattrs are never set anyway. /// Don't set xattr values. Currently xattrs are never set anyway.
@@ -16,10 +18,16 @@ verbose: bool = false,
/// Where to print verbose log. /// Where to print verbose log.
verbose_writer: ?*Writer = null, verbose_writer: ?*Writer = null,
pub const Default: ExtractionOptions = .{}; pub const SingleThreadedDefault: ExtractionOptions = .{};
pub fn VerboseDefault(wrt: *Writer) ExtractionOptions { pub fn Default() !ExtractionOptions {
return .{
.threads = try std.Thread.getCpuCount(),
};
}
pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions {
return .{ return .{
.verbose = true, .verbose = true,
.verbose_writer = wrt, .verbose_writer = wrt,
.threads = try std.Thread.getCpuCount(),
}; };
} }
+1
View File
@@ -1 +1,2 @@
pub const Archive = @import("archive.zig"); pub const Archive = @import("archive.zig");
pub const ExtractionOptions = @import("options.zig");