Belac/zig-squashfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-set & re-write of several things

Browse Source 
Implemented new decompression cache to prevent decompressing the same block multiple times
This commit is contained in:
Caleb Gardner
2026-05-27 06:03:54 -05:00
parent 4b2b7021c7
commit 7c4089f826
42 changed files with 566 additions and 2585 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -2,3 +2,4 @@ testing/
.zig-cache/
zig-out/
zig-pkg/
.zed/settings.json
+22
View File
@@ -0,0 +1,22 @@
// Folder-specific settings
//
// For a full list of overridable settings, and general information on folder-specific settings,
// see the documentation: https://zed.dev/docs/configuring-zed#settings-files
{
"lsp": {
"zls": {
"initialization_options": {
"usePlaceholders": false,
},
"settings": {
"build_on_save": true,
"use_placeholders": false,
"build_on_save_args": [
"-fincremental",
"-Dallow_lzo=true",
"-Ddebug=true",
],
},
},
},
}
build.zig
+131 -67
View File
@@ -1,99 +1,122 @@
const std = @import("std");
const Build = std.Build;
const Step = Build.Step;
pub fn build(b: *std.Build) !void {
const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
// const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
const debug = b.option(bool, "debug", "Enable options to make debugging easier.") orelse false;
const version_string_option = b.option([]const u8, "version", "Version of the library/binary");
const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
var debug = b.option(bool, "debug", "Enable options to make debugging easier.") orelse false;
const dynamic = b.option(bool, "dynamic", "Use dynamic linking for C libraries (if used).") orelse false;
var version_string = b.option([]const u8, "version", "Version of the library/binary") orelse "0.0.0-testing";
const target = b.standardTargetOptions(.{});
var optimize = b.standardOptimizeOption(.{});
const zig_squashfs_options = b.addOptions();
zig_squashfs_options.addOption(bool, "use_zig_decomp", use_zig_decomp);
// zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo);
zig_squashfs_options.addOption(bool, "allow_lzo", allow_lzo);
const target = b.standardTargetOptions(.{});
const optimize = b.standardOptimizeOption(.{});
const mod = b.addModule("zig_squashfs", .{
.root_source_file = b.path("src/root.zig"),
.target = target,
.optimize = if (debug == true) .Debug else optimize,
.link_libc = !use_zig_decomp,
.valgrind = debug,
.error_tracing = debug,
.strip = if (debug == true) false else null,
});
mod.addOptions("config", zig_squashfs_options);
if (!use_zig_decomp) {
var zlib_ng = b.dependency("zlib_ng", .{
.target = target,
.optimize = optimize,
});
mod.linkLibrary(zlib_ng.artifact("zng"));
mod.linkSystemLibrary("lzma", .{ .preferred_link_mode = .static });
var minilzo = b.dependency("minilzo", .{
.target = target,
.optimize = optimize,
});
mod.linkLibrary(minilzo.artifact("minilzo"));
var lz4 = b.dependency("lz4", .{
.target = target,
.optimize = optimize,
});
mod.linkLibrary(lz4.artifact("lz4"));
var zstd = b.dependency("zstd", .{
.target = target,
.optimize = optimize,
});
mod.linkLibrary(zstd.artifact("zstd"));
}
var version = version_string_option orelse "0.0.0-testing";
if (version[0] == 'v') version = version[1..];
version_string = std.mem.trimStart(u8, version_string, "v");
const version = try std.SemanticVersion.parse(version_string);
const unsquashfs_options = b.addOptions();
unsquashfs_options.addOption(
std.SemanticVersion,
"version",
try std.SemanticVersion.parse(version),
version,
);
var exe_mod = b.createModule(.{
.root_source_file = b.path("src/bin/unsquashfs.zig"),
if (debug) optimize = .Debug;
if (optimize == .Debug) debug = true;
const c_import = b.addTranslateC(.{
.root_source_file = b.path("src/c.h"),
.target = target,
.optimize = if (debug == true) .Debug else optimize,
.link_libc = !use_zig_decomp,
.imports = &.{
.{ .name = "zig_squashfs", .module = mod },
},
.valgrind = debug,
.error_tracing = debug,
.strip = if (debug == true) false else null,
.optimize = optimize,
});
exe_mod.addOptions("config", unsquashfs_options);
const exe = b.addExecutable(.{
.name = "unsquashfs",
.root_module = exe_mod,
if (allow_lzo) c_import.defineCMacro("ALLOW_LZO", null);
if (dynamic) {
c_import.linkSystemLibrary("zlib-ng", .{});
c_import.linkSystemLibrary("lzma", .{});
if (allow_lzo)
c_import.linkSystemLibrary("minilzo", .{});
c_import.linkSystemLibrary("lz4", .{});
c_import.linkSystemLibrary("zstd", .{});
}
var lib = b.addLibrary(.{
.name = "squashfs",
.root_module = b.addModule("squashfs", .{
.root_source_file = b.path("src/root.zig"),
.target = target,
.optimize = optimize,
.valgrind = debug,
.error_tracing = debug,
.strip = debug,
.imports = &.{
.{ .name = "config", .module = zig_squashfs_options.createModule() },
.{ .name = "c", .module = c_import.createModule() },
},
}),
.use_llvm = debug,
.version = version,
});
const lib = b.addLibrary(.{
.name = "squashfs",
.root_module = mod,
const deps = try getDependencies(b, target, optimize, allow_lzo, dynamic);
for (deps) |d|
lib.root_module.linkLibrary(d);
const exe = b.addExecutable(.{
.name = "unsquashfs",
.root_module = b.createModule(.{
.root_source_file = b.path("src/bin/unsquashfs.zig"),
.target = target,
.optimize = optimize,
.imports = &.{
.{ .name = "config", .module = unsquashfs_options.createModule() },
.{ .name = "squashfs", .module = lib.root_module },
},
.valgrind = debug,
.error_tracing = debug,
.strip = if (debug == true) false else null,
}),
.use_llvm = debug,
.version = version,
});
b.installArtifact(lib);
b.installArtifact(exe);
const mod_tests = b.addTest(.{
.root_module = mod,
.root_module = b.createModule(.{
.root_source_file = b.path("src/root.zig"),
.target = target,
.optimize = optimize,
.valgrind = debug,
.error_tracing = debug,
.strip = debug,
.imports = &.{
.{ .name = "config", .module = zig_squashfs_options.createModule() },
.{ .name = "c", .module = c_import.createModule() },
},
}),
.use_llvm = true, // Helps with lldb degugging
.test_runner = .{
.mode = .simple,
.path = b.path("src/test.zig"),
},
});
for (deps) |d|
mod_tests.root_module.linkLibrary(d);
if (dynamic) {
mod_tests.root_module.linkSystemLibrary("zlib-ng", .{});
mod_tests.root_module.linkSystemLibrary("lzma", .{});
mod_tests.root_module.linkSystemLibrary("minilzo", .{});
mod_tests.root_module.linkSystemLibrary("lz4", .{});
}
const run_mod_tests = b.addRunArtifact(mod_tests);
const test_step = b.step("test", "Run tests");
test_step.dependOn(&run_mod_tests.step);
@@ -101,13 +124,54 @@ pub fn build(b: *std.Build) !void {
// zls build check steps
const lib_check = b.addLibrary(.{
.name = "squashfs",
.root_module = mod,
.root_module = lib.root_module,
});
const exe_check = b.addExecutable(.{
.name = "unsquashfs",
.root_module = exe_mod,
.root_module = exe.root_module,
});
const check = b.step("check", "Check if unsquashfs compiles");
check.dependOn(&lib_check.step);
check.dependOn(&exe_check.step);
}
fn getDependencies(b: *Build, target: Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, allow_lzo: bool, dynamic: bool) ![]*Step.Compile {
if (dynamic) return &.{};
var list: std.ArrayList(*Step.Compile) = .empty;
errdefer list.clearAndFree(b.allocator);
var zlib_ng = b.dependency("zlib_ng", .{
.target = target,
.optimize = optimize,
});
try list.append(b.allocator, zlib_ng.artifact("zng"));
var xz = b.dependency("xz", .{
.target = target,
.optimize = optimize,
});
try list.append(b.allocator, xz.artifact("lzma"));
if (allow_lzo) {
var minilzo = b.dependency("minilzo", .{
.target = target,
.optimize = optimize,
});
try list.append(b.allocator, minilzo.artifact("minilzo"));
}
var lz4 = b.dependency("lz4", .{
.target = target,
.optimize = optimize,
});
try list.append(b.allocator, lz4.artifact("lz4"));
var zstd = b.dependency("zstd", .{
.target = target,
.optimize = optimize,
});
try list.append(b.allocator, zstd.artifact("zstd"));
return list.toOwnedSlice(b.allocator);
}
build.zig.zon
+5 -1
View File
@@ -2,7 +2,7 @@
.name = .squashfs,
.version = "0.0.6",
.fingerprint = 0x37ba29474b87f145, // Changing this has security and trust implications.
.minimum_zig_version = "0.15.2",
.minimum_zig_version = "0.16.1",
.dependencies = .{
.zlib_ng = .{
.url = "git+https://github.com/CalebQ42/zig-zlib-ng#5f2f02dfb28acca2517dacbbd09e9b987f57b133",
@@ -20,6 +20,10 @@
.url = "git+https://github.com/CalebQ42/zig-minilzo.git#7cbae997b91a44d74b7cd6c073584dc9562a6c90",
.hash = "minilzo-2.10.0-Ij7BO8wLAADeWI4Pe4jp8XTDsDaquZR14oZ7_9yKKDWP",
},
.xz = .{
.url = "git+https://github.com/akunaakwei/zig-xz.git#e2d389262c8291907e3e4c6fb119819141c16c0f",
.hash = "xz-5.8.2-6v47_JYeAABSL-jonprpL5-E_YaaGc4B5xrbe93WsJ3G",
},
},
.paths = .{
"build.zig",
mise.toml
+2
View File
@@ -0,0 +1,2 @@
[tools]
zig = "master" # 0.16.0 compilation errors with SEGV
src/archive.zig
+24 -86
View File
@@ -1,105 +1,43 @@
//! A squashfs archive read from a file.
//! Can be used to directly access File's contents or extract to the filesystem.
const std = @import("std");
const File = std.fs.File;
const builtin = @import("builtin");
const Io = std.Io;
const MemoryMap = Io.File.MemoryMap;
const c = @import("c");
const config = @import("config");
const cDecomp = @import("decomp/misc_c.zig");
const Decomp = @import("decomp/zig_decomp.zig");
const ExtractionOptions = @import("options.zig");
const Inode = @import("inode.zig");
const InodeRef = Inode.Ref;
const BlockSize = @import("inode_data/file.zig").BlockSize;
const SfsFile = @import("file.zig");
const Superblock = @import("super.zig").Superblock;
const Tables = @import("tables.zig");
const MetadataReader = @import("util/metadata.zig");
const OffsetFile = @import("util/offset_file.zig");
const XattrTable = @import("xattr.zig");
const DecompCache = @import("util/decomp_cache.zig");
const CompressionType = @import("util/decompress.zig").CompressionType;
const Archive = @This();
alloc: std.mem.Allocator,
fil: OffsetFile,
map: MemoryMap,
cache: DecompCache,
super: Superblock,
tables: ?Tables = null,
pub fn init(alloc: std.mem.Allocator, io: Io, file: Io.File, offset: u64, max_cache_size: u64) !Archive {
var rdr = file.reader(io, &[0]u8{});
try rdr.seekTo(offset);
decomp: @import("decomp/types.zig").Decomp,
/// Default settings using std.Thread.getCpuCount() threads and the minimum of 4gb or half of system memory for memory usage.
pub fn init(alloc: std.mem.Allocator, fil: File, offset: u64) !Archive {
var super: Superblock = undefined;
const red = try fil.pread(@ptrCast(&super), offset);
std.debug.assert(red == @sizeOf(Superblock));
try rdr.interface.readSliceEndian(Superblock, @ptrCast(&super), .little);
try super.validate();
const off_fil: OffsetFile = .init(fil, offset);
if (!config.use_zig_decomp and config.allow_lzo)
_ = c.lzo_init();
const map = try file.createMemoryMap(
io,
.{ .offset = offset, .len = super.size, .protection = .{ .read = true } },
);
return .{
.alloc = alloc,
.fil = off_fil,
.decomp = if (config.use_zig_decomp)
switch (super.compression) {
.lz4 => return error.Lz4Unsupported,
.lzo => return error.LzoUnsupported,
.gzip => .{ .gzip = .{} },
.lzma => .{ .lzma = .{} },
.xz => .{ .xz = .{} },
.zstd => .{ .zstd = .{} },
}
else switch (super.compression) {
.gzip => .{ .gzip = .init(alloc) },
.zstd => .{ .zstd = .init(alloc) },
.xz => .{ .xz = .init(alloc) },
.lzma => .{ .lzma = .init(alloc) },
.lzo => .{ .lzo = .{} },
.lz4 => .{ .lz4 = .{} },
},
.map = map,
.cache = try .init(alloc, map, super.compression, max_cache_size),
.super = super,
};
}
pub fn deinit(self: *Archive) void {
if (self.tables != null)
self.tables.?.deinit();
self.decomp.deinit();
}
pub fn inode(self: *Archive, alloc: std.mem.Allocator, num: u32) !Inode {
if (self.tables == null)
self.tables = try .init(alloc, self);
const ref = try self.export_table.get(num - 1);
var rdr = try self.fil.readerAt(ref.block_start + self.super.inode_start, &[0]u8{});
var meta: MetadataReader = .init(alloc, &rdr.interface, &self.decomp);
try meta.interface.discardAll(ref.block_offset);
return try .read(alloc, &meta.interface, self.super.block_size);
}
pub fn root(self: *Archive, alloc: std.mem.Allocator) !SfsFile {
if (self.tables == null)
self.tables = try .init(alloc, self);
var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{});
var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
try meta.interface.discardAll(self.super.root_ref.block_offset);
const in: Inode = try .read(alloc, &meta.interface, self.super.block_size);
return .init(self, in, "");
}
pub fn open(self: *Archive, alloc: std.mem.Allocator, path: []const u8) !SfsFile {
if (self.tables == null)
self.tables = try .init(alloc, self);
var root_fil = try self.root(alloc);
defer if (!SfsFile.pathIsSelf(path)) root_fil.deinit();
return root_fil.open(path);
}
pub fn extract(self: Archive, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void {
var rdr = try self.fil.readerAt(self.super.root_ref.block_start + self.super.inode_start, &[0]u8{});
var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
try meta.interface.discardAll(self.super.root_ref.block_offset);
const in: Inode = try .read(self.alloc, &meta.interface, self.super.block_size);
try in.extractTo(alloc, self, path, options);
pub fn deinit(self: *Archive, io: Io) void {
self.map.destroy(io);
}
src/bin/unsquashfs.zig
+24 -20
View File
@@ -1,9 +1,10 @@
const std = @import("std");
const Io = std.Io;
const Writer = std.Io.Writer;
const builtin = @import("builtin");
const config = @import("config");
const squashfs = @import("zig_squashfs");
const squashfs = @import("squashfs");
//TODO: Add more options
const help_mgs =
@@ -38,40 +39,43 @@ var ignore_xattrs: bool = false;
var ignore_permissions: bool = false;
var force: bool = false;
pub fn main() !void {
const alloc = std.heap.smp_allocator;
var stdout = std.fs.File.stdout();
var out = stdout.writer(&[0]u8{});
pub fn main(init: std.process.Init) !void {
const alloc = init.gpa;
const io = init.io;
var stdout = Io.File.stdout();
var out = stdout.writer(io, &[0]u8{});
defer out.interface.flush() catch {};
try handleArgs(alloc, &out.interface);
try handleArgs(&out.interface, init.minimal.args);
if (archive.len == 0) {
try out.interface.print("You must provide a squashfs archive\n", .{});
try out.interface.print(help_mgs, .{});
return;
}
var fil: std.fs.File = try std.fs.cwd().openFile(archive, .{}); //TODO: Handle error gracefully.
defer fil.close();
var arc: squashfs.Archive = try .init(alloc, fil, offset); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
defer arc.deinit();
var fil: std.Io.File = try Io.Dir.cwd().openFile(io, archive, .{}); //TODO: Handle error gracefully.
defer fil.close(io);
var arc: squashfs.Archive = try .init(alloc, io, fil, offset, 1 * 1024 * 1024 * 1024); //TODO: Update when memory size matters. //TODO: Handle error gracefully.
defer arc.deinit(io);
const options: squashfs.ExtractionOptions = .{
.threads = if (threads == 0) try std.Thread.getCpuCount() else threads,
.single_threaded = threads == 1,
.verbose = verbose,
.verbose_writer = if (verbose) &out.interface else null,
.ignore_xattr = ignore_xattrs,
.ignore_permissions = ignore_permissions,
};
if (force)
try std.fs.cwd().deleteTree(extLoc);
try Io.Dir.cwd().deleteTree(io, extLoc);
try arc.extract(alloc, extLoc, options); //TODO: Handle error gracefully.
}
fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
var args = try std.process.argsWithAllocator(alloc);
defer args.deinit();
_ = args.next(); // args[0] is the application launch command.
while (args.next()) |arg| {
fn handleArgs(out: *Writer, args: std.process.Args) !void {
var arg_iter = args.iterate();
_ = arg_iter.next(); // args[0] is the application launch command.
while (arg_iter.next()) |arg| {
if (std.mem.eql(u8, arg, "-o")) {
const nxt = args.next();
const nxt = arg_iter.next();
if (nxt == null or nxt.?.len == 0) {
try out.print("-o must be followed by a number\n", .{});
return errors.InvalidArguments;
@@ -82,7 +86,7 @@ fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
};
continue;
} else if (std.mem.eql(u8, arg, "-d")) {
const nxt = args.next();
const nxt = arg_iter.next();
if (nxt == null or nxt.?.len == 0) {
try out.print("-d must be followed by a location\n", .{});
return errors.InvalidArguments;
@@ -90,7 +94,7 @@ fn handleArgs(alloc: std.mem.Allocator, out: *Writer) !void {
extLoc = nxt.?;
continue;
} else if (std.mem.eql(u8, arg, "-p")) {
const nxt = args.next();
const nxt = arg_iter.next();
if (nxt == null or nxt.?.len == 0) {
try out.print("-p must be followed by a number\n", .{});
return errors.InvalidArguments;
src/c.h
+7
View File
@@ -0,0 +1,7 @@
#ifdef ALLOW_LZO
#include <lzo/minilzo.h>
#endif
#include <zlib-ng.h>
#include <zstd.h>
#include <lz4.h>
#include <lzma.h>
src/c.zig
-8
View File
@@ -1,8 +0,0 @@
pub const c = @cImport({
@cInclude("zlib-ng.h");
@cInclude("lzma.h");
@cInclude("lz4.h");
@cInclude("zstd.h");
@cInclude("zstd_errors.h");
@cInclude("lzo/minilzo.h");
});
src/decomp.zig
-25
View File
@@ -1,25 +0,0 @@
const std = @import("std");
const Decompressor = @This();
pub const Error = error{
OutOfMemory,
BadInput,
OutputTooSmall,
ReadFailed,
WriteFailed,
EndOfStream,
};
vtable: *const struct {
decompress: *const fn (*Decompressor, []u8, []u8) Error!usize = DefaultDecompress,
stateless: *const fn (std.mem.Allocator, []u8, []u8) Error!usize,
},
pub fn decompress(self: *Decompressor, in: []u8, out: []u8) Error!usize {
return self.vtable.decompress(self, in, out);
}
fn DefaultDecompress(self: *Decompressor, in: []u8, out: []u8) Error!usize {
return self.vtable.stateless(std.heap.smp_allocator, in, out);
}
src/decomp/c/lz4.zig
-14
View File
@@ -1,14 +0,0 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
const res = c.LZ4_decompress_fast(in.ptr, out.ptr, @intCast(out.len));
if (res < 0) return Decompressor.Error.ReadFailed;
return @abs(res);
}
src/decomp/c/lzma.zig
-127
View File
@@ -1,127 +0,0 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
alloc: std.mem.Allocator,
streams: std.AutoHashMap(std.Thread.Id, c.lzma_stream),
interface: Decompressor,
pub fn init(alloc: std.mem.Allocator) Self {
return .{
.alloc = alloc,
.streams = .init(alloc),
.interface = .{
.vtable = &.{
.decompress = decompress,
.stateless = stateless,
},
},
};
}
pub fn deinit(self: *Self) void {
self.streams.deinit();
}
fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
var self: *Self = @fieldParentPtr("interface", decomp);
var strm = try self.getOrCreate();
strm.next_in = in.ptr;
strm.avail_in = in.len;
strm.next_out = out.ptr;
strm.avail_out = out.len;
var res = c.lzma_alone_decoder(strm, out.len * 2);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
while (res == c.LZMA_OK)
res = c.lzma_code(strm, c.LZMA_RUN);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
return strm.total_out;
}
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var strm: c.lzma_stream = .{
.allocator = &.{
.alloc = lzmaAlloc,
.free = lzmaFree,
.@"opaque" = @ptrCast(@constCast(&alloc)),
},
.next_in = in.ptr,
.avail_in = in.len,
.next_out = out.ptr,
.avail_out = out.len,
};
var res = c.lzma_alone_decoder(&strm, out.len * 2);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
while (res == c.LZMA_OK)
res = c.lzma_code(&strm, c.LZMA_RUN);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
return strm.total_out;
}
inline fn getOrCreate(self: *Self) !*c.lzma_stream {
const res = try self.streams.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) return res.value_ptr;
res.value_ptr.* = .{ .allocator = &.{
.alloc = lzmaAlloc,
.free = lzmaFree,
.@"opaque" = @ptrCast(&self.alloc),
} };
return res.value_ptr;
}
inline fn decodeResult(res: usize) Error!void {
return switch (res) {
c.LZMA_OK, c.LZMA_STREAM_END => {},
c.LZMA_NO_CHECK => Error.NoCheck,
c.LZMA_UNSUPPORTED_CHECK => Error.UnsupportedCheck,
c.LZMA_GET_CHECK => Error.GetCheck,
c.LZMA_MEM_ERROR, c.LZMA_MEMLIMIT_ERROR => Error.OutOfMemory,
c.LZMA_FORMAT_ERROR => Error.Format,
c.LZMA_OPTIONS_ERROR => Error.Options,
c.LZMA_DATA_ERROR => Error.Data,
c.LZMA_BUF_ERROR => Error.Buffer,
c.LZMA_PROG_ERROR => Error.Program,
c.LZMA_SEEK_NEEDED => Error.SeekNeeded,
else => Error.Unknown,
};
}
inline fn lzmaErrToDecompErr(err: Error) Decompressor.Error {
return switch (err) {
Error.OutOfMemory => Decompressor.Error.OutOfMemory,
Error.NoCheck => Decompressor.Error.ReadFailed,
Error.UnsupportedCheck => Decompressor.Error.ReadFailed,
Error.GetCheck => Decompressor.Error.ReadFailed,
Error.Format => Decompressor.Error.ReadFailed,
Error.Options => Decompressor.Error.ReadFailed,
Error.Data => Decompressor.Error.ReadFailed,
Error.Buffer => Decompressor.Error.WriteFailed,
Error.Program => Decompressor.Error.ReadFailed,
Error.SeekNeeded => Decompressor.Error.ReadFailed,
else => Decompressor.Error.ReadFailed,
};
}
fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
return alloc.rawAlloc(size, .@"1", 0);
}
fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0);
}
const Error = error{
OutOfMemory,
NoCheck,
UnsupportedCheck,
GetCheck,
Format,
Options,
Data,
Buffer,
Program,
SeekNeeded,
Unknown,
};
src/decomp/c/lzo.zig
-67
View File
@@ -1,67 +0,0 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var out_len: usize = out.len;
const res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &out_len, null);
decodeError(res) catch |err| return lzoErrToDecompErr(err);
return out_len;
}
inline fn decodeError(res: c_int) Error!void {
return switch (res) {
c.LZO_E_OK => {},
c.LZO_E_EOF_NOT_FOUND => Error.EofNotFound,
c.LZO_E_INPUT_NOT_CONSUMED => Error.InputNotConsumed,
c.LZO_E_INPUT_OVERRUN => Error.InputOverrun,
c.LZO_E_INTERNAL_ERROR => Error.InternalError,
c.LZO_E_INVALID_ALIGNMENT => Error.InvalidAlignment,
c.LZO_E_INVALID_ARGUMENT => Error.InvalidArgument,
c.LZO_E_LOOKBEHIND_OVERRUN => Error.LookbehindOverrun,
c.LZO_E_NOT_COMPRESSIBLE => Error.NotCompressible,
c.LZO_E_NOT_YET_IMPLEMENTED => Error.NotYetImplemented,
c.LZO_E_OUTPUT_NOT_CONSUMED => Error.OutputNotConsumed,
c.LZO_E_OUTPUT_OVERRUN => Error.OutputOverrun,
c.LZO_E_OUT_OF_MEMORY => Error.OutOfMemory,
else => Error.Unknown,
};
}
inline fn lzoErrToDecompErr(err: Error) Decompressor.Error {
return switch (err) {
Error.EofNotFound => Decompressor.Error.ReadFailed,
Error.InputNotConsumed => Decompressor.Error.ReadFailed,
Error.InputOverrun => Decompressor.Error.ReadFailed,
Error.InternalError => Decompressor.Error.ReadFailed,
Error.InvalidAlignment => Decompressor.Error.ReadFailed,
Error.InvalidArgument => Decompressor.Error.ReadFailed,
Error.LookbehindOverrun => Decompressor.Error.ReadFailed,
Error.NotCompressible => Decompressor.Error.ReadFailed,
Error.NotYetImplemented => Decompressor.Error.ReadFailed,
Error.OutputNotConsumed => Decompressor.Error.WriteFailed,
Error.OutputOverrun => Decompressor.Error.WriteFailed,
Error.OutOfMemory => Decompressor.Error.OutOfMemory,
else => Decompressor.Error.ReadFailed,
};
}
const Error = error{
EofNotFound,
InputNotConsumed,
InputOverrun,
InternalError,
InvalidAlignment,
InvalidArgument,
LookbehindOverrun,
NotCompressible,
NotYetImplemented,
OutputNotConsumed,
OutputOverrun,
OutOfMemory,
Unknown,
};
src/decomp/c/xz.zig
-127
View File
@@ -1,127 +0,0 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
alloc: std.mem.Allocator,
streams: std.AutoHashMap(std.Thread.Id, c.lzma_stream),
interface: Decompressor,
pub fn init(alloc: std.mem.Allocator) Self {
return .{
.alloc = alloc,
.streams = .init(alloc),
.interface = .{
.vtable = &.{
.decompress = decompress,
.stateless = stateless,
},
},
};
}
pub fn deinit(self: *Self) void {
self.streams.deinit();
}
fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
var self: *Self = @fieldParentPtr("interface", decomp);
var strm = try self.getOrCreate();
strm.next_in = in.ptr;
strm.avail_in = in.len;
strm.next_out = out.ptr;
strm.avail_out = out.len;
var res = c.lzma_stream_decoder(strm, out.len * 2, 0);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
while (res == c.LZMA_OK)
res = c.lzma_code(strm, c.LZMA_RUN);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
return strm.total_out;
}
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var strm: c.lzma_stream = .{
.allocator = &.{
.alloc = lzmaAlloc,
.free = lzmaFree,
.@"opaque" = @ptrCast(@constCast(&alloc)),
},
.next_in = in.ptr,
.avail_in = in.len,
.next_out = out.ptr,
.avail_out = out.len,
};
var res = c.lzma_stream_decoder(&strm, out.len * 2, 0);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
while (res == c.LZMA_OK)
res = c.lzma_code(&strm, c.LZMA_RUN);
decodeResult(res) catch |err| return lzmaErrToDecompErr(err);
return strm.total_out;
}
inline fn getOrCreate(self: *Self) !*c.lzma_stream {
const res = try self.streams.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) return res.value_ptr;
res.value_ptr.* = .{ .allocator = &.{
.alloc = lzmaAlloc,
.free = lzmaFree,
.@"opaque" = @ptrCast(&self.alloc),
} };
return res.value_ptr;
}
inline fn decodeResult(res: usize) Error!void {
return switch (res) {
c.LZMA_OK, c.LZMA_STREAM_END => {},
c.LZMA_NO_CHECK => Error.NoCheck,
c.LZMA_UNSUPPORTED_CHECK => Error.UnsupportedCheck,
c.LZMA_GET_CHECK => Error.GetCheck,
c.LZMA_MEM_ERROR, c.LZMA_MEMLIMIT_ERROR => Error.OutOfMemory,
c.LZMA_FORMAT_ERROR => Error.Format,
c.LZMA_OPTIONS_ERROR => Error.Options,
c.LZMA_DATA_ERROR => Error.Data,
c.LZMA_BUF_ERROR => Error.Buffer,
c.LZMA_PROG_ERROR => Error.Program,
c.LZMA_SEEK_NEEDED => Error.SeekNeeded,
else => Error.Unknown,
};
}
inline fn lzmaErrToDecompErr(err: Error) Decompressor.Error {
return switch (err) {
Error.OutOfMemory => Decompressor.Error.OutOfMemory,
Error.NoCheck => Decompressor.Error.ReadFailed,
Error.UnsupportedCheck => Decompressor.Error.ReadFailed,
Error.GetCheck => Decompressor.Error.ReadFailed,
Error.Format => Decompressor.Error.ReadFailed,
Error.Options => Decompressor.Error.ReadFailed,
Error.Data => Decompressor.Error.ReadFailed,
Error.Buffer => Decompressor.Error.WriteFailed,
Error.Program => Decompressor.Error.ReadFailed,
Error.SeekNeeded => Decompressor.Error.ReadFailed,
else => Decompressor.Error.ReadFailed,
};
}
fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
return alloc.rawAlloc(size, .@"1", 0);
}
fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0);
}
const Error = error{
OutOfMemory,
NoCheck,
UnsupportedCheck,
GetCheck,
Format,
Options,
Data,
Buffer,
Program,
SeekNeeded,
Unknown,
};
src/decomp/c/zlib.zig
-112
View File
@@ -1,112 +0,0 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
alloc: std.mem.Allocator,
streams: std.AutoHashMap(std.Thread.Id, c.zng_stream),
interface: Decompressor,
pub fn init(alloc: std.mem.Allocator) Self {
return .{
.alloc = alloc,
.streams = .init(alloc),
.interface = .{
.vtable = &.{
.decompress = decompress,
.stateless = stateless,
},
},
};
}
pub fn deinit(self: *Self) void {
self.streams.deinit();
}
fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
const self: *Self = @fieldParentPtr("interface", decomp);
var strm = try self.getOrCreate();
strm.next_in = in.ptr;
strm.avail_in = @truncate(in.len);
strm.next_out = out.ptr;
strm.total_out = out.len;
var res = c.zng_inflateReset(strm);
decodeError(res) catch |err| return zlibErrToDecompErr(err);
res = c.zng_inflate(strm, c.Z_FULL_FLUSH);
decodeError(res) catch |err| return zlibErrToDecompErr(err);
return strm.total_out;
}
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var strm: c.zng_stream = .{
.zalloc = zalloc,
.zfree = zfree,
.@"opaque" = @constCast(&alloc),
.next_in = in.ptr,
.avail_in = @truncate(in.len),
.next_out = out.ptr,
.total_out = out.len,
};
var res = c.zng_inflateInit(&strm);
decodeError(res) catch |err| return zlibErrToDecompErr(err);
res = c.zng_inflate(&strm, c.Z_FULL_FLUSH);
decodeError(res) catch |err| return zlibErrToDecompErr(err);
return strm.total_out;
}
fn getOrCreate(self: *Self) !*c.zng_stream {
const res = try self.streams.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) return res.value_ptr;
res.value_ptr.* = .{
.zalloc = zalloc,
.zfree = zfree,
.@"opaque" = &self.alloc,
};
return res.value_ptr;
}
fn zalloc(ptr: ?*anyopaque, size: c_uint, len: c_uint) callconv(.c) ?*anyopaque {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
return alloc.rawAlloc(size * len, .@"1", 0);
}
fn zfree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void {
var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr)));
alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0);
}
inline fn decodeError(res: i32) Error!void {
if (res >= 0) return;
return switch (res) {
c.Z_STREAM_ERROR => Error.Stream,
c.Z_DATA_ERROR => Error.Data,
c.Z_MEM_ERROR => Error.OutOfMemory,
c.Z_BUF_ERROR => Error.Buffer,
c.Z_VERSION_ERROR => Error.Version,
else => Error.Misc,
};
}
inline fn zlibErrToDecompErr(err: Error) Decompressor.Error {
return switch (err) {
Error.OutOfMemory => Decompressor.Error.OutOfMemory,
Error.Misc => Decompressor.Error.ReadFailed,
Error.Stream => Decompressor.Error.ReadFailed,
Error.Data => Decompressor.Error.ReadFailed,
Error.Buffer => Decompressor.Error.WriteFailed,
Error.Version => Decompressor.Error.ReadFailed,
};
}
const Error = error{
OutOfMemory,
Misc,
Stream,
Data,
Buffer,
Version,
};
src/decomp/c/zstd.zig
-165
View File
@@ -1,165 +0,0 @@
const std = @import("std");
const c = @import("../../c.zig").c;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
alloc: std.mem.Allocator,
ctx: std.AutoHashMap(std.Thread.Id, ?*c.ZSTD_DCtx),
interface: Decompressor,
pub fn init(alloc: std.mem.Allocator) Self {
return .{
.alloc = alloc,
.ctx = .init(alloc),
.interface = .{
.vtable = &.{
.decompress = decompress,
.stateless = stateless,
},
},
};
}
pub fn deinit(self: *Self) void {
self.ctx.deinit();
}
fn decompress(decomp: *Decompressor, in: []u8, out: []u8) Decompressor.Error!usize {
var self: *Self = @fieldParentPtr("interface", decomp);
const ctx = try self.getOrCreate();
const res = c.ZSTD_decompressDCtx(ctx, out.ptr, out.len, in.ptr, in.len);
decodeError(res) catch |err| return zstdErrToDecompErr(err);
return res;
}
fn stateless(_: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
decodeError(res) catch |err| return zstdErrToDecompErr(err);
return res;
}
inline fn getOrCreate(self: *Self) !?*c.ZSTD_DCtx {
const res = try self.ctx.getOrPut(std.Thread.getCurrentId());
if (res.found_existing) return res.value_ptr.*;
res.value_ptr.* = c.ZSTD_createDCtx();
return res.value_ptr.*;
}
inline fn decodeError(res: usize) Error!void {
if (c.ZSTD_isError(res) == 0) return;
return switch (c.ZSTD_getErrorCode(res)) {
c.ZSTD_error_prefix_unknown => Error.PrefixUnknown,
c.ZSTD_error_version_unsupported => Error.VersionUnsupported,
c.ZSTD_error_frameParameter_unsupported => Error.FrameParameterUnsupported,
c.ZSTD_error_frameParameter_windowTooLarge => Error.FrameParameterWindowTooLarge,
c.ZSTD_error_corruption_detected => Error.CorruptionDetected,
c.ZSTD_error_checksum_wrong => Error.ChecksumWrong,
c.ZSTD_error_literals_headerWrong => Error.LiteralsHeaderWrong,
c.ZSTD_error_dictionary_corrupted => Error.DictionaryCorrupted,
c.ZSTD_error_dictionary_wrong => Error.DictionaryWrong,
c.ZSTD_error_dictionaryCreation_failed => Error.DictionaryCreationFailed,
c.ZSTD_error_parameter_unsupported => Error.ParameterUnsupported,
c.ZSTD_error_parameter_combination_unsupported => Error.ParameterCombinationUnsupported,
c.ZSTD_error_parameter_outOfBound => Error.ParameterOutOfBound,
c.ZSTD_error_tableLog_tooLarge => Error.TableLogTooLarge,
c.ZSTD_error_maxSymbolValue_tooLarge => Error.MaxSymbolValueTooLarge,
c.ZSTD_error_maxSymbolValue_tooSmall => Error.MaxSymbolValueTooSmall,
c.ZSTD_error_cannotProduce_uncompressedBlock => Error.CannotProduceUncompressedBlock,
c.ZSTD_error_stabilityCondition_notRespected => Error.StabilityConditionNotRespected,
c.ZSTD_error_stage_wrong => Error.StageWrong,
c.ZSTD_error_init_missing => Error.InitMissing,
c.ZSTD_error_memory_allocation => Error.MemoryAllocation,
c.ZSTD_error_workSpace_tooSmall => Error.WorkSpaceTooSmall,
c.ZSTD_error_dstSize_tooSmall => Error.DstSizeTooSmall,
c.ZSTD_error_srcSize_wrong => Error.SrcSizeWrong,
c.ZSTD_error_dstBuffer_null => Error.DstBufferNull,
c.ZSTD_error_noForwardProgress_destFull => Error.NoForwardProgressDestFull,
c.ZSTD_error_noForwardProgress_inputEmpty => Error.NoForwardProgressInputEmpty,
c.ZSTD_error_frameIndex_tooLarge => Error.FrameIndexTooLarge,
c.ZSTD_error_seekableIO => Error.SeekableIo,
c.ZSTD_error_dstBuffer_wrong => Error.DstBufferWrong,
c.ZSTD_error_srcBuffer_wrong => Error.SrcBufferWrong,
c.ZSTD_error_sequenceProducer_failed => Error.SequenceProducerFailed,
c.ZSTD_error_externalSequences_invalid => Error.ExternalSequencesInvalid,
else => Error.Generic,
};
}
inline fn zstdErrToDecompErr(err: Error) Decompressor.Error {
return switch (err) {
Error.OutOfMemory => Decompressor.Error.OutOfMemory,
Error.Generic => Decompressor.Error.ReadFailed,
Error.PrefixUnknown => Decompressor.Error.ReadFailed,
Error.VersionUnsupported => Decompressor.Error.ReadFailed,
Error.FrameParameterUnsupported => Decompressor.Error.ReadFailed,
Error.FrameParameterWindowTooLarge => Decompressor.Error.ReadFailed,
Error.CorruptionDetected => Decompressor.Error.ReadFailed,
Error.ChecksumWrong => Decompressor.Error.ReadFailed,
Error.LiteralsHeaderWrong => Decompressor.Error.ReadFailed,
Error.DictionaryCorrupted => Decompressor.Error.ReadFailed,
Error.DictionaryWrong => Decompressor.Error.ReadFailed,
Error.DictionaryCreationFailed => Decompressor.Error.ReadFailed,
Error.ParameterUnsupported => Decompressor.Error.ReadFailed,
Error.ParameterCombinationUnsupported => Decompressor.Error.ReadFailed,
Error.ParameterOutOfBound => Decompressor.Error.ReadFailed,
Error.TableLogTooLarge => Decompressor.Error.ReadFailed,
Error.MaxSymbolValueTooLarge => Decompressor.Error.ReadFailed,
Error.MaxSymbolValueTooSmall => Decompressor.Error.ReadFailed,
Error.CannotProduceUncompressedBlock => Decompressor.Error.ReadFailed,
Error.StabilityConditionNotRespected => Decompressor.Error.ReadFailed,
Error.StageWrong => Decompressor.Error.ReadFailed,
Error.InitMissing => Decompressor.Error.ReadFailed,
Error.MemoryAllocation => Decompressor.Error.OutOfMemory,
Error.WorkSpaceTooSmall => Decompressor.Error.WriteFailed,
Error.DstSizeTooSmall => Decompressor.Error.WriteFailed,
Error.SrcSizeWrong => Decompressor.Error.ReadFailed,
Error.DstBufferNull => Decompressor.Error.WriteFailed,
Error.NoForwardProgressDestFull => Decompressor.Error.WriteFailed,
Error.NoForwardProgressInputEmpty => Decompressor.Error.ReadFailed,
Error.FrameIndexTooLarge => Decompressor.Error.ReadFailed,
Error.SeekableIo => Decompressor.Error.ReadFailed,
Error.DstBufferWrong => Decompressor.Error.WriteFailed,
Error.SrcBufferWrong => Decompressor.Error.ReadFailed,
Error.SequenceProducerFailed => Decompressor.Error.ReadFailed,
Error.ExternalSequencesInvalid => Decompressor.Error.ReadFailed,
};
}
const Error = error{
OutOfMemory,
Generic,
PrefixUnknown,
VersionUnsupported,
FrameParameterUnsupported,
FrameParameterWindowTooLarge,
CorruptionDetected,
ChecksumWrong,
LiteralsHeaderWrong,
DictionaryCorrupted,
DictionaryWrong,
DictionaryCreationFailed,
ParameterUnsupported,
ParameterCombinationUnsupported,
ParameterOutOfBound,
TableLogTooLarge,
MaxSymbolValueTooLarge,
MaxSymbolValueTooSmall,
CannotProduceUncompressedBlock,
StabilityConditionNotRespected,
StageWrong,
InitMissing,
MemoryAllocation,
WorkSpaceTooSmall,
DstSizeTooSmall,
SrcSizeWrong,
DstBufferNull,
NoForwardProgressDestFull,
NoForwardProgressInputEmpty,
FrameIndexTooLarge,
SeekableIo,
DstBufferWrong,
SrcBufferWrong,
SequenceProducerFailed,
ExternalSequencesInvalid,
};
src/decomp/types.zig
-43
View File
@@ -1,43 +0,0 @@
const config = @import("config");
const Decompressor = @import("../decomp.zig");
const cLz4 = @import("c/lz4.zig");
const cLzma = @import("c/lzma.zig");
const cLzo = @import("c/lzo.zig");
const cXz = @import("c/xz.zig");
const cZlib = @import("c/zlib.zig");
const cZstd = @import("c/zstd.zig");
const zigLzma = @import("zig/lzma.zig");
const zigXz = @import("zig/xz.zig");
const zigZlib = @import("zig/zstd.zig");
const zigZstd = @import("zig/zstd.zig");
pub const Decomp = union(enum) {
gzip: if (config.use_zig_decomp) zigZlib else cZlib,
lzma: if (config.use_zig_decomp) zigLzma else cLzma,
lzo: if (config.use_zig_decomp) void else cLzo,
xz: if (config.use_zig_decomp) zigXz else cXz,
lz4: if (config.use_zig_decomp) void else cLz4,
zstd: if (config.use_zig_decomp) zigZstd else cZstd,
pub fn deinit(self: *Decomp) void {
switch (self) {
.gzip => self.gzip.deinit(),
.lzma => self.lzma.deinit(),
.xz => self.xz.deinit(),
.zstd => self.zstd.deinit(),
else => {},
}
}
pub fn decompressor(self: *Decomp) *Decompressor {
return switch (self) {
.gzip => &self.gzip.interface,
.lzma => &self.lzma.interface,
.lzo => &self.lzo.interface,
.xz => &self.xz.interface,
.lz4 => &self.lz4.interface,
.zstd => &self.zstd.interface,
};
}
};
src/decomp/zig/lzma.zig
-19
View File
@@ -1,19 +0,0 @@
const std = @import("std");
const lzma = std.compress.lzma;
const Reader = std.Io.Reader;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var rdr: Reader = .fixed(in);
var decomp = try lzma.decompress(alloc, rdr.adaptToOldInterface());
defer decomp.deinit();
return decomp.read(out) catch |err| switch (err) {
error.CorruptInput, error.EndOfStream, error.Overflow => return Decompressor.Error.ReadFailed,
else => return err,
};
}
src/decomp/zig/xz.zig
-25
View File
@@ -1,25 +0,0 @@
const std = @import("std");
const xz = std.compress.xz;
const Reader = std.Io.Reader;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
var rdr: Reader = .fixed(in);
var decomp = try xz.decompress(alloc, rdr.adaptToOldInterface());
defer decomp.deinit();
return decomp.read(out) catch |err| switch (err) {
error.CorruptInput,
error.EndOfStream,
error.EndOfStreamWithNoError,
error.WrongChecksum,
error.Unsupported,
error.Overflow,
=> Decompressor.Error.ReadFailed,
else => return err,
};
}
src/decomp/zig/zlib.zig
-18
View File
@@ -1,18 +0,0 @@
const std = @import("std");
const Reader = std.Io.Reader;
const flate = std.compress.flate;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
const buf = try alloc.alloc(u8, out.len);
defer alloc.free(buf);
var rdr: Reader = .fixed(in);
var decomp = flate.Decompress.init(&rdr, .zlib, buf);
return decomp.reader.readSliceShort(out);
}
src/decomp/zig/zstd.zig
-18
View File
@@ -1,18 +0,0 @@
const std = @import("std");
const Reader = std.Io.Reader;
const zstd = std.compress.zstd;
const Decompressor = @import("../../decomp.zig");
const Self = @This();
interface: Decompressor = .{ .vtable = &.{ .stateless = stateless } },
fn stateless(alloc: std.mem.Allocator, in: []u8, out: []u8) Decompressor.Error!usize {
const buf = try alloc.alloc(u8, out.len * 2);
defer alloc.free(buf);
var rdr: Reader = .fixed(in);
var decomp = zstd.Decompress.init(&rdr, buf, .{ .window_len = @min(out.len, zstd.default_window_len) });
return decomp.reader.readSliceShort(out);
}
src/dir_entry.zig
-60
View File
@@ -1,60 +0,0 @@
//! Directory entry from the directory table.
const std = @import("std");
const Reader = std.Io.Reader;
const InodeType = @import("inode.zig").InodeType;
const Entry = @This();
const Header = extern struct { // use extern due to bad alignment with packed.
count: u32,
block_start: u32,
num: u32,
};
const RawEntry = packed struct {
offset: u16,
inode_offset: i16,
inode_type: InodeType,
name_size: u16,
};
block_start: u32,
block_offset: u16,
num: u32,
inode_type: InodeType,
name: []const u8,
pub fn readDir(alloc: std.mem.Allocator, rdr: *Reader, size: u32) ![]Entry {
var cur_red: u32 = 3; // start at 3 due to "." & ".." being counted in the dir size.
var hdr: Header = undefined;
var raw: RawEntry = undefined;
var out: std.ArrayList(Entry) = try .initCapacity(alloc, 100); // Start out with a decent capacity instead of needing to allocate per header.
errdefer out.deinit(alloc);
while (cur_red < size) {
try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
cur_red += @sizeOf(Header);
try out.ensureUnusedCapacity(alloc, hdr.count + 1);
for (0..hdr.count + 1) |_| {
try rdr.readSliceEndian(RawEntry, @ptrCast(&raw), .little);
const name = try alloc.alloc(u8, raw.name_size + 1);
errdefer alloc.free(name);
try rdr.readSliceEndian(u8, name, .little);
const val = out.addOneAssumeCapacity();
val.* = .{
.block_start = hdr.block_start,
.block_offset = raw.offset,
.num = @abs(hdr.num + raw.offset),
.inode_type = raw.inode_type,
.name = name,
};
cur_red += @sizeOf(RawEntry) + raw.name_size + 1;
}
}
return out.toOwnedSlice(alloc);
}
pub fn deinit(self: Entry, alloc: std.mem.Allocator) void {
alloc.free(self.name);
}
src/file.zig
-202
View File
@@ -1,202 +0,0 @@
//! A file/directory within the squashfs archive.
const std = @import("std");
const File = std.fs.File;
const WaitGroup = std.Thread.WaitGroup;
const Mutex = std.Thread.Mutex;
const Archive = @import("archive.zig");
const DirEntry = @import("dir_entry.zig");
const ExtractionOptions = @import("options.zig");
const Inode = @import("inode.zig");
const BlockSize = @import("inode_data/file.zig").BlockSize;
const DataReader = @import("util/data.zig");
const MetadataReader = @import("util/metadata.zig");
const FileError = error{
NotDirectory,
NotRegularFile,
NotSymlink,
NotDevice,
NotFound,
ExtractionPathExists,
};
const SfsFile = @This();
archive: *Archive,
inode: Inode,
name: []const u8,
/// Initialize a new File.
/// name is copied to the File so can be safely freed afterwards.
pub fn init(archive: *Archive, inode: Inode, name: []const u8) !SfsFile {
const new_name = try archive.allocator().alloc(u8, name.len);
@memcpy(new_name, name);
return .{
.archive = archive,
.inode = inode,
.name = new_name,
};
}
pub fn fromEntry(archive: *Archive, entry: DirEntry) !SfsFile {
var rdr = try archive.fil.readerAt(entry.block_start + archive.super.inode_start, &[0]u8{});
var meta: MetadataReader = .init(archive.allocator(), &rdr.interface, archive.decomp);
try meta.interface.discardAll(entry.block_offset);
const inode: Inode = try .read(archive.allocator(), &meta.interface, archive.super.block_size);
errdefer inode.deinit(archive.allocator());
return .init(archive, inode, entry.name);
}
pub fn deinit(self: SfsFile) void {
var alloc = self.archive.allocator();
alloc.free(self.name);
self.inode.deinit(alloc);
}
fn getEntries(self: SfsFile) ![]DirEntry {
return self.inode.dirEntries(self.archive.allocator(), self.archive.*);
}
pub fn ownerUid(self: SfsFile) !u16 {
return self.archive.id(self.inode.hdr.uid_idx);
}
pub fn ownerGid(self: SfsFile) !u16 {
return self.archive.id(self.inode.hdr.gid_idx);
}
pub fn permissions(self: SfsFile) u16 {
return self.inode.hdr.permissions;
}
pub fn isRegular(self: SfsFile) bool {
return switch (self.inode.hdr.inode_type) {
.file, .ext_file => true,
else => false,
};
}
/// The returned DataReader will no longer work if the File's deinit function is called
/// or, more specifically, it's inode's deinit function is called.
pub fn dataReader(self: SfsFile) !DataReader {
return self.inode.dataReader(self.archive);
}
pub fn isDir(self: SfsFile) bool {
return switch (self.inode.hdr.inode_type) {
.dir, .ext_dir => true,
else => false,
};
}
pub fn iterate(self: SfsFile) !Iterator {
if (!self.isDir()) return FileError.NotDirectory;
return .{
.entries = try self.getEntries(),
.archive = self.archive,
};
}
/// Open a sub-file/folder within a directory at the given path.
/// If path is "", ".", "/", or "./", this File is returned.
pub fn open(self: SfsFile, alloc: std.mem.Allocator, path: []const u8) !SfsFile {
if (!self.isDir()) return FileError.NotDirectory;
if (pathIsSelf(path)) return self;
// Recursively stip ending & leading path separators.
if (path[0] == '/') return self.open(path[1..]);
if (path[path.len - 1] == '/') return self.open(path[0 .. path.len - 1]);
const idx = std.mem.indexOf(u8, path, "/") orelse path.len;
const first_element = path[0..idx];
if (std.mem.eql(u8, first_element, ".")) return self.open(path[idx + 1 ..]);
const entries = try self.getEntries();
defer {
for (entries) |e| {
e.deinit(alloc);
}
alloc.free(entries);
}
var cur_slice = entries;
var split = cur_slice.len / 2;
while (cur_slice.len > 0) {
split = cur_slice.len / 2;
const comp = std.mem.order(u8, first_element, cur_slice[split].name);
switch (comp) {
.eq => {
var fil: SfsFile = try .fromEntry(self.archive, cur_slice[split]);
if (idx == path.len) {
return fil;
}
defer fil.deinit();
return fil.open(alloc, path[idx + 1 ..]);
},
.lt => cur_slice = cur_slice[0..split],
.gt => cur_slice = cur_slice[split + 1 ..],
}
}
return FileError.NotFound;
}
pub fn isSymlink(self: SfsFile) bool {
return switch (self.inode.hdr.inode_type) {
.symlink, .ext_symlink => true,
else => false,
};
}
pub fn symlinkPath(self: SfsFile) ![]const u8 {
if (!self.isSymlink()) return FileError.NotSymlink;
return switch (self.inode.data) {
.symlink => |s| s.target,
.ext_symlink => |s| s.target,
else => unreachable,
};
}
/// Check if the File is a block or character device.
pub fn isDevice(self: SfsFile) bool {
return switch (self.inode.hdr.inode_type) {
.block_dev, .char_dev, .ext_block_dev, .ext_char_dev => true,
else => false,
};
}
/// If the File is a block or character device, get's it's device number.
pub fn devNum(self: SfsFile) !u32 {
if (!self.isDevice()) return FileError.NotDevice;
return switch (self.inode.data) {
.block_dev, .char_dev => |d| d.dev,
.ext_block_dev, .ext_char_dev => |d| d.dev,
else => unreachable,
};
}
/// Extract the given File to the path. If File is a regular file, the path must be a directory or not exist.
/// If the gievn path is a folder, the File's contents will be extracted within.
pub fn extract(self: *SfsFile, alloc: std.mem.Allocator, path: []const u8, options: ExtractionOptions) !void {
return self.inode.extractToThreaded(alloc, self.archive, path, options);
}
/// Utility function.
pub fn pathIsSelf(path: []const u8) bool {
if (path.len == 0) return true;
if (path.len == 1 and (path[0] == '/' or path[0] == '.')) return true;
if (path.len == 2 and (path[0] == '.' and path[1] == '/')) return true;
return false;
}
pub const Iterator = struct {
entries: []DirEntry,
archive: *Archive,
idx: u32 = 0,
pub fn next(self: *Iterator) !?SfsFile {
if (self.idx >= self.entries.len) return null;
defer self.idx += 1;
return try SfsFile.fromEntry(self.archive, self.entries[self.idx]);
}
pub fn deinit(self: Iterator) void {
var alloc = self.archive.allocator();
for (self.entries) |e| {
e.deinit(alloc);
}
alloc.free(self.entries);
}
};
src/inode.zig
+1 -132
View File
@@ -1,26 +1,12 @@
//! A file-system object. Represents a File or directory.
const std = @import("std");
const Reader = std.Io.Reader;
const WaitGroup = std.Thread.WaitGroup;
const Pool = std.Thread.Pool;
const Mutex = std.Thread.Mutex;
const Archive = @import("archive.zig");
const DirEntry = @import("dir_entry.zig");
const ExtractionOptions = @import("options.zig");
const dir = @import("inode_data/dir.zig");
const file = @import("inode_data/file.zig");
const misc = @import("inode_data/misc.zig");
const Tables = @import("tables.zig");
const DataReader = @import("util/data.zig");
const ThreadedDataReader = @import("util/data_threaded.zig");
const InodeExtract = @import("util/extract.zig");
const InodeFinish = @import("util/inode_finish.zig");
const FinishUnion = InodeFinish.FinishUnion;
const MetadataReader = @import("util/metadata.zig");
pub const Ref = packed struct {
pub const Ref = packed struct(u64) {
block_offset: u16,
block_start: u32,
_: u16,
@@ -70,120 +56,3 @@ pub const Header = packed struct {
};
const Inode = @This();
hdr: Header,
data: InodeData,
pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode {
var hdr: Header = undefined;
try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little);
return .{
.hdr = hdr,
.data = switch (hdr.inode_type) {
.dir => .{ .dir = try .read(rdr) },
.file => .{ .file = try .read(alloc, rdr, block_size) },
.symlink => .{ .symlink = try .read(alloc, rdr) },
.block_dev => .{ .block_dev = try .read(rdr) },
.char_dev => .{ .char_dev = try .read(rdr) },
.fifo => .{ .fifo = try .read(rdr) },
.socket => .{ .socket = try .read(rdr) },
.ext_dir => .{ .ext_dir = try .read(rdr) },
.ext_file => .{ .ext_file = try .read(alloc, rdr, block_size) },
.ext_symlink => .{ .ext_symlink = try .read(alloc, rdr) },
.ext_block_dev => .{ .ext_block_dev = try .read(rdr) },
.ext_char_dev => .{ .ext_char_dev = try .read(rdr) },
.ext_fifo => .{ .ext_fifo = try .read(rdr) },
.ext_socket => .{ .ext_socket = try .read(rdr) },
},
};
}
pub fn readFromEntry(alloc: std.mem.Allocator, archive: Archive, entry: DirEntry) !Inode {
var rdr = try archive.fil.readerAt(archive.super.inode_start + entry.block_start, &[0]u8{});
var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp);
try meta.interface.discardAll(entry.block_offset);
return read(alloc, &meta.interface, archive.super.block_size);
}
pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
switch (self.data) {
.file => |f| alloc.free(f.block_sizes),
.ext_file => |f| alloc.free(f.block_sizes),
.symlink => |s| alloc.free(s.target),
.ext_symlink => |s| alloc.free(s.target),
else => {},
}
}
/// Get the data reader for a file inode.
pub fn dataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive, tables: *Tables) !DataReader {
return switch (self.hdr.inode_type) {
.file => readerFromData(alloc, archive, tables, self.data.file),
.ext_file => readerFromData(alloc, archive, tables, self.data.ext_file),
else => error.NotRegularFile,
};
}
fn readerFromData(alloc: std.mem.Allocator, archive: Archive, tables: *Tables, data: anytype) !DataReader {
var out: DataReader = .init(alloc, archive, data.block_sizes, data.block_start, data.size);
if (data.frag_idx != 0xFFFFFFFF)
out.addFragment(try tables.frag_table.get(data.frag_idx), data.frag_block_offset);
return out;
}
/// Get the directory entries for a directory inode.
pub fn dirEntries(self: Inode, alloc: std.mem.Allocator, archive: Archive) ![]DirEntry {
return switch (self.hdr.inode_type) {
.dir => entriesFromData(alloc, archive, self.data.dir),
.ext_dir => entriesFromData(alloc, archive, self.data.ext_dir),
else => error.NotDirectory,
};
}
fn entriesFromData(alloc: std.mem.Allocator, archive: Archive, data: anytype) ![]DirEntry {
var rdr = try archive.fil.readerAt(archive.super.dir_start + data.block_start, &[0]u8{});
var meta: MetadataReader = .init(alloc, &rdr.interface, archive.decomp);
try meta.interface.discardAll(data.block_offset);
return DirEntry.readDir(alloc, &meta.interface, data.size);
}
/// Returns the xattr index for the given inode. If the inode isn't an extended variant or doesn't have any, the u32 max is returned (0xFFFFFFFF).
pub fn xattrIdx(self: Inode) u32 {
return switch (self.data) {
.ext_dir => |d| d.xattr_id,
.ext_file => |f| f.xattr_idx,
.ext_symlink => |s| s.xattr_idx,
.ext_block_dev, .ext_char_dev => |d| d.xattr_idx,
.ext_fifo, .ext_socket => |i| i.xattr_idx,
else => 0xFFFFFFFF,
};
}
/// Applies the Inode's metadata to the given File.
/// Mod time is always set, but permissions and xattrs are set based on the given ExtractionOptions.
pub fn setMetadata(self: Inode, alloc: std.mem.Allocator, tables: *Tables, fil: std.fs.File, options: ExtractionOptions) !void {
const time = @as(i128, self.hdr.mod_time) * 1000000000;
try fil.updateTimes(time, time);
if (!options.ignore_permissions) {
try fil.chmod(self.hdr.permissions);
try fil.chown(try tables.id_table.get(self.hdr.uid_idx), try tables.id_table.get(self.hdr.gid_idx));
}
if (!options.ignore_xattr) {
const idx = self.xattrIdx();
if (idx == 0xFFFFFFFF) return;
const xattrs = try tables.xattr_table.get(alloc, idx);
defer alloc.free(xattrs);
for (xattrs) |kv| {
const res = std.os.linux.fsetxattr(fil.handle, kv.key, kv.value.ptr, kv.value.len, 0);
alloc.free(kv.key);
alloc.free(kv.value);
if (res != 0) {
if (options.verbose)
options.verbose_writer.?.print("fsetxattr has result of: {}\n", .{res}) catch {};
return error.SetXattr;
}
}
}
}
/// Extract the inode to the given path.
pub fn extractTo(self: Inode, alloc: std.mem.Allocator, archive: Archive, path: []const u8, options: ExtractionOptions) !void {
return InodeExtract.extractTo(alloc, self, archive, path, options);
}
src/inode_data/dir.zig
+2 -2
View File
@@ -1,6 +1,6 @@
const Reader = @import("std").Io.Reader;
pub const Dir = packed struct {
pub const Dir = extern struct {
block_start: u32,
hard_links: u32,
size: u16,
@@ -14,7 +14,7 @@ pub const Dir = packed struct {
}
};
pub const ExtDir = packed struct {
pub const ExtDir = extern struct {
hard_links: u32,
size: u32,
block_start: u32,
src/inode_data/file.zig
+38 -25
View File
@@ -10,25 +10,30 @@ pub const BlockSize = packed struct {
pub const File = struct {
block_start: u32, // bytes 0-3
frag_idx: u32, // bytes 4-7
frag_block_offset: u32, // bytes 8-11
frag_offset: u32, // bytes 8-11
size: u32, // bytes 12-15
block_sizes: []BlockSize,
pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !File {
var start: [16]u8 = undefined;
try rdr.readSliceAll(&start);
const frag_idx: u32 = std.mem.readInt(u32, start[4..8], .little);
const size: u32 = std.mem.readInt(u32, start[12..16], .little);
var num_blocks: u32 = size / block_size;
if (size % block_size != 0 and frag_idx == 0xFFFFFFFF) num_blocks += 1;
var values = struct {
block_start: u32, // bytes 0-3
frag_idx: u32, // bytes 4-7
frag_offset: u32, // bytes 8-11
size: u32, // bytes 12-15
};
try rdr.readSliceEndian(@TypeOf(values), @ptrCast(&values), .little);
var num_blocks: u32 = values.size / block_size;
if (values.size % block_size != 0 and values.frag_idx == 0xFFFFFFFF) num_blocks += 1;
const sizes = try alloc.alloc(BlockSize, num_blocks);
errdefer alloc.free(sizes);
try rdr.readSliceEndian(BlockSize, sizes, .little);
return .{
.block_start = std.mem.readInt(u32, start[0..4], .little),
.frag_idx = frag_idx,
.frag_block_offset = std.mem.readInt(u32, start[8..12], .little),
.size = size,
.block_start = values.block_start,
.frag_idx = values.frag_idx,
.frag_offset = values.frag_offset,
.size = values.size,
.block_sizes = sizes,
};
}
@@ -44,28 +49,36 @@ pub const ExtFile = struct {
sparse: u64, // bytes 16-23
hard_links: u32, // bytes 24-27
frag_idx: u32, // bytes 28-31
frag_block_offset: u32, // bytes 32-35
frag_offset: u32, // bytes 32-35
xattr_idx: u32, // bytes 36-39
block_sizes: []BlockSize,
pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !ExtFile {
var start: [40]u8 = undefined;
try rdr.readSliceAll(&start);
const frag_idx: u32 = std.mem.readInt(u32, start[28..32], .little);
const size: u64 = std.mem.readInt(u64, start[8..16], .little);
var num_blocks: u32 = @truncate(size / block_size);
if (size % block_size != 0 and frag_idx == 0xFFFFFFFF) num_blocks += 1;
var values = struct {
block_start: u64, // bytes 0-7
size: u64, // bytes 8-15
sparse: u64, // bytes 16-23
hard_links: u32, // bytes 24-27
frag_idx: u32, // bytes 28-31
frag_offset: u32, // bytes 32-35
xattr_idx: u32, // bytes 36-39
};
try rdr.readSliceEndian(@TypeOf(values), @ptrCast(&values), .little);
var num_blocks: u32 = values.size / block_size;
if (values.size % block_size != 0 and values.frag_idx == 0xFFFFFFFF) num_blocks += 1;
const sizes = try alloc.alloc(BlockSize, num_blocks);
errdefer alloc.free(sizes);
try rdr.readSliceEndian(BlockSize, sizes, .little);
return .{
.block_start = std.mem.readInt(u64, start[0..8], .little),
.size = size,
.sparse = std.mem.readInt(u64, start[16..24], .little),
.hard_links = std.mem.readInt(u32, start[24..28], .little),
.frag_idx = frag_idx,
.frag_block_offset = std.mem.readInt(u32, start[32..36], .little),
.xattr_idx = std.mem.readInt(u32, start[36..40], .little),
.block_start = values.block_start,
.size = values.size,
.sparse = values.sparse,
.hard_links = values.hard_links,
.frag_idx = values.frag_idx,
.frag_offset = values.frag_offset,
.xattr_idx = values.xattr_idx,
.block_sizes = sizes,
};
}
src/inode_data/misc.zig
+4 -4
View File
@@ -50,7 +50,7 @@ pub const ExtSymlink = struct {
};
/// A block or character device.
pub const Dev = packed struct {
pub const Dev = extern struct {
hard_links: u32,
dev: u32,
@@ -62,7 +62,7 @@ pub const Dev = packed struct {
};
/// An extended block or character device.
pub const ExtDev = packed struct {
pub const ExtDev = extern struct {
hard_links: u32,
dev: u32,
xattr_idx: u32,
@@ -75,7 +75,7 @@ pub const ExtDev = packed struct {
};
/// A socket or FIFO file.
pub const IPC = packed struct {
pub const IPC = extern struct {
hard_links: u32,
pub fn read(rdr: *Reader) !IPC {
@@ -86,7 +86,7 @@ pub const IPC = packed struct {
};
/// An extended socket or FIFO file.
pub const ExtIPC = packed struct {
pub const ExtIPC = extern struct {
hard_links: u32,
xattr_idx: u32,
src/options.zig
+8 -11
View File
@@ -5,25 +5,22 @@ const Writer = std.Io.Writer;
const ExtractionOptions = @This();
/// The number of threads used for extraction. 0 implies single threaded.
threads: usize = 1,
/// Don't set the file's owner & permissions after extraction
/// Force single-threaded extraction. Io.Threaded.global_single_threaded also works.
single_threaded: bool = false,
/// Don't set the file's owner, permissions, & modify time after extraction.
ignore_permissions: bool = false,
/// Don't set xattr values. Currently xattrs are never set anyway.
/// Don't set xattr values.
ignore_xattr: bool = false,
/// Replace symlinks with their target.
/// Replace symlinks with their target. Currently doesn't do anything.
dereference_symlinks: bool = false,
/// Verbose logging. If true, verbose_writer must be set
verbose: bool = false,
/// Where to print verbose log.
verbose_writer: ?*Writer = null,
pub const SingleThreadedDefault: ExtractionOptions = .{};
pub fn Default() !ExtractionOptions {
return .{
.threads = try std.Thread.getCpuCount(),
};
}
pub const defaultSingleThreaded: ExtractionOptions = .{ .single_threaded = true };
pub const default: ExtractionOptions = .{};
pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions {
return .{
.verbose = true,
src/super.zig
+4 -10
View File
@@ -2,6 +2,7 @@ const std = @import("std");
const math = std.math;
const InodeRef = @import("inode.zig").Ref;
const CompressionType = @import("util/decompress.zig").CompressionType;
const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little);
@@ -13,22 +14,15 @@ const SuperblockError = error{
};
/// A squashfs Superblock
pub const Superblock = packed struct {
pub const Superblock = extern struct {
magic: u32,
inode_count: u32,
mod_time: u32,
block_size: u32,
frag_count: u32,
compression: enum(u16) {
gzip = 1,
lzma,
lzo,
xz,
lz4,
zstd,
},
compression: CompressionType,
block_log: u16,
flags: packed struct {
flags: packed struct(u16) {
inode_uncompressed: bool,
data_uncompressed: bool,
check: bool,
src/tables.zig
-108
View File
@@ -1,108 +0,0 @@
const std = @import("std");
const Mutex = std.Thread.Mutex;
const Archive = @import("archive.zig");
const BlockSize = @import("inode_data/file.zig").BlockSize;
const InodeRef = @import("inode.zig").Ref;
const Superblock = @import("super.zig").Superblock;
const MetadataReader = @import("util/metadata.zig");
const OffsetFile = @import("util/offset_file.zig");
const XattrTable = @import("xattr.zig");
const Decompressor = @import("decomp.zig");
/// Information about a fragment section. Multiple fragments are contained in the block described by a single FragEntry.
/// The offset into the block and fragment size is stored in the file's inode.
pub const FragEntry = packed struct {
start: u64,
size: BlockSize,
_: u32,
};
const Tables = @This();
frag_table: Table(FragEntry),
id_table: Table(u16),
export_table: Table(InodeRef),
xattr_table: XattrTable,
pub fn init(alloc: std.mem.Allocator, archive: Archive) !Tables {
return .{
.frag_table = try .init(alloc, archive.fil, archive.decomp, archive.super.frag_start, archive.super.frag_count),
.id_table = try .init(alloc, archive.fil, archive.decomp, archive.super.id_start, archive.super.id_count),
.export_table = try .init(alloc, archive.fil, archive.decomp, archive.super.export_start, archive.super.inode_count),
.xattr_table = try .init(alloc, archive.fil, archive.decomp, archive.super.xattr_start),
};
}
pub fn deinit(self: *Tables) void {
self.frag_table.deinit();
self.id_table.deinit();
self.export_table.deinit();
self.xattr_table.deinit();
}
/// A two-layer metadata table.
pub fn Table(T: anytype) type {
return struct {
const Self = @This();
const VALS_PER_BLOCK = 8192 / @sizeOf(T);
alloc: std.mem.Allocator,
fil: OffsetFile,
decomp: *Decompressor,
tab_start: u64,
tab: std.AutoHashMap(u32, []T),
values: u32,
mut: Mutex = .{},
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, tab_start: u64, values: u32) !Self {
return .{
.alloc = alloc,
.fil = fil,
.decomp = decomp,
.tab_start = tab_start,
.tab = .init(alloc),
.values = values,
};
}
pub fn deinit(self: *Self) void {
var iter = self.tab.valueIterator();
while (iter.next()) |s| {
self.alloc.free(s.*);
}
self.tab.deinit();
}
pub fn get(self: *Self, idx: u32) !T {
if (idx >= self.values) return error.InvalidIndex;
const block_num = idx / VALS_PER_BLOCK;
const idx_offset = idx - (block_num * VALS_PER_BLOCK);
if (self.tab.contains(block_num)) {
const block = self.tab.get(block_num).?;
return block[idx_offset];
}
self.mut.lock();
defer self.mut.unlock();
// Double check in case of race condition..
if (self.tab.contains(block_num)) {
const block = self.tab.get(block_num).?;
return block[idx_offset];
}
const is_last = (self.values - 1) / VALS_PER_BLOCK == block_num;
const slice_size = if (is_last) self.values - (block_num * VALS_PER_BLOCK) else VALS_PER_BLOCK;
const slice = try self.alloc.alloc(T, slice_size);
var rdr = try self.fil.readerAt(self.tab_start + (8 * block_num), &[0]u8{});
var offset: u64 = 0;
try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
rdr = try self.fil.readerAt(offset, &[0]u8{});
var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
try meta.interface.readSliceEndian(T, @ptrCast(slice), .little);
try self.tab.put(block_num, slice);
return slice[idx_offset];
}
};
}
src/util/c_decomp.zig
+61
View File
@@ -0,0 +1,61 @@
const std = @import("std");
const c = @import("c");
const Error = @import("decompress.zig").DecompressionError;
pub fn zlibDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var strem: c.zng_stream = .{
.next_in = in.ptr,
.avail_in = @truncate(in.len),
.next_out = out.ptr,
.avail_out = @truncate(out.len),
};
var res = c.zng_inflateInit(&strem);
if (res != c.Z_OK) return Error.ReadFailed;
defer _ = c.zng_inflateEnd(&strem);
res = c.zng_inflate(&strem, c.Z_FULL_FLUSH);
if (res != c.Z_OK) return Error.ReadFailed;
return strem.total_out;
}
pub fn lzmaDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var strem: c.lzma_stream = .{
.next_in = in.ptr,
.avail_in = in.len,
.next_out = out.ptr,
.avail_out = out.len,
};
var res = c.lzma_auto_decoder(&strem, out.len * 2, 0);
if (res != c.LZMA_OK) return Error.ReadFailed;
defer c.lzma_end(&strem);
while (res == c.LZMA_OK)
res = c.lzma_code(&strem, c.LZMA_RUN);
if (res != c.LZMA_FINISH) return Error.ReadFailed;
return strem.total_out;
}
pub fn lzoDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var out_len = out.len;
const res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &out_len, null);
if (res != c.LZO_E_OK) return Error.ReadFailed;
return out_len;
}
pub fn lz4Decompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
const res = c.LZ4_decompress_safe(
in.ptr,
out.ptr,
@bitCast(@as(u32, @truncate(in.len))),
@bitCast(@as(u32, @truncate(out.len))),
);
if (res < 0) return Error.ReadFailed;
return @abs(res);
}
pub fn zstdDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
if (c.ZSTD_isError(res) != 0)
return Error.ReadFailed;
return res;
}
src/util/data.zig
-174
View File
@@ -1,174 +0,0 @@
//! A reader for a regular file.
const std = @import("std");
const Reader = std.Io.Reader;
const Writer = std.Io.Writer;
const Limit = std.Io.Limit;
const Archive = @import("../archive.zig");
const DecompFn = @import("../decomp.zig").DecompFn;
const BlockSize = @import("../inode_data/file.zig").BlockSize;
const FragEntry = @import("../tables.zig").FragEntry;
const OffsetFile = @import("offset_file.zig");
const DataReader = @This();
alloc: std.mem.Allocator,
fil: OffsetFile,
decomp: DecompFn,
block_size: u32,
blocks: []BlockSize,
frag: ?FragEntry = null, // TODO: do something better?
frag_offset: u32 = 0,
size: u64,
interface: Reader,
cur_offset: u64,
block_idx: u32 = 0,
pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64) DataReader {
return .{
.alloc = alloc,
.fil = archive.fil,
.decomp = archive.decomp,
.block_size = archive.super.block_size,
.blocks = blocks,
.size = size,
.cur_offset = start,
.interface = .{
.end = 0,
.seek = 0,
.buffer = &[0]u8{},
.vtable = &.{
.stream = stream,
.discard = discard,
.readVec = readVec,
},
},
};
}
pub fn deinit(self: *DataReader) void {
self.alloc.free(self.interface.buffer);
self.interface.end = 0;
self.interface.seek = 0;
}
pub fn addFragment(self: *DataReader, entry: FragEntry, frag_offset: u32) void {
self.frag = entry;
self.frag_offset = frag_offset;
}
fn numBlocks(self: DataReader) usize {
var res = self.blocks.len;
if (self.frag != null) res += 1;
return res;
}
fn advance(self: *DataReader) !void {
if (self.block_idx > self.blocks.len or (self.block_idx == self.blocks.len and self.frag == null)) {
if (self.interface.buffer.len > 0) {
self.alloc.free(self.interface.buffer);
self.interface.buffer = &[0]u8{};
self.interface.end = 0;
self.interface.seek = 0;
}
return Reader.Error.EndOfStream;
}
defer self.block_idx += 1;
const cur_block_size = if (self.block_idx == self.numBlocks() - 1) self.size % self.block_size else self.block_size;
try self.resizeBuffer(cur_block_size);
self.interface.seek = 0;
self.interface.end = cur_block_size;
if (self.block_idx == self.blocks.len) { // fragment
var rdr = try self.fil.readerAt(self.frag.?.start, &[0]u8{});
if (self.frag.?.size.uncompressed) {
try rdr.interface.discardAll(self.frag_offset);
try rdr.interface.readSliceAll(self.interface.buffer);
return;
}
const tmp_buf = try self.alloc.alloc(u8, self.frag.?.size.size);
defer self.alloc.free(tmp_buf);
try rdr.interface.readSliceAll(tmp_buf);
const needed_block = try self.alloc.alloc(u8, self.block_size);
defer self.alloc.free(needed_block);
_ = try self.decomp(self.alloc, tmp_buf, needed_block);
@memcpy(self.interface.buffer, needed_block[self.frag_offset .. self.frag_offset + cur_block_size]);
return;
}
const block = self.blocks[self.block_idx];
if (block.size == 0) {
@memset(self.interface.buffer, 0);
return;
}
var rdr = try self.fil.readerAt(self.cur_offset, &[0]u8{});
self.cur_offset += block.size;
if (block.uncompressed) {
try rdr.interface.readSliceAll(self.interface.buffer);
return;
}
const tmp_buf = try self.alloc.alloc(u8, block.size);
defer self.alloc.free(tmp_buf);
try rdr.interface.readSliceAll(tmp_buf);
_ = try self.decomp(self.alloc, tmp_buf, self.interface.buffer);
}
/// Does not guarentee that data currently in the buffer is retained.
fn resizeBuffer(self: *DataReader, size: usize) !void {
if (self.interface.buffer.len == size) return;
if (!self.alloc.resize(self.interface.buffer, size)) {
self.alloc.free(self.interface.buffer);
self.interface.buffer = self.alloc.alloc(u8, size) catch |err| {
self.interface.buffer = &[0]u8{};
return err;
};
} else {
self.interface.buffer.len = size;
}
}
fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) Reader.StreamError!usize {
var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr));
if (rdr.seek >= rdr.end) self.advance() catch |err| {
if (err == error.EndOfStream) return error.EndOfStream;
std.log.err("Error advancing data reader: {}\n", .{err});
return Reader.Error.ReadFailed;
};
if (limit == .nothing) return 0;
const to_read = @min(rdr.end - rdr.seek, @intFromEnum(limit));
const res = try wrt.write(rdr.buffer[rdr.seek .. rdr.seek + to_read]);
rdr.seek += res;
return res;
}
fn discard(rdr: *Reader, limit: Limit) Reader.Error!usize {
var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr));
if (rdr.seek >= rdr.end) self.advance() catch |err| {
if (err == error.EndOfStream) return error.EndOfStream;
std.log.err("Error advancing data reader: {}\n", .{err});
return Reader.Error.ReadFailed;
};
if (limit == .nothing) return 0;
const to_adv = @min(rdr.end - rdr.seek, @intFromEnum(limit));
rdr.seek += to_adv;
return to_adv;
}
fn readVec(rdr: *Reader, vec: [][]u8) Reader.Error!usize {
var self: *DataReader = @alignCast(@fieldParentPtr("interface", rdr));
if (rdr.seek >= rdr.end) self.advance() catch |err| {
if (err == error.EndOfStream) return error.EndOfStream;
std.log.err("Error advancing data reader: {}\n", .{err});
return Reader.Error.ReadFailed;
};
var cur_red: usize = 0;
for (vec) |s| {
const to_copy: usize = @min(rdr.end - rdr.seek, s.len);
@memcpy(s[0..to_copy], rdr.buffer[rdr.seek .. rdr.seek + to_copy]);
rdr.seek += to_copy;
cur_red += to_copy;
if (rdr.end == rdr.seek) break;
}
return cur_red;
}
src/util/data_threaded.zig
-207
View File
@@ -1,207 +0,0 @@
//! Similiar to DataReader, but set-up for threaded writing to files.
const std = @import("std");
const Reader = std.Io.Reader;
const Writer = std.Io.Writer;
const Limit = std.Io.Limit;
const WaitGroup = std.Thread.WaitGroup;
const Pool = std.Thread.Pool;
const Archive = @import("../archive.zig");
const DecompFn = @import("../decomp.zig").DecompFn;
const BlockSize = @import("../inode_data/file.zig").BlockSize;
const FragEntry = @import("../tables.zig").FragEntry;
const InodeFinish = @import("inode_finish.zig");
const OffsetFile = @import("offset_file.zig");
const ThreadedDataReader = @This();
alloc: std.mem.Allocator,
fil: OffsetFile,
decomp: DecompFn,
block_size: u32,
blocks: []BlockSize,
frag: ?FragEntry = null, // TODO: do something better?
frag_offset: u32 = 0,
size: u64,
num_blocks: usize,
start_offset: u64,
pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, start: u64, size: u64) ThreadedDataReader {
return .{
.alloc = alloc,
.fil = archive.fil,
.decomp = archive.decomp,
.block_size = archive.super.block_size,
.blocks = blocks,
.size = size,
.num_blocks = blocks.len,
.start_offset = start,
};
}
pub fn addFragment(self: *ThreadedDataReader, entry: FragEntry, frag_offset: u32) void {
self.frag = entry;
self.frag_offset = frag_offset;
self.num_blocks = self.blocks.len + 1;
}
/// Extract the data to the file threadedly, using pool to spawn threads.
/// If errors occur, they are set to finish.out_err.
pub fn extractThreaded(self: ThreadedDataReader, file: std.fs.File, pool: *Pool, finish: *InodeFinish) void {
var cur_write_offset: u64 = 0;
var cur_read_offset: u64 = self.start_offset;
for (0..self.blocks.len) |i| {
const cur_block_size = if (i == self.num_blocks - 1) self.size % self.block_size else self.block_size;
pool.spawn(workThreadBlocks, .{ self, file, cur_write_offset, cur_read_offset, self.blocks[i], cur_block_size, finish }) catch |res_err| {
finish.logError("Can't spawn pool task: {}", .{res_err});
finish.out_err.* = res_err;
finish.finish();
};
cur_write_offset += cur_block_size;
cur_read_offset += self.blocks[i].size;
}
if (self.frag != null)
pool.spawn(workThreadFragment, .{ self, file, cur_write_offset, finish }) catch |res_err| {
finish.logError("Can't spawn pool task: {}", .{res_err});
finish.out_err.* = res_err;
finish.finish();
};
}
fn workThreadBlocks(
self: ThreadedDataReader,
fil: std.fs.File,
write_offset: u64,
read_offset: u64,
block: BlockSize,
cur_block_size: u64,
finish: *InodeFinish,
) void {
defer finish.finish();
var wrt = fil.writer(&[0]u8{});
wrt.seekTo(write_offset) catch |err| {
finish.logError("Error seeking file writer: {}", .{err});
finish.out_err.* = err;
return;
};
defer wrt.interface.flush() catch |err| {
finish.logError("Error flushing file writer: {}", .{err});
finish.out_err.* = err;
};
if (block.size == 0) {
wrt.interface.splatByteAll(0, cur_block_size) catch |err| {
finish.logError("Error writing zeroes: {}", .{err});
finish.out_err.* = err;
return;
};
return;
}
var rdr = self.fil.readerAt(read_offset, &[0]u8{}) catch |err| {
finish.logError("Error creating file reader: {}", .{err});
finish.out_err.* = err;
return;
};
if (block.uncompressed) {
rdr.interface.streamExact(&wrt.interface, block.size) catch |err| {
finish.logError("Error streaming data: {}", .{err});
finish.out_err.* = err;
return;
};
return;
}
// TODO: shared buffers
const read_buf = self.alloc.alloc(u8, block.size) catch |err| {
finish.logError("Error creating reader buffer: {}", .{err});
finish.out_err.* = err;
return;
};
defer self.alloc.free(read_buf);
rdr.interface.readSliceAll(read_buf) catch |err| {
finish.logError("Error reading data into reader buffer: {}", .{err});
finish.out_err.* = err;
return;
};
// TODO: shared buffers
const res_buf = self.alloc.alloc(u8, cur_block_size) catch |err| {
finish.logError("Error creating result buffer: {}", .{err});
finish.out_err.* = err;
return;
};
defer self.alloc.free(res_buf);
_ = self.decomp(self.alloc, read_buf, res_buf) catch |err| {
finish.logError("Error decompressing data block: {}", .{err});
finish.out_err.* = err;
return;
};
wrt.interface.writeAll(res_buf) catch |err| {
finish.logError("Error writing to file: {}", .{err});
finish.out_err.* = err;
return;
};
}
fn workThreadFragment(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, finish: *InodeFinish) void {
defer finish.finish();
var wrt = fil.writer(&[0]u8{});
wrt.seekTo(write_offset) catch |err| {
finish.logError("Error seeking file writer for file fragment: {}", .{err});
finish.out_err.* = err;
return;
};
defer wrt.interface.flush() catch |err| {
finish.out_err.* = err;
};
var rdr = self.fil.readerAt(self.frag.?.start, &[0]u8{}) catch |err| {
finish.logError("Error creating file reader for file fragment: {}", .{err});
finish.out_err.* = err;
return;
};
if (self.frag.?.size.uncompressed) {
rdr.interface.discardAll(self.frag_offset) catch |err| {
finish.logError("Error discarding useless fragment data: {}", .{err});
finish.out_err.* = err;
return;
};
rdr.interface.streamExact(&wrt.interface, self.size % self.block_size) catch |err| {
finish.logError("Error streaming fragment data: {}", .{err});
finish.out_err.* = err;
return;
};
return;
}
const tmp_buf = self.alloc.alloc(u8, self.frag.?.size.size) catch |err| {
finish.logError("Error creating a temporary buffer for a file fragment: {}", .{err});
finish.out_err.* = err;
return;
};
defer self.alloc.free(tmp_buf);
rdr.interface.readSliceAll(tmp_buf) catch |err| {
finish.logError("Error reading data into fragment buffer: {}", .{err});
finish.out_err.* = err;
return;
};
const needed_block = self.alloc.alloc(u8, self.block_size) catch |err| {
finish.logError("Error allocating fragment decompression results: {}", .{err});
finish.out_err.* = err;
return;
};
defer self.alloc.free(needed_block);
_ = self.decomp(self.alloc, tmp_buf, needed_block) catch |err| {
finish.logError("Error decompressing fragment: {}", .{err});
finish.out_err.* = err;
return;
};
wrt.interface.writeAll(needed_block[self.frag_offset .. self.frag_offset + (self.size % self.block_size)]) catch |err| {
finish.logError("Error writing fragment: {}", .{err});
finish.out_err.* = err;
return;
};
}
src/util/decomp_cache.zig
+108
View File
@@ -0,0 +1,108 @@
const std = @import("std");
const Io = std.Io;
const ArrayHashMap = std.array_hash_map.Auto;
const Atomic = std.atomic.Value;
const Decompress = @import("decompress.zig");
const DecompressFn = Decompress.DecompressFn;
const DecompressType = Decompress.CompressionType;
const DecompCache = @This();
const Cache = struct {
cache: []u8,
usage: Atomic(u32) = .init(0),
};
arena: std.heap.ArenaAllocator,
decomp: DecompressFn,
map: Io.File.MemoryMap,
cache: ArrayHashMap(u64, Cache),
mut: Io.RwLock = .init,
cond: Io.Condition = .init,
max_size: u64,
cur_size: u64 = 0,
pub fn init(alloc: std.mem.Allocator, map: Io.File.MemoryMap, decomp_type: DecompressType, max_size: u64) !DecompCache {
return .{
.arena = .init(alloc),
.decomp = try Decompress.getDecompressFn(decomp_type),
.map = map,
.cache = .empty,
.max_size = max_size,
};
}
pub fn deinit(self: *DecompCache, io: Io) void {
self.mut.lockUncancelable(io);
self.cache.deinit();
self.arena.deinit();
}
fn makeRoom(self: *DecompCache, size: u32) !void {
if (size + self.cur_size < self.max_size) return;
var iter = self.cache.iterator();
while (iter.next()) |ent| {
const val = ent.value_ptr;
if (val.usage.load(.unordered) == 0) {
self.cur_size -= val.cache.len;
_ = self.cache.orderedRemove(ent.key_ptr.*);
}
if (size + self.cur_size < self.max_size) return;
}
self.cond.wait(Io, &self.mut.mutex);
try self.makeRoom(size);
}
pub fn checkinBlock(self: *DecompCache, io: Io, offset: u64) void {
self.mut.lockSharedUncancelable(io);
defer self.mut.unlockShared(io);
const get = self.cache.getPtr(offset);
if (get == null) return;
const res = get.?.usage.fetchSub(1, .unordered);
if (res == 0) self.cond.broadcast(io);
}
pub fn checkoutBlock(self: *DecompCache, io: Io, offset: u64, block_size: u32, max_result_size: u32) ![]u8 {
{
try self.mut.lockShared(io);
defer self.mut.unlockShared(io);
const get = self.cache.getPtr(offset);
if (get != null) {
_ = get.?.usage.fetchAdd(1, .unordered);
return get.?.cache;
}
}
try self.mut.lock(io);
defer self.mut.unlock(io);
try self.makeRoom(max_result_size);
var alloc = self.arena.allocator();
const buf_alloc = self.arena.child_allocator;
var out = try alloc.alloc(u8, max_result_size);
errdefer alloc.free(out);
const out_size = try self.decomp(buf_alloc, self.map[offset..][0..block_size], out);
if (out_size != max_result_size) {
if (alloc.resize(out, out_size)) {
out.len = out_size;
} else {
const new_out = try alloc.alloc(u8, out_size);
@memcpy(new_out, out[0..out_size]);
alloc.free(out);
out = new_out;
}
}
try self.cache.put(alloc, offset, out);
return out;
}
src/util/decompress.zig
+38
View File
@@ -0,0 +1,38 @@
const std = @import("std");
const Io = std.Io;
const config = @import("config");
const c_decomp = @import("c_decomp.zig");
const zig_decomp = @import("zig_decomp.zig");
pub const DecompressionError = Io.Reader.Error || std.mem.Allocator.Error;
pub const DecompressFn = *const fn (alloc: std.mem.Allocator, in: []u8, out: []u8) DecompressionError!usize;
pub const CompressionType = enum(u16) {
gzip = 1,
lzma,
lzo,
xz,
lz4,
zstd,
};
pub fn getDecompressFn(t: CompressionType) !DecompressFn {
return if (config.use_zig_decomp) switch (t) {
.lzo => error.LzoUnsupported,
.lz4 => error.Lz4Unsupported,
.gzip => zig_decomp.zlibDecompress,
.lzma => zig_decomp.lzmaDecompress,
.xz => zig_decomp.xzDecompress,
.zstd => zig_decomp.zstdDecompress,
} else switch (t) {
.gzip => c_decomp.zlibDecompress,
.lzma => c_decomp.lzmaDecompress,
.lzo => if (config.allow_lzo) c_decomp.lzoDecompress else error.LzoUnsupported,
.xz => c_decomp.lzmaDecompress,
.lz4 => c_decomp.lz4Decompress,
.zstd => c_decomp.zstdDecompress,
};
}
src/util/extract.zig
-353
View File
@@ -1,353 +0,0 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const Pool = std.Thread.Pool;
const WaitGroup = std.Thread.WaitGroup;
const Archive = @import("../archive.zig");
const DirEntry = @import("../dir_entry.zig");
const Inode = @import("../inode.zig");
const ExtractionOptions = @import("../options.zig");
const Tables = @import("../tables.zig");
const InodeFinish = @import("inode_finish.zig");
const FinishUnion = InodeFinish.FinishUnion;
const ThreadedDataReader = @import("data_threaded.zig");
// 1 MB
const STACK_ALLOC_SIZE = 1024 * 1024;
pub fn extractTo(
allocator: Allocator,
inode: Inode,
archive: Archive,
path: []const u8,
options: ExtractionOptions,
) !void {
if (path[path.len - 1] == '/')
return extractTo(allocator, inode, archive, path[0 .. path.len - 2], options);
var stack_alloc = std.heap.stackFallback(STACK_ALLOC_SIZE, allocator);
var arena: std.heap.ArenaAllocator = .init(stack_alloc.get());
defer arena.deinit();
if (options.threads <= 1) {
const alloc = arena.allocator();
var tables: Tables = try .init(alloc, archive);
return extractSingleThread(arena.allocator(), inode, archive, &tables, path, options);
}
var thread_alloc = std.heap.ThreadSafeAllocator{ .child_allocator = arena.allocator() };
const alloc = thread_alloc.allocator();
var tables: Tables = try .init(alloc, archive);
var pool_alloc = std.heap.stackFallback(10 * 1024, alloc);
var pool: Pool = undefined;
try pool.init(.{ .allocator = pool_alloc.get(), .n_jobs = options.threads - 1 });
var wg: WaitGroup = .{};
var err: ?anyerror = null;
wg.start();
try pool.spawn(extractMultiThread, .{
alloc,
inode,
archive,
&tables,
path,
options,
&pool,
FinishUnion{ .wg = &wg },
&err,
});
pool.waitAndWork(&wg);
if (err != null) return err.?;
}
fn extractSingleThread(
alloc: Allocator,
inode: Inode,
archive: Archive,
tables: *Tables,
path: []const u8,
options: ExtractionOptions,
) !void {
switch (inode.hdr.inode_type) {
.dir, .ext_dir => {
_ = std.fs.cwd().makeDir(path) catch |err| switch (err) {
std.fs.Dir.MakeError.PathAlreadyExists => {},
else => return err,
};
// Currently we are ignoring any deinit or free calls since we know we are under an ArenaAllocator.
// Possibly in the future, do some simple math to see if it would be safe to ONLY deinit via Arena,
// otherwise be more conscientious about freeing memory.
// For now, this is good enough.
const entries = try inode.dirEntries(alloc, archive);
for (entries) |ent| {
const sub_inode: Inode = try .readFromEntry(alloc, archive, ent);
const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", ent.name });
try extractSingleThread(alloc, sub_inode, archive, tables, new_path, options);
}
const fil = try std.fs.cwd().openFile(path, .{});
defer fil.close();
try inode.setMetadata(alloc, tables, fil, options);
},
.file, .ext_file => {
var fil = try std.fs.cwd().createFile(path, .{ .exclusive = true });
defer fil.close();
var wrt = fil.writer(&[0]u8{});
var dat_rdr = try inode.dataReader(alloc, archive, tables);
defer dat_rdr.deinit();
_ = try dat_rdr.interface.streamRemaining(&wrt.interface);
try wrt.interface.flush();
try inode.setMetadata(alloc, tables, fil, options);
},
.symlink, .ext_symlink => return extractSymlink(inode, path),
else => return extractDeviceAndIPC(inode, alloc, tables, path, options),
}
}
fn extractMultiThread(
alloc: Allocator,
inode: Inode,
archive: Archive,
tables: *Tables,
path: []const u8,
options: ExtractionOptions,
pool: *Pool,
fin: FinishUnion,
err: *?anyerror,
) void {
if (err.* != null) {
fin.finish();
return;
}
switch (inode.hdr.inode_type) {
.dir, .ext_dir => {
_ = std.fs.cwd().makeDir(path) catch |res_err| switch (res_err) {
std.fs.Dir.MakeError.PathAlreadyExists => {},
else => {
err.* = res_err;
fin.finish();
return;
},
};
// Currently we are ignoring any deinit or free calls since we know we are under an ArenaAllocator.
// Possibly in the future, do some simple math to see if it would be safe to ONLY deinit via Arena,
// otherwise be more conscientious about freeing memory.
// For now, this is good enough.
const entries = inode.dirEntries(alloc, archive) catch |res_err| {
err.* = res_err;
fin.finish();
return;
};
if (entries.len == 0) {
fin.finish();
return;
}
var dir_fin = InodeFinish.create(
alloc,
inode,
path,
tables,
options,
fin,
err,
null,
entries.len,
) catch |res_err| {
err.* = res_err;
fin.finish();
return;
};
for (entries) |ent| {
if (ent.inode_type == .dir) {
extractEntry(
alloc,
ent,
archive,
tables,
path,
options,
pool,
.{ .fin = dir_fin },
err,
);
continue;
}
pool.spawn(
extractEntry,
.{ alloc, ent, archive, tables, path, options, pool, FinishUnion{ .fin = dir_fin }, err },
) catch |res_err| {
err.* = res_err;
dir_fin.finish();
return;
};
}
},
.file, .ext_file => {
const fil = std.fs.cwd().createFile(path, .{ .exclusive = true }) catch |res_err| {
if (options.verbose)
options.verbose_writer.?.print("Can't create file at {s}: {}\n", .{ path, res_err }) catch {};
err.* = res_err;
fin.finish();
return;
};
var data_rdr = threadedDataReader(inode, alloc, archive, tables) catch |res_err| {
if (options.verbose)
options.verbose_writer.?.print("Can't create data reader for inode #{} (extracting to {s}): {}\n", .{ inode.hdr.num, path, res_err }) catch {};
err.* = res_err;
fin.finish();
return;
};
if (data_rdr == null) {
inode.setMetadata(alloc, tables, fil, options) catch |res_err| {
if (options.verbose)
options.verbose_writer.?.print("Can't set metadata to {s}: {}\n", .{ path, res_err }) catch {};
err.* = res_err;
};
fin.finish();
return;
}
const file_fin = InodeFinish.create(
alloc,
inode,
path,
tables,
options,
fin,
err,
fil,
data_rdr.?.num_blocks,
) catch |res_err| {
if (options.verbose)
options.verbose_writer.?.print("Can't create callback for inode #{} (extracting to {s}): {}\n", .{ inode.hdr.num, path, res_err }) catch {};
err.* = res_err;
fin.finish();
return;
};
data_rdr.?.extractThreaded(fil, pool, file_fin);
},
.symlink, .ext_symlink => {
extractSymlink(inode, path) catch |res_err| {
err.* = res_err;
};
fin.finish();
},
else => {
extractDeviceAndIPC(inode, alloc, tables, path, options) catch |res_err| {
err.* = res_err;
};
fin.finish();
},
}
}
fn extractEntry(
alloc: Allocator,
ent: DirEntry,
archive: Archive,
tables: *Tables,
path: []const u8,
options: ExtractionOptions,
pool: *Pool,
fin: FinishUnion,
err: *?anyerror,
) void {
const new_path = std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", ent.name }) catch |res_err| {
err.* = res_err;
fin.finish();
return;
};
const inode = Inode.readFromEntry(alloc, archive, ent) catch |res_err| {
err.* = res_err;
fin.finish();
return;
};
extractMultiThread(alloc, inode, archive, tables, new_path, options, pool, fin, err);
}
/// Get a threaded data reader for a file inode.
fn threadedDataReader(self: Inode, alloc: std.mem.Allocator, archive: Archive, tables: *Tables) !?ThreadedDataReader {
return switch (self.hdr.inode_type) {
.file => threadedReaderFromData(alloc, archive, tables, self.data.file),
.ext_file => threadedReaderFromData(alloc, archive, tables, self.data.ext_file),
else => error.NotRegularFile,
};
}
fn threadedReaderFromData(alloc: std.mem.Allocator, archive: Archive, tables: *Tables, data: anytype) !?ThreadedDataReader {
if (data.block_sizes.len == 0 and data.frag_idx == 0xFFFFFFFF) return null;
var out: ThreadedDataReader = .init(alloc, archive, data.block_sizes, data.block_start, data.size);
if (data.frag_idx != 0xFFFFFFFF)
out.addFragment(try tables.frag_table.get(data.frag_idx), data.frag_block_offset);
return out;
}
/// Creates the symlink described by the inode.
/// Sets metadata.
fn extractSymlink(self: Inode, path: []const u8) !void {
const target = switch (self.data) {
.symlink => |s| s.target,
.ext_symlink => |s| s.target,
else => unreachable,
};
try std.fs.cwd().symLink(target, path, .{});
}
/// Creates the device described by the inode.
/// Sets metadata.
fn extractDeviceAndIPC(self: Inode, alloc: std.mem.Allocator, tables: *Tables, path: []const u8, options: ExtractionOptions) !void {
var mode: u32 = undefined;
var dev: u32 = 0;
switch (self.data) {
.char_dev => |d| {
mode = std.posix.S.IFCHR;
dev = d.dev;
},
.ext_char_dev => |d| {
mode = std.posix.S.IFCHR;
dev = d.dev;
},
.block_dev => |d| {
mode = std.posix.S.IFBLK;
dev = d.dev;
},
.ext_block_dev => |d| {
mode = std.posix.S.IFBLK;
dev = d.dev;
},
.fifo, .ext_fifo => mode = std.posix.S.IFIFO,
.socket, .ext_socket => mode = std.posix.S.IFSOCK,
else => unreachable,
}
const res: std.os.linux.E = @enumFromInt(std.os.linux.mknod(@ptrCast(path), mode, dev));
switch (res) {
.SUCCESS => {},
.ACCES => return std.fs.Dir.MakeError.AccessDenied,
.DQUOT => return std.fs.Dir.MakeError.DiskQuota,
.EXIST => return std.fs.Dir.MakeError.PathAlreadyExists,
.FAULT, .NOENT => return std.fs.Dir.MakeError.BadPathName,
.LOOP => return std.fs.Dir.MakeError.SymLinkLoop,
.NAMETOOLONG => return std.fs.Dir.MakeError.NameTooLong,
.NOMEM => return std.fs.Dir.MakeError.SystemResources,
.NOSPC => return std.fs.Dir.MakeError.NoSpaceLeft,
.NOTDIR => return std.fs.Dir.MakeError.NotDir,
.PERM => return std.fs.Dir.MakeError.PermissionDenied,
.ROFS => return std.fs.Dir.MakeError.ReadOnlyFileSystem,
else => return blk: {
std.debug.print("unhandled mknod result: {}\n", .{res});
break :blk std.fs.Dir.MakeError.Unexpected;
},
}
var fil = try std.fs.cwd().openFile(path, .{});
defer fil.close();
try self.setMetadata(alloc, tables, fil, options);
}
src/util/inode_finish.zig
-101
View File
@@ -1,101 +0,0 @@
const std = @import("std");
const WaitGroup = std.Thread.WaitGroup;
const Mutex = std.Thread.Mutex;
const Archive = @import("../archive.zig");
const Inode = @import("../inode.zig");
const ExtractionOptions = @import("../options.zig");
const Tables = @import("../tables.zig");
const InodeFinish = @This();
const FinishEnum = enum {
wg,
fin,
};
pub const FinishUnion = union(FinishEnum) {
wg: *WaitGroup,
fin: *InodeFinish,
pub fn finish(self: FinishUnion) void {
switch (self) {
.wg => |wg| wg.finish(),
.fin => |fin| fin.finish(),
}
}
};
alloc: std.mem.Allocator,
inode: Inode,
path: []const u8,
tables: *Tables,
options: ExtractionOptions,
parent_finish: FinishUnion,
fil: ?std.fs.File,
out_err: *?anyerror,
wg: WaitGroup = .{},
mut: Mutex = .{},
pub fn create(
alloc: std.mem.Allocator,
inode: Inode,
path: []const u8,
tables: *Tables,
options: ExtractionOptions,
parent_finish: FinishUnion,
out_err: *?anyerror,
fil: ?std.fs.File,
work_size: usize,
) !*InodeFinish {
if (work_size == 0)
return error.InvalidWorkSize;
const out = try alloc.create(InodeFinish);
errdefer alloc.destroy(out);
out.* = .{
.alloc = alloc,
.inode = inode,
.path = path,
.tables = tables,
.options = options,
.parent_finish = parent_finish,
.out_err = out_err,
.fil = fil,
};
out.wg.startMany(work_size);
return out;
}
pub fn logError(self: *InodeFinish, comptime fmt: []const u8, args: anytype) void {
if (self.options.verbose)
self.options.verbose_writer.?.print(fmt, args) catch {};
}
pub fn finish(self: *InodeFinish) void {
self.mut.lock();
{
defer self.mut.unlock();
self.wg.finish();
if (!self.wg.isDone()) return;
}
defer {
self.parent_finish.finish();
self.alloc.destroy(self);
}
if (self.fil == null)
self.fil = std.fs.cwd().openFile(self.path, .{}) catch |err| {
if (self.options.verbose)
self.options.verbose_writer.?.print("Error opening {s} to set metadata: {}\n", .{ self.path, err }) catch {};
self.out_err.* = err;
return;
};
defer self.fil.?.close();
self.inode.setMetadata(self.alloc, self.tables, self.fil.?, self.options) catch |err| {
if (self.options.verbose)
self.options.verbose_writer.?.print("Error setting metadata to {s}: {}\n", .{ self.path, err }) catch {};
self.out_err.* = err;
return;
};
}
src/util/meta_cache.zig
-41
View File
@@ -1,41 +0,0 @@
const std = @import("std");
const OffsetFile = @import("offset_file.zig");
const MetadataCache = @This();
alloc: std.mem.Allocator,
buf: []u8,
fixed_alloc: std.heap.FixedBufferAllocator,
cache: std.AutoArrayHashMap(u64, [8192]u8),
mut: std.Thread.Mutex = .{},
cache_mut: std.AutoArrayHashMap(u64, std.Thread.Mutex),
fil: OffsetFile,
pub fn init(alloc: std.mem.Allocator, cache_size: u64) !MetadataCache {}
pub fn deinit(self: *MetadataCache) void {
self.mut.lock();
defer self.mut.unlock();
self.cache.deinit();
self.cache_mut.deinit();
self.alloc.free(self.buf);
}
pub fn getChunk(self: *MetadataCache, offset: u64) ![8192]u8 {
var res = self.cache.get(offset);
if (res != null) return res.?;
var offset_mut = blk: {
self.mut.lock();
defer self.mut.unlock();
const mut = try self.cache_mut.getOrPut(offset);
if (!mut.found_existing)
mut.value_ptr.* = .{};
break :blk mut.value_ptr;
};
offset_mut.lock();
defer offset_mut.unlock();
}
src/util/metadata.zig
+46 -76
View File
@@ -1,97 +1,67 @@
const std = @import("std");
const Reader = std.Io.Reader;
const Writer = std.Io.Writer;
const Limit = std.Io.Limit;
const StreamError = std.Io.Reader.StreamError;
const Io = std.Io;
const Reader = Io.Reader;
const Writer = Io.Writer;
const Limit = Io.Limit;
const DecompFn = @import("../decomp.zig").DecompFn;
const DecompCache = @import("decomp_cache.zig");
const BlockHeader = packed struct {
const MetadataReader = @This();
const BlockHeader = packed struct(u16) {
size: u15,
uncompressed: bool,
};
const This = @This();
io: Io,
alloc: std.mem.Allocator,
rdr: *Reader,
decomp: DecompFn,
cur_offset: u64 = 0,
next_offset: u64,
buf: [8192]u8 = undefined,
cache: *DecompCache,
interface: Reader,
err: ?anyerror = null,
interface: Reader = .{
.buffer = &[0]u8{},
.end = 0,
.seek = 0,
.vtable = &.{
.stream = stream,
.discard = discard,
.readVec = readVec,
},
},
pub fn init(alloc: std.mem.Allocator, rdr: *Reader, decomp: DecompFn) This {
pub fn init(io: Io, cache: *DecompCache, offset: u64) void {
return .{
.alloc = alloc,
.rdr = rdr,
.decomp = decomp,
.interface = .{
.buffer = &[0]u8{},
.end = 0,
.seek = 0,
.vtable = &.{
.stream = stream,
.discard = discard,
.readVec = readVec,
},
},
.io = io,
.next_offset = offset,
.cache = cache,
};
}
pub fn deinit(self: *MetadataReader) void {
if (self.cur_block_offset != 0)
self.cache.checkinBlock(self.io, self.cur_block_offset);
}
fn advance(self: *This) !void {
self.interface.seek = 0;
var hdr: BlockHeader = undefined;
try self.rdr.readSliceEndian(BlockHeader, @ptrCast(&hdr), .little);
fn advance(self: *MetadataReader) !void {
if (self.interface.buffer.len > 0)
self.cache.checkinBlock(self.io, self.cur_offset);
const hdr = std.mem.readInt(BlockHeader, self.cache.map[self.next_offset..][0..2], .little);
self.cur_offset = self.next_offset + 2;
self.next_offset += hdr.size;
if (hdr.uncompressed) {
try self.rdr.readSliceEndian(u8, self.buf[0..hdr.size], .little);
self.interface.buffer = self.cache.map[self.cur_offset..][0..hdr.size];
self.interface.end = hdr.size;
self.interface.buffer = self.buf[0..hdr.size];
self.interface.seek = 0;
return;
}
var tmp_buf: [8192]u8 = undefined;
try self.rdr.readSliceAll(tmp_buf[0..hdr.size]);
self.interface.end = try self.decomp(self.alloc, tmp_buf[0..hdr.size], &self.buf);
self.interface.buffer = self.buf[0..self.interface.end];
self.interface.buffer = try self.cache.checkoutBlock(self.io, self.cur_offset, hdr.size, 8192);
self.interface.end = self.interface.buffer.len;
self.interface.seek = 0;
}
fn stream(rdr: *Reader, wrt: *Writer, limit: Limit) StreamError!usize {
const self: *This = @fieldParentPtr("interface", rdr);
if (rdr.end == rdr.seek) self.advance() catch |err| {
self.err = err;
return StreamError.ReadFailed;
};
if (@intFromEnum(limit) == 0) return 0;
const to_write = @min(rdr.end - rdr.seek, @intFromEnum(limit));
const wrote = try wrt.write(self.buf[rdr.seek .. rdr.seek + to_write]);
self.interface.seek += wrote;
return wrote;
}
fn discard(rdr: *Reader, limit: Limit) error{ EndOfStream, ReadFailed }!usize {
const self: *This = @fieldParentPtr("interface", rdr);
if (rdr.end == rdr.seek) self.advance() catch |err| {
self.err = err;
return StreamError.ReadFailed;
};
if (@intFromEnum(limit) == 0) return 0;
const to_skip = @min(rdr.end - rdr.seek, @intFromEnum(limit));
rdr.seek += to_skip;
return to_skip;
}
fn readVec(rdr: *Reader, vec: [][]u8) error{ EndOfStream, ReadFailed }!usize {
const self: *This = @fieldParentPtr("interface", rdr);
if (rdr.end == rdr.seek) self.advance() catch |err| {
self.err = err;
return StreamError.ReadFailed;
};
var cur_red: usize = 0;
for (vec) |s| {
const to_copy: usize = @min(rdr.end - rdr.seek, s.len);
@memcpy(s[0..to_copy], self.buf[rdr.seek .. rdr.seek + to_copy]);
rdr.seek += to_copy;
cur_red += to_copy;
if (rdr.end == rdr.seek) break;
}
return cur_red;
}
fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {}
fn discard(r: *Reader, limit: Limit) Reader.Error!usize {}
fn readVec(r: *Reader, vec: [][]u8) Reader.Error!usize {}
src/util/offset_file.zig
-20
View File
@@ -1,20 +0,0 @@
//! A File where it's meaningful (to us) content starts at a given offset.
const std = @import("std");
const File = std.fs.File;
const Reader = std.fs.File.Reader;
const OffsetFile = @This();
fil: File,
offset: u64,
pub fn init(fil: File, init_offset: u64) OffsetFile {
return .{ .fil = fil, .offset = init_offset };
}
pub fn readerAt(self: OffsetFile, offset: u64, buffer: []u8) !Reader {
var rdr = self.fil.reader(buffer);
try rdr.seekTo(self.offset + offset);
return rdr;
}
src/util/zig_decomp.zig
+40
View File
@@ -0,0 +1,40 @@
const std = @import("std");
const Reader = std.Io.Reader;
const flate = std.compress.flate;
const zstd = std.compress.zstd;
const xz = std.compress.xz;
const lzma = std.compress.lzma;
const Error = @import("decompress.zig").DecompressionError;
pub fn zlibDecompress(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var buf: [flate.max_window_len]u8 = undefined;
var rdr: Reader = .fixed(in);
var decomp: flate.Decompress = .init(&rdr, .zlib, &buf);
return decomp.reader.readSliceShort(out);
}
pub fn zstdDecompress(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
const buf = try alloc.alloc(u8, in.len + zstd.block_size_max);
defer alloc.free(buf);
var rdr: Reader = .fixed(in);
var decomp: zstd.Decompress = .init(&rdr, buf, .{ .window_len = in.len });
return decomp.reader.readSliceShort(out);
}
pub fn lzmaDecompress(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var rdr: Reader = .fixed(in);
var decomp: lzma.Decompress = .initOptions(&rdr, alloc, &[0]u8{}, .{}, 2 * out.len);
defer decomp.deinit();
return decomp.reader.readSliceShort(out);
}
pub fn xzDecompress(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var rdr: Reader = .fixed(in);
var decomp: xz.Decompress = .init(&rdr, alloc, &[0]u8{});
defer decomp.deinit();
return decomp.reader.readSliceShort(out);
}
src/xattr.zig
-117
View File
@@ -1,117 +0,0 @@
const std = @import("std");
const Decompressor = @import("decomp.zig");
const Table = @import("tables.zig").Table;
const MetadataReader = @import("util/metadata.zig");
const OffsetFile = @import("util/offset_file.zig");
const Ref = packed struct {
block_offset: u16,
block_start: u32,
_: u16,
};
const Entry = packed struct {
ref: Ref,
count: u32,
size: u32,
};
const KeyPrefix = enum(u8) {
user,
trusted,
security,
};
const KeyRaw = packed struct {
type: packed struct {
prefix: KeyPrefix,
out_of_line: bool,
_: u7,
},
name_size: u16,
};
pub const KeyValue = struct {
key: [:0]u8,
value: []u8,
};
const XattrTable = @This();
alloc: std.mem.Allocator,
fil: OffsetFile,
decomp: *Decompressor,
count: u32,
start: u64,
table: Table(Entry),
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *Decompressor, table_start: u64) !XattrTable {
var info = packed struct {
start: u64 = undefined,
count: u32 = undefined,
_: u32 = undefined,
}{};
var rdr = try fil.readerAt(table_start, &[0]u8{});
try rdr.interface.readSliceEndian(@TypeOf(info), @ptrCast(&info), .little);
return .{
.alloc = alloc,
.fil = fil,
.decomp = decomp,
.count = info.count,
.start = info.start,
.table = try .init(alloc, fil, decomp, table_start + 16, info.count),
};
}
pub fn deinit(self: *XattrTable) void {
self.table.deinit();
}
pub fn get(self: *XattrTable, alloc: std.mem.Allocator, idx: u32) ![]KeyValue {
const entry: Entry = try self.table.get(idx);
const out = try alloc.alloc(KeyValue, entry.count);
for (out) |*kv| {
var rdr = try self.fil.readerAt(self.start + entry.ref.block_start, &[0]u8{});
var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp);
try meta.interface.discardAll(entry.ref.block_offset);
var key_raw: KeyRaw = undefined;
try meta.interface.readSliceEndian(KeyRaw, @ptrCast(&key_raw), .little);
switch (key_raw.type.prefix) {
.user => {
kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 5 + 1));
@memcpy(kv.key[0..5], "user.");
try meta.interface.readSliceAll(kv.key[5 .. kv.key.len - 1]);
kv.key[kv.key.len - 1] = 0;
},
.security => {
kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 9 + 1));
@memcpy(kv.key[0..9], "security.");
try meta.interface.readSliceAll(kv.key[9 .. kv.key.len - 1]);
kv.key[kv.key.len - 1] = 0;
},
.trusted => {
kv.key = @ptrCast(try alloc.alloc(u8, key_raw.name_size + 8 + 1));
@memcpy(kv.key[0..8], "trusted.");
try meta.interface.readSliceAll(kv.key[8 .. kv.key.len - 1]);
kv.key[kv.key.len - 1] = 0;
},
}
if (key_raw.type.out_of_line) {
try meta.interface.discardAll(4);
var ref: Ref = undefined;
try meta.interface.readSliceEndian(Ref, @ptrCast(&ref), .little);
rdr = try self.fil.readerAt(self.start + ref.block_start, &[0]u8{});
meta = .init(alloc, &rdr.interface, self.decomp);
try meta.interface.discardAll(ref.block_offset);
}
var value_size: u32 = undefined;
try meta.interface.readSliceEndian(u32, @ptrCast(&value_size), .little);
kv.value = try alloc.alloc(u8, value_size);
try meta.interface.readSliceAll(kv.value);
}
return out;
}