diff --git a/benchmark.sh b/benchmark.sh index 5d85dc1..4f2b971 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -5,4 +5,13 @@ ARCHIVE="testing/LinuxPATest.sfs" REF_EXT_LOC="testing/LinuxPAReference" PROG_EXT_LOC="testing/LinuxPABinTest" +echo "Testing Multi-threaded Performance" +echo "" + hyperfine --warmup 5 --prepare "rm -rf $REF_EXT_LOC && rm -rf $PROG_EXT_LOC" "unsquashfs -d $REF_EXT_LOC $ARCHIVE" "zig-out/bin/unsquashfs -d $PROG_EXT_LOC $ARCHIVE" + +echo "" +echo "Testing Single-threaded Performance" +echo "" + +hyperfine --warmup 5 --prepare "rm -rf $REF_EXT_LOC && rm -rf $PROG_EXT_LOC" "unsquashfs -p 1 -d $REF_EXT_LOC $ARCHIVE" "zig-out/bin/unsquashfs -p 1 -d $PROG_EXT_LOC $ARCHIVE" diff --git a/build.zig b/build.zig index 531f849..9f6ee51 100644 --- a/build.zig +++ b/build.zig @@ -3,6 +3,10 @@ const std = @import("std"); pub fn build(b: *std.Build) !void { const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false; const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false; +<<<<<<< HEAD +======= + const dynamic = b.option(bool, "dynamic", "Dynamicly link C decompression libraries") orelse false; +>>>>>>> dfbfbda (Build is working again (on Zig master branch)) var debug = b.option(bool, "debug", "Enable options to make debugging easier."); const version_string_option = b.option([]const u8, "version", "Version of the library/binary"); @@ -28,18 +32,24 @@ pub fn build(b: *std.Build) !void { .target = target, .valgrind = debug, .root_source_file = b.path("src/root.zig"), +<<<<<<< HEAD // .link_libc = true, .imports = &.{ .{ .name = "options", .module = zig_squashfs_options.createModule() }, .{ .name = "c", .module = c.createModule() }, +======= + .imports = &.{ + .{ .name = "options", .module = zig_squashfs_options.createModule() }, +>>>>>>> dfbfbda (Build is working again (on Zig master branch)) }, }), .use_llvm = debug, }); - const zstd = b.dependency("zstd", .{ .optimize = optimize, .target = target }); - lib.root_module.linkLibrary(zstd.artifact("zstd")); + const deps = try dependencies(b, optimize, target, use_zig_decomp, allow_lzo, dynamic); + defer b.allocator.free(deps); +<<<<<<< HEAD const zng = b.dependency("zlib_ng", .{ .optimize = optimize, .target = target }); lib.root_module.linkLibrary(zng.artifact("zng")); @@ -51,6 +61,23 @@ pub fn build(b: *std.Build) !void { const lz4 = b.dependency("lz4", .{ .optimize = optimize, .target = target }); lib.root_module.linkLibrary(lz4.artifact("lz4")); +======= + for (deps) |d| + lib.root_module.linkLibrary(d); + + if (!use_zig_decomp) { + const c = b.addTranslateC(.{ + .optimize = optimize, + .target = target, + .root_source_file = b.path("src/c.h"), + }); + if (allow_lzo) c.defineCMacro("ALLOW_LZO", null); + lib.root_module.addImport("c", c.createModule()); + + if (dynamic) + dynamicLinkLibraries(c, allow_lzo); + } +>>>>>>> dfbfbda (Build is working again (on Zig master branch)) var version = version_string_option orelse "0.0.0-testing"; if (version[0] == 'v') version = version[1..]; @@ -80,17 +107,32 @@ pub fn build(b: *std.Build) !void { const mod_tests = b.addTest(.{ .root_module = b.createModule(.{ - .optimize = .Debug, + .optimize = optimize, .target = target, - .root_source_file = b.path("src/test.zig"), + .root_source_file = b.path("src/root.zig"), .imports = &.{ - .{ .name = "c", .module = c.createModule() }, + .{ .name = "options", .module = zig_squashfs_options.createModule() }, }, - .valgrind = true, + .valgrind = debug, }), - .use_llvm = true, + .use_llvm = debug, }); - mod_tests.root_module.linkLibrary(zstd.artifact("zstd")); + + for (deps) |d| + mod_tests.root_module.linkLibrary(d); + + if (!use_zig_decomp) { + const c = b.addTranslateC(.{ + .optimize = optimize, + .target = target, + .root_source_file = b.path("src/c.h"), + }); + mod_tests.root_module.addImport("c", c.createModule()); + if (allow_lzo) c.defineCMacro("ALLOW_LZO", null); + + if (dynamic) + dynamicLinkLibraries(c, allow_lzo); + } const run_mod_tests = b.addRunArtifact(mod_tests); const test_step = b.step("test", "Run tests"); @@ -109,3 +151,42 @@ pub fn build(b: *std.Build) !void { check.dependOn(&lib_check.step); check.dependOn(&exe_check.step); } + +pub fn dynamicLinkLibraries(mod: *std.Build.Step.TranslateC, allow_lzo: bool) void { + mod.linkSystemLibrary("zstd", .{}); + mod.linkSystemLibrary("zlib-ng", .{}); + mod.linkSystemLibrary("lzma", .{}); + mod.linkSystemLibrary("lz4", .{}); + if (allow_lzo) + mod.linkSystemLibrary("minilzo", .{}); +} +fn dependencies( + b: *std.Build, + optimize: std.builtin.OptimizeMode, + target: std.Build.ResolvedTarget, + use_zig_decomp: bool, + allow_lzo: bool, + dynamic: bool, +) ![]*std.Build.Step.Compile { + if (use_zig_decomp or dynamic) return &.{}; + + var list: std.ArrayList(*std.Build.Step.Compile) = .empty; + + const zstd = b.dependency("zstd", .{ .optimize = optimize, .target = target }); + try list.append(b.allocator, zstd.artifact("zstd")); + + const zng = b.dependency("zlib_ng", .{ .optimize = optimize, .target = target }); + try list.append(b.allocator, zng.artifact("zng")); + + const xz = b.dependency("xz", .{ .optimize = optimize, .target = target }); + try list.append(b.allocator, xz.artifact("lzma")); + + const lz4 = b.dependency("lz4", .{ .optimize = optimize, .target = target }); + try list.append(b.allocator, lz4.artifact("lz4")); + + if (allow_lzo) { + const minilzo = b.dependency("minilzo", .{ .optimize = optimize, .target = target }); + try list.append(b.allocator, minilzo.artifact("minilzo")); + } + return list.toOwnedSlice(b.allocator); +} diff --git a/src/archive.zig b/src/archive.zig index 33588b9..a09df6c 100644 --- a/src/archive.zig +++ b/src/archive.zig @@ -23,6 +23,7 @@ pub fn init(io: Io, file: std.Io.File, offset: u64) !Archive { try rdr.seekTo(offset); var super: Superblock = undefined; try rdr.interface.readSliceEndian(Superblock, @ptrCast(&super), .little); + try super.validate(); return .{ .file = try .init(io, file, super.size, offset), @@ -31,15 +32,14 @@ pub fn init(io: Io, file: std.Io.File, offset: u64) !Archive { .stateless_decomp = try Decomp.StatelessDecomp(super.compression), }; } -pub fn deinit(self: Archive, io: Io) void { +pub fn deinit(self: *Archive, io: Io) void { self.file.deinit(io); } /// The root folder of the Archive. Used to open other Files. -pub fn root(self: Archive, alloc: std.mem.Allocator, io: Io) !File { +pub fn root(self: Archive, alloc: std.mem.Allocator) !File { const root_inode = try Utils.inodeFromRef( alloc, - io, self.file, self.stateless_decomp, self.super.inode_start, @@ -50,7 +50,7 @@ pub fn root(self: Archive, alloc: std.mem.Allocator, io: Io) !File { } /// Opens a File within the archive. pub fn open(self: Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File { - const root_file = try self.root(alloc, io); + const root_file = try self.root(alloc); const path = std.mem.trim(u8, filepath, "/"); if (Utils.pathIsSelf(path)) return root_file; @@ -95,6 +95,19 @@ pub fn idTable(self: Archive, alloc: std.mem.Allocator, io: Io, idx: u32) !u16 { ); } +/// Extract the entire archive contents to the given directory. +pub fn extract(self: Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void { + const root_inode = try Utils.inodeFromRef( + alloc, + self.file, + self.stateless_decomp, + self.super.inode_start, + self.super.block_size, + self.super.root_ref, + ); + return root_inode.extract(alloc, io, self.file, self.super, extract_dir, options); +} + // Superblock const SQUASHFS_MAGIC: u32 = std.mem.readInt(u32, "hsqs", .little); @@ -155,17 +168,111 @@ pub const Superblock = extern struct { } }; -// Extraction +// Tests -/// Extract the entire archive contents to the given directory. -pub fn extract(self: Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void { - const root_inode = try Utils.inodeFromRef( - alloc, - self.file, - self.stateless_decomp, - self.super.inode_start, - self.super.block_size, - self.super.root_ref, - ); - return root_inode.extract(alloc, io, self.file, self.super, extract_dir, options); +const TestArchive = "testing/LinuxPATest.sfs"; + +test "Basics" { + std.debug.print("Starting test: Basics...\n", .{}); + + const alloc = std.testing.allocator; + const io = std.testing.io; + + var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); + defer fil.close(io); + var sfs: Archive = try .init(io, fil, 0); + defer sfs.deinit(io); + try std.testing.expectEqualDeep(sfs.super, LinuxPATestCorrectSuperblock); + const root_file = try sfs.root(alloc); + defer root_file.deinit(); } + +const TestFile = "Start.exe"; +const TestFileExtractLocation = "testing/Start.exe"; + +test "ExtractSingleFile" { + std.debug.print("Starting test: ExtractSingleFile...\n", .{}); + + const alloc = std.testing.allocator; + const io = std.testing.io; + + Io.Dir.cwd().deleteFile(io, TestFileExtractLocation) catch {}; + var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); + defer fil.close(io); + var sfs: Archive = try .init(io, fil, 0); + defer sfs.deinit(io); + var test_fil = try sfs.open(alloc, io, TestFile); + defer test_fil.deinit(); + try test_fil.extract(alloc, io, TestFileExtractLocation, .default); + //TODO: validate extracted file. +} + +const TestFullExtractLocation = "testing/TestExtract"; + +test "ExtractCompleteArchive" { + std.debug.print("Starting test: ExtractCompleteArchive...\n", .{}); + + const alloc = std.testing.allocator; + const io = std.testing.io; + + Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {}; + var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); + defer fil.close(io); + var sfs: Archive = try .init(io, fil, 0); + defer sfs.deinit(io); + try sfs.extract(alloc, io, TestFullExtractLocation, .default); +} + +test "ExtractCompleteArchiveSingleThreaded" { + std.debug.print("Starting test: ExtractCompleteArchive...\n", .{}); + + const alloc = std.testing.allocator; + const io = Io.Threaded.global_single_threaded.io(); + + Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {}; + var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); + defer fil.close(io); + var sfs: Archive = try .init(io, fil, 0); + defer sfs.deinit(io); + try sfs.extract(alloc, io, TestFullExtractLocation, .default); +} + +const LinuxPATestCorrectSuperblock: Superblock = .{ + .magic = std.mem.readInt(u32, "hsqs", .little), + .inode_count = 2974, + .mod_time = 1632696724, + .block_size = 131072, + .frag_count = 264, + .compression = .zstd, + .block_log = 17, + .flags = .{ + .inode_uncompressed = false, + .data_uncompressed = false, + .check = false, + .frag_uncompressed = false, + .fragment_never = false, + .fragment_always = false, + .duplicates = true, + .exportable = true, + .xattr_uncompressed = false, + .xattr_never = false, + .compression_options = false, + .ids_uncompressed = false, + ._ = 0, + }, + .id_count = 1, + .ver_maj = 4, + .ver_min = 0, + .root_ref = .{ + .block_offset = 1363, + .block_start = 29237, + ._ = 0, + }, + .size = 106841744, + .id_start = 106841632, + .xattr_start = 106841720, + .inode_start = 106778274, + .dir_start = 106807998, + .frag_start = 106837747, + .export_start = 106841602, +}; diff --git a/src/bin/unsquashfs.zig b/src/bin/unsquashfs.zig index c278ec3..e0bdcf5 100644 --- a/src/bin/unsquashfs.zig +++ b/src/bin/unsquashfs.zig @@ -68,7 +68,13 @@ pub fn main(init: std.process.Init) !void { if (force) try Io.Dir.cwd().deleteTree(io, extLoc); - try arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully. + if (threads != 0) { + if (threads == 1) + return arc.extract(alloc, Io.Threaded.global_single_threaded.io(), extLoc, options); //TODO: Handle error gracefully. + var limited_io = Io.Threaded.init(alloc, .{ .async_limit = .limited(threads - 1), .concurrent_limit = .limited(threads - 1) }); + return arc.extract(alloc, limited_io.io(), extLoc, options); //TODO: Handle error gracefully. + } + return arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully. } fn handleArgs(args: std.process.Args, out: *Writer) !void { diff --git a/src/decomp.zig b/src/decomp.zig index 948fb80..40c3b74 100644 --- a/src/decomp.zig +++ b/src/decomp.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const Io = std.Io; const options = @import("options"); @@ -7,7 +8,13 @@ const xz = @import("decomp/zig_xz.zig"); const Decompressor = @import("util/decompressor.zig"); const zlib = if (options.use_zig_decomp) @import("decomp/zig_zlib.zig") else @import("decomp/c_zlib.zig"); +<<<<<<< HEAD const lzo = if (options.use_zig_decomp or !options.allow_lzo) void else @import("decomp/c_lzo.zig"); +======= +const lzma = if (options.use_zig_decomp) @import("decomp/zig_lzma.zig") else @import("decomp/c_lzma.zig"); +const lzo = if (options.use_zig_decomp or options.allow_lzo) void else @import("decomp/c_lzo.zig"); +const xz = if (options.use_zig_decomp) @import("decomp/zig_xz.zig") else @import("decomp/c_xz.zig"); +>>>>>>> dfbfbda (Build is working again (on Zig master branch)) const lz4 = if (options.use_zig_decomp) void else @import("decomp/c_lz4.zig"); const zstd = if (options.use_zig_decomp) @import("decomp/zig_zstd.zig") else @import("decomp/c_zstd.zig"); @@ -45,6 +52,7 @@ pub const Decomp = union(enum) { lz4: lz4, zstd: zstd, +<<<<<<< HEAD pub fn init(val: Enum, alloc: std.mem.Allocator, io: std.Io, block_size: u32) !Decomp { return switch (val) { .gzip => .{ .gzip = zlib.init(alloc, io, block_size) }, @@ -53,13 +61,33 @@ pub const Decomp = union(enum) { .xz => .{ .xz = .{} }, .lz4 => .{ .lz4 = .{} }, .zstd => .{ .zstd = zstd.init(alloc, io, block_size) }, +======= + pub fn init(val: Enum, alloc: std.mem.Allocator, io: Io, block_size: u32) !Decomp { + return switch (val) { + .gzip => .{ .gzip = if (options.use_zig_decomp) try zlib.init(alloc, io, block_size) else try zlib.init(alloc, io) }, + .lzma => .{ .lzma = if (options.use_zig_decomp) try lzma.init(alloc, io, block_size) else .{} }, + .lzo => if (options.use_zig_decomp or !options.allow_lzo) error.LzoUnsupported else .{ .lzo = .{} }, + .xz => .{ .xz = if (options.use_zig_decomp) try zlib.init(alloc, io, block_size) else .{} }, + .lz4 => if (options.use_zig_decomp) error.Lz4Unsupported else .{ .lz4 = .{} }, + .zstd => .{ .zstd = if (options.use_zig_decomp) try zstd.init(alloc, io, block_size) else try zstd.init(alloc, io) }, +>>>>>>> dfbfbda (Build is working again (on Zig master branch)) }; } - pub fn deinit(self: *Decomp) void { - switch (self.*) { - .gzip => self.gzip.deinit(), - .zstd => self.zstd.deinit(), - else => {}, + pub fn deinit(self: *Decomp, alloc: std.mem.Allocator) void { + if (options.use_zig_decomp) { + switch (self.*) { + .gzip => self.gzip.deinit(), + .lzma => self.lzma.deinit(), + .xz => self.xz.deinit(), + .zstd => self.zstd.deinit(), + else => {}, + } + } else { + switch (self.*) { + .gzip => self.gzip.deinit(alloc), + .zstd => self.zstd.deinit(alloc), + else => {}, + } } } @@ -67,9 +95,9 @@ pub const Decomp = union(enum) { return switch (self.*) { .gzip => &self.gzip.interface, .lzma => &lzma.stateless_decompressor, - .lzo => &lzo.stateless_decompressor, + .lzo => if (options.use_zig_decomp or !options.allow_lzo) unreachable else &lzo.stateless_decompressor, .xz => &xz.stateless_decompressor, - .lz4 => &lz4.stateless_decompressor, + .lz4 => if (options.use_zig_decomp) unreachable else &lz4.stateless_decompressor, .zstd => &self.zstd.interface, }; } diff --git a/src/decomp/c_lz4.zig b/src/decomp/c_lz4.zig index fb81735..a558998 100644 --- a/src/decomp/c_lz4.zig +++ b/src/decomp/c_lz4.zig @@ -8,18 +8,12 @@ const Error = Decompressor.Error; pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize { +<<<<<<< HEAD const res = c.LZ4_decompress_fast(in.ptr, out.ptr, @truncate(out.len)); +======= + const out_len: c_int = @bitCast(@as(u32, @truncate(out.len))); + const res = c.LZ4_decompress_fast(in.ptr, out.ptr, out_len); +>>>>>>> dfbfbda (Build is working again (on Zig master branch)) if (res < 0) return Error.ReadFailed; return @abs(res); } - -// lzma_allocator - -fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - return alloc.rawAlloc(size, .@"1", 0); -} -fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); -} diff --git a/src/decomp/c_lzma.zig b/src/decomp/c_lzma.zig index 6e890b2..497cc4b 100644 --- a/src/decomp/c_lzma.zig +++ b/src/decomp/c_lzma.zig @@ -15,13 +15,8 @@ const Self = @This(); pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; -fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { +fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize { var stream: c.lzma_stream = .{ - .allocator = &.{ - .alloc = lzmaAlloc, - .free = lzmaFree, - .@"opaque" = @constCast(&alloc), - }, .next_in = in.ptr, .avail_in = in.len, .next_out = out.ptr, @@ -38,11 +33,12 @@ fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, // lzma_allocator -fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - return alloc.rawAlloc(size, .@"1", 0); -} -fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); -} +// fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { +// var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr)); +// return alloc.rawAlloc(size, .@"1", 0); +// } +// fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { +// if (mem_ptr == null) return; +// var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr)); +// alloc.free(@as([*]u8, @ptrCast(mem_ptr.?))); +// } diff --git a/src/decomp/c_lzo.zig b/src/decomp/c_lzo.zig index 851519f..106b512 100644 --- a/src/decomp/c_lzo.zig +++ b/src/decomp/c_lzo.zig @@ -22,14 +22,3 @@ fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: if (res != c.LZO_E_OK) return Error.ReadFailed; return out_len; } - -// lzma_allocator - -fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - return alloc.rawAlloc(size, .@"1", 0); -} -fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); -} diff --git a/src/decomp/c_xz.zig b/src/decomp/c_xz.zig index 6e890b2..d88325e 100644 --- a/src/decomp/c_xz.zig +++ b/src/decomp/c_xz.zig @@ -15,13 +15,8 @@ const Self = @This(); pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; -fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { +fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize { var stream: c.lzma_stream = .{ - .allocator = &.{ - .alloc = lzmaAlloc, - .free = lzmaFree, - .@"opaque" = @constCast(&alloc), - }, .next_in = in.ptr, .avail_in = in.len, .next_out = out.ptr, @@ -38,11 +33,13 @@ fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, // lzma_allocator -fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - return alloc.rawAlloc(size, .@"1", 0); -} -fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { - var alloc: *std.mem.Allocator = @ptrCast(@alignCast(@constCast(ptr))); - alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); -} +// fn lzmaAlloc(ptr: ?*anyopaque, size: usize, _: usize) callconv(.c) ?*anyopaque { +// var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr)); +// const mem = alloc.alloc(u8, size) catch return null; +// return mem.ptr; +// } +// fn lzmaFree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { +// if (mem_ptr == null) return; +// var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr)); +// alloc.free(@as([*]u8, @ptrCast(mem_ptr.?))); +// } diff --git a/src/decomp/c_zlib.zig b/src/decomp/c_zlib.zig index a007740..5a8c579 100644 --- a/src/decomp/c_zlib.zig +++ b/src/decomp/c_zlib.zig @@ -20,20 +20,15 @@ io: Io, ctx: []c.zng_stream, ctx_queue: Queue, -pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self { +pub fn init(alloc: std.mem.Allocator, io: Io) !Self { const buf = try alloc.alloc(c.zng_stream, 20); // TODO: Choose a better number instead of a random one. var queue: Queue = .init(buf); for (0..20) |_| - try queue.putOne(io, .{ - .zalloc = zalloc, - .zfree = zfree, - }); + try queue.putOne(io, .{}); return .{ - .alloc = alloc, .io = io, - .block_size = block_size, .ctx = buf, .ctx_queue = queue, }; @@ -52,13 +47,12 @@ fn decomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8 var stream = self.ctx_queue.getOne(self.io) catch return Error.ReadFailed; defer self.ctx_queue.putOne(self.io, stream) catch {}; - stream.@"opaque" = @constCast(&alloc); stream.next_in = in.ptr; stream.avail_in = @truncate(in.len); stream.next_out = out.ptr; stream.avail_out = @truncate(out.len); - try zlibDecomp(&stream, in, out); + try zlibDecomp(&stream); return stream.total_out; } @@ -74,9 +68,8 @@ inline fn zlibDecomp(stream: *c.zng_stream) !void { pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; -fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { +fn statelessDecomp(_: ?*const Decompressor, _: std.mem.Allocator, in: []u8, out: []u8) Error!usize { var stream: c.zng_stream = .{ - .@"opaque" = @constCast(&alloc), .next_in = in.ptr, .avail_in = @truncate(in.len), .next_out = out.ptr, @@ -93,6 +86,12 @@ fn zalloc(ptr: ?*anyopaque, size: c_uint, len: c_uint) callconv(.c) ?*anyopaque return alloc.rawAlloc(size * len, .@"1", 0); } fn zfree(ptr: ?*anyopaque, mem_ptr: ?*anyopaque) callconv(.c) void { +<<<<<<< HEAD var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr)); alloc.rawFree(@ptrCast(mem_ptr), .@"1", 0); +======= + if (mem_ptr == null) return; + var alloc: *std.mem.Allocator = @ptrCast(@alignCast(ptr)); + alloc.free(@as([*]u8, @ptrCast(mem_ptr.?))); +>>>>>>> dfbfbda (Build is working again (on Zig master branch)) } diff --git a/src/decomp/c_zstd.zig b/src/decomp/c_zstd.zig index 3622639..005080b 100644 --- a/src/decomp/c_zstd.zig +++ b/src/decomp/c_zstd.zig @@ -20,17 +20,15 @@ io: Io, ctx: []?*c.ZSTD_DCtx, ctx_queue: Queue, -pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self { +pub fn init(alloc: std.mem.Allocator, io: Io) !Self { const buf = try alloc.alloc(?*c.ZSTD_DCtx, 20); // TODO: Choose a better number instead of a random one. var queue: Queue = .init(buf); for (0..20) |_| try queue.putOne(io, c.ZSTD_createDCtx()); return .{ - .alloc = alloc, .io = io, - .block_size = block_size, .ctx = buf, .ctx_queue = queue, }; @@ -38,7 +36,7 @@ pub fn init(alloc: std.mem.Allocator, io: Io, block_size: u32) !Self { pub fn deinit(self: *Self, alloc: std.mem.Allocator) void { self.ctx_queue.close(self.io); for (self.ctx) |ctx| - c.ZSTD_freeDCtx(ctx); + _ = c.ZSTD_freeDCtx(ctx); alloc.free(self.ctx); } diff --git a/src/file.zig b/src/file.zig index 878d2cf..bfbab86 100644 --- a/src/file.zig +++ b/src/file.zig @@ -37,7 +37,7 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, in: Inode, name: []const } pub fn fromDirEntry(alloc: std.mem.Allocator, archive: Archive, ent: DirEntry) !File { var rdr = archive.file.readerAt(archive.super.inode_start + ent.block_start); - var meta: MetadataReader = .init(alloc, &rdr.interface, archive.stateless_decomp); + var meta: MetadataReader = .init(alloc, &rdr, archive.stateless_decomp); try meta.interface.discardAll(ent.block_offset); var in: Inode = try .read(alloc, &meta.interface, archive.super.block_size); @@ -52,7 +52,6 @@ pub fn deinit(self: File) void { pub fn open(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File { const entries = try self.inode.readDirectory( alloc, - io, self.archive.file, self.archive.stateless_decomp, self.archive.super.dir_start, @@ -77,7 +76,7 @@ pub fn open(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) } } else return Error.FileNotFound; - const first_elem_file = try fromDirEntry(alloc, io, self.archive, search_slice[idx]); + const first_elem_file = try fromDirEntry(alloc, self.archive, search_slice[idx]); if (first_element.len == path.len) return first_elem_file; defer first_elem_file.deinit(); diff --git a/src/inode.zig b/src/inode.zig index 45dc0ca..87938b0 100644 --- a/src/inode.zig +++ b/src/inode.zig @@ -276,6 +276,7 @@ pub fn extract( const path = std.mem.trimEnd(u8, filepath, "/"); var decomp_base: Decomp = try .init(super.compression, alloc, io, super.block_size); + defer decomp_base.deinit(alloc); const decomp = decomp_base.decompressor(); var frag_mgr: FragManager = try .init(alloc, fil, decomp, super.frag_start, super.frag_count, super.block_size); diff --git a/src/options.zig b/src/options.zig index 1f6ba00..64cdb45 100644 --- a/src/options.zig +++ b/src/options.zig @@ -5,8 +5,6 @@ const Writer = std.Io.Writer; const ExtractionOptions = @This(); -// /// The number of threads used for extraction. 0 implies single threaded. -// threads: usize = 1, // As of Zig 0.16 this should no longer be necessary, instead this should be set by the io instance used. /// Don't set the file's owner & permissions after extraction ignore_permissions: bool = false, /// Don't set xattr values. Currently xattrs are never set anyway. @@ -18,16 +16,11 @@ verbose: bool = false, /// Where to print verbose log. verbose_writer: ?*Writer = null, -pub const SingleThreadedDefault: ExtractionOptions = .{}; -pub fn Default() !ExtractionOptions { - return .{ - .threads = try std.Thread.getCpuCount(), - }; -} +pub const default: ExtractionOptions = .{}; + pub fn VerboseDefault(wrt: *Writer) !ExtractionOptions { return .{ .verbose = true, .verbose_writer = wrt, - .threads = try std.Thread.getCpuCount(), }; } diff --git a/src/root.zig b/src/root.zig index 29fe47b..36e0cba 100644 --- a/src/root.zig +++ b/src/root.zig @@ -1,2 +1,6 @@ pub const Archive = @import("archive.zig"); pub const ExtractionOptions = @import("options.zig"); + +test { + @import("std").testing.refAllDecls(@This()); +} diff --git a/src/test.zig b/src/test.zig deleted file mode 100644 index 4e8b379..0000000 --- a/src/test.zig +++ /dev/null @@ -1,92 +0,0 @@ -const std = @import("std"); -const Io = std.Io; -const io = std.testing.io; -const alloc = std.testing.allocator; -const stuff = @import("builtin"); - -const Archive = @import("archive.zig"); -const Superblock = Archive.Superblock; - -const TestArchive = "testing/LinuxPATest.sfs"; - -test "Basics" { - std.debug.print("Starting test: Basics...\n", .{}); - - var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); - defer fil.close(io); - var sfs: Archive = try .init(io, fil, 0); - defer sfs.deinit(io); - try std.testing.expectEqualDeep(sfs.super, LinuxPATestCorrectSuperblock); - const root_file = try sfs.root(alloc, io); - defer root_file.deinit(); -} - -const TestFile = "Start.exe"; -const TestFileExtractLocation = "testing/Start.exe"; - -test "ExtractSingleFile" { - std.debug.print("Starting test: ExtractSingleFile...\n", .{}); - - Io.Dir.cwd().deleteFile(io, TestFileExtractLocation) catch {}; - var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); - defer fil.close(io); - var sfs: Archive = try .init(io, fil, 0); - defer sfs.deinit(io); - var test_fil = try sfs.open(alloc, io, TestFile); - defer test_fil.deinit(); - try test_fil.extract(alloc, io, TestFileExtractLocation, try .Default()); - //TODO: validate extracted file. -} - -const TestFullExtractLocation = "testing/TestExtract"; - -test "ExtractCompleteArchive" { - std.debug.print("Starting test: ExtractCompleteArchive...\n", .{}); - - Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {}; - var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); - defer fil.close(io); - var sfs: Archive = try .init(io, fil, 0); - defer sfs.deinit(io); - try sfs.extract(alloc, io, TestFullExtractLocation, try .Default()); -} - -const LinuxPATestCorrectSuperblock: Superblock = .{ - .magic = std.mem.readInt(u32, "hsqs", .little), - .inode_count = 2974, - .mod_time = 1632696724, - .block_size = 131072, - .frag_count = 264, - .compression = .zstd, - .block_log = 17, - .flags = .{ - .inode_uncompressed = false, - .data_uncompressed = false, - .check = false, - .frag_uncompressed = false, - .fragment_never = false, - .fragment_always = false, - .duplicates = true, - .exportable = true, - .xattr_uncompressed = false, - .xattr_never = false, - .compression_options = false, - .ids_uncompressed = false, - ._ = 0, - }, - .id_count = 1, - .ver_maj = 4, - .ver_min = 0, - .root_ref = .{ - .block_offset = 1363, - .block_start = 29237, - ._ = 0, - }, - .size = 106841744, - .id_start = 106841632, - .xattr_start = 106841720, - .inode_start = 106778274, - .dir_start = 106807998, - .frag_start = 106837747, - .export_start = 106841602, -}; diff --git a/src/util/offset_file.zig b/src/util/offset_file.zig index 45e3688..8ce28e8 100644 --- a/src/util/offset_file.zig +++ b/src/util/offset_file.zig @@ -18,7 +18,7 @@ pub fn init(io: Io, fil: File, archive_size: u64, init_offset: u64) !OffsetFile }), }; } -pub fn deinit(self: @This(), io: Io) void { +pub fn deinit(self: *OffsetFile, io: Io) void { self.map.destroy(io); }