From ba2f069a4a22fb5c50c37f80d812eb12f3a76572 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Wed, 27 May 2026 11:59:47 -0500 Subject: [PATCH] More work on re-implementing everything Finished MetadataReader Finished DirEntry Started work on File Added most important things to Inode Fixed test.zig Fixed some build issues causing SEGV --- .zed/settings.json | 3 +- build.zig | 16 ++--- mise.toml | 2 +- src/archive.zig | 58 ++++++++++++++++- src/bin/unsquashfs.zig | 8 ++- src/dir_entry.zig | 65 +++++++++++++++++++ src/file.zig | 118 ++++++++++++++++++++++++++++++++++ src/inode.zig | 131 ++++++++++++++++++++++++++++++++------ src/inode_data/dir.zig | 2 +- src/inode_data/file.zig | 8 ++- src/root.zig | 5 ++ src/test.zig | 49 ++++++++------ src/util/decomp_cache.zig | 23 ++++--- src/util/metadata.zig | 52 ++++++++++++--- 14 files changed, 460 insertions(+), 80 deletions(-) create mode 100644 src/dir_entry.zig create mode 100644 src/file.zig diff --git a/.zed/settings.json b/.zed/settings.json index e8f253c..25f8c6e 100644 --- a/.zed/settings.json +++ b/.zed/settings.json @@ -9,12 +9,13 @@ "usePlaceholders": false, }, "settings": { - "build_on_save": true, + // "build_on_save": true, "use_placeholders": false, "build_on_save_args": [ "-fincremental", "-Dallow_lzo=true", "-Ddebug=true", + "test", ], }, }, diff --git a/build.zig b/build.zig index a364edc..9a21606 100644 --- a/build.zig +++ b/build.zig @@ -51,7 +51,7 @@ pub fn build(b: *std.Build) !void { .optimize = optimize, .valgrind = debug, .error_tracing = debug, - .strip = debug, + .strip = !debug, .imports = &.{ .{ .name = "config", .module = zig_squashfs_options.createModule() }, .{ .name = "c", .module = c_import.createModule() }, @@ -78,7 +78,7 @@ pub fn build(b: *std.Build) !void { }, .valgrind = debug, .error_tracing = debug, - .strip = if (debug == true) false else null, + .strip = !debug, }), .use_llvm = debug, .version = version, @@ -92,19 +92,15 @@ pub fn build(b: *std.Build) !void { .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, - .valgrind = debug, - .error_tracing = debug, - .strip = debug, + .valgrind = true, + .error_tracing = true, + .strip = false, .imports = &.{ .{ .name = "config", .module = zig_squashfs_options.createModule() }, .{ .name = "c", .module = c_import.createModule() }, }, }), - .use_llvm = true, // Helps with lldb degugging - .test_runner = .{ - .mode = .simple, - .path = b.path("src/test.zig"), - }, + .use_llvm = debug, // Helps with lldb degugging }); for (deps) |d| diff --git a/mise.toml b/mise.toml index a7b766c..3b9ae66 100644 --- a/mise.toml +++ b/mise.toml @@ -1,2 +1,2 @@ [tools] -zig = "master" # 0.16.0 compilation errors with SEGV +zig = "master" diff --git a/src/archive.zig b/src/archive.zig index 9d7d1d3..9e60d8e 100644 --- a/src/archive.zig +++ b/src/archive.zig @@ -5,18 +5,28 @@ const MemoryMap = Io.File.MemoryMap; const c = @import("c"); const config = @import("config"); +const ExtractionOptions = @import("options.zig"); +const File = @import("file.zig"); const Superblock = @import("super.zig").Superblock; const DecompCache = @import("util/decomp_cache.zig"); const CompressionType = @import("util/decompress.zig").CompressionType; const Archive = @This(); +const CACHE_MIN = 16 * 1024 * 1024; +const CACHE_MAX = 1 * 1024 * 1024 * 1024; + map: MemoryMap, cache: DecompCache, super: Superblock, -pub fn init(alloc: std.mem.Allocator, io: Io, file: Io.File, offset: u64, max_cache_size: u64) !Archive { +/// Open a squashfs archive from an Io.File. +pub fn init(alloc: std.mem.Allocator, io: Io, fil: Io.File) !Archive { + return initAdvanced(alloc, io, fil, 0, 0); +} +/// If max_cache_size is zero, a size is selected based on system ram, up to 1GB with a minimum of 16MB. +pub fn initAdvanced(alloc: std.mem.Allocator, io: Io, file: Io.File, offset: u64, max_cache_size: u64) !Archive { var rdr = file.reader(io, &[0]u8{}); try rdr.seekTo(offset); @@ -29,15 +39,57 @@ pub fn init(alloc: std.mem.Allocator, io: Io, file: Io.File, offset: u64, max_ca const map = try file.createMemoryMap( io, - .{ .offset = offset, .len = super.size, .protection = .{ .read = true } }, + .{ + .offset = offset, + .len = super.size, + .protection = .{ .read = true }, + }, ); + const cache_size = blk: { + if (max_cache_size > CACHE_MIN) break :blk CACHE_MIN; + const sys_mem = std.process.totalSystemMemory() catch break :blk CACHE_MIN; + var min = @min(CACHE_MAX, sys_mem / 4); + if (min < CACHE_MIN and sys_mem > CACHE_MIN) + min = CACHE_MIN; + break :blk min; + }; return .{ .map = map, - .cache = try .init(alloc, map, super.compression, max_cache_size), + .cache = try .init( + alloc, + map, + super.compression, + cache_size, + ), .super = super, }; } pub fn deinit(self: *Archive, io: Io) void { + self.cache.deinit(io); self.map.destroy(io); } + +pub fn root(self: *Archive, alloc: std.mem.Allocator, io: Io) !File { + return .fromRef(alloc, io, self, "", self.super.root_ref); +} + +pub fn open(self: *Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File { + const path = std.mem.trim(u8, filepath, "/"); + + var root_file = try self.root(alloc, io); + + if (path.len == 0 or std.mem.eql(u8, path, ".")) return root_file; + defer root_file.deinit(); + + return root_file.open(alloc, io, path); +} + +pub fn extract(self: *Archive, alloc: std.mem.Allocator, io: Io, ext_dir: []const u8, options: ExtractionOptions) !void { + _ = self; + _ = alloc; + _ = io; + _ = ext_dir; + _ = options; + return error.TODO; +} diff --git a/src/bin/unsquashfs.zig b/src/bin/unsquashfs.zig index 98a0dcd..d2a1850 100644 --- a/src/bin/unsquashfs.zig +++ b/src/bin/unsquashfs.zig @@ -56,7 +56,7 @@ pub fn main(init: std.process.Init) !void { var fil: std.Io.File = try Io.Dir.cwd().openFile(io, archive, .{}); //TODO: Handle error gracefully. defer fil.close(io); - var arc: squashfs.Archive = try .init(alloc, io, fil, offset, 1 * 1024 * 1024 * 1024); //TODO: Update when memory size matters. //TODO: Handle error gracefully. + var arc: squashfs.Archive = try .initAdvanced(alloc, io, fil, offset, 0); //TODO: Update when memory size matters. //TODO: Handle error gracefully. defer arc.deinit(io); const options: squashfs.ExtractionOptions = .{ .single_threaded = threads == 1, @@ -67,7 +67,7 @@ pub fn main(init: std.process.Init) !void { }; if (force) try Io.Dir.cwd().deleteTree(io, extLoc); - try arc.extract(alloc, extLoc, options); //TODO: Handle error gracefully. + try arc.extract(alloc, io, extLoc, options); //TODO: Handle error gracefully. } fn handleArgs(out: *Writer, args: std.process.Args) !void { @@ -135,3 +135,7 @@ fn handleArgs(out: *Writer, args: std.process.Args) !void { archive = arg; } } + +test { + std.testing.refAllDecls(squashfs.Archive); +} diff --git a/src/dir_entry.zig b/src/dir_entry.zig new file mode 100644 index 0000000..c54430b --- /dev/null +++ b/src/dir_entry.zig @@ -0,0 +1,65 @@ +const std = @import("std"); +const Reader = std.Io.Reader; + +const Inode = @import("inode.zig"); + +const Header = extern struct { + count: u32, + block_start: u32, + num: u32, +}; +const Entry = extern struct { + block_offset: u16, + num_offset: i16, + inode_type: Inode.Type, + name_size: u16, +}; + +const DirEntry = @This(); + +inode_type: Inode.Type, +name: []const u8, + +block_start: u32, +block_offset: u32, +num: u32, + +pub fn deinit(self: DirEntry, alloc: std.mem.Allocator) void { + alloc.free(self.name); +} + +pub fn readEntries(alloc: std.mem.Allocator, rdr: *Reader, size: u32) ![]DirEntry { + var out: std.ArrayList(DirEntry) = try .initCapacity(alloc, 50); + errdefer out.deinit(alloc); + + var tot_read: u32 = 3; + while (tot_read < size) { + var hdr: Header = undefined; + try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little); + tot_read += @sizeOf(Header); + + try out.ensureUnusedCapacity(alloc, hdr.count + 1); + + for (0..hdr.count + 1) |_| { + var ent: Entry = undefined; + try rdr.readSliceEndian(Entry, @ptrCast(&ent), .little); + tot_read += @sizeOf(Entry) + ent.name_size + 1; + + const name = try alloc.alloc(u8, ent.name_size + 1); + errdefer alloc.free(name); + + try rdr.readSliceEndian(u8, name, .little); + + out.appendAssumeCapacity(.{ + .inode_type = ent.inode_type, + .name = name, + + .block_offset = ent.block_offset, + .block_start = hdr.block_start, + .num = @intCast(@as(i64, @intCast(hdr.num)) + ent.num_offset), + }); + } + } + + return out.toOwnedSlice(alloc); +} diff --git a/src/file.zig b/src/file.zig new file mode 100644 index 0000000..46e62c9 --- /dev/null +++ b/src/file.zig @@ -0,0 +1,118 @@ +//! A wrapper around an Inode to make common activities easier. + +const std = @import("std"); +const Io = std.Io; + +const Archive = @import("archive.zig"); +const DirEntry = @import("dir_entry.zig"); +const ExtractionOptions = @import("options.zig"); +const Inode = @import("inode.zig"); +const MetadataReader = @import("util/metadata.zig"); + +pub const Error = error{ + NotFound, +}; + +const File = @This(); + +alloc: std.mem.Allocator, +archive: *Archive, + +name: []const u8, +inode: Inode, + +pub fn fromEntry(alloc: std.mem.Allocator, io: Io, archive: *Archive, entry: DirEntry) !File { + var meta: MetadataReader = .init(io, &archive.cache, archive.super.inode_start + entry.block_start); + defer meta.deinit(); + try meta.interface.discardAll(entry.block_offset); + + const new_name = try alloc.alloc(u8, entry.name.len); + errdefer alloc.free(new_name); + @memcpy(new_name, entry.name); + + return .{ + .alloc = alloc, + .archive = archive, + + .name = new_name, + .inode = try .fromReader(alloc, &meta.interface, archive.super.block_size), + }; +} +/// Create a File from an Inode.Ref. name should be created using the alloc given. +pub fn fromRef(alloc: std.mem.Allocator, io: Io, archive: *Archive, name: []const u8, ref: Inode.Ref) !File { + return .{ + .alloc = alloc, + .archive = archive, + + .name = name, + .inode = try .fromRef( + alloc, + io, + &archive.cache, + archive.super.inode_start, + archive.super.block_size, + ref, + ), + }; +} +pub fn copy(alloc: std.mem.Allocator, from: File) !File { + const new_name = try alloc.alloc(u8, from.name.len); + errdefer alloc.free(new_name); + @memcpy(new_name, from.name); + + return .{ + .alloc = alloc, + .archive = from.archive, + + .inode = try .copy(alloc, from.inode), + .name = new_name, + }; +} +pub fn deinit(self: File) void { + self.alloc.free(self.name); + self.inode.deinit(self.alloc); +} + +pub fn open(self: File, alloc: std.mem.Allocator, io: Io, filepath: []const u8) !File { + const path = std.mem.trim(u8, filepath, "/"); + + if (path.len == 0 or std.mem.eql(u8, path, ".")) return .copy(alloc, self); + + const first_element = std.mem.sliceTo(path, '/'); + + const entries = try self.inode.readDirectory(alloc, io, &self.archive.cache, self.archive.super.dir_start); + defer { + for (entries) |entry| + entry.deinit(alloc); + alloc.free(entries); + } + + // Potentially I could use linear searching on small dir tables... + var search_slice = entries; + var idx = search_slice.len / 2; + while (search_slice.len > 0) { + const order = std.mem.order(u8, first_element, search_slice[idx].name); + switch (order) { + .eq => break, + .gt => search_slice = search_slice[idx..], + .lt => search_slice = search_slice[0..idx], + } + idx = search_slice.len / 2; + } + if (search_slice.len == 0) return Error.NotFound; + + var fil: File = try .fromEntry(alloc, io, self.archive, search_slice[idx]); + if (path.len == first_element.len) return fil; + defer fil.deinit(); + + return fil.open(alloc, io, filepath[first_element.len..]); +} + +pub fn extract(self: File, alloc: std.mem.Allocator, io: Io, ext_loc: []const u8, options: ExtractionOptions) !void { + _ = self; + _ = alloc; + _ = io; + _ = ext_loc; + _ = options; + return error.TODO; +} diff --git a/src/inode.zig b/src/inode.zig index 159f444..4ae910e 100644 --- a/src/inode.zig +++ b/src/inode.zig @@ -1,10 +1,15 @@ //! A file-system object. Represents a File or directory. const std = @import("std"); +const Io = std.Io; +const Reader = Io.Reader; -const dir = @import("inode_data/dir.zig"); -const file = @import("inode_data/file.zig"); -const misc = @import("inode_data/misc.zig"); +const DirEntry = @import("dir_entry.zig"); +const DirTypes = @import("inode_data/dir.zig"); +const FileTypes = @import("inode_data/file.zig"); +const MiscTypes = @import("inode_data/misc.zig"); +const DecompCache = @import("util/decomp_cache.zig"); +const MetadataReader = @import("util/metadata.zig"); pub const Ref = packed struct(u64) { block_offset: u16, @@ -12,7 +17,7 @@ pub const Ref = packed struct(u64) { _: u16, }; -pub const InodeType = enum(u16) { +pub const Type = enum(u16) { dir = 1, file, symlink, @@ -29,25 +34,25 @@ pub const InodeType = enum(u16) { ext_socket, }; -pub const InodeData = union(InodeType) { - dir: dir.Dir, - file: file.File, - symlink: misc.Symlink, - block_dev: misc.Dev, - char_dev: misc.Dev, - fifo: misc.IPC, - socket: misc.IPC, - ext_dir: dir.ExtDir, - ext_file: file.ExtFile, - ext_symlink: misc.ExtSymlink, - ext_block_dev: misc.ExtDev, - ext_char_dev: misc.ExtDev, - ext_fifo: misc.ExtIPC, - ext_socket: misc.ExtIPC, +pub const Data = union(Type) { + dir: DirTypes.Dir, + file: FileTypes.File, + symlink: MiscTypes.Symlink, + block_dev: MiscTypes.Dev, + char_dev: MiscTypes.Dev, + fifo: MiscTypes.IPC, + socket: MiscTypes.IPC, + ext_dir: DirTypes.ExtDir, + ext_file: FileTypes.ExtFile, + ext_symlink: MiscTypes.ExtSymlink, + ext_block_dev: MiscTypes.ExtDev, + ext_char_dev: MiscTypes.ExtDev, + ext_fifo: MiscTypes.ExtIPC, + ext_socket: MiscTypes.ExtIPC, }; pub const Header = packed struct { - inode_type: InodeType, + inode_type: Type, permissions: u16, uid_idx: u16, gid_idx: u16, @@ -55,4 +60,90 @@ pub const Header = packed struct { num: u32, }; +pub const Error = error{ + NotDirectory, +}; + const Inode = @This(); + +hdr: Header, +data: Data, + +pub fn fromRef(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, inode_start: u64, block_size: u32, ref: Ref) !Inode { + var meta: MetadataReader = .init(io, cache, ref.block_start + inode_start); + defer meta.deinit(); + try meta.interface.discardAll(ref.block_offset); + return fromReader(alloc, &meta.interface, block_size); +} +pub fn fromReader(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !Inode { + var hdr: Header = undefined; + try rdr.readSliceEndian(Header, @ptrCast(&hdr), .little); + return .{ + .hdr = hdr, + .data = switch (hdr.inode_type) { + .dir => .{ .dir = try .read(rdr) }, + .file => .{ .file = try .read(alloc, rdr, block_size) }, + .symlink => .{ .symlink = try .read(alloc, rdr) }, + .block_dev => .{ .block_dev = try .read(rdr) }, + .char_dev => .{ .char_dev = try .read(rdr) }, + .fifo => .{ .fifo = try .read(rdr) }, + .socket => .{ .socket = try .read(rdr) }, + .ext_dir => .{ .ext_dir = try .read(rdr) }, + .ext_file => .{ .ext_file = try .read(alloc, rdr, block_size) }, + .ext_symlink => .{ .ext_symlink = try .read(alloc, rdr) }, + .ext_block_dev => .{ .ext_block_dev = try .read(rdr) }, + .ext_char_dev => .{ .ext_char_dev = try .read(rdr) }, + .ext_fifo => .{ .ext_fifo = try .read(rdr) }, + .ext_socket => .{ .ext_socket = try .read(rdr) }, + }, + }; +} +pub fn copy(alloc: std.mem.Allocator, from: Inode) !Inode { + var new = from; + switch (from.data) { + .file => |f| { + new.data.file.block_sizes = try alloc.alloc(FileTypes.BlockSize, f.block_sizes.len); + @memcpy(new.data.file.block_sizes, f.block_sizes); + }, + .ext_file => |f| { + new.data.ext_file.block_sizes = try alloc.alloc(FileTypes.BlockSize, f.block_sizes.len); + @memcpy(new.data.ext_file.block_sizes, f.block_sizes); + }, + .symlink => |s| { + const new_target = try alloc.alloc(u8, s.target.len); + @memcpy(new_target, s.target); + new.data.symlink.target = new_target; + }, + .ext_symlink => |s| { + const new_target = try alloc.alloc(u8, s.target.len); + @memcpy(new_target, s.target); + new.data.ext_symlink.target = new_target; + }, + else => {}, + } + return new; +} +pub fn deinit(self: Inode, alloc: std.mem.Allocator) void { + switch (self.data) { + .file => |f| alloc.free(f.block_sizes), + .ext_file => |f| alloc.free(f.block_sizes), + .symlink => |s| alloc.free(s.target), + .ext_symlink => |s| alloc.free(s.target), + else => {}, + } +} + +pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, io: Io, cache: *DecompCache, dir_start: u64) ![]DirEntry { + return switch (self.data) { + .dir => |d| readDirectoryFromData(alloc, io, cache, dir_start, d), + .ext_dir => |d| readDirectoryFromData(alloc, io, cache, dir_start, d), + else => Error.NotDirectory, + }; +} +fn readDirectoryFromData(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, dir_start: u64, d: anytype) ![]DirEntry { + var meta: MetadataReader = .init(io, cache, dir_start + d.block_start); + defer meta.deinit(); + try meta.interface.discardAll(d.block_offset); + + return DirEntry.readEntries(alloc, &meta.interface, d.size); +} diff --git a/src/inode_data/dir.zig b/src/inode_data/dir.zig index 68e58f3..6b45db0 100644 --- a/src/inode_data/dir.zig +++ b/src/inode_data/dir.zig @@ -26,7 +26,7 @@ pub const ExtDir = extern struct { pub fn read(rdr: *Reader) !ExtDir { var d: ExtDir = undefined; - try rdr.readSliceEndian(Dir, @ptrCast(&d), .little); + try rdr.readSliceEndian(ExtDir, @ptrCast(&d), .little); return d; } }; diff --git a/src/inode_data/file.zig b/src/inode_data/file.zig index 61d414f..d5c3626 100644 --- a/src/inode_data/file.zig +++ b/src/inode_data/file.zig @@ -15,12 +15,13 @@ pub const File = struct { block_sizes: []BlockSize, pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !File { - var values = struct { + const raw_values = struct { block_start: u32, // bytes 0-3 frag_idx: u32, // bytes 4-7 frag_offset: u32, // bytes 8-11 size: u32, // bytes 12-15 }; + var values: raw_values = undefined; try rdr.readSliceEndian(@TypeOf(values), @ptrCast(&values), .little); var num_blocks: u32 = values.size / block_size; @@ -54,7 +55,7 @@ pub const ExtFile = struct { block_sizes: []BlockSize, pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !ExtFile { - var values = struct { + const raw_values = struct { block_start: u64, // bytes 0-7 size: u64, // bytes 8-15 sparse: u64, // bytes 16-23 @@ -63,9 +64,10 @@ pub const ExtFile = struct { frag_offset: u32, // bytes 32-35 xattr_idx: u32, // bytes 36-39 }; + var values: raw_values = undefined; try rdr.readSliceEndian(@TypeOf(values), @ptrCast(&values), .little); - var num_blocks: u32 = values.size / block_size; + var num_blocks: u32 = @truncate(values.size / block_size); if (values.size % block_size != 0 and values.frag_idx == 0xFFFFFFFF) num_blocks += 1; const sizes = try alloc.alloc(BlockSize, num_blocks); errdefer alloc.free(sizes); diff --git a/src/root.zig b/src/root.zig index 29fe47b..3aeee40 100644 --- a/src/root.zig +++ b/src/root.zig @@ -1,2 +1,7 @@ pub const Archive = @import("archive.zig"); pub const ExtractionOptions = @import("options.zig"); +const Test = @import("test.zig"); + +test { + @import("std").testing.refAllDecls(Test); +} diff --git a/src/test.zig b/src/test.zig index 4bb58cb..43d3d14 100644 --- a/src/test.zig +++ b/src/test.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const Io = std.Io; const stuff = @import("builtin"); const Archive = @import("archive.zig"); @@ -7,40 +8,46 @@ const Superblock = @import("super.zig").Superblock; const TestArchive = "testing/LinuxPATest.sfs"; test "Basics" { - var fil = try std.fs.cwd().openFile(TestArchive, .{}); - defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - if (sfs.super != LinuxPATestCorrectSuperblock) { - std.debug.print("Superblock wrong\nShould be: {}\n\nis: {}\n", .{ LinuxPATestCorrectSuperblock, sfs.super }); - return error.BadSuperblock; - } + const io = std.testing.io; + const alloc = std.testing.allocator; + + var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); + defer fil.close(io); + var sfs: Archive = try .init(alloc, io, fil); + defer sfs.deinit(io); + try std.testing.expectEqualDeep(sfs.super, LinuxPATestCorrectSuperblock); } const TestFile = "Start.exe"; const TestFileExtractLocation = "testing/Start.exe"; test "ExtractSingleFile" { - std.fs.cwd().deleteFile(TestFileExtractLocation) catch {}; - var fil = try std.fs.cwd().openFile(TestArchive, .{}); - defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - var test_fil = try sfs.open(TestFile); + const io = std.testing.io; + const alloc = std.testing.allocator; + + Io.Dir.cwd().deleteFile(io, TestFileExtractLocation) catch {}; + var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); + defer fil.close(io); + var sfs: Archive = try .init(alloc, io, fil); + defer sfs.deinit(io); + var test_fil = try sfs.open(alloc, io, TestFile); defer test_fil.deinit(); - try test_fil.extract(TestFileExtractLocation, .Default); + try test_fil.extract(alloc, io, TestFileExtractLocation, .default); //TODO: validate extracted file. } const TestFullExtractLocation = "testing/TestExtract"; test "ExtractCompleteArchive" { - std.fs.cwd().deleteTree(TestFullExtractLocation) catch {}; - var fil = try std.fs.cwd().openFile(TestArchive, .{}); - defer fil.close(); - var sfs: Archive = try .init(std.testing.allocator, fil); - defer sfs.deinit(); - try sfs.extract(TestFullExtractLocation, .Default); + const io = std.testing.io; + const alloc = std.testing.allocator; + + Io.Dir.cwd().deleteTree(io, TestFullExtractLocation) catch {}; + var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); + defer fil.close(io); + var sfs: Archive = try .init(alloc, io, fil); + defer sfs.deinit(io); + try sfs.extract(alloc, io, TestFullExtractLocation, .default); } const LinuxPATestCorrectSuperblock: Superblock = .{ diff --git a/src/util/decomp_cache.zig b/src/util/decomp_cache.zig index 36ed295..481d759 100644 --- a/src/util/decomp_cache.zig +++ b/src/util/decomp_cache.zig @@ -11,7 +11,7 @@ const DecompCache = @This(); const Cache = struct { cache: []u8, - usage: Atomic(u32) = .init(0), + usage: Atomic(u32), }; arena: std.heap.ArenaAllocator, @@ -40,11 +40,11 @@ pub fn init(alloc: std.mem.Allocator, map: Io.File.MemoryMap, decomp_type: Decom } pub fn deinit(self: *DecompCache, io: Io) void { self.mut.lockUncancelable(io); - self.cache.deinit(); + self.cache.deinit(self.arena.child_allocator); self.arena.deinit(); } -fn makeRoom(self: *DecompCache, size: u32) !void { +fn makeRoom(self: *DecompCache, io: Io, size: u32) !void { if (size + self.cur_size < self.max_size) return; var iter = self.cache.iterator(); while (iter.next()) |ent| { @@ -55,8 +55,8 @@ fn makeRoom(self: *DecompCache, size: u32) !void { } if (size + self.cur_size < self.max_size) return; } - self.cond.wait(Io, &self.mut.mutex); - try self.makeRoom(size); + try self.cond.wait(io, &self.mut.mutex); + return self.makeRoom(io, size); } pub fn checkinBlock(self: *DecompCache, io: Io, offset: u64) void { @@ -65,7 +65,7 @@ pub fn checkinBlock(self: *DecompCache, io: Io, offset: u64) void { const get = self.cache.getPtr(offset); if (get == null) return; - const res = get.?.usage.fetchSub(1, .unordered); + const res = get.?.usage.fetchSub(1, .acq_rel); if (res == 0) self.cond.broadcast(io); } pub fn checkoutBlock(self: *DecompCache, io: Io, offset: u64, block_size: u32, max_result_size: u32) ![]u8 { @@ -75,14 +75,14 @@ pub fn checkoutBlock(self: *DecompCache, io: Io, offset: u64, block_size: u32, m const get = self.cache.getPtr(offset); if (get != null) { - _ = get.?.usage.fetchAdd(1, .unordered); + _ = get.?.usage.fetchAdd(1, .acq_rel); return get.?.cache; } } try self.mut.lock(io); defer self.mut.unlock(io); - try self.makeRoom(max_result_size); + try self.makeRoom(io, max_result_size); var alloc = self.arena.allocator(); const buf_alloc = self.arena.child_allocator; @@ -90,7 +90,7 @@ pub fn checkoutBlock(self: *DecompCache, io: Io, offset: u64, block_size: u32, m var out = try alloc.alloc(u8, max_result_size); errdefer alloc.free(out); - const out_size = try self.decomp(buf_alloc, self.map[offset..][0..block_size], out); + const out_size = try self.decomp(buf_alloc, self.map.memory[offset..][0..block_size], out); if (out_size != max_result_size) { if (alloc.resize(out, out_size)) { out.len = out_size; @@ -102,7 +102,10 @@ pub fn checkoutBlock(self: *DecompCache, io: Io, offset: u64, block_size: u32, m } } - try self.cache.put(alloc, offset, out); + try self.cache.put(buf_alloc, offset, .{ + .cache = out, + .usage = .init(1), + }); return out; } diff --git a/src/util/metadata.zig b/src/util/metadata.zig index c067dd3..bbc7b52 100644 --- a/src/util/metadata.zig +++ b/src/util/metadata.zig @@ -1,3 +1,5 @@ +//! A cache for decompressed blocks. Used for Metadata & fragments. + const std = @import("std"); const Io = std.Io; const Reader = Io.Reader; @@ -31,7 +33,7 @@ interface: Reader = .{ }, }, -pub fn init(io: Io, cache: *DecompCache, offset: u64) void { +pub fn init(io: Io, cache: *DecompCache, offset: u64) MetadataReader { return .{ .io = io, @@ -41,18 +43,18 @@ pub fn init(io: Io, cache: *DecompCache, offset: u64) void { }; } pub fn deinit(self: *MetadataReader) void { - if (self.cur_block_offset != 0) - self.cache.checkinBlock(self.io, self.cur_block_offset); + if (self.cur_offset != 0) + self.cache.checkinBlock(self.io, self.cur_offset); } fn advance(self: *MetadataReader) !void { if (self.interface.buffer.len > 0) self.cache.checkinBlock(self.io, self.cur_offset); - const hdr = std.mem.readInt(BlockHeader, self.cache.map[self.next_offset..][0..2], .little); + const hdr: BlockHeader = @bitCast(std.mem.readInt(u16, self.cache.map.memory[self.next_offset..][0..2], .little)); self.cur_offset = self.next_offset + 2; self.next_offset += hdr.size; if (hdr.uncompressed) { - self.interface.buffer = self.cache.map[self.cur_offset..][0..hdr.size]; + self.interface.buffer = self.cache.map.memory[self.cur_offset..][0..hdr.size]; self.interface.end = hdr.size; self.interface.seek = 0; return; @@ -62,6 +64,40 @@ fn advance(self: *MetadataReader) !void { self.interface.seek = 0; } -fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {} -fn discard(r: *Reader, limit: Limit) Reader.Error!usize {} -fn readVec(r: *Reader, vec: [][]u8) Reader.Error!usize {} +fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize { + if (r.seek == r.end) { + var self: *MetadataReader = @fieldParentPtr("interface", r); + self.advance() catch return Reader.Error.ReadFailed; + } + if (limit == .nothing) return 0; + const to_write = @min(r.end - r.seek, @intFromEnum(limit)); + const wrote = try w.write(r.buffer[r.seek..][0..to_write]); + r.seek += wrote; + return wrote; +} +fn discard(r: *Reader, limit: Limit) Reader.Error!usize { + if (r.seek == r.end) { + var self: *MetadataReader = @fieldParentPtr("interface", r); + self.advance() catch return Reader.Error.ReadFailed; + } + if (limit == .nothing) return 0; + const to_skip = @min(r.end - r.seek, @intFromEnum(limit)); + r.seek += to_skip; + return to_skip; +} +fn readVec(r: *Reader, vec: [][]u8) Reader.Error!usize { + if (r.seek == r.end) { + var self: *MetadataReader = @fieldParentPtr("interface", r); + self.advance() catch return Reader.Error.ReadFailed; + } + if (vec.len == 0) return 0; + var total_copied: usize = 0; + for (vec) |v| { + const to_cpy = @min(r.end - r.seek, v.len); + @memcpy(v[0..to_cpy], r.buffer[r.seek..][0..to_cpy]); + r.seek += to_cpy; + total_copied += to_cpy; + if (r.seek == r.end) break; + } + return total_copied; +}