2 Commits

Author SHA1 Message Date
Caleb Gardner 3239bf0e01 IT WORKS AND IS FAST 2026-05-22 15:45:44 -05:00
Caleb Gardner 0df14b8adc Moved to File.MemoryMap instead of direct file I/O 2026-05-22 12:49:07 -05:00
16 changed files with 311 additions and 389 deletions
Executable
+8
View File
@@ -0,0 +1,8 @@
#! /usr/bin/env bash
ARCHIVE="testing/LinuxPATest.sfs"
REF_EXT_LOC="testing/LinuxPAReference"
PROG_EXT_LOC="testing/LinuxPABinTest"
hyperfine --warmup 5 --prepare "rm -rf $REF_EXT_LOC && rm -rf $PROG_EXT_LOC" "unsquashfs -d $REF_EXT_LOC $ARCHIVE" "zig-out/bin/unsquashfs -d $PROG_EXT_LOC $ARCHIVE"
+4 -1
View File
@@ -3,7 +3,7 @@ const std = @import("std");
pub fn build(b: *std.Build) !void { pub fn build(b: *std.Build) !void {
// const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false; // const use_zig_decomp = b.option(bool, "use_zig_decomp", "Use zig standard library for decompression.") orelse false;
// const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false; // const allow_lzo = b.option(bool, "allow_lzo", "Compile with lzo support") orelse false;
const debug = b.option(bool, "debug", "Enable options to make debugging easier."); var debug = b.option(bool, "debug", "Enable options to make debugging easier.");
const version_string_option = b.option([]const u8, "version", "Version of the library/binary"); const version_string_option = b.option([]const u8, "version", "Version of the library/binary");
// const zig_squashfs_options = b.addOptions(); // const zig_squashfs_options = b.addOptions();
@@ -13,6 +13,8 @@ pub fn build(b: *std.Build) !void {
const target = b.standardTargetOptions(.{}); const target = b.standardTargetOptions(.{});
const optimize = b.standardOptimizeOption(.{}); const optimize = b.standardOptimizeOption(.{});
if (optimize == .Debug) debug = true;
const lib = b.addLibrary(.{ const lib = b.addLibrary(.{
.name = "squashfs", .name = "squashfs",
.root_module = b.createModule(.{ .root_module = b.createModule(.{
@@ -20,6 +22,7 @@ pub fn build(b: *std.Build) !void {
.target = target, .target = target,
.valgrind = debug, .valgrind = debug,
.root_source_file = b.path("src/root.zig"), .root_source_file = b.path("src/root.zig"),
.link_libc = true,
}), }),
.use_llvm = debug, .use_llvm = debug,
}); });
-10
View File
@@ -1,10 +0,0 @@
#!/bin/sh
zig test \
-lc \
-lz \
-llzma \
-lminilzo \
-llz4 \
-lzstd \
src/test.zig
+17 -14
View File
@@ -23,8 +23,9 @@ pub fn init(io: Io, file: std.Io.File, offset: u64) !Archive {
try rdr.seekTo(offset); try rdr.seekTo(offset);
var super: Superblock = undefined; var super: Superblock = undefined;
try rdr.interface.readSliceEndian(Superblock, @ptrCast(&super), .little); try rdr.interface.readSliceEndian(Superblock, @ptrCast(&super), .little);
return .{ return .{
.file = .init(file, offset), .file = try .init(io, file, super.size, offset),
.super = super, .super = super,
.stateless_decomp = try Decomp.StatelessDecomp(super.compression), .stateless_decomp = try Decomp.StatelessDecomp(super.compression),
@@ -53,19 +54,6 @@ pub fn open(self: Archive, alloc: std.mem.Allocator, io: Io, filepath: []const u
defer root_file.deinit(); defer root_file.deinit();
return root_file.open(alloc, io, filepath); return root_file.open(alloc, io, filepath);
} }
/// Extract the entire archive contents to the given directory.
pub fn extract(self: Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void {
const root_inode = try Utils.inodeFromRef(
alloc,
io,
self.file,
self.stateless_decomp,
self.super.inode_start,
self.super.block_size,
self.super.root_ref,
);
return root_inode.extract(alloc, io, self.file, self.super, extract_dir, options);
}
/// Returns the inode with the given inode number. /// Returns the inode with the given inode number.
/// Requires that the archive is exportable (has an export lookup table). /// Requires that the archive is exportable (has an export lookup table).
@@ -163,3 +151,18 @@ pub const Superblock = extern struct {
return SuperblockError.InvalidBlockLog; return SuperblockError.InvalidBlockLog;
} }
}; };
// Extraction
/// Extract the entire archive contents to the given directory.
pub fn extract(self: Archive, alloc: std.mem.Allocator, io: Io, extract_dir: []const u8, options: ExtractionOptions) !void {
const root_inode = try Utils.inodeFromRef(
alloc,
self.file,
self.stateless_decomp,
self.super.inode_start,
self.super.block_size,
self.super.root_ref,
);
return root_inode.extract(alloc, io, self.file, self.super, extract_dir, options);
}
+6 -9
View File
@@ -48,21 +48,18 @@ pub fn main(init: std.process.Init) !void {
var out = stdout.writer(io, &[0]u8{}); var out = stdout.writer(io, &[0]u8{});
defer out.interface.flush() catch {}; defer out.interface.flush() catch {};
// try handleArgs(init.minimal.args, &out.interface); try handleArgs(init.minimal.args, &out.interface);
// if (archive.len == 0) { if (archive.len == 0) {
// try out.interface.print("You must provide a squashfs archive\n", .{}); try out.interface.print("You must provide a squashfs archive\n", .{});
// try out.interface.print(help_mgs, .{}); try out.interface.print(help_mgs, .{});
// return; return;
// } }
archive = "testing/LinuxPATest.sfs";
extLoc = "testing/LinuxPABinTest";
var fil = try Io.Dir.cwd().openFile(io, archive, .{}); //TODO: Handle error gracefully. var fil = try Io.Dir.cwd().openFile(io, archive, .{}); //TODO: Handle error gracefully.
defer fil.close(io); defer fil.close(io);
var arc: squashfs.Archive = try .init(io, fil, offset); //TODO: Handle error gracefully. var arc: squashfs.Archive = try .init(io, fil, offset); //TODO: Handle error gracefully.
const options: squashfs.ExtractionOptions = .{ const options: squashfs.ExtractionOptions = .{
.threads = if (threads == 0) try std.Thread.getCpuCount() else threads,
.verbose = verbose, .verbose = verbose,
.verbose_writer = if (verbose) &out.interface else null, .verbose_writer = if (verbose) &out.interface else null,
.ignore_xattr = ignore_xattrs, .ignore_xattr = ignore_xattrs,
+1 -2
View File
@@ -69,8 +69,7 @@ inline fn zstdDecomp(buffer: []u8, in: []u8, out: []u8) !usize {
pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp }; pub const stateless_decompressor: Decompressor = .{ .decomp_fn = statelessDecomp };
fn statelessDecomp(d: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize { fn statelessDecomp(_: ?*const Decompressor, alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
_ = d;
_ = alloc; _ = alloc;
const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len); const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
if (c.ZSTD_isError(res) == 1) if (c.ZSTD_isError(res) == 1)
+2 -2
View File
@@ -35,8 +35,8 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, in: Inode, name: []const
.name = new_name, .name = new_name,
}; };
} }
pub fn fromDirEntry(alloc: std.mem.Allocator, io: Io, archive: Archive, ent: DirEntry) !File { pub fn fromDirEntry(alloc: std.mem.Allocator, archive: Archive, ent: DirEntry) !File {
var rdr = try archive.file.readerAt(io, archive.super.inode_start + ent.block_start, &[0]u8{}); var rdr = archive.file.readerAt(archive.super.inode_start + ent.block_start);
var meta: MetadataReader = .init(alloc, &rdr.interface, archive.stateless_decomp); var meta: MetadataReader = .init(alloc, &rdr.interface, archive.stateless_decomp);
try meta.interface.discardAll(ent.block_offset); try meta.interface.discardAll(ent.block_offset);
+16 -15
View File
@@ -23,16 +23,13 @@ block_size: u32,
entries: []FragEntry, entries: []FragEntry,
frag_cache: std.array_hash_map.Auto(u32, []u8), frag_cache: std.array_hash_map.Auto(u32, []u8),
cache_mut: std.Io.Mutex = .init, cache_mut: std.Io.RwLock = .init,
pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, frag_start: u64, frag_num: u32, block_size: u32) !FragManager { pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, frag_start: u64, frag_num: u32, block_size: u32) !FragManager {
var buf: [8 * 1024]u8 = undefined; const first_offset: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[frag_start .. frag_start + 8]), .little);
var rdr = try fil.readerAt(io, frag_start, &buf);
var first_offset: u64 = undefined;
try rdr.interface.readSliceEndian(u64, @ptrCast(&first_offset), .little);
rdr = try fil.readerAt(io, first_offset, &buf); var rdr = fil.readerAt(first_offset);
var meta: MetadataReader = .init(alloc, &rdr.interface, decomp); var meta: MetadataReader = .init(alloc, &rdr, decomp);
const entries = try alloc.alloc(FragEntry, frag_num); const entries = try alloc.alloc(FragEntry, frag_num);
errdefer alloc.free(entries); errdefer alloc.free(entries);
@@ -59,24 +56,28 @@ pub fn deinit(self: *FragManager, io: Io) void {
} }
pub fn get(self: *FragManager, io: Io, idx: u32) ![]u8 { pub fn get(self: *FragManager, io: Io, idx: u32) ![]u8 {
if (self.frag_cache.contains(idx)) {
return self.frag_cache.get(idx).?; try self.cache_mut.lockShared(io);
defer self.cache_mut.unlockShared(io);
if (self.frag_cache.contains(idx))
return self.frag_cache.get(idx).?;
}
try self.cache_mut.lock(io); try self.cache_mut.lock(io);
defer self.cache_mut.unlock(io); defer self.cache_mut.unlock(io);
if (self.frag_cache.contains(idx))
return self.frag_cache.get(idx).?;
const entry = self.entries[idx]; const entry = self.entries[idx];
const out = try self.alloc.alloc(u8, if (entry.size.uncompressed) entry.size.size else self.block_size); const out = try self.alloc.alloc(u8, if (entry.size.uncompressed) entry.size.size else self.block_size);
var buf: [1024 * 1024]u8 = undefined;
var rdr = try self.fil.readerAt(io, entry.start, &buf);
if (entry.size.uncompressed) { if (entry.size.uncompressed) {
try rdr.interface.readSliceAll(out); @memcpy(out, self.fil.map.memory[entry.start .. entry.start + entry.size.size]);
} else { } else {
@branchHint(.likely); @branchHint(.likely);
try rdr.interface.fill(entry.size.size); _ = try self.decomp.Decompress(self.alloc, self.fil.map.memory[entry.start .. entry.start + entry.size.size], out);
_ = try self.decomp.Decompress(self.alloc, buf[0..entry.size.size], out);
} }
try self.frag_cache.put(self.alloc, idx, out); try self.frag_cache.put(self.alloc, idx, out);
+152 -231
View File
@@ -64,16 +64,16 @@ pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
// Utility Functions // Utility Functions
/// Read the directory entries /// Read the directory entries
pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64) ![]DirEntry { pub fn readDirectory(self: Inode, alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64) ![]DirEntry {
return switch (self.data) { return switch (self.data) {
.dir => |d| readDirFromData(alloc, io, fil, decomp, dir_offset, d), .dir => |d| readDirFromData(alloc, fil, decomp, dir_offset, d),
.ext_dir => |d| readDirFromData(alloc, io, fil, decomp, dir_offset, d), .ext_dir => |d| readDirFromData(alloc, fil, decomp, dir_offset, d),
else => Error.NotDirectory, else => Error.NotDirectory,
}; };
} }
fn readDirFromData(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64, d: anytype) ![]DirEntry { fn readDirFromData(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, dir_offset: u64, d: anytype) ![]DirEntry {
var rdr = try fil.readerAt(io, dir_offset + d.block_start, &[0]u8{}); var rdr = fil.readerAt(dir_offset + d.block_start);
var meta: MetadataReader = .init(alloc, &rdr.interface, decomp); var meta: MetadataReader = .init(alloc, &rdr, decomp);
try meta.interface.discardAll(d.block_offset); try meta.interface.discardAll(d.block_offset);
return DirEntry.readDirectory(alloc, &meta.interface, d.size); return DirEntry.readDirectory(alloc, &meta.interface, d.size);
@@ -137,9 +137,9 @@ pub fn xattrIndex(self: Inode) !u32 {
.ext_char_dev => |e| e.xattr_idx, .ext_char_dev => |e| e.xattr_idx,
.ext_fifo => |e| e.xattr_idx, .ext_fifo => |e| e.xattr_idx,
.ext_socket => |e| e.xattr_idx, .ext_socket => |e| e.xattr_idx,
else => Error.NoXattr, else => return error.NoXattr,
}; };
if (idx == 0xFFFFFFFF) return Error.NoXattr; if (idx == 0xFFFFFFFF) return error.NoXattr;
return idx; return idx;
} }
// Get an inode's xattr values. If the inode does not have xattr values (including if the inode is not an extended type), an empty slice is returned. // Get an inode's xattr values. If the inode does not have xattr values (including if the inode is not an extended type), an empty slice is returned.
@@ -208,16 +208,46 @@ pub const Header = extern struct {
// Extract // Extract
const ExtractError = error{ MknodFailed, CannotSetXattr, ConcurrencyUnavailable } || DataExtractor.Error || Io.Dir.CreateFileAtomicError || LookupTable.Error || const ExtractError = error{ MknodFailed, CannotSetXattr } || DataExtractor.Error || DirEntry.Error ||
Io.File.Reader.SeekError || Io.File.Atomic.LinkError || Io.Dir.CreateDirError || Io.File.OpenError || Decompressor.Error || Io.File.Atomic.InitError || Io.File.Atomic.LinkError || Io.Dir.SymLinkError;
Io.File.SetPermissionsError || Io.File.SetOwnerError || Io.Dir.SymLinkError || Io.Dir.CreateDirPathError;
const PathRet = struct { const PathRet = struct {
path: []const u8, path: []const u8,
inode: Inode, inode: Inode,
xattr_idx: ?u32 = null, xattr_idx: ?u32 = null,
};
fn DirCompare(_: void, a: PathRet, b: PathRet) std.math.Order{
fn setMetadata(path_ret: PathRet, alloc: std.mem.Allocator, io: Io, id_table: *CachedTable(u16), xattr_table: ?*XattrTable, options: ExtractionOptions) !void {
var fil = Io.Dir.cwd().openFile(io, path_ret.path, .{}) catch |err| {
std.debug.print("{s}: {}\n", .{ path_ret.path, err });
return err;
};
defer fil.close(io);
if (!options.ignore_permissions) {
try fil.setPermissions(io, @enumFromInt(path_ret.inode.hdr.permissions));
try fil.setOwner(io, try id_table.get(io, path_ret.inode.hdr.uid_idx), try id_table.get(io, path_ret.inode.hdr.gid_idx));
}
if (xattr_table != null) {
const idx = path_ret.inode.xattrIndex() catch return;
const xattrs = try xattr_table.?.get(alloc, io, idx);
defer {
for (xattrs) |x|
x.deinit(alloc);
alloc.free(xattrs);
}
const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path_ret.path}, 0);
defer alloc.free(sentinel_path);
for (xattrs) |x| {
const xattr_ret = std.os.linux.fsetxattr(fil.handle, x.key, x.value.ptr, x.value.len, 0);
if (xattr_ret != 0)
return ExtractError.CannotSetXattr;
}
}
}
};
fn DirCompare(_: void, a: PathRet, b: PathRet) std.math.Order {
return std.math.order(std.mem.count(u8, a.path, "/"), std.mem.count(u8, b.path, "/"));
} }
const ExtractReturnUnion = union(enum) { const ExtractReturnUnion = union(enum) {
path_ret: ExtractError!PathRet, path_ret: ExtractError!PathRet,
@@ -241,263 +271,154 @@ pub fn extract(
) !void { ) !void {
const path = std.mem.trimEnd(u8, filepath, "/"); const path = std.mem.trimEnd(u8, filepath, "/");
var decomp_base: Decomp = switch (super.compression) { const decomp = try @import("decomp.zig").StatelessDecomp(super.compression);
.gzip => .{ .gzip = try .init(alloc, super.block_size) },
.lzma => .{ .lzma = try .init(alloc, super.block_size) },
.xz => .{ .xz = try .init(alloc, super.block_size) },
.zstd => .{ .zstd = try .init(alloc, io, super.block_size) },
else => unreachable,
};
defer decomp_base.deinit();
const decomp = decomp_base.decompressor();
var frags: FragManager = try .init(alloc, io, fil, decomp, super.frag_start, super.frag_count, super.block_size); var frag_mgr: FragManager = try .init(alloc, fil, decomp, super.frag_start, super.frag_count, super.block_size);
defer frags.deinit(io); defer frag_mgr.deinit(io);
var sel_buf = [1]ExtractReturnUnion{undefined} ** 10; var arena: std.heap.ArenaAllocator = .init(alloc);
defer arena.deinit();
var sel_buf: [10]ExtractReturnUnion = undefined;
var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf); var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf);
defer sel.cancelDiscard(); defer sel.cancelDiscard();
var fold_queu: std.PriorityDequeue(PathRet, void, comptime compareFn: fn (Context, T, T) Order) sel.async(.path_ret, extractReal, .{ self, alloc, io, fil, super, decomp, &arena, &sel, &frag_mgr, path });
try sel.concurrent(.path_ret, extractReal, .{ self, alloc, io, fil, decomp, super, &frags, &sel, path }); var id_table: CachedTable(u16) = .init(alloc, fil, decomp, super.id_start, super.id_count);
defer id_table.deinit(io);
var xattr_table: ?XattrTable = if (!options.ignore_xattr) var xattr_table: ?XattrTable = if (super.flags.xattr_never or options.ignore_xattr or !@hasField(std.os, "linux"))
try .init(alloc, io, fil, decomp, super.xattr_start) null
else else
null; try .init(alloc, fil, decomp, super.xattr_start);
defer if (!options.ignore_xattr) xattr_table.?.deinit(io); defer if (xattr_table != null) xattr_table.?.deinit(io);
if (xattr_table != null) try xattr_table.?.table.fill(io);
var id_table: ?CachedTable(u16) = if (!options.ignore_xattr) var dir_queue: std.PriorityDequeue(PathRet, void, DirCompare) = .empty;
.init(alloc, fil, decomp, super.id_start, super.id_count) defer dir_queue.deinit(alloc);
else
null;
defer if (!options.ignore_xattr) id_table.?.deinit(io);
if (id_table != null) try id_table.?.fill(io);
while (true) { while (true) {
const group_token = sel.group.token.load(.acquire); if (sel.group.token.load(.unordered) == null) break;
if (group_token == null) break;
// std.debug.print("{any}\n", .{sel.group.state});
// std.debug.print("Waiting for return...", .{});
const ret = try sel.await(); const ret = try sel.await();
defer sel.queue.putOneUncancelable(io, ret) catch {};
// std.debug.print("Got One...\n", .{});
const path_ret = try ret.path_ret; const path_ret = try ret.path_ret;
if (options.ignore_permissions and options.ignore_xattr) continue; if (options.ignore_permissions and xattr_table == null) continue;
if (options.ignore_permissions and path_ret.xattr_idx == null) continue;
var ret_file = try Io.Dir.cwd().openFile(io, path_ret.path, .{}); if (path_ret.inode.hdr.inode_type == .dir or path_ret.inode.hdr.inode_type == .ext_dir) {
defer ret_file.close(io); try dir_queue.push(alloc, path_ret);
continue;
if (!options.ignore_permissions) {
try ret_file.setPermissions(io, @enumFromInt(path_ret.inode.hdr.permissions));
try ret_file.setOwner(
io,
try id_table.?.get(io, path_ret.inode.hdr.uid_idx),
try id_table.?.get(io, path_ret.inode.hdr.gid_idx),
);
} }
if (@hasField(std.os, "linux") and !options.ignore_xattr and path_ret.xattr_idx != null) {
const xattrs = try xattr_table.?.get(alloc, io, path_ret.xattr_idx.?);
defer {
for (xattrs) |x|
alloc.free(x.key);
alloc.free(xattrs);
}
for (xattrs) |x| { try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options);
const res = std.os.linux.fsetxattr(ret_file.handle, x.key, x.value.ptr, x.value.len, 0);
if (res != 0)
return error.CannotSetXattr;
}
}
} }
var iter = dir_queue.iterator();
while (iter.next()) |path_ret|
try path_ret.setMetadata(alloc, io, &id_table, if (xattr_table == null) null else &xattr_table.?, options);
} }
pub fn extractReal( pub fn extractReal(
self: Inode, self: Inode,
alloc: std.mem.Allocator, alloc: std.mem.Allocator,
io: Io, io: Io,
fil: OffsetFile, fil: OffsetFile,
decomp: *const Decompressor,
super: Archive.Superblock, super: Archive.Superblock,
frags: *FragManager, decomp: *const Decompressor,
inode_arena: *std.heap.ArenaAllocator,
sel: *Io.Select(ExtractReturnUnion), sel: *Io.Select(ExtractReturnUnion),
frag_mgr: *FragManager,
path: []const u8, path: []const u8,
) ExtractError!PathRet { ) ExtractError!PathRet {
return switch (self.hdr.inode_type) { switch (self.hdr.inode_type) {
.dir, .ext_dir => extractDir(self, alloc, io, fil, decomp, super, sel, frags, path), .dir, .ext_dir => {
.file, .ext_file => extractFile(self, alloc, io, fil, decomp, frags, super.block_size, path), try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777));
.symlink, .ext_symlink => extractSymlink(self, io, path),
else => extractDevOrIPC(self, alloc, path),
};
}
fn extractDir(
self: Inode,
alloc: std.mem.Allocator,
io: Io,
fil: OffsetFile,
decomp: *const Decompressor,
super: Archive.Superblock,
parent_select: *Io.Select(ExtractReturnUnion),
frags: *FragManager,
path: []const u8,
) ExtractError!PathRet {
try Io.Dir.cwd().createDirPath(io, path);
var sel_buf = [1]ExtractReturnUnion{undefined} ** 10; const entries = self.readDirectory(alloc, fil, decomp, super.dir_start) catch |err| switch (err) {
var sel: Io.Select(ExtractReturnUnion) = .init(io, &sel_buf); Error.NotDirectory, Error.NotExtended, Error.NotRegularFile, Error.NotSymlink => unreachable,
defer sel.cancelDiscard(); else => |e| return e,
};
defer {
for (entries) |e|
e.deinit(alloc);
alloc.free(entries);
}
var num: usize = 0; const inode_alloc = inode_arena.allocator();
{
const dir_entries = self.readDirectory(alloc, io, fil, decomp, super.dir_start) catch |err| switch (err) {
Error.NotDirectory => unreachable,
else => return @errorCast(err),
};
num = dir_entries.len;
defer {
for (dir_entries) |d|
d.deinit(alloc);
alloc.free(dir_entries);
}
for (dir_entries) |d| { for (entries) |e| {
var rdr = try fil.readerAt(io, d.block_start + super.inode_start, &[0]u8{}); const new_path = try std.mem.concat(inode_alloc, u8, &[_][]const u8{ path, "/", e.name });
var meta_rdr: MetadataReader = .init(alloc, &rdr.interface, decomp);
try meta_rdr.interface.discardAll(d.block_offset);
const inode = try read(alloc, &meta_rdr.interface, super.block_size);
errdefer inode.deinit(alloc);
const new_path = try std.mem.concat(alloc, u8, &[_][]const u8{ path, "/", d.name }); var rdr = fil.readerAt(super.inode_start + e.block_start);
errdefer alloc.free(new_path); var meta: MetadataReader = .init(alloc, &rdr, decomp);
try meta.interface.discardAll(e.block_offset);
try sel.concurrent(.path_ret, extractReal, .{ self, alloc, io, fil, decomp, super, frags, &sel, new_path }); const new_inode = try read(inode_alloc, &meta.interface, super.block_size);
}
}
while (num > 0) { sel.async(.path_ret, extractReal, .{ new_inode, alloc, io, fil, super, decomp, inode_arena, sel, frag_mgr, new_path });
const ret = sel.await() catch break; }
num -= 1; },
.file, .ext_file => {
var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{ .make_path = true });
defer atomic.deinit(io);
parent_select.queue.putOne(io, ret) catch |err| switch (err) { var ext: DataExtractor = switch (self.data) {
error.Canceled => return error.Canceled, .file => |f| blk: {
else => break, var ext: DataExtractor = .init(fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
}; if (f.frag_idx != 0xFFFFFFFF)
ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
break :blk ext;
},
.ext_file => |f| blk: {
var ext: DataExtractor = .init(fil, decomp, super.block_size, f.size, f.block_start, f.block_sizes);
if (f.frag_idx != 0xFFFFFFFF)
ext.addFrag(f.frag_block_offset, try frag_mgr.get(io, f.frag_idx));
break :blk ext;
},
else => unreachable,
};
try ext.extractAsync(alloc, io, atomic.file);
try atomic.link(io);
},
.symlink, .ext_symlink => try Io.Dir.cwd().symLink(io, self.symlinkTarget() catch unreachable, path, .{}),
else => {
var mode: u32 = undefined;
var dev: u32 = 0;
const DT = std.posix.DT;
switch (self.data) {
.char_dev => |d| {
dev = d.dev;
mode = DT.CHR;
},
.ext_char_dev => |d| {
dev = d.dev;
mode = DT.CHR;
},
.block_dev => |d| {
dev = d.dev;
mode = DT.BLK;
},
.ext_block_dev => |d| {
dev = d.dev;
mode = DT.BLK;
},
.fifo, .ext_fifo => mode = DT.FIFO,
.socket, .ext_socket => mode = DT.SOCK,
else => unreachable,
}
const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
const res = std.os.linux.mknod(sentinel_path, mode, dev);
alloc.free(sentinel_path);
if (res != 0)
return ExtractError.MknodFailed;
},
} }
return .{ return .{
.path = path, .path = path,
.inode = self, .inode = self,
}; };
} }
fn extractFile(
self: Inode,
alloc: std.mem.Allocator,
io: Io,
fil: OffsetFile,
decomp: *const Decompressor,
frag: *FragManager,
block_size: u32,
path: []const u8,
) ExtractError!PathRet {
var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{});
defer atomic.deinit(io);
var ret: PathRet = .{
.inode = self,
.path = path,
};
const data: DataExtractor = blk: {
switch (self.data) {
.file => |f| {
var data: DataExtractor = .init(fil, decomp, block_size, f.size, f.block_start, f.block_sizes);
if (f.frag_idx != 0xFFFFFFFF)
data.addFrag(f.frag_block_offset, try frag.get(io, f.frag_idx));
break :blk data;
},
.ext_file => |f| {
if (f.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = f.xattr_idx;
var data: DataExtractor = .init(fil, decomp, block_size, f.size, f.block_start, f.block_sizes);
if (f.frag_idx != 0xFFFFFFFF)
data.addFrag(f.frag_block_offset, try frag.get(io, f.frag_idx));
break :blk data;
},
else => unreachable,
}
};
try data.extractAsync(alloc, io, atomic.file);
try atomic.link(io);
return ret;
}
fn extractSymlink(self: Inode, io: Io, path: []const u8) ExtractError!PathRet {
const target = switch (self.data) {
.symlink => |s| s.target,
.ext_symlink => |s| s.target,
else => unreachable,
};
try Io.Dir.cwd().symLink(io, target, path, .{});
return .{
.path = path,
.inode = self,
};
}
fn extractDevOrIPC(self: Inode, alloc: std.mem.Allocator, path: []const u8) ExtractError!PathRet {
var dev_num: u32 = 0;
var mode: u32 = 0;
const DT = std.posix.DT;
var ret: PathRet = .{
.inode = self,
.path = path,
};
switch (self.data) {
.block_dev => |d| {
dev_num = d.dev;
mode = DT.BLK;
},
.ext_block_dev => |d| {
dev_num = d.dev;
mode = DT.BLK;
if (d.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = d.xattr_idx;
},
.char_dev => |d| {
dev_num = d.dev;
mode = DT.CHR;
},
.ext_char_dev => |d| {
dev_num = d.dev;
mode = DT.CHR;
if (d.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = d.xattr_idx;
},
.fifo => mode = DT.FIFO,
.ext_fifo => |f| {
mode = DT.FIFO;
if (f.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = f.xattr_idx;
},
.socket => mode = DT.SOCK,
.ext_socket => |s| {
mode = DT.SOCK;
if (s.xattr_idx != 0xFFFFFFFF) ret.xattr_idx = s.xattr_idx;
},
else => unreachable,
}
const sentinel_path = try std.mem.concatWithSentinel(alloc, u8, &[_][]const u8{path}, 0);
defer alloc.free(sentinel_path);
const res = std.os.linux.mknod(sentinel_path, mode, dev_num);
if (res != 0) return ExtractError.MknodFailed;
return ret;
}
+22 -19
View File
@@ -5,18 +5,17 @@ const Decompressor = @import("util/decompressor.zig");
const MetadataReader = @import("util/metadata.zig"); const MetadataReader = @import("util/metadata.zig");
const OffsetFile = @import("util/offset_file.zig"); const OffsetFile = @import("util/offset_file.zig");
pub fn lookupValue(comptime T: anytype, alloc: std.mem.Allocator, io: Io, decomp: *const Decompressor, file: OffsetFile, table_start: u64, idx: u32) !T { pub fn lookupValue(comptime T: anytype, alloc: std.mem.Allocator, decomp: *const Decompressor, file: OffsetFile, table_start: u64, idx: u32) !T {
const T_PER_BLOCK: u16 = 8192 / @sizeOf(T); const T_PER_BLOCK: u16 = 8192 / @sizeOf(T);
const block = idx / T_PER_BLOCK; const block = idx / T_PER_BLOCK;
const block_offset = idx % T_PER_BLOCK; const block_offset = idx % T_PER_BLOCK;
var rdr = try file.readerAt(io, table_start + (8 * block), &[0]u8{}); const offset_pos = table_start + (8 * block);
var offset: u64 = undefined; const offset: u64 = std.mem.readInt(u64, @ptrCast(file.map.memory[offset_pos .. offset_pos + 8]), .little);
try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
rdr = try file.readerAt(io, offset, &[0]u8{}); var rdr = file.readerAt(offset);
var meta: MetadataReader = .init(alloc, &rdr.interface, decomp); var meta: MetadataReader = .init(alloc, &rdr, decomp);
try meta.interface.discardAll(@sizeOf(T) * block_offset); try meta.interface.discardAll(@sizeOf(T) * block_offset);
var out: T = undefined; var out: T = undefined;
@@ -41,7 +40,7 @@ pub fn CachedTable(comptime T: anytype) type {
table: std.AutoHashMap(u32, []T), table: std.AutoHashMap(u32, []T),
mut: Io.Mutex = .init, mut: Io.RwLock = .init,
pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, offset: u64, total_num: u32) Table { pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, offset: u64, total_num: u32) Table {
return .{ return .{
@@ -72,16 +71,15 @@ pub fn CachedTable(comptime T: anytype) type {
num_blocks += 1; num_blocks += 1;
for (0..num_blocks) |block| { for (0..num_blocks) |block| {
var rdr = try self.fil.readerAt(io, self.table_start + (8 * block), &[0]u8{}); const offset_pos = self.table_start + (8 * block);
var offset: u64 = undefined; const offset: u64 = std.mem.readInt(u64, @ptrCast(self.fil.map.memory[offset_pos .. offset_pos + 8]), .little);
try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK) const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK)
@truncate(self.total_num % T_PER_BLOCK) @truncate(self.total_num % T_PER_BLOCK)
else else
T_PER_BLOCK; T_PER_BLOCK;
rdr = try self.fil.readerAt(io, offset, &[0]u8{}); var rdr = self.fil.readerAt(offset);
var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp); var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp);
const slice = try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little); const slice = try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little);
@@ -92,8 +90,14 @@ pub fn CachedTable(comptime T: anytype) type {
pub fn get(self: *Table, io: Io, idx: u32) Error!T { pub fn get(self: *Table, io: Io, idx: u32) Error!T {
const block = idx / T_PER_BLOCK; const block = idx / T_PER_BLOCK;
const block_offset = idx % T_PER_BLOCK; const block_offset = idx % T_PER_BLOCK;
if (self.table.contains(block))
return self.table.get(block).?[block_offset]; {
try self.mut.lockShared(io);
defer self.mut.unlockShared(io);
if (self.table.contains(block))
return self.table.get(block).?[block_offset];
}
try self.mut.lock(io); try self.mut.lock(io);
defer self.mut.unlock(io); defer self.mut.unlock(io);
@@ -101,20 +105,19 @@ pub fn CachedTable(comptime T: anytype) type {
if (self.table.contains(block)) if (self.table.contains(block))
return self.table.get(block).?[block_offset]; return self.table.get(block).?[block_offset];
var rdr = try self.fil.readerAt(io, self.table_start + (8 * block), &[0]u8{}); const offset_pos = self.table_start + (8 * block);
var offset: u64 = undefined; const offset: u64 = std.mem.readInt(u64, @ptrCast(self.fil.map.memory[offset_pos .. offset_pos + 8]), .little);
try rdr.interface.readSliceEndian(u64, @ptrCast(&offset), .little);
const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK) const len: u16 = if (self.total_num % T_PER_BLOCK != 0 and block == (self.total_num - 1) / T_PER_BLOCK)
@truncate(self.total_num % T_PER_BLOCK) @truncate(self.total_num % T_PER_BLOCK)
else else
T_PER_BLOCK; T_PER_BLOCK;
rdr = try self.fil.readerAt(io, offset, &[0]u8{}); var rdr = self.fil.readerAt(offset);
var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp); var meta: MetadataReader = .init(self.alloc, &rdr, self.decomp);
const slice = try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little); const slice = try meta.interface.readSliceEndianAlloc(self.alloc, T, len, .little);
try self.table.put(block, slice); try self.table.put(@truncate(block), slice);
return slice[block_offset]; return slice[block_offset];
} }
+2 -2
View File
@@ -5,8 +5,8 @@ const Writer = std.Io.Writer;
const ExtractionOptions = @This(); const ExtractionOptions = @This();
/// The number of threads used for extraction. 0 implies single threaded. // /// The number of threads used for extraction. 0 implies single threaded.
threads: usize = 1, // TODO: Update to better integrate with zig 0.16 Io. Maybe limit to only single or multi-threaded. // threads: usize = 1, // As of Zig 0.16 this should no longer be necessary, instead this should be set by the io instance used.
/// Don't set the file's owner & permissions after extraction /// Don't set the file's owner & permissions after extraction
ignore_permissions: bool = false, ignore_permissions: bool = false,
/// Don't set xattr values. Currently xattrs are never set anyway. /// Don't set xattr values. Currently xattrs are never set anyway.
+33 -39
View File
@@ -10,7 +10,7 @@ const OffsetFile = @import("offset_file.zig");
// const SharedCache = @import("shared_cache.zig"); // const SharedCache = @import("shared_cache.zig");
pub const Error = error{OutOfMemory} || Io.File.Reader.SeekError || Io.Writer.Error || Io.File.Writer.Error; pub const Error = Decompressor.Error || Io.File.MemoryMap.CreateError || Io.File.WritePositionalError;
const DataExtractor = @This(); const DataExtractor = @This();
@@ -74,13 +74,14 @@ fn blockThread(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.Fi
else else
self.block_size; self.block_size;
const write_offset = self.block_size * idx;
var wrt = fil.writer(io, &[0]u8{}); var wrt = fil.writer(io, &[0]u8{});
wrt.seekTo(self.block_size * idx) catch |err| { wrt.seekTo(write_offset) catch |err| {
ret_err.* = err; ret_err.* = err;
if (err == error.Canceled) io.recancel(); if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled; return Io.Cancelable.Canceled;
}; };
defer wrt.flush() catch {};
if (block.size == 0) { if (block.size == 0) {
wrt.interface.splatByteAll(0, cur_block_size) catch |err| { wrt.interface.splatByteAll(0, cur_block_size) catch |err| {
@@ -88,56 +89,49 @@ fn blockThread(self: DataExtractor, alloc: std.mem.Allocator, io: Io, fil: Io.Fi
if (err == error.Canceled) io.recancel(); if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled; return Io.Cancelable.Canceled;
}; };
return; } else {
} if (block.uncompressed) {
wrt.interface.writeAll(self.fil.map.memory[read_offset..][0..cur_block_size]) catch |err| {
ret_err.* = err;
if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled;
};
} else {
@branchHint(.likely);
var rdr = self.fil.readerAt(io, read_offset, &[0]u8{}) catch |err| { var tmp: [1024 * 1024]u8 = undefined;
_ = self.decomp.Decompress(alloc, self.fil.map.memory[read_offset..][0..block.size], tmp[0..cur_block_size]) catch |err| {
ret_err.* = err;
return Io.Cancelable.Canceled;
};
wrt.interface.writeAll(tmp[0..cur_block_size]) catch |err| {
ret_err.* = err;
if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled;
};
}
}
wrt.flush() catch |err| {
ret_err.* = err; ret_err.* = err;
if (err == error.Canceled) io.recancel(); if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled; return Io.Cancelable.Canceled;
}; };
if (block.uncompressed) {
rdr.interface.streamExact(&wrt.interface, cur_block_size) catch |err| {
ret_err.* = err;
if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled;
};
return;
} else {
@branchHint(.likely);
var cache: [1024 * 1024]u8 = undefined;
var tmp: [1024 * 1024]u8 = undefined;
rdr.interface.readSliceAll(cache[0..block.size]) catch |err| {
ret_err.* = err;
if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled;
};
_ = self.decomp.Decompress(alloc, cache[0..block.size], tmp[0..cur_block_size]) catch |err| {
ret_err.* = err;
if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled;
};
wrt.interface.writeAll(tmp[0..cur_block_size]) catch |err| {
ret_err.* = err;
if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled;
};
}
} }
fn fragThread(self: DataExtractor, io: Io, fil: Io.File, ret_err: *?Error) Io.Cancelable!void { fn fragThread(self: DataExtractor, io: Io, fil: Io.File, ret_err: *?Error) Io.Cancelable!void {
const cur_block_size = self.file_size % self.block_size; const cur_block_size = self.file_size % self.block_size;
var write_buf: [10 * 1024]u8 = undefined; const write_offset = self.blocks.len * self.block_size;
var wrt = fil.writer(io, &write_buf);
wrt.seekTo(self.blocks.len * self.block_size) catch |err| { var wrt = fil.writer(io, &[0]u8{});
wrt.seekTo(write_offset) catch |err| {
ret_err.* = err; ret_err.* = err;
if (err == error.Canceled) io.recancel(); if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled; return Io.Cancelable.Canceled;
}; };
wrt.interface.writeAll(self.frag_block.?[self.frag_offset .. self.frag_offset + cur_block_size]) catch |err| { wrt.interface.writeAll(self.frag_block.?[self.frag_offset..][0..cur_block_size]) catch |err| {
ret_err.* = err; ret_err.* = err;
if (err == error.Canceled) io.recancel(); if (err == error.Canceled) io.recancel();
return Io.Cancelable.Canceled; return Io.Cancelable.Canceled;
+6 -12
View File
@@ -19,7 +19,6 @@ alloc: std.mem.Allocator,
fil: OffsetFile, fil: OffsetFile,
io: Io, io: Io,
decomp: *const Decompressor, decomp: *const Decompressor,
cache: *Io.Queue([]u8),
block_size: u32, block_size: u32,
file_size: u64, file_size: u64,
@@ -34,14 +33,13 @@ sparse_block: bool = false,
interface: Io.Reader, interface: Io.Reader,
pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, cache: *Io.Queue([]u8), block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) !DataReader { pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, block_size: u32, file_size: u64, data_start: u64, blocks: []BlockSize) !DataReader {
return .{ return .{
.alloc = alloc, .alloc = alloc,
.fil = fil, .fil = fil,
.io = io, .io = io,
.decomp = decomp, .decomp = decomp,
.cache = cache,
.block_size = block_size, .block_size = block_size,
.file_size = file_size, .file_size = file_size,
@@ -74,9 +72,10 @@ fn numBlocks(self: DataReader) usize {
return num; return num;
} }
fn advanceBuffer(self: *DataReader) !void { fn advanceBuffer(self: *DataReader) !void {
if (self.block_idx >= self.numBlocks()) { if (self.block_idx >= self.numBlocks())
return Reader.Error.EndOfStream; return Reader.Error.EndOfStream;
}
errdefer self.interface.end = 0;
defer self.block_idx += 1; defer self.block_idx += 1;
self.interface.end = if (self.block_idx == self.numBlocks() - 1) self.interface.end = if (self.block_idx == self.numBlocks() - 1)
@@ -101,18 +100,13 @@ fn advanceBuffer(self: *DataReader) !void {
self.sparse_block = false; self.sparse_block = false;
} }
if (block.uncompressed) { if (block.uncompressed) {
try self.fil.readAt(self.io, self.cur_offset, self.interface.buffer[0..self.interface.end]); @memcpy(self.interface.buffer[0..self.interface.end], self.fil.map.memory[self.cur_offset .. self.cur_offset + self.interface.end]);
self.cur_offset += self.interface.end; self.cur_offset += self.interface.end;
} else { } else {
@branchHint(.likely); @branchHint(.likely);
const tmp = try self.cache.getOne(self.io);
defer self.cache.putOne(self.io, tmp) catch {};
var rdr_buf: [50 * 1024]u8 = undefined; _ = try self.decomp.Decompress(self.alloc, self.fil.map.memory[self.cur_offset .. self.cur_offset + block.size], self.interface.buffer[0..self.interface.end]);
var rdr = try self.fil.readerAt(self.io, self.cur_offset, &rdr_buf);
try rdr.interface.readSliceAll(tmp[0..block.size]);
self.cur_offset += block.size; self.cur_offset += block.size;
_ = try self.decomp.Decompress(self.alloc, tmp[0..block.size], self.interface.buffer[0..self.interface.end]);
} }
self.interface.seek = 0; self.interface.seek = 0;
} }
+3 -3
View File
@@ -16,9 +16,9 @@ pub fn pathIsSelf(path: []const u8) bool {
return path[0] == '.'; return path[0] == '.';
} }
/// Creates an Inode from an Inode.Ref. /// Creates an Inode from an Inode.Ref.
pub fn inodeFromRef(alloc: std.mem.Allocator, io: Io, file: OffsetFile, decomp: *const Decompressor, inode_start: u64, block_size: u32, ref: Inode.Ref) !Inode { pub fn inodeFromRef(alloc: std.mem.Allocator, file: OffsetFile, decomp: *const Decompressor, inode_start: u64, block_size: u32, ref: Inode.Ref) !Inode {
var rdr = try file.readerAt(io, inode_start + ref.block_start, &[0]u8{}); var rdr = file.readerAt(inode_start + ref.block_start);
var meta: MetadataReader = .init(alloc, &rdr.interface, decomp); var meta: MetadataReader = .init(alloc, &rdr, decomp);
try meta.interface.discardAll(ref.block_offset); try meta.interface.discardAll(ref.block_offset);
return .read(alloc, &meta.interface, block_size); return .read(alloc, &meta.interface, block_size);
+15 -12
View File
@@ -3,22 +3,25 @@
const std = @import("std"); const std = @import("std");
const Io = std.Io; const Io = std.Io;
const File = Io.File; const File = Io.File;
const Reader = File.Reader; const Reader = Io.Reader;
const OffsetFile = @This(); const OffsetFile = @This();
fil: File, map: Io.File.MemoryMap,
offset: u64,
pub fn init(fil: File, init_offset: u64) OffsetFile { pub fn init(io: Io, fil: File, archive_size: u64, init_offset: u64) !OffsetFile {
return .{ .fil = fil, .offset = init_offset }; return .{
.map = try fil.createMemoryMap(io, .{
.protection = .{ .read = true, .write = false, .execute = false },
.len = archive_size,
.offset = init_offset,
}),
};
}
pub fn deinit(self: @This(), io: Io) void {
self.map.destroy(io);
} }
pub fn readerAt(self: OffsetFile, io: Io, offset: u64, buffer: []u8) Reader.SeekError!Reader { pub fn readerAt(self: OffsetFile, offset: u64) Reader {
var rdr = self.fil.reader(io, buffer); return .fixed(self.map.memory[offset..]);
try rdr.seekTo(self.offset + offset);
return rdr;
}
pub fn readAt(self: OffsetFile, io: Io, offset: u64, buf: []u8) File.ReadPositionalError!void {
_ = try self.fil.readPositionalAll(io, buf, self.offset + offset);
} }
+24 -18
View File
@@ -18,15 +18,11 @@ kv_start: u64,
table: LookupTable.CachedTable(TableValue), table: LookupTable.CachedTable(TableValue),
value_cache: std.AutoHashMap(InodeRef, []const u8), value_cache: std.AutoHashMap(InodeRef, []const u8),
value_mut: Io.Mutex = .init, value_mut: Io.RwLock = .init,
pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const Decompressor, xattr_start: u64) !XattrCachedTable { pub fn init(alloc: std.mem.Allocator, fil: OffsetFile, decomp: *const Decompressor, xattr_start: u64) !XattrCachedTable {
var rdr = try fil.readerAt(io, xattr_start, &[0]u8{}); const start: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[xattr_start .. xattr_start + 8]), .little);
const num: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[xattr_start + 8 .. xattr_start + 16]), .little);
var start: u64 = undefined;
try rdr.interface.readSliceEndian(u64, @ptrCast(&start), .little);
var num: u32 = undefined;
try rdr.interface.readSliceEndian(u32, @ptrCast(&num), .little);
return .{ return .{
.alloc = alloc, .alloc = alloc,
@@ -41,6 +37,7 @@ pub fn init(alloc: std.mem.Allocator, io: Io, fil: OffsetFile, decomp: *const De
}; };
} }
pub fn deinit(self: *XattrCachedTable, io: Io) void { pub fn deinit(self: *XattrCachedTable, io: Io) void {
self.value_mut.lockUncancelable(io);
self.table.deinit(io); self.table.deinit(io);
self.value_cache.deinit(); self.value_cache.deinit();
} }
@@ -48,8 +45,8 @@ pub fn deinit(self: *XattrCachedTable, io: Io) void {
pub fn get(self: *XattrCachedTable, alloc: std.mem.Allocator, io: Io, idx: u32) ![]XattrSemiOwned { pub fn get(self: *XattrCachedTable, alloc: std.mem.Allocator, io: Io, idx: u32) ![]XattrSemiOwned {
const lookup = try self.table.get(io, idx); const lookup = try self.table.get(io, idx);
var rdr = try self.fil.readerAt(io, self.kv_start + lookup.ref.block_start, &[0]u8{}); var rdr = self.fil.readerAt(self.kv_start + lookup.ref.block_start);
var meta: MetadataReader = .init(alloc, &rdr.interface, self.decomp); var meta: MetadataReader = .init(alloc, &rdr, self.decomp);
try meta.interface.discardAll(lookup.ref.block_offset); try meta.interface.discardAll(lookup.ref.block_offset);
const out = try alloc.alloc(XattrSemiOwned, lookup.count); const out = try alloc.alloc(XattrSemiOwned, lookup.count);
@@ -99,8 +96,20 @@ pub fn get(self: *XattrCachedTable, alloc: std.mem.Allocator, io: Io, idx: u32)
} }
const val_ref: InodeRef = .{ .block_start = meta.cur_block_start, .block_offset = @truncate(meta.interface.seek) }; const val_ref: InodeRef = .{ .block_start = meta.cur_block_start, .block_offset = @truncate(meta.interface.seek) };
{
try self.value_mut.lockShared(io);
defer self.value_mut.unlockShared(io);
if (self.value_cache.contains(val_ref)) {
out[i] = .{
.key = key,
.value = try self.valueAt(io, val_ref),
};
continue;
}
}
try self.value_mut.lock(io); try self.value_mut.lock(io);
defer self.value_mut.unlock(io); defer self.value_mut.unlock(io);
if (self.value_cache.contains(val_ref)) { if (self.value_cache.contains(val_ref)) {
out[i] = .{ out[i] = .{
.key = key, .key = key,
@@ -131,8 +140,8 @@ fn valueAt(self: *XattrCachedTable, io: Io, ref: InodeRef) ![]const u8 {
if (self.value_cache.contains(ref)) return self.value_cache.get(ref).?; if (self.value_cache.contains(ref)) return self.value_cache.get(ref).?;
var rdr = try self.fil.readerAt(io, self.kv_start + ref.block_start, &[0]u8{}); var rdr = self.fil.readerAt(self.kv_start + ref.block_start);
var meta: MetadataReader = .init(self.alloc, &rdr.interface, self.decomp); var meta: MetadataReader = .init(self.alloc, &rdr, self.decomp);
try meta.interface.discardAll(ref.block_offset); try meta.interface.discardAll(ref.block_offset);
var val_size: u32 = undefined; var val_size: u32 = undefined;
@@ -204,14 +213,11 @@ const XattrPrefix = packed struct(u16) {
// Stateless // Stateless
pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *const Decompressor, fil: OffsetFile, table_start: u64, idx: u16) ![]XattrOwned { pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *const Decompressor, fil: OffsetFile, table_start: u64, idx: u16) ![]XattrOwned {
var rdr = try fil.readerAt(io, table_start, &[0]u8{}); const kv_start: u64 = std.mem.readInt(u64, @ptrCast(fil.map.memory[table_start .. table_start + 8]), .little);
var kv_start: u64 = undefined;
try rdr.interface.readSliceEndian(u64, @ptrCast(&kv_start), .little);
const lookup = try LookupTable.lookupValue(TableValue, alloc, io, decomp, fil, table_start + 16, idx); const lookup = try LookupTable.lookupValue(TableValue, alloc, io, decomp, fil, table_start + 16, idx);
rdr = try fil.readerAt(io, kv_start + lookup.ref.block_start, &[0]u8{}); var rdr = fil.readerAt(kv_start + lookup.ref.block_start);
var meta: MetadataReader = .init(alloc, &rdr.interface, decomp); var meta: MetadataReader = .init(alloc, &rdr.interface, decomp);
try meta.interface.discardAll(lookup.ref.block_offset); try meta.interface.discardAll(lookup.ref.block_offset);
@@ -252,7 +258,7 @@ pub fn statelessLookup(alloc: std.mem.Allocator, io: Io, decomp: *const Decompre
const value: ValueOutOfLineEntry = undefined; const value: ValueOutOfLineEntry = undefined;
try meta.interface.readSliceEndian(ValueOutOfLineEntry, @ptrCast(&value), .little); try meta.interface.readSliceEndian(ValueOutOfLineEntry, @ptrCast(&value), .little);
var ool_rdr = try fil.readerAt(io, kv_start + value.ref.block_start, &[0]u8{}); var ool_rdr = fil.readerAt(kv_start + value.ref.block_start);
var ool_meta: MetadataReader = .init(alloc, &ool_rdr.interface, decomp); var ool_meta: MetadataReader = .init(alloc, &ool_rdr.interface, decomp);
try ool_meta.interface.discardAll(value.ref.block_offset); try ool_meta.interface.discardAll(value.ref.block_offset);