Further work on making everything work

Directory table reading
Finished MetadataReader
This commit is contained in:
Caleb J. Gardner
2026-08-29 06:40:34 -05:00
parent e0b697dd0b
commit c5d75d3839
8 changed files with 545 additions and 63 deletions
+46 -11
View File
@@ -1,9 +1,10 @@
const std = @import("std");
const Io = std.Io;
const util = @import("utils/util.zig");
const File = @import("file.zig");
const Inode = @import("inode.zig");
const Util = @import("utils/util.zig");
const Decomp = @import("decomp.zig");
const Options = @import("options.zig");
@@ -13,6 +14,7 @@ const Archive = @This();
map: Io.File.MemoryMap,
super: Super,
root_inode_ref: Inode.Reference,
pub fn open(io: Io, file: Io.File, offset: u64) !Archive {
var map = try file.createMemoryMap(io, .{
@@ -21,22 +23,41 @@ pub fn open(io: Io, file: Io.File, offset: u64) !Archive {
.protection = .{ .read = true },
});
var superblock = Util.readValue(Superblock, map.memory[0..@sizeOf(Superblock)]);
const super = try superblock.check();
var superblock = util.readValue(Superblock, map.memory[0..@sizeOf(Superblock)]);
const super = try superblock.checkAndMinimize();
return .{
.map = map,
.super = super,
.root_inode_ref = superblock.root_inode_ref,
};
}
pub fn close(self: *Archive, io: Io) void {
self.map.destroy(io);
}
pub fn root(self: *Archive) !File {
_ = self;
return error.TODO;
pub fn root(self: *Archive, alloc: std.mem.Allocator) !File {
return .{
.alloc = alloc,
.data = self.map.memory,
.super = self.super,
.inode = try .fromRef(
alloc,
self.map.memory,
self.root_inode_ref,
self.super,
),
.name = "",
};
}
pub fn openFile(self: *Archive, alloc: std.mem.Allocator, filepath: []const u8) !File {
var root_file = try self.root(alloc);
defer root_file.deinit();
return root.open(alloc, filepath);
}
pub fn extract(self: *Archive, alloc: std.mem.Allocator, io: Io, ext_loc: []const u8, options: Options) !void {
_ = self;
@@ -59,11 +80,25 @@ const Superblock = extern struct {
frag_count: u32,
compression: Decomp.Enum,
block_log: u16,
flags: u16, // TODO: break out into packed struct.
flags: packed struct(u16) {
inode_uncompressed: bool,
data_uncompressed: bool,
check: bool,
fragment_uncompressed: bool,
fragment_never: bool,
fragment_always: bool,
de_duplicate: bool,
exportable: bool,
xattr_uncompressed: bool,
xattr_never: bool,
compression_options: bool,
id_uncompressed: bool,
_: u4,
},
id_count: u16,
version_major: u16,
version_minor: u16,
root_inode_ref: Inode.Reference, // TODO: Change to inode reference packed struct.
root_inode_ref: Inode.Reference,
size: u64,
id_table_start: u64,
xattr_table_start: u64,
@@ -72,20 +107,21 @@ const Superblock = extern struct {
frag_table_start: u64,
export_table_start: u64,
fn check(self: Superblock) !Super {
fn checkAndMinimize(self: Superblock) !Super {
if (self.magic != MAGIC)
return error.InvalidMagic;
if (self.version_major != 4 or self.version_minor != 0)
return error.IncompatibleVersion;
if (std.math.log2(self.block_size) != self.block_log)
return error.BadBlockLog;
if (self.flags.check)
return error.BadCheckFlag;
return .{
.block_size = self.block_size,
.frag_count = self.frag_count,
.decomp_fn = try self.compression.func(),
.id_count = self.id_count,
.root_inode_ref = self.root_inode_ref,
.id_table_start = self.id_table_start,
.xattr_table_start = self.xattr_table_start,
.inode_table_start = self.inode_table_start,
@@ -100,7 +136,6 @@ pub const Super = struct {
frag_count: u32,
decomp_fn: Decomp.Fn,
id_count: u16,
root_inode_ref: Inode.Reference,
id_table_start: u64,
xattr_table_start: u64,
inode_table_start: u64,
+79 -33
View File
@@ -16,7 +16,7 @@ pub const Enum = enum(u16) {
return switch (self) {
.gzip => if (build.zig_decomp) zigZlib else cZlib,
.lzma => if (build.zig_decomp) zigLzma else cLzma,
.lzo => if (build.zig_decomp) error.LzoUnsupported else cLzo,
.lzo => if (build.zig_decomp or !build.allow_lzo) error.LzoUnsupported else cLzo,
.xz => if (build.zig_decomp) zigXz else cXz,
.lz4 => if (build.zig_decomp) error.Lz4Unsupported else cLz4,
.zstd => if (build.zig_decomp) zigZstd else cZstd,
@@ -25,7 +25,7 @@ pub const Enum = enum(u16) {
};
pub const Fn = *const fn (alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize;
pub const Error = Io.Reader.Error;
pub const Error = Io.Reader.Error || std.mem.Allocator.Error;
// Actual decomp functions
@@ -45,56 +45,102 @@ fn cZlib(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
const res = c.inflate(&stream, c.Z_FULL_FLUSH);
if (res != c.Z_OK)
return Error.ReadFailed;
return 0;
return stream.total_out;
}
fn zigLzma(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var rdr: Io.Reader = .fixed(in);
var decomp: std.compress.lzma.Decompress = .initOptions(&rdr, alloc, out, .{}, 0);
var decomp: std.compress.lzma.Decompress = .initOptions(&rdr, alloc, &[0]u8{}, .{}, 0);
defer decomp.deinit();
return decomp.reader.readSliceShort(out);
}
fn cLzma(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
_ = alloc;
_ = in;
_ = out;
fn cLzma(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var stream: c.lzma_stream = .{
.avail_in = in.len,
.next_in = in.ptr,
.avail_out = out.len,
.next_out = out.ptr,
};
defer c.lzma_end(&stream);
var res = c.lzma_alone_decoder(&stream, 0);
if (res != c.LZMA_OK)
return Error.ReadFailed;
while (res == c.LZMA_OK)
res = c.lzma_code(&stream, c.LZMA_RUN);
if (res != c.LZMA_FINISH)
return Error.ReadFailed;
return stream.total_out;
}
fn cLzo(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
_ = alloc;
_ = in;
_ = out;
fn cLzo(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var res = c.lzo_init();
if (res != c.LZO_E_OK)
return Error.ReadFailed;
var len = out.len;
res = c.lzo1x_decompress(in.ptr, in.len, out.ptr, &len, null);
if (res != c.LZO_E_OK)
return Error.ReadFailed;
return Error.ReadFailed;
}
fn zigXz(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
_ = alloc;
_ = in;
_ = out;
return Error.ReadFailed;
var rdr: Io.Reader = .fixed(in);
var decomp: std.compress.xz.Decompress = .init(&rdr, alloc, &[0]u8{});
defer decomp.deinit();
return decomp.reader.readSliceShort(out);
}
fn cXz(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
_ = alloc;
_ = in;
_ = out;
fn cXz(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
var stream: c.lzma_stream = .{
.avail_in = in.len,
.next_in = in.ptr,
.avail_out = out.len,
.next_out = out.ptr,
};
defer c.lzma_end(&stream);
var res = c.lzma_stream_decoder(&stream, 0, 0);
if (res != c.LZMA_OK)
return Error.ReadFailed;
while (res == c.LZMA_OK)
res = c.lzma_code(&stream, c.LZMA_RUN);
if (res != c.LZMA_FINISH)
return Error.ReadFailed;
return stream.total_out;
}
fn cLz4(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
_ = alloc;
_ = in;
_ = out;
return Error.ReadFailed;
fn cLz4(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
const res = c.LZ4_decompress_safe(in.ptr, out.ptr, @intCast(in.len), @intCast(out.len));
if (res < 0) return Error.ReadFailed;
return @abs(res);
}
fn zigZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
_ = alloc;
_ = in;
_ = out;
return Error.ReadFailed;
var rdr: Io.Reader = .fixed(in);
const buf = try alloc.alloc(u8, 1024 * 1024);
defer alloc.free(buf);
var decomp: std.compress.zstd.Decompress = .init(&rdr, buf, .{
.window_len = 1024 * 1024,
});
return decomp.reader.readSliceShort(out);
}
fn cZstd(alloc: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
_ = alloc;
_ = in;
_ = out;
fn cZstd(_: std.mem.Allocator, in: []u8, out: []u8) Error!usize {
const res = c.ZSTD_decompress(out.ptr, out.len, in.ptr, in.len);
if (c.ZSTD_isError(res) == 1)
return Error.ReadFailed;
return res;
}
+74
View File
@@ -0,0 +1,74 @@
const std = @import("std");
const Io = std.Io;
const utils = @import("utils/util.zig");
const Inode = @import("inode.zig");
const Directory = @This();
entries: []Entry,
pub fn read(alloc: std.mem.Allocator, rdr: *Io.Reader, size: u32) !Directory {
var red: u32 = 3;
var out: std.ArrayList(Entry) = try .initCapacity(alloc, 10);
errdefer out.deinit(alloc);
while (red < size) {
const hdr: Header = try utils.readValueRdr(Header, rdr);
red += @sizeOf(Header);
try out.ensureUnusedCapacity(alloc, hdr.count + 1);
for (hdr.count + 1) |_| {
const raw: RawEntry = try utils.readValueRdr(RawEntry, rdr);
const name = try alloc.alloc(u8, raw.name_size + 1);
errdefer alloc.free(name);
try rdr.readSliceAll(name);
out.addOneAssumeCapacity().* = .{
.start = hdr.start,
.offset = raw.offset,
.type = raw.type,
.name = name,
};
red += @sizeOf(RawEntry + raw.name_size + 1);
}
}
return out.toOwnedSlice(alloc);
}
pub fn deinit(self: Directory, alloc: std.mem.Allocator) void {
for (self.entries) |entry|
entry.deinit(alloc);
alloc.free(self.entries);
}
// Types
pub const Entry = struct {
start: u32,
offset: u16,
type: Inode.Type,
name: []const u8,
pub fn deinit(self: Entry, alloc: std.mem.Allocator) void {
alloc.free(self.name);
}
};
const Header = extern struct {
count: u32,
start: u32,
inode_num: u32,
};
const RawEntry = extern struct {
offset: u16,
inode_num_offset: i16,
type: Inode.Type,
name_size: u16,
};
+61 -1
View File
@@ -2,15 +2,75 @@ const std = @import("std");
const Io = std.Io;
const Inode = @import("inode.zig");
const Directory = @import("dir.zig");
const MetadataReader = @import("utils/meta.zig");
const Super = @import("archive.zig").Super;
const File = @This();
alloc: std.mem.Allocator,
data: []u8,
super: Super,
name: []const u8,
inode: Inode,
pub fn close(self: File) void {
pub fn copy(self: File, alloc: std.mem.Allocator) !File {
const new_name = try alloc.dupe(u8, self.name);
errdefer alloc.free(new_name);
return .{
.alloc = alloc,
.data = self.data,
.super = self.super,
.name = new_name,
.inode = try self.inode.copy(alloc),
};
}
pub fn deinit(self: File) void {
self.inode.deinit(self.alloc);
self.alloc.free(self.name);
}
pub fn open(self: File, alloc: std.mem.Allocator, filepath: []const u8) !File {
switch (self.inode.header.type) {
.dir, .ext_dir => {},
else => return error.NotDirectory,
}
const path = std.mem.trim(u8, filepath, "/");
if (path.len == 0 or (path.len == 1 and path[0] == '.'))
return self.copy(alloc);
const first_element: []u8 = std.mem.sliceTo(path, '/');
var dir_size: u32 = undefined;
var dir_rdr: MetadataReader = undefined;
switch (self.inode.data) {
.dir => |d| {
dir_rdr = .init(alloc, self.data[d.block..], self.super.decomp_fn);
try dir_rdr.interface.discardAll(d.offset);
dir_size = d.size;
},
.ext_dir => |d| {
dir_rdr = .init(alloc, self.data[d.block..], self.super.decomp_fn);
try dir_rdr.interface.discardAll(d.offset);
dir_size = d.size;
},
else => unreachable,
}
const inode: Inode = {
var dir: Directory = try .read(alloc, &dir_rdr.interface, dir_size);
defer dir.deinit(alloc);
};
return error.TODO;
}
+173 -13
View File
@@ -1,11 +1,58 @@
const std = @import("std");
const Io = std.Io;
const util = @import("utils/util.zig");
const MetadataReader = @import("utils/meta.zig");
const Super = @import("archive.zig").Super;
const Inode = @This();
header: Header,
data: Data,
pub fn fromRef(alloc: std.mem.Allocator, data: []u8, ref: Reference, super: Super) !Inode {
var meta: MetadataReader = .init(alloc, data[super.inode_table_start + ref.start ..], super.decomp_fn);
try meta.interface.discardAll(ref.offset);
return read(alloc, &meta.interface, super.block_size);
}
pub fn read(alloc: std.mem.Allocator, rdr: *Io.Reader, block_size: u32) !Inode {
const hdr = try util.readValueRdr(Header, rdr);
return .{
.header = hdr,
.data = switch (hdr.type) {
.dir => .{ .dir = try .read(rdr) },
.file => .{ .file = try .read(rdr, alloc, block_size) },
.symlink => .{ .symlink = try .read(rdr, alloc) },
.block_dev => .{ .block_dev = try .read(rdr) },
.char_dev => .{ .char_dev = try .read(rdr) },
.fifo => .{ .fifo = try .read(rdr) },
.socket => .{ .socket = try .read(rdr) },
.ext_dir => .{ .ext_dir = try .read(rdr) },
.ext_file => .{ .ext_file = try .read(rdr, alloc, block_size) },
.ext_symlink => .{ .ext_symlink = try .read(rdr, alloc) },
.ext_block_dev => .{ .ext_block_dev = try .read(rdr) },
.ext_char_dev => .{ .ext_char_dev = try .read(rdr) },
.ext_fifo => .{ .ext_fifo = try .read(rdr) },
.ext_socket => .{ .ext_socket = try .read(rdr) },
},
};
}
pub fn copy(self: Inode, alloc: std.mem.Allocator) !Inode {
var new_inode = self;
switch (new_inode.data) {
.file => |*f| f.blocks = try alloc.dupe(BlockSize, f.blocks),
.ext_file => |*f| f.blocks = try alloc.dupe(BlockSize, f.blocks),
.symlink => |*f| f.target = try alloc.dupe(u8, f.target),
.ext_symlink => |*f| f.target = try alloc.dupe(u8, f.target),
else => {},
}
return new_inode;
}
pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
switch (self.data) {
.file => |f| alloc.free(f.blocks),
@@ -20,10 +67,16 @@ pub fn deinit(self: Inode, alloc: std.mem.Allocator) void {
pub const Reference = packed struct(u64) {
offset: u16,
block: u32,
start: u32,
_: u16,
};
pub const BlockSize = packed struct(u32) {
size: u23,
uncompressed: bool,
_: u8,
};
pub const Type = enum(u16) {
dir,
file,
@@ -42,7 +95,7 @@ pub const Type = enum(u16) {
};
pub const Header = extern struct {
type: Type, // TODO: enum
type: Type,
permission: u16,
uid_idx: u16,
gid_idx: u16,
@@ -60,7 +113,7 @@ pub const Data = union(Type) {
socket: Fifo,
ext_dir: ExtDir,
ext_file: ExtFile,
ext_symlink: ExtSymlink,
ext_symlink: Symlink,
ext_block_dev: ExtDev,
ext_char_dev: ExtDev,
ext_fifo: ExtFifo,
@@ -70,38 +123,145 @@ pub const Data = union(Type) {
// Inode data types
pub const Dir = extern struct {
block: u32,
start: u32,
hard_links: u32,
size: u16,
offset: u16,
parent_inode_num: u32,
pub fn read(rdr: *Io.Reader) !Dir {
return util.readValueRdr(Dir, rdr);
}
};
pub const ExtDir = extern struct {
hard_links: u32,
size: u32,
block: u32,
start: u32,
parent_inode_num: u32,
idx_count: u16,
offset: u16,
xattr_idx: u32,
pub fn read(rdr: *Io.Reader) !ExtDir {
return util.readValueRdr(ExtDir, rdr);
}
};
pub const File = struct {
blocks: []u64,
start: u32,
frag_idx: u32,
frag_offset: u32,
size: u32,
blocks: []BlockSize,
pub fn read(rdr: *Io.Reader, alloc: std.mem.Allocator, block_size: u32) !File {
var raw: [16]u8 = undefined;
try rdr.readSliceAll(&raw);
const frag_idx = std.mem.readInt(u32, raw[4..8], .little);
const size = std.mem.readInt(u32, raw[12..16], .little);
var num = size / block_size;
if (frag_idx == 0xFFFFFFFF and size % block_size > 0)
num += 1;
const blocks = try alloc.alloc(BlockSize, num);
try rdr.readSliceEndian(BlockSize, blocks, .little);
return .{
.start = std.mem.readInt(u32, raw[0..4], .little),
.frag_idx = frag_idx,
.frag_offset = std.mem.readInt(u32, raw[8..12], .little),
.size = size,
.blocks = blocks,
};
}
};
pub const ExtFile = struct {
blocks: []u64,
start: u64,
size: u64,
hard_links: u32,
frag_idx: u32,
frag_offset: u32,
xattr_idx: u32,
blocks: []BlockSize,
pub fn read(rdr: *Io.Reader, alloc: std.mem.Allocator, block_size: u32) !ExtFile {
var raw: [40]u8 = undefined;
try rdr.readSliceAll(&raw);
const frag_idx = std.mem.readInt(u32, raw[28..32], .little);
const size = std.mem.readInt(u64, raw[8..16], .little);
var num = size / block_size;
if (frag_idx == 0xFFFFFFFF and size % block_size > 0)
num += 1;
const blocks = try alloc.alloc(BlockSize, num);
try rdr.readSliceEndian(BlockSize, blocks, .little);
return .{
.start = std.mem.readInt(u64, raw[0..8], .little),
.size = size,
.hard_links = std.mem.readInt(u32, raw[24..28], .little),
.frag_idx = frag_idx,
.frag_offset = std.mem.readInt(u32, raw[32..36], .little),
.xattr_idx = std.mem.readInt(u32, raw[36..], .little),
.blocks = blocks,
};
}
};
pub const Symlink = struct {
hard_links: u32,
target: []const u8,
pub fn read(rdr: *Io.Reader, alloc: std.mem.Allocator) !Symlink {
var raw: [8]u8 = undefined;
try rdr.readSliceAll(&raw);
const target = try alloc.alloc(u8, std.mem.readInt(u32, raw[4..], .little));
errdefer alloc.free(target);
try rdr.readSliceEndian(u8, target, .little);
return .{
.hard_links = std.mem.readInt(u32, raw[0..4], .little),
.target = target,
};
pub const ExtSymlink = struct {
target: []const u8,
}
};
pub const Dev = extern struct {};
pub const ExtDev = extern struct {};
pub const Dev = extern struct {
hard_links: u32,
device: u32,
pub const Fifo = extern struct {};
pub const ExtFifo = extern struct {};
pub fn read(rdr: *Io.Reader) !Dev {
return util.readValueRdr(Dev, rdr);
}
};
pub const ExtDev = extern struct {
hard_links: u32,
device: u32,
xattr_idx: u32,
pub fn read(rdr: *Io.Reader) !ExtDev {
return util.readValue(ExtDev, rdr);
}
};
pub const Fifo = extern struct {
hard_links: u32,
pub fn read(rdr: *Io.Reader) !Fifo {
return util.readValueRdr(Fifo, rdr);
}
};
pub const ExtFifo = extern struct {
hard_links: u32,
xattr_idx: u32,
pub fn read(rdr: *Io.Reader) !ExtFifo {
return util.readValueRdr(ExtFifo, rdr);
}
};
+1 -1
View File
@@ -17,6 +17,6 @@ test "Open" {
std.debug.print("{}\n", .{arc.super});
var root = try arc.root();
var root = try arc.root(alloc);
defer root.close();
}
+101
View File
@@ -1,8 +1,109 @@
const std = @import("std");
const Io = std.Io;
const util = @import("util.zig");
const Decomp = @import("../decomp.zig");
const MetadataReader = @This();
alloc: std.mem.Allocator,
data: []u8,
decomp: Decomp.Fn,
decomp_block: [8192]u8 = undefined,
interface: Io.Reader = .{
.buffer = &[0]u8,
.end = 0,
.seek = 0,
.vtable = &.{
.stream = stream,
.discard = discard,
.readVec = readVec,
},
},
pub fn init(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn) MetadataReader {
return .{
.alloc = alloc,
.data = data,
.decomp = decomp,
};
}
fn advance(self: *MetadataReader) Io.Reader.Error!void {
self.interface.seek = 0;
errdefer self.interface.end = 0;
const hdr = util.readValue(Header, self.data[0..2]);
const block = self.data[2..][0..hdr.size];
self.data = self.data[hdr.size + 2 ..];
if (hdr.uncompressed) {
self.interface.end = hdr.size;
self.interface.buffer = block;
return;
}
self.interface.end = self.decomp(self.alloc, block, &self.decomp_block) catch return error.ReadFailed;
self.interface.buffer = self.decomp_block[0..self.interface.end];
}
fn stream(rdr: *Io.Reader, wrt: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize {
if (rdr.seek >= rdr.end) {
var meta: *MetadataReader = @fieldParentPtr("interface", rdr);
try meta.advance();
}
if (limit == .nothing) return 0;
const size = @min(@intFromEnum(limit), rdr.end - rdr.seek);
const wrote = try wrt.write(rdr.buffer[rdr.seek..][0..size]);
rdr.seek += wrote;
return wrote;
}
fn discard(rdr: *Io.Reader, limit: Io.Limit) Io.Reader.Error!usize {
if (rdr.seek >= rdr.end) {
var meta: *MetadataReader = @fieldParentPtr("interface", rdr);
try meta.advance();
}
if (limit == .nothing) return 0;
const size = @min(@intFromEnum(limit), rdr.end - rdr.seek);
rdr.seek += size;
return size;
}
fn readVec(rdr: *Io.Reader, vec: [][]u8) Io.Reader.Error!usize {
if (rdr.seek >= rdr.end) {
var meta: *MetadataReader = @fieldParentPtr("interface", rdr);
try meta.advance();
}
var wrote: usize = 0;
for (vec) |s| {
const size = @min(rdr.end - rdr.seek, s.len);
@memcpy(s[0..size], rdr.buffer[rdr.seek..][0..size]);
wrote += size;
rdr.seek += size;
if (rdr.seek >= rdr.end) break;
}
return wrote;
}
// Types
const Header = packed struct(u16) {
size: u15,
uncompressed: bool,
};
+6
View File
@@ -20,3 +20,9 @@ pub fn readValue(comptime T: type, bytes: []u8) T {
}
return res;
}
pub fn readValueRdr(comptime T: type, rdr: *std.Io.Reader) !T {
var tmp: [@sizeOf(T)]u8 = undefined;
try rdr.readSliceAll(&tmp);
return readValue(T, &tmp);
}