Belac/zig-squashfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More work on data reader

Browse Source
This commit is contained in:
Caleb Gardner
2025-08-07 20:47:04 -05:00
parent b3f4a02b72
commit 1269d3e30d
3 changed files with 64 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/file.zig
+6 -24
View File
@@ -63,15 +63,7 @@ pub fn File(comptime T: type) type {
out.entries = try dir.readDirectory(rdr.alloc, &meta, d.size); out.entries = try dir.readDirectory(rdr.alloc, &meta, d.size);
}, },
.file => |f| { .file => |f| {
out.data_reader = try .init( out.data_reader = try .init(rdr, inode);
rdr.alloc,
rdr.rdr,
rdr.super.comp,
f.block,
f.size,
f.block_sizes,
rdr.super.block_size,
);
if (f.hasFragment()) { if (f.hasFragment()) {
try out.data_reader.?.addFragment( try out.data_reader.?.addFragment(
try rdr.frag_table.get(f.frag_idx), try rdr.frag_table.get(f.frag_idx),
@@ -80,15 +72,7 @@ pub fn File(comptime T: type) type {
} }
}, },
.ext_file => |f| { .ext_file => |f| {
out.data_reader = try .init( out.data_reader = try .init(rdr, inode);
rdr.alloc,
rdr.rdr,
rdr.super.comp,
f.block,
f.size,
f.block_sizes,
rdr.super.block_size,
);
if (f.hasFragment()) { if (f.hasFragment()) {
try out.data_reader.?.addFragment( try out.data_reader.?.addFragment(
try rdr.frag_table.get(f.frag_idx), try rdr.frag_table.get(f.frag_idx),
@@ -112,7 +96,7 @@ pub fn File(comptime T: type) type {
const inode: Inode = try .init(&meta, rdr.alloc, rdr.super.block_size); const inode: Inode = try .init(&meta, rdr.alloc, rdr.super.block_size);
return .init(rdr, inode, ent.name); return .init(rdr, inode, ent.name);
} }
pub fn deinit(self: Self) void { pub fn deinit(self: *Self) void {
self.rdr.alloc.free(self.name); self.rdr.alloc.free(self.name);
self.inode.deinit(self.rdr.alloc); self.inode.deinit(self.rdr.alloc);
if (self.entries != null) { if (self.entries != null) {
@@ -190,7 +174,7 @@ pub fn File(comptime T: type) type {
pub const ExtractError = error{FileExists}; pub const ExtractError = error{FileExists};
pub fn extract(self: Self, op: ExtractionOptions, path: []const u8) !void { pub fn extract(self: *Self, op: ExtractionOptions, path: []const u8) !void {
var exists = true; var exists = true;
var stat: ?std.fs.File.Stat = null; var stat: ?std.fs.File.Stat = null;
if (std.fs.cwd().statFile(path)) |s| { if (std.fs.cwd().statFile(path)) |s| {
@@ -224,7 +208,7 @@ pub fn File(comptime T: type) type {
if (errs.items.len > 0) return errs.items[0]; if (errs.items.len > 0) return errs.items[0];
} }
fn extractReal( fn extractReal(
self: Self, self: *Self,
op: ExtractionOptions, op: ExtractionOptions,
path: []const u8, path: []const u8,
errs: *std.ArrayList(anyerror), errs: *std.ArrayList(anyerror),
@@ -365,9 +349,7 @@ pub fn File(comptime T: type) type {
defer if (!complete) self.rdr.alloc.destroy(fil_errs); defer if (!complete) self.rdr.alloc.destroy(fil_errs);
fil_errs.* = .init(self.rdr.alloc); fil_errs.* = .init(self.rdr.alloc);
defer if (!complete) fil_errs.deinit(); defer if (!complete) fil_errs.deinit();
@constCast(&self.data_reader.?).setPool(pol);
self.data_reader.?.writeToNoBlock( self.data_reader.?.writeToNoBlock(
errs,
ext_fil, ext_fil,
extractRegFinish, extractRegFinish,
.{ .{
@@ -409,7 +391,7 @@ pub fn File(comptime T: type) type {
}; };
} }
fn extractDirWait( fn extractDirWait(
self: Self, self: *Self,
op: ExtractionOptions, op: ExtractionOptions,
path: []const u8, path: []const u8,
dir_wg: *WaitGroup, dir_wg: *WaitGroup,
src/reader/data.zig
+44 -53
View File
@@ -15,7 +15,7 @@ const DataReaderError = error{
const DecompCompletion = struct { const DecompCompletion = struct {
errs: std.ArrayList(anyerror), errs: std.ArrayList(anyerror),
map: std.ArrayHashMap(usize, []u8), map: std.AutoArrayHashMap(usize, []u8),
mut: std.Thread.Mutex = .{}, mut: std.Thread.Mutex = .{},
cond: std.Thread.Condition = .{}, cond: std.Thread.Condition = .{},
@@ -25,13 +25,14 @@ const DecompCompletion = struct {
.map = .init(alloc), .map = .init(alloc),
}; };
} }
fn deinit(self: *DecompCompletion) !void { fn deinit(self: *DecompCompletion) void {
self.active = false;
self.errs.deinit(); self.errs.deinit();
self.map.deinit(); self.map.deinit();
} }
fn clear(self: *DecompCompletion) void { fn clear(self: *DecompCompletion) void {
self.mut.lock();
defer self.mut.unlock();
self.errs.clearAndFree(); self.errs.clearAndFree();
self.map.clearAndFree(); self.map.clearAndFree();
} }
@@ -39,20 +40,29 @@ const DecompCompletion = struct {
fn add(self: *DecompCompletion, idx: usize, data: []u8) !void { fn add(self: *DecompCompletion, idx: usize, data: []u8) !void {
self.mut.lock(); self.mut.lock();
defer self.mut.unlock(); defer self.mut.unlock();
defer self.cond.signal();
try self.map.put(idx, data); try self.map.put(idx, data);
} }
fn addErr(self: *DecompCompletion, err: anyerror) void { fn addErr(self: *DecompCompletion, err: anyerror) void {
self.mut.lock();
defer self.mut.unlock();
defer self.cond.signal();
self.errs.append(err) catch {}; self.errs.append(err) catch {};
} }
fn getBlock(self: *DecompCompletion, idx: usize) ?[]u8 { fn getBlock(self: *DecompCompletion, idx: usize) ?[]u8 {
self.mut.lock();
defer self.mut.unlock();
const res = self.map.fetchSwapRemove(idx); const res = self.map.fetchSwapRemove(idx);
if(res == null) return null; if (res == null) return null;
return res.?.value; return res.?.value;
} }
fn hasErrs(self: DecompCompletion) bool{ fn hasErrs(self: DecompCompletion) bool {
return self.errs.items.len > 0; return self.errs.items.len > 0;
} }
fn condWait(self: *DecompCompletion) void {
self.cond.wait(self.mut);
}
}; };
pub fn DataReader(comptime T: type) type { pub fn DataReader(comptime T: type) type {
@@ -101,7 +111,7 @@ pub fn DataReader(comptime T: type) type {
.block_size = rdr.super.block_size, .block_size = rdr.super.block_size,
.sizes = sizes, .sizes = sizes,
.offsets = offsets, .offsets = offsets,
.files_size = file_size, .file_size = file_size,
.completion = .init(rdr.alloc), .completion = .init(rdr.alloc),
}; };
} }
@@ -133,34 +143,25 @@ pub fn DataReader(comptime T: type) type {
thr.detach(); thr.detach();
} }
write_thr.join(); write_thr.join();
if () return errs.items[0]; if (self.completion.hasErrs()) return self.completion.errs.items[0];
} }
pub fn writeToNoBlock(self: Self, wrt: anytype, comptime finish: anytype, finish_args: anytype) !void { pub fn writeToNoBlock(self: *Self, wrt: anytype, comptime finish: anytype, finish_args: anytype) !void {
comptime std.debug.assert(std.meta.hasFn(@TypeOf(wrt), "write") or std.meta.hasFn(@TypeOf(wrt), "pwrite")); comptime std.debug.assert(std.meta.hasFn(@TypeOf(wrt), "write") or std.meta.hasFn(@TypeOf(wrt), "pwrite"));
var map: DecompCompletion = .init(self.alloc); errdefer self.completions.clear();
errdefer map.deinit();
var mut = try self.alloc.create(std.Thread.Mutex);
errdefer self.alloc.destroy(mut);
mut.* = .{};
var cond = try self.alloc.create(std.Thread.Condition);
errdefer self.alloc.destroy(cond);
cond.* = .{};
var errs: std.ArrayList(anyerror) = .init(self.alloc);
errdefer errs.deinit();
var write_thr = try std.Thread.spawn( var write_thr = try std.Thread.spawn(
.{ .allocator = self.alloc }, .{ .allocator = self.alloc },
writeThread, writeThread,
.{ self, wrt, &errs, &map, &mut, &cond, finish, finish_args }, .{ self, wrt, finish, finish_args },
); );
write_thr.detach(); write_thr.detach();
for (0..self.numBlocks()) |i| { for (0..self.numBlocks()) |i| {
var thr = std.Thread.spawn( var thr = std.Thread.spawn(
.{ .allocator = self.alloc }, .{ .allocator = self.alloc },
decompThread, decompThread,
.{ self, i, &errs, &map, &mut, &cond }, .{ self, i },
) catch |err| { ) catch |err| {
errs.append(err) catch {}; self.completion.addErr(err) catch {};
}; };
thr.detach(); thr.detach();
} }
@@ -199,79 +200,69 @@ pub fn DataReader(comptime T: type) type {
} }
fn writeThread( fn writeThread(
self: Self, self: *Self,
wrt: anytype, wrt: anytype,
errs: *std.ArrayList(anyerror),
map: *DecompCompletion,
mut: *std.Thread.Mutex,
cond: *std.Thread.Condition,
comptime finish: anytype, comptime finish: anytype,
finish_args: anytype, finish_args: anytype,
) void { ) void {
var cur_idx: usize = 0; var cur_idx: usize = 0;
mut.lock(); self.completion.mut.lock();
defer mut.unlock(); defer self.completion.mut.unlock();
while (cur_idx < self.numBlocks() and errs.items.len == 0) { while (cur_idx < self.numBlocks() and !self.completion.hasErrs()) {
cond.wait(mut); self.completion.condWait();
if (errs.items.len > 0) break; if (self.completion.hasErrs()) break;
if (comptime std.meta.hasFn(@TypeOf(wrt), "pwrite")) { if (comptime std.meta.hasFn(@TypeOf(wrt), "pwrite")) {
for (map.keys()) |k| { for (self.completion.map.keys()) |k| {
const blk = map.fetchSwapRemove(k).?.value; const blk = self.completion.getBlock(k).?;
defer self.alloc.free(blk); defer self.alloc.free(blk);
if (blk.len > 0) { if (blk.len > 0) {
_ = wrt.pwrite(map.fetchSwapRemove(k).?.value, self.block_size * k) catch |err| { _ = wrt.pwrite(blk, self.block_size * k) catch |err| {
errs.append(err) catch {}; self.completion.addErr(err);
break; break;
}; };
} else { } else {
_ = wrt.pwrite(&[1]u8{0}, (self.block_size * (k + 1)) - 1) catch |err| { _ = wrt.pwrite(&[1]u8{0}, (self.block_size * (k + 1)) - 1) catch |err| {
errs.append(err) catch {}; self.completion.addErr(err);
break; break;
}; };
} }
cur_idx += 1; cur_idx += 1;
} }
continue; continue;
} } else {
while (map.contains(cur_idx)) { while (self.completion.getBlock(cur_idx)) |blk| {
const blk = map.fetchSwapRemove(cur_idx).?.value;
defer self.alloc.free(blk); defer self.alloc.free(blk);
if (blk.len > 0) { if (blk.len > 0) {
_ = wrt.write(blk) catch |err| { _ = wrt.write(blk) catch |err| {
errs.append(err) catch {}; self.completion.addErr(err);
break; break;
}; };
} else { } else {
const blank: [1024 * 1024]u8 = [1]u8{0} ** (1024 * 1024); const blank: [1024 * 1024]u8 = [1]u8{0} ** (1024 * 1024);
_ = wrt.write(blank[0..self.block_size]) catch |err| { _ = wrt.write(blank[0..self.block_size]) catch |err| {
errs.append(err) catch {}; self.completion.addErr(err);
break; break;
}; };
} }
cur_idx += 1; cur_idx += 1;
} }
} }
}
if (comptime @TypeOf(finish) != @TypeOf(null) and @TypeOf(finish_args) != @TypeOf(null)) @call(.auto, finish, finish_args); if (comptime @TypeOf(finish) != @TypeOf(null) and @TypeOf(finish_args) != @TypeOf(null)) @call(.auto, finish, finish_args);
} }
fn decompThread( fn decompThread(
self: Self, self: *Self,
idx: usize, idx: usize,
errs: *std.ArrayList(anyerror),
map: *DecompCompletion,
mut: *std.Thread.Mutex,
cond: *std.Thread.Condition,
) void { ) void {
if (errs.items.len > 0) return; if (self.completion.hasErrs()) return;
defer self.completion.cond.signal();
const block = self.blockAt(idx) catch |err| { const block = self.blockAt(idx) catch |err| {
errs.append(err) catch {}; self.completion.addErr(err);
return; return;
}; };
mut.lock(); self.completion.add(idx, block) catch |err| {
defer mut.unlock(); self.completion.addErr(err);
map.put(idx, block) catch |err| {
errs.append(err) catch {};
}; };
cond.signal();
} }
}; };
} }
src/root.zig
+3 -3
View File
@@ -16,10 +16,10 @@ test "OpenFile" {
_ = try rdr.id_table.get(rdr.super.id_count - 1); _ = try rdr.id_table.get(rdr.super.id_count - 1);
_ = try rdr.export_table.get(rdr.super.inode_count - 1); _ = try rdr.export_table.get(rdr.super.inode_count - 1);
std.debug.print("{}\n", .{rdr.super}); std.debug.print("{}\n", .{rdr.super});
const root = try rdr.root(); var root = try rdr.root();
defer root.deinit(); defer root.deinit();
var iter = root.iterate(); var iter = root.iterate();
while (try iter.next()) |f| { while (try iter.next()) |*f| {
defer f.deinit(); defer f.deinit();
std.debug.print("{s}\n", .{f.name}); std.debug.print("{s}\n", .{f.name});
} }
@@ -34,7 +34,7 @@ test "ExtractSingleFile" {
defer sfs_fil.close(); defer sfs_fil.close();
var rdr: SfsFile = try .init(std.testing.allocator, sfs_fil, 0); var rdr: SfsFile = try .init(std.testing.allocator, sfs_fil, 0);
defer rdr.deinit(); defer rdr.deinit();
const fil = try rdr.open(single_file); var fil = try rdr.open(single_file);
defer fil.deinit(); defer fil.deinit();
var op: ExtractionOptions = try .init(); var op: ExtractionOptions = try .init();
op.verbose = true; op.verbose = true;