Belac/zig-squashfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expirementation with a new way to finish threads. Currently not working.

Browse Source
This commit is contained in:
Caleb J. Gardner
2026-03-04 06:39:44 -06:00
parent 4515610082
commit edfe919c1b
4 changed files with 224 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/util/data_threaded.zig
+45 -48
View File
@@ -11,6 +11,7 @@ const Archive = @import("../archive.zig");
const FragEntry = Archive.FragEntry;
const DecompFn = @import("../decomp.zig").DecompFn;
const BlockSize = @import("../inode_data/file.zig").BlockSize;
const InodeFinish = @import("inode_finish.zig");
const OffsetFile = @import("offset_file.zig");
const ThreadedDataReader = @This();
@@ -25,6 +26,7 @@ blocks: []BlockSize,
frag: ?FragEntry = null, // TODO: do something better?
frag_offset: u32 = 0,
size: u64,
num_blocks: usize,
start_offset: u64,
@@ -34,8 +36,12 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, sta
.fil = archive.fil,
.decomp = archive.decomp,
.block_size = archive.super.block_size,
.blocks = blocks,
.size = size,
.num_blocks = blocks.len,
.start_offset = start,
};
}
@@ -43,141 +49,132 @@ pub fn init(alloc: std.mem.Allocator, archive: Archive, blocks: []BlockSize, sta
pub fn addFragment(self: *ThreadedDataReader, entry: FragEntry, frag_offset: u32) void {
self.frag = entry;
self.frag_offset = frag_offset;
}
fn numBlocks(self: ThreadedDataReader) usize {
var res = self.blocks.len;
if (self.frag != null) res += 1;
return res;
self.num_blocks = self.blocks.len + 1;
}
/// Extract the data to the file threadedly, using pool to spawn threads.
/// If multiple errors occur, thread spawning errors will have, then the last decompression error that occurs;
///
/// The function must be called from an unused DataReader. The DataReader is still usable afterwards.
/// If only extractThreaded is used, there is no need to call deinit() afterwards.
///
/// The file will always be written to starting at 0.
pub fn extractThreaded(self: ThreadedDataReader, file: std.fs.File, pool: *Pool) !void {
var wg: WaitGroup = .{};
wg.startMany(self.numBlocks());
var out_err: ?anyerror = null;
/// This function only returns an error if pool.spawn fails. For actual extraction errors finish.out_err will be set.
pub fn extractThreaded(self: ThreadedDataReader, file: std.fs.File, pool: *Pool, finish: *InodeFinish) !void {
var cur_write_offset: u64 = 0;
var cur_read_offset: u64 = self.start_offset;
for (0..self.blocks.len) |i| {
const cur_block_size = if (i == self.numBlocks() - 1) self.size % self.block_size else self.block_size;
try pool.spawn(workThreadBlocks, .{ self, file, cur_write_offset, cur_read_offset, self.blocks[i], cur_block_size, &wg, &out_err });
const cur_block_size = if (i == self.num_blocks - 1) self.size % self.block_size else self.block_size;
try pool.spawn(workThreadBlocks, .{ self, file, cur_write_offset, cur_read_offset, self.blocks[i], cur_block_size, finish });
cur_write_offset += cur_block_size;
cur_read_offset += self.blocks[i].size;
}
if (self.frag != null) {
try pool.spawn(workThreadFragment, .{ self, file, cur_write_offset, &wg, &out_err });
}
pool.waitAndWork(&wg);
if (out_err != null) return out_err.?;
if (self.frag != null)
try pool.spawn(workThreadFragment, .{ self, file, cur_write_offset, finish });
}
fn workThreadBlocks(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, read_offset: u64, block: BlockSize, cur_block_size: u64, wg: *WaitGroup, out_err: *?anyerror) void {
defer wg.finish();
fn workThreadBlocks(
self: ThreadedDataReader,
fil: std.fs.File,
write_offset: u64,
read_offset: u64,
block: BlockSize,
cur_block_size: u64,
finish: *InodeFinish,
) void {
defer finish.finish();
var wrt = fil.writer(&[0]u8{});
wrt.seekTo(write_offset) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
defer wrt.interface.flush() catch |err| {
out_err.* = err;
finish.out_err.* = err;
};
if (block.size == 0) {
wrt.interface.splatByteAll(0, cur_block_size) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
return;
}
var rdr = self.fil.readerAt(read_offset, &[0]u8{}) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
if (block.uncompressed) {
rdr.interface.streamExact(&wrt.interface, block.size) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
return;
}
// TODO: shared buffers
const read_buf = self.alloc.alloc(u8, block.size) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
defer self.alloc.free(read_buf);
rdr.interface.readSliceAll(read_buf) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
// TODO: shared buffers
const res_buf = self.alloc.alloc(u8, cur_block_size) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
defer self.alloc.free(res_buf);
_ = self.decomp(self.alloc, read_buf, res_buf) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
wrt.interface.writeAll(res_buf) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
}
fn workThreadFragment(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, wg: *WaitGroup, out_err: *?anyerror) void {
defer wg.finish();
fn workThreadFragment(self: ThreadedDataReader, fil: std.fs.File, write_offset: u64, finish: *InodeFinish) void {
defer finish.finish();
var wrt = fil.writer(&[0]u8{});
wrt.seekTo(write_offset) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
defer wrt.interface.flush() catch |err| {
out_err.* = err;
finish.out_err.* = err;
};
var rdr = self.fil.readerAt(self.frag.?.start, &[0]u8{}) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
if (self.frag.?.size.uncompressed) {
rdr.interface.discardAll(self.frag_offset) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
rdr.interface.streamExact(&wrt.interface, self.size % self.block_size) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
return;
}
const tmp_buf = self.alloc.alloc(u8, self.frag.?.size.size) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
defer self.alloc.free(tmp_buf);
rdr.interface.readSliceAll(tmp_buf) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
const needed_block = self.alloc.alloc(u8, self.block_size) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
defer self.alloc.free(needed_block);
_ = self.decomp(self.alloc, tmp_buf, needed_block) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
wrt.interface.writeAll(needed_block[self.frag_offset .. self.frag_offset + (self.size % self.block_size)]) catch |err| {
out_err.* = err;
finish.out_err.* = err;
return;
};
}