Work on extraction

src/archive.zig

@@ -29,8 +29,9 @@ const Archive = @This();
 const DEFAULT_MEM_SIZE = 4 * 1024 * 1024 * 1024;
 
 parent_alloc: std.mem.Allocator,
-alloc: std.heap.FixedBufferAllocator,
+alloc: std.heap.ThreadSafeAllocator,
-fixed_buf: []u8,
+// alloc: std.heap.FixedBufferAllocator,
+// fixed_buf: []u8,
 thread_count: usize,
 
 fil: OffsetFile,
@@ -59,15 +60,16 @@ pub fn init(alloc: std.mem.Allocator, fil: File) !Archive {
 /// If trying to extract a full archive, a large memory size & thread count could help.
 /// If you're planning on only interacting with a small number of files, it should be fine to use few threads and a small memory size.
 pub fn initAdvanced(alloc: std.mem.Allocator, fil: File, offset: u64, threads: usize, mem: usize) !Archive {
+    _ = mem;
     var super: Superblock = undefined;
     const red = try fil.pread(@ptrCast(&super), offset);
     std.debug.assert(red == @sizeOf(Superblock));
     try super.validate();
-    const fixed_buf = try alloc.alloc(u8, mem);
+    // const fixed_buf = try alloc.alloc(u8, mem);
     return .{
         .parent_alloc = alloc,
-        .alloc = .init(fixed_buf),
+        .alloc = .{ .child_allocator = alloc },
-        .fixed_buf = fixed_buf,
+        // .fixed_buf = fixed_buf,
         .thread_count = threads,
         .fil = .init(fil, offset),
 
@@ -75,7 +77,7 @@ pub fn initAdvanced(alloc: std.mem.Allocator, fil: File, offset: u64, threads: u
     };
 }
 pub fn deinit(self: *Archive) void {
-    self.parent_alloc.free(self.fixed_buf);
+    // self.parent_alloc.free(self.fixed_buf);
     if (self.setup) {
         self.decomp.deinit();
         self.frag_table.deinit();
@@ -85,14 +87,15 @@ pub fn deinit(self: *Archive) void {
 }
 
 pub fn allocator(self: *Archive) std.mem.Allocator {
-    return self.alloc.threadSafeAllocator();
+    return self.alloc.allocator();
 }
 
 fn setupValues(self: *Archive) !void {
-    self.decomp = try .init(self.allocator(), self.super.compression, self.super.block_size, self.thread_count);
+    const alloc = self.allocator();
-    self.frag_table = try .init(self.allocator(), self.fil, &self.decomp, self.super.frag_start, self.super.frag_count);
+    self.decomp = try .init(alloc, self.super.compression, self.super.block_size, self.thread_count);
-    self.id_table = try .init(self.allocator(), self.fil, &self.decomp, self.super.id_start, self.super.id_count);
+    self.frag_table = try .init(alloc, self.fil, &self.decomp, self.super.frag_start, self.super.frag_count);
-    self.export_table = try .init(self.allocator(), self.fil, &self.decomp, self.super.export_start, self.super.inode_count);
+    self.id_table = try .init(alloc, self.fil, &self.decomp, self.super.id_start, self.super.id_count);
+    self.export_table = try .init(alloc, self.fil, &self.decomp, self.super.export_start, self.super.inode_count);
     self.setup = true;
 }
 

src/decomp.zig

@@ -59,6 +59,7 @@ pub const DecompThread = struct {
         self.status.store(3, .release);
         Futex.wake(&self.status, 1);
         self.thr.join();
+        self.mgr.alloc.free(self.buf);
     }
 
     pub fn submitData(self: *DecompThread, dat: []u8, res: []u8) anyerror!usize {

src/file.zig

@@ -46,9 +46,7 @@ pub fn fromEntry(archive: *Archive, entry: DirEntry) !SfsFile {
     try meta.interface.discardAll(entry.block_offset);
     const inode: Inode = try .read(archive.allocator(), &meta.interface, archive.super.block_size);
     errdefer inode.deinit(archive.allocator());
-    const new_name = try archive.allocator().alloc(u8, entry.name.len);
+    return .init(archive, inode, entry.name);
-    @memcpy(new_name, entry.name);
-    return .init(archive, inode, new_name);
 }
 
 pub fn deinit(self: SfsFile) void {
@@ -144,18 +142,26 @@ pub fn iterate(self: SfsFile) !Iterator {
     };
 }
 /// Open a sub-file/folder within a directory at the given path.
-/// If path is ".", "/", or "./", this File is returned.
+/// If path is "", ".", "/", or "./", this File is returned.
 pub fn open(self: SfsFile, path: []const u8) !SfsFile {
     if (!self.isDir()) return FileError.NotDirectory;
     if (pathIsSelf(path)) return self;
+
     // Recursively stip ending & leading path separators.
-    // TODO: potentially do this more efficiently or have stricter path requirements.
     if (path[0] == '/') return self.open(path[1..]);
     if (path[path.len - 1] == '/') return self.open(path[0 .. path.len - 1]);
+
     const idx = std.mem.indexOf(u8, path, "/") orelse path.len;
     const first_element = path[0..idx];
     if (std.mem.eql(u8, first_element, ".")) return self.open(path[idx + 1 ..]);
     const entries = try self.getEntries();
+    defer {
+        var alloc = self.archive.allocator();
+        for (entries) |e| {
+            e.deinit(alloc);
+        }
+        alloc.free(entries);
+    }
     var cur_slice = entries;
     var split = cur_slice.len / 2;
     while (cur_slice.len > 0) {
@@ -240,7 +246,7 @@ pub fn extract(self: *SfsFile, path: []const u8, options: ExtractionOptions) !vo
     }
     defer if (ext_path.len > path.len) alloc.free(ext_path);
     var pool: std.Thread.Pool = undefined;
-    try pool.init(.{ .allocator = alloc });
+    try pool.init(.{ .allocator = alloc, .n_jobs = 16 });
     var wg: WaitGroup = .{};
     defer pool.deinit();
     var err: ?anyerror = null;
@@ -266,6 +272,7 @@ const ParentInfo = struct {
             return;
         }
         self.mut.unlock();
+        std.debug.print("finishing dir {}: {s}\n", .{ self.sfs_fil.inode.hdr.num, self.sfs_fil.name });
         self.sfs_fil.archive.allocator().destroy(self.mut);
         self.sfs_fil.archive.allocator().destroy(self.dir_wg);
         defer self.parent_wg.finish();
@@ -393,6 +400,7 @@ fn extractReal(self: SfsFile, path: []const u8, options: ExtractionOptions, pol:
             };
         },
         .dir, .ext_dir => {
+            std.debug.print("starting dir {}: {s}\n", .{ self.inode.hdr.num, self.name });
             if (std.fs.cwd().statFile(path)) |stat| {
                 if (stat.kind != .directory) {
                     std.log.err("{s} exists and is not a folder\n", .{path});
@@ -457,20 +465,24 @@ fn extractReal(self: SfsFile, path: []const u8, options: ExtractionOptions, pol:
                 @memcpy(new_path[0..path.len], path);
                 @memcpy(new_path[new_path.len - fil.name.len ..], fil.name);
                 if (!path_has_end_sep) new_path[path.len] = '/';
-                pol.spawn(extractReal, .{
+                if (fil.isDir()) {
-                    fil,
+                    fil.extractReal(new_path, options, pol, wg, out_err, parent_info);
-                    new_path,
+                } else {
-                    options,
+                    pol.spawn(extractReal, .{
-                    pol,
+                        fil,
-                    wg,
+                        new_path,
-                    out_err,
+                        options,
-                    parent_info,
+                        pol,
-                }) catch |err| {
+                        wg,
-                    std.log.err("Error starting sub-file extraction thread: {}\n", .{err});
+                        out_err,
-                    out_err.* = err;
+                        parent_info,
-                    dir_wg.finish();
+                    }) catch |err| {
-                    break;
+                        std.log.err("Error starting sub-file extraction thread: {}\n", .{err});
-                };
+                        out_err.* = err;
+                        dir_wg.finish();
+                        break;
+                    };
+                }
             }
         },
         .socket, .ext_socket => {

src/test.zig

@@ -27,6 +27,7 @@ test "ExtractSingleFile" {
     var sfs: Archive = try .init(std.testing.allocator, fil);
     defer sfs.deinit();
     var test_fil = try sfs.open(TestFile);
+    defer test_fil.deinit();
     try test_fil.extract(TestFileExtractLocation, .VerboseDefault);
     //TODO: validate extracted file.
 }

src/util/data.zig

@@ -104,6 +104,7 @@ fn advance(self: *DataReader) !void {
         return;
     }
     var rdr = try self.fil.readerAt(self.cur_offset, &[0]u8{});
+    self.cur_offset += block.size;
     if (block.uncompressed) {
         try rdr.interface.readSliceAll(self.interface.buffer);
         return;