diff --git a/src/inode.zig b/src/inode.zig index 843e100..cc5aff4 100644 --- a/src/inode.zig +++ b/src/inode.zig @@ -154,61 +154,6 @@ fn readDirectoryFromData(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, // Extraction -pub const ExtractionError = error{ SetXattr, Mknod, Canceled } || DirEntry.Error || Io.Dir.CreateFileAtomicError || DataExtract.Error || Io.File.Atomic.LinkError || - Io.Dir.SymLinkError; - -const ExtractReturn = struct { - path: []const u8, - inode: Inode, - - fn deinit(self: ExtractReturn, alloc: std.mem.Allocator) void { - self.inode.deinit(alloc); - alloc.free(self.path); - } - fn setMetadata(self: ExtractReturn, alloc: std.mem.Allocator, io: Io, cache: *DecompCache, id_start: u64, xattr_start: u64, options: ExtractionOptions) !void { - defer self.deinit(alloc); - if (options.ignore_permissions and options.ignore_xattr) return; - - var fil = try Io.Dir.cwd().openFile(io, self.path, .{}); - defer fil.close(io); - - if (!options.ignore_permissions) { - try fil.setTimestamps(io, .{ .modify_timestamp = .{ - .new = .{ .nanoseconds = self.inode.hdr.mod_time * std.time.ns_per_s }, - } }); - try fil.setPermissions(io, @enumFromInt(self.inode.hdr.permissions)); - try fil.setOwner( - io, - try LookupTable.lookup(u16, io, cache, id_start, self.inode.hdr.uid_idx), - try LookupTable.lookup(u16, io, cache, id_start, self.inode.hdr.gid_idx), - ); - } - if (options.ignore_xattr or @hasField(std.os, "linux")) return; - const xattr_idx: u32 = switch (self.inode.data) { - .ext_dir => |d| d.xattr_idx, - .ext_file => |f| f.xattr_idx, - .ext_symlink => |s| s.xattr_idx, - .ext_block_dev, .ext_char_dev => |d| d.xattr_idx, - .ext_fifo, .ext_socket => |i| i.xattr_idx, - else => return, - }; - if (xattr_idx == 0xFFFFFFFF) return; - const xattrs = try LookupTable.xattrLookup(alloc, io, cache, xattr_start, xattr_idx); - defer { - for (xattrs) |kv| - kv.deinit(alloc); - alloc.free(xattrs); - } - - for (xattrs) |kv| { - const res = std.os.linux.fsetxattr(fil.handle, kv.key.ptr, kv.value.ptr, kv.value.len, 0); - if (res != 0) - return ExtractionError.SetXattr; - } - } -}; -const ExtractUnion = union { ret: ExtractionError!ExtractReturn }; - pub fn extract( self: Inode, alloc: std.mem.Allocator, @@ -225,13 +170,16 @@ pub fn extract( ) !void { const path = std.mem.trimEnd(u8, ext_loc, "/"); + var sel_val: std.atomic.Value(usize) = .init(1); + var sel_buf: [5]ExtractUnion = undefined; var sel: Io.Select(ExtractUnion) = .init(io, &sel_buf); defer sel.cancelDiscard(); - var meta_loop = io.async(metadataLoop, .{ alloc, io, cache, id_start, xattr_start, &sel, options }); + var meta_loop = io.async(metadataLoop, .{ alloc, io, cache, id_start, xattr_start, &sel, &sel_val, options }); + defer _ = meta_loop.cancel(io) catch {}; - sel.async(.ret, extractReal, .{ self, alloc, io, cache, dir_start, inode_start, frag_start, block_size, &sel, path, true }); + sel.async(.ret, extractReal, .{ self, alloc, io, cache, dir_start, inode_start, frag_start, block_size, &sel, &sel_val, path, true }); try meta_loop.await(io); } @@ -244,7 +192,8 @@ fn extractReal( inode_start: u64, frag_start: u64, block_size: u32, - master_sel: *Io.Select(ExtractUnion), + sel: *Io.Select(ExtractUnion), + sel_val: *std.atomic.Value(usize), path: []const u8, origin: bool, ) ExtractionError!ExtractReturn { @@ -254,6 +203,8 @@ fn extractReal( }; switch (self.hdr.inode_type) { .dir, .ext_dir => { + try Io.Dir.cwd().createDir(io, path, @enumFromInt(0o777)); + const entries = self.readDirectory(alloc, io, cache, dir_start) catch |err| switch (err) { error.NotDirectory => unreachable, else => |e| return e, @@ -264,68 +215,67 @@ fn extractReal( alloc.free(entries); } - var sel_buf: [5]ExtractUnion = undefined; - var sel: Io.Select(ExtractUnion) = .init(io, &sel_buf); - defer sel.cancelDiscard(); + if (entries.len != 0) { + _ = sel_val.fetchAdd(entries.len, .acq_rel); - var dir_loop = io.async(dirLoop, .{ alloc, io, &sel, master_sel }); + for (entries) |entry| { + var meta: MetadataReader = .init(io, cache, inode_start + entry.block_start); + defer meta.deinit(); + try meta.interface.discardAll(entry.block_offset); - for (entries) |entry| { - var meta: MetadataReader = .init(io, cache, inode_start + entry.block_start); - defer meta.deinit(); - try meta.interface.discardAll(entry.block_offset); + var new_inode: Inode = try .fromReader(alloc, &meta.interface, block_size); + errdefer new_inode.deinit(alloc); - var new_inode: Inode = try .fromReader(alloc, &meta.interface, block_size); - errdefer new_inode.deinit(alloc); - - const new_path = try std.mem.concat(alloc, u8, &.{ path, "/", entry.name }); - errdefer alloc.free(new_path); - - sel.async(.ret, extractReal, .{ new_inode, alloc, io, cache, dir_start, inode_start, frag_start, block_size, master_sel, new_path, false }); + const new_path = try std.mem.concat(alloc, u8, &.{ path, "/", entry.name }); + errdefer alloc.free(new_path); + sel.async( + .ret, + extractReal, + .{ new_inode, alloc, io, cache, dir_start, inode_start, frag_start, block_size, sel, sel_val, new_path, false }, + ); + } } - - try dir_loop.await(io); }, .file, .ext_file => { - var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{}); - defer atomic.deinit(io); + std.debug.print("{s} {}\n", .{ path, self.data }); + // var atomic = try Io.Dir.cwd().createFileAtomic(io, path, .{}); + // defer atomic.deinit(io); - var data: DataExtract = undefined; - var frag_offset: ?u64 = null; - switch (self.data) { - .file => |f| { - data = .init(cache.decomp, cache.map, block_size, f.block_start, f.size, f.block_sizes); - if (f.frag_idx != 0xFFFFFFFF) { - const entry: FragEntry = try LookupTable.lookup(FragEntry, io, cache, frag_start, f.frag_idx); - if (entry.size.uncompressed) { - data.addFrag(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset); - } else { - frag_offset = entry.start; - const block = try cache.checkoutBlock(io, entry.start, entry.size.size, block_size); - data.addFrag(block, f.frag_offset); - } - } - }, - .ext_file => |f| { - data = .init(cache.decomp, cache.map, block_size, f.block_start, f.size, f.block_sizes); - if (f.frag_idx != 0xFFFFFFFF) { - const entry: FragEntry = try LookupTable.lookup(FragEntry, io, cache, frag_start, f.frag_idx); - if (entry.size.uncompressed) { - data.addFrag(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset); - } else { - frag_offset = entry.start; - const block = try cache.checkoutBlock(io, entry.start, entry.size.size, block_size); - data.addFrag(block, f.frag_offset); - } - } - }, - else => unreachable, - } - defer if (frag_offset != null) cache.checkinBlock(io, frag_offset.?); + // var data: DataExtract = undefined; + // var frag_offset: ?u64 = null; + // switch (self.data) { + // .file => |f| { + // data = .init(cache.decomp, cache.map, block_size, f.block_start, f.size, f.block_sizes); + // if (f.frag_idx != 0xFFFFFFFF) { + // const entry: FragEntry = try LookupTable.lookup(FragEntry, io, cache, frag_start, f.frag_idx); + // if (entry.size.uncompressed) { + // data.addFrag(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset); + // } else { + // frag_offset = entry.start; + // const block = try cache.checkoutBlock(io, entry.start, entry.size.size, block_size); + // data.addFrag(block, f.frag_offset); + // } + // } + // }, + // .ext_file => |f| { + // data = .init(cache.decomp, cache.map, block_size, f.block_start, f.size, f.block_sizes); + // if (f.frag_idx != 0xFFFFFFFF) { + // const entry: FragEntry = try LookupTable.lookup(FragEntry, io, cache, frag_start, f.frag_idx); + // if (entry.size.uncompressed) { + // data.addFrag(cache.map.memory[entry.start..][0..entry.size.size], f.frag_offset); + // } else { + // frag_offset = entry.start; + // const block = try cache.checkoutBlock(io, entry.start, entry.size.size, block_size); + // data.addFrag(block, f.frag_offset); + // } + // } + // }, + // else => unreachable, + // } + // defer if (frag_offset != null) cache.checkinBlock(io, frag_offset.?) catch {}; + // try data.asyncExtract(alloc, io, atomic.file); - try data.asyncExtract(alloc, io, atomic.file); - - try atomic.link(io); + // try atomic.link(io); }, .symlink, .ext_symlink => { const target = switch (self.data) { @@ -374,32 +324,111 @@ fn extractReal( return .{ .path = path, .inode = self, + .origin = origin, }; } -fn metadataLoop(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, id_start: u64, xattr_start: u64, sel: *Io.Select(ExtractUnion), options: ExtractionOptions) !void { - defer { - while (sel.cancel()) |ret| { + +const ExtractUnion = union { ret: ExtractionError!ExtractReturn }; +pub const ExtractionError = error{ SetXattr, Mknod, Canceled } || DirEntry.Error || Io.Dir.CreateFileAtomicError || DataExtract.Error || Io.File.Atomic.LinkError || + Io.Dir.SymLinkError; + +const ExtractReturn = struct { + path: []const u8, + inode: Inode, + origin: bool, + + fn deinit(self: ExtractReturn, alloc: std.mem.Allocator) void { + if (self.origin) return; + alloc.free(self.path); + self.inode.deinit(alloc); + } + fn setMetadata(self: ExtractReturn, alloc: std.mem.Allocator, io: Io, cache: *DecompCache, id_start: u64, xattr_start: u64, options: ExtractionOptions) !void { + if (options.ignore_permissions and options.ignore_xattr) return; + + var fil = try Io.Dir.cwd().openFile(io, self.path, .{}); + defer fil.close(io); + + if (!options.ignore_permissions) { + try fil.setTimestamps(io, .{ .modify_timestamp = .{ + .new = .{ .nanoseconds = @as(i96, @intCast(self.inode.hdr.mod_time)) * std.time.ns_per_s }, + } }); + try fil.setPermissions(io, @enumFromInt(self.inode.hdr.permissions)); + try fil.setOwner( + io, + try LookupTable.lookup(u16, io, cache, id_start, self.inode.hdr.uid_idx), + try LookupTable.lookup(u16, io, cache, id_start, self.inode.hdr.gid_idx), + ); + } + if (options.ignore_xattr) return; + const xattr_idx: u32 = switch (self.inode.data) { + .ext_dir => |d| d.xattr_idx, + .ext_file => |f| f.xattr_idx, + .ext_symlink => |s| s.xattr_idx, + .ext_block_dev, .ext_char_dev => |d| d.xattr_idx, + .ext_fifo, .ext_socket => |i| i.xattr_idx, + else => return, + }; + if (xattr_idx == 0xFFFFFFFF) return; + const xattrs = try LookupTable.xattrLookup(alloc, io, cache, xattr_start, xattr_idx); + defer { + for (xattrs) |kv| + kv.deinit(alloc); + alloc.free(xattrs); + } + + for (xattrs) |kv| { + const res = std.os.linux.fsetxattr(fil.handle, kv.key.ptr, kv.value.ptr, kv.value.len, 0); + if (res != 0) + return ExtractionError.SetXattr; + } + } +}; + +fn metadataLoop( + alloc: std.mem.Allocator, + io: Io, + cache: *DecompCache, + id_start: u64, + xattr_start: u64, + sel: *Io.Select(ExtractUnion), + sel_val: *std.atomic.Value(usize), + options: ExtractionOptions, +) !void { + errdefer { + while (sel.group.token.load(.unordered) != null) { + const ret = sel.queue.getOne(io) catch break; + const res = ret.ret catch continue; res.deinit(alloc); } } - while (sel.group.token.load(.unordered) != null) { - const ret = try sel.await(); + var dir_queue: std.PriorityDequeue(ExtractReturn, void, dirReturnQueueOrder) = .empty; + defer { + while (dir_queue.popMax()) |ret| + ret.deinit(alloc); + dir_queue.deinit(alloc); + } + + while (sel_val.load(.unordered) > 0) { + defer _ = sel_val.fetchSub(1, .acq_rel); + const ret = try sel.queue.getOne(io); const res = try ret.ret; + if (res.inode.hdr.inode_type == .dir or res.inode.hdr.inode_type == .ext_dir) { + try dir_queue.push(alloc, res); + } else { + defer res.deinit(alloc); + try res.setMetadata(alloc, io, cache, id_start, xattr_start, options); + } + } + + while (dir_queue.popMax()) |res| { + defer res.deinit(alloc); try res.setMetadata(alloc, io, cache, id_start, xattr_start, options); } } -fn dirLoop(alloc: std.mem.Allocator, io: Io, dir_sel: *Io.Select(ExtractUnion), master_sel: *Io.Select(ExtractUnion)) ExtractionError!void { - while (dir_sel.group.token.load(.unordered) != null) { - const ret = try dir_sel.await(); - master_sel.queue.putOne(io, ret) catch |err| switch (err) { - error.Closed => { - const res = try ret.ret; - res.deinit(alloc); - }, - else => |e| return e, - }; - } + +fn dirReturnQueueOrder(_: void, a: ExtractReturn, b: ExtractReturn) std.math.Order { + return std.math.order(std.mem.count(u8, a.path, "/"), std.mem.count(u8, b.path, "/")); } diff --git a/src/inode_data/file.zig b/src/inode_data/file.zig index 7c46d71..e489205 100644 --- a/src/inode_data/file.zig +++ b/src/inode_data/file.zig @@ -15,7 +15,7 @@ pub const File = struct { block_sizes: []BlockSize, pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !File { - const raw_values = struct { + const raw_values = extern struct { block_start: u32, // bytes 0-3 frag_idx: u32, // bytes 4-7 frag_offset: u32, // bytes 8-11 @@ -55,7 +55,7 @@ pub const ExtFile = struct { block_sizes: []BlockSize, pub fn read(alloc: std.mem.Allocator, rdr: *Reader, block_size: u32) !ExtFile { - const raw_values = struct { + const raw_values = extern struct { block_start: u64, // bytes 0-7 size: u64, // bytes 8-15 sparse: u64, // bytes 16-23 diff --git a/src/lookup_table.zig b/src/lookup_table.zig index 37361ff..5a2d628 100644 --- a/src/lookup_table.zig +++ b/src/lookup_table.zig @@ -11,6 +11,7 @@ pub fn lookup(comptime T: anytype, io: Io, cache: *DecompCache, table_start: u64 const block_idx = idx / PER_BLOCK; const block_offset = idx % PER_BLOCK; + if (table_start + (block_idx * 8) > cache.map.memory.len) return error.ReadFailed; const offset: u64 = std.mem.readInt(u64, cache.map.memory[table_start + (block_idx * 8) ..][0..8], .little); var meta: MetadataReader = .init(io, cache, offset); @@ -47,7 +48,7 @@ const KeyEntry = extern struct { out_of_line: bool, _: u7, }, - name_size: u8, + name_size: u16, }; pub fn xattrLookup(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, xattr_start: u64, idx: u32) ![]XattrKV { @@ -72,17 +73,23 @@ pub fn xattrLookup(alloc: std.mem.Allocator, io: Io, cache: *DecompCache, xattr_ var key_entry: KeyEntry = undefined; try meta.interface.readSliceEndian(KeyEntry, @ptrCast(&key_entry), .little); - var key_len = key_entry.name_size; - key_len += switch (key_entry.prefix.prefix) { + const prefix_len: u16 = switch (key_entry.prefix.prefix) { .user => 5, .trusted => 8, .security => 9, }; + var key_len = key_entry.name_size; + key_len += prefix_len; kv.key = try alloc.allocSentinel(u8, key_len, 0); errdefer alloc.free(kv.key); - try meta.interface.readSliceEndian(u8, kv.key, .little); + try meta.interface.readSliceEndian(u8, kv.key[prefix_len..], .little); + switch (key_entry.prefix.prefix) { + .user => @memcpy(kv.key[0..prefix_len], "user."), + .trusted => @memcpy(kv.key[0..prefix_len], "trusted."), + .security => @memcpy(kv.key[0..prefix_len], "security."), + } if (key_entry.prefix.out_of_line) { try meta.interface.discardAll(8); diff --git a/src/test.zig b/src/test.zig index 43d3d14..97853dd 100644 --- a/src/test.zig +++ b/src/test.zig @@ -36,6 +36,24 @@ test "ExtractSingleFile" { //TODO: validate extracted file. } +const TestDir = "Documents"; +const TestDirExtractLocation = "testing/Documents"; + +test "ExtractSmallDir" { + const io = std.testing.io; + const alloc = std.testing.allocator; + + Io.Dir.cwd().deleteTree(io, TestDirExtractLocation) catch {}; + var fil = try Io.Dir.cwd().openFile(io, TestArchive, .{}); + defer fil.close(io); + var sfs: Archive = try .init(alloc, io, fil); + defer sfs.deinit(io); + var test_fil = try sfs.open(alloc, io, TestDir); + defer test_fil.deinit(); + try test_fil.extract(alloc, io, TestDirExtractLocation, .default); + //TODO: validate extracted file. +} + const TestFullExtractLocation = "testing/TestExtract"; test "ExtractCompleteArchive" { diff --git a/src/util/data_extract.zig b/src/util/data_extract.zig index b658c92..0e9eda6 100644 --- a/src/util/data_extract.zig +++ b/src/util/data_extract.zig @@ -33,27 +33,30 @@ pub fn addFrag(self: *DataExtract, frag_block: []u8, frag_offset: u32) void { self.frag_offset = frag_offset; } -pub const Error = error{} || Io.File.MemoryMap.CreateError || Io.File.WritePositionalError || Decompress.Error; +pub const Error = error{} || Io.File.MemoryMap.CreateError || Io.File.WritePositionalError || Decompress.Error || Io.File.MemoryMap.SetLengthError; pub fn asyncExtract(self: DataExtract, alloc: std.mem.Allocator, io: Io, fil: Io.File) Error!void { - var err: ?Error = null; + if (self.size == 0) return; + try fil.writePositionalAll(io, &.{0}, self.size - 1); var map = try fil.createMemoryMap(io, .{ .len = self.size, .protection = .{ .write = true }, .undefined_contents = true }); defer map.destroy(io); var group: Io.Group = .init; defer group.cancel(io); + var ret_err: ?Error = null; + var offset: u64 = self.block_start; for (0..self.blocks.len) |i| { - group.async(io, blockThread, .{ self, alloc, map, offset, i, &err }); + group.async(io, blockThread, .{ self, alloc, map, offset, i, &ret_err }); offset += self.blocks[i].size; } if (self.frag_data != null) group.async(io, fragThread, .{ self, map }); try group.await(io); - if (err != null) return err.?; + if (ret_err != null) return ret_err.?; return map.write(io); } fn blockThread(self: DataExtract, alloc: std.mem.Allocator, map: Io.File.MemoryMap, read_offset: u64, idx: usize, ret_err: *?Error) error{Canceled}!void { @@ -69,14 +72,12 @@ fn blockThread(self: DataExtract, alloc: std.mem.Allocator, map: Io.File.MemoryM @memset(map.memory[write_offset..][0..size], 0); return; } else if (block.uncompressed) { - @memcpy(self.map.memory[read_offset..][0..size], map.memory[write_offset..][0..size]); + @memcpy(map.memory[write_offset..][0..size], self.map.memory[read_offset..][0..block.size]); } - var tmp: [1024 * 1024]u8 = undefined; - _ = self.decomp(alloc, self.map.memory[read_offset..][0..block.size], tmp[0..size]) catch |err| { + _ = self.decomp(alloc, self.map.memory[read_offset..][0..block.size], map.memory[write_offset..][0..size]) catch |err| { ret_err.* = err; return error.Canceled; }; - @memcpy(map.memory[write_offset..][0..size], tmp[0..size]); } fn fragThread(self: DataExtract, map: Io.File.MemoryMap) error{Canceled}!void { const size = self.size % self.block_size; diff --git a/src/util/decomp_cache.zig b/src/util/decomp_cache.zig index 78b4313..84a2e9a 100644 --- a/src/util/decomp_cache.zig +++ b/src/util/decomp_cache.zig @@ -60,12 +60,12 @@ fn makeRoom(self: *DecompCache, io: Io, size: u32) !void { return self.makeRoom(io, size); } -pub fn checkinBlock(self: *DecompCache, io: Io, offset: u64) void { +pub fn checkinBlock(self: *DecompCache, io: Io, offset: u64) !void { self.mut.lockSharedUncancelable(io); defer self.mut.unlockShared(io); const get = self.cache.getPtr(offset); - if (get == null) return; + if (get == null) return error.NotACachedBlock; const res = get.?.usage.fetchSub(1, .acq_rel); if (res == 0) self.cond.broadcast(io); } diff --git a/src/util/metadata.zig b/src/util/metadata.zig index bbc7b52..4986935 100644 --- a/src/util/metadata.zig +++ b/src/util/metadata.zig @@ -22,6 +22,8 @@ next_offset: u64, cache: *DecompCache, +buf_uncompress: bool = false, + interface: Reader = .{ .buffer = &[0]u8{}, .end = 0, @@ -43,16 +45,21 @@ pub fn init(io: Io, cache: *DecompCache, offset: u64) MetadataReader { }; } pub fn deinit(self: *MetadataReader) void { - if (self.cur_offset != 0) - self.cache.checkinBlock(self.io, self.cur_offset); + if (self.cur_offset != 0 and !self.buf_uncompress) + self.cache.checkinBlock(self.io, self.cur_offset) catch {}; } fn advance(self: *MetadataReader) !void { - if (self.interface.buffer.len > 0) - self.cache.checkinBlock(self.io, self.cur_offset); + if (self.interface.buffer.len > 0 and !self.buf_uncompress) + self.cache.checkinBlock(self.io, self.cur_offset) catch |err| { + std.debug.print("UH OH! {}\n", .{err}); + return error.ReadFailed; + }; const hdr: BlockHeader = @bitCast(std.mem.readInt(u16, self.cache.map.memory[self.next_offset..][0..2], .little)); self.cur_offset = self.next_offset + 2; - self.next_offset += hdr.size; + self.next_offset = self.cur_offset + hdr.size; + + self.buf_uncompress = hdr.uncompressed; if (hdr.uncompressed) { self.interface.buffer = self.cache.map.memory[self.cur_offset..][0..hdr.size]; self.interface.end = hdr.size;