diff --git a/src/data/extractor.zig b/src/data/extractor.zig index e69de29..daf3e27 100644 --- a/src/data/extractor.zig +++ b/src/data/extractor.zig @@ -0,0 +1,103 @@ +const std = @import("std"); +const Io = std.Io; + +const Decomp = @import("../decomp.zig"); +const DataBlock = @import("../inode.zig").DataBlock; + +const Extractor = @This(); + +data: []u8, +decomp: Decomp.Fn, +block_size: u32, + +blocks: []DataBlock, + +start: u64, +size: u64, + +frag_data: ?[]u8 = null, +frag_offset: u32 = 0, + +pub fn init(data: []u8, decomp: Decomp.Fn, block_size: u32, blocks: []DataBlock, start: u64, size: u64) Extractor { + return .{ + .data = data, + .decomp = decomp, + .block_size = block_size, + + .blocks = blocks, + + .start = start, + .size = size, + }; +} +pub fn addFrag(self: *Extractor, frag_data: []u8, frag_offset: u32) void { + self.frag_data = frag_data; + self.frag_offset = frag_offset; +} + +pub fn extractAsync(self: Extractor, alloc: std.mem.Allocator, io: Io, file: Io.File) Error!void { + if (self.size == 0) return; + + try file.writePositionalAll(io, &[0]u8{}, self.size); + + var map = try file.createMemoryMap(io, .{ + .len = self.size, + .protection = .{ .write = true }, + }); + defer map.destroy(io); + + var err: ?Error = null; + var group: Io.Group = .init; + + var read_offset: u64 = self.start; + for (0.., self.blocks) |i, block| { + group.async(io, blockThread, .{ self, alloc, map.memory, read_offset, i, &err }); + read_offset += block.size; + } + if (self.frag_data != null) + group.async(io, fragThread, .{ self, map.memory }); + + try group.await(io); + + if (err != null) + return err; + + try map.write(io); +} + +fn blockThread(self: Extractor, alloc: std.mem.Allocator, map_data: []u8, read_offset: u64, block_idx: u32, err: *?Error) error{Canceled}!void { + const size = if (self.frag_data == null and block_idx == (self.size - 1 / self.block_size)) + self.size % self.block_size + else + self.block_size; + const offset = self.block_size * block_idx; + + const block = self.blocks[block_idx]; + + if (block.size == 0) { + @memset(map_data[offset..][0..size], 0); + return; + } + + const data = self.data[read_offset..][0..block.size]; + + if (block.uncompressed) { + @memcpy(map_data[offset..][0..block.size], data); + return; + } + + _ = self.decomp(alloc, data, map_data[offset..][0..size]) catch |inner_err| { + err.* = inner_err; + return; + }; +} +fn fragThread(self: Extractor, map_data: []u8) error{Canceled}!void { + const size = self.size % self.block_size; + const offset = self.blocks.len * self.block_size; + + @memcpy(map_data[offset..][0..size], self.frag_data.?[self.frag_offset..][0..size]); +} + +// Types + +const Error = Io.File.WritePositionalError || Io.File.MemoryMap.CreateError || Decomp.Error; diff --git a/src/data/reader.zig b/src/data/reader.zig index e69de29..e48473e 100644 --- a/src/data/reader.zig +++ b/src/data/reader.zig @@ -0,0 +1,155 @@ +const std = @import("std"); +const Io = std.Io; + +const Decomp = @import("../decomp.zig"); +const DataBlock = @import("../inode.zig").DataBlock; + +const Reader = @This(); + +alloc: std.mem.Allocator, + +data: []u8, +decomp: Decomp.Fn, +block_size: u32, + +blocks: []DataBlock, + +size: u64, + +offset: u64, +block_idx: u32 = 0, +sparse_block: bool = false, + +frag_data: ?[]u8 = null, +frag_offset: u32 = 0, + +block: [1024 * 1024]u8 = undefined, + +interface: Io.Reader = .{ + .buffer = &[0]u8{}, + .end = 0, + .seek = 0, + .vtable = &.{ + .stream = stream, + .discard = discard, + .readVec = readVec, + }, +}, + +pub fn init(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, block_size: u32, blocks: []DataBlock, size: u64, data_start: u64) Reader { + return .{ + .alloc = alloc, + + .data = data, + .decomp = decomp, + .block_size = block_size, + + .blocks = blocks, + + .size = size, + + .offset = data_start, + }; +} +pub fn addFrag(self: *Reader, frag_data: []u8, frag_offset: u32) void { + self.frag_data = frag_data; + self.frag_offset = frag_offset; +} + +fn advance(self: *Reader) Io.Reader.Error!void { + if (self.block_idx > self.blocks.len) return error.EndOfStream; + defer self.block_idx += 1; + + self.interface.seek = 0; + errdefer self.interface.end = 0; + + if (self.block_idx == self.blocks.len) { + if (self.frag_data == null) return error.EndOfStream; + + self.sparse_block = false; + + const size = self.size % self.block_size; + + self.interface.buffer = self.frag_data.?[self.frag_offset..][0..size]; + + self.interface.end = size; + return; + } + + const size = if (self.frag_data == null and self.block_idx == self.blocks.len - 1) + self.size % self.block_size + else + self.block_size; + + const block = self.blocks[self.block_idx]; + defer self.offset += block.size; + + if (block.size == 0) { + self.sparse_block = true; + self.end = size; + return; + } else { + self.sparse_block = false; + } + + if (block.uncompressed) { + self.interface.buffer = self.data[self.offset..][0..block.size]; + self.interface.end = block.size; + return; + } + + _ = self.decomp(self.alloc, self.data[self.offset..][0..block.size], self.block[0..size]) catch return error.ReadFailed; + self.interface.buffer = self.block[0..size]; + self.interface.end = size; +} + +fn stream(r: *Io.Reader, w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize { + const self: *Reader = @fieldParentPtr("interface", r); + if (r.seek >= r.end) + try self.advance(); + + if (limit == .nothing) return; + + const to_write = @min(@intFromEnum(limit), r.end - r.seek); + + const wrote = if (self.sparse_block) + try w.splatByte(0, to_write) + else + try w.write(r.buffer[r.seek..][0..to_write]); + + r.seek += wrote; + return wrote; +} +fn discard(r: *Io.Reader, limit: Io.Limit) Io.Reader.Error!usize { + if (r.seek >= r.end) { + const self: *Reader = @fieldParentPtr("interface", r); + try self.advance(); + } + if (limit == .nothing) return; + + const to_discard = @min(@intFromEnum(limit), r.end - r.seek); + + r.seek += to_discard; + return to_discard; +} +fn readVec(r: *Io.Reader, vec: [][]u8) Io.Reader.Error!usize { + const self: *Reader = @fieldParentPtr("interface", r); + if (r.seek >= r.end) + try self.advance(); + + var copied: usize = 0; + for (vec) |v| { + const to_cpy = @min(v.len, r.end - r.seek); + + if (self.sparse_block) + @memset(v[0..to_cpy], 0) + else + @memcpy(v[0..to_cpy], r.buffer[r.seek..][0..to_cpy]); + + copied += to_cpy; + + if (r.seek >= r.end) break; + } + + return copied; +} diff --git a/src/extract-multi.zig b/src/extract-multi.zig index 7ede70a..ad7b9bb 100644 --- a/src/extract-multi.zig +++ b/src/extract-multi.zig @@ -1,10 +1,12 @@ const std = @import("std"); const Io = std.Io; -const Superblock = @import("archive.zig").Superblock; -const Inode = @import("inode.zig"); const Decomp = @import("decomp.zig"); const ExtractionOptions = @import("options.zig"); +const Inode = @import("inode.zig"); +const Lookup = @import("lookup.zig"); +const Superblock = @import("archive.zig").Superblock; +const XattrTable = @import("xattr.zig"); pub fn extract( alloc: std.mem.Allocator, @@ -16,3 +18,15 @@ pub fn extract( ext_loc: []const u8, options: ExtractionOptions, ) !void {} + +pub fn extractReal( + alloc: std.mem.Allocator, + io: Io, + super: Superblock, + data: []u8, + decomp: Decomp.Fn, + frag_table: Lookup.Table(u64), + inode: Inode, + ext_loc: []const u8, + options: ExtractionOptions, +) !void {} diff --git a/src/extract-single.zig b/src/extract-single.zig index 7ede70a..e88758f 100644 --- a/src/extract-single.zig +++ b/src/extract-single.zig @@ -1,10 +1,12 @@ const std = @import("std"); const Io = std.Io; -const Superblock = @import("archive.zig").Superblock; -const Inode = @import("inode.zig"); const Decomp = @import("decomp.zig"); const ExtractionOptions = @import("options.zig"); +const Inode = @import("inode.zig"); +const Lookup = @import("lookup.zig"); +const Superblock = @import("archive.zig").Superblock; +const XattrTable = @import("xattr.zig"); pub fn extract( alloc: std.mem.Allocator, @@ -16,3 +18,17 @@ pub fn extract( ext_loc: []const u8, options: ExtractionOptions, ) !void {} + +pub fn extractReal( + alloc: std.mem.Allocator, + io: Io, + super: Superblock, + data: []u8, + decomp: Decomp.Fn, + frag_table: Lookup.Table(u64), + id_table: Lookup.Table(u16), + xattr_table: XattrTable, + inode: Inode, + ext_loc: []const u8, + options: ExtractionOptions, +) !void {} diff --git a/src/lookup.zig b/src/lookup.zig index c849813..4de234b 100644 --- a/src/lookup.zig +++ b/src/lookup.zig @@ -2,6 +2,8 @@ const std = @import("std"); const Io = std.Io; const Decomp = @import("decomp.zig"); +const MetadataReader = @import("meta_rdr.zig"); +const ProtectedMap = @import("util/protected_map.zig"); pub fn Table(comptime T: anytype) type { return struct { @@ -15,91 +17,43 @@ pub fn Table(comptime T: anytype) type { table_start: u64, table_num: u32, - pub fn init(alloc: std.mem.Allocator, table_start: u64, table_num: u32) Self {} - pub fn deinit(self: *Self) void {} + table: ProtectedMap(u32, []T, getBlock), - pub fn get(self: *Self, io: Io, idx: u32) !T {} - }; -} - -pub fn ProtectedMap(comptime K: anytype, comptime T: anytype, comptime create_fn: anytype) type { - std.debug.assert(std.meta.activeTag(@typeInfo(create_fn)) == .@"fn"); - - const ret_info = @typeInfo(create_fn).@"fn".return_type; - - std.debug.assert(ret_info == T or - (std.meta.activeTag(@typeInfo(ret_info)) == .error_union and @typeInfo(ret_info).error_union.payload == T)); - - return struct { - const Map = @This(); - - alloc: std.mem.Allocator, - - map: std.AutoHashMap(K, ProtectedValue), - mut: Io.RwLock = .init, - - pub fn init(alloc: std.mem.Allocator, default_value: T) Map { - return .{ - .alloc = alloc, - - .map = .init(alloc), - - .default_value = default_value, - }; + pub fn init(alloc: std.mem.Allocator, data: []u8, decomp: Decomp.Fn, table_start: u64, table_num: u32) Self { + return .{ .data = data, .decomp = decomp, .table_start = table_start, .table_num = table_num, .table = .init(alloc) }; } - pub fn deinit(self: *Map) void {} + pub fn deinit(self: *Self) void { + var iter = self.table.map.valueIterator(); - pub fn getOrPut(self: *Map, io: Io, key: K, create_fn_args: std.meta.ArgsTuple(create_fn)) !*T { - { - try self.mut.lockShared(io); - defer self.mut.unlockShared(io); + while (iter.next()) |val| { + if (val.err != null) continue; - const value = self.map.getPtr(key); - if (value != null) { - if (!value.?.filled.isSet()) - value.?.filled.wait(io); - if (value.?.err != null) return value.?.err != null; - return value.?.value; - } - } - try self.mut.lock(io); - - const res = self.map.getOrPut(key) catch |err| { - self.mut.unlock(io); - return err; - }; - if (res.found_existing) { - self.mut.unlock(io); - return self.getOrPut(io, key, create_fn_args); - } - res.value_ptr.* = .{}; - defer res.value_ptr.filled.set(io); - - res.mut.unlock(io); - - self.mut.lockSharedUncancelable(io); - defer self.mut.unlockShared(io); - - if (@TypeOf(CreateError) == void) { - res.value_ptr.value = @call(.auto, create_fn, create_fn_args); - } else { - res.value_ptr.value = @call(.auto, create_fn, create_fn_args) catch |err| { - res.value_ptr.err = err; - }; + self.table.alloc.free(val.value); } + self.table.deinit(); } - // Map Types + pub fn get(self: *Self, io: Io, idx: u32) !T { + const block = idx / VALUES_PER_BLOCK; + const block_idx = idx % VALUES_PER_BLOCK; - const CreateError: type = switch (@typeInfo(@typeInfo(create_fn).@"fn".return_type)) { - .error_union => |e| e.error_set, - else => void, - }; + const values = try self.table.getOrPut(io, block, .{ self.*, block }); - const ProtectedValue = struct { - value: T = undefined, - err: ?CreateError = null, - filled: Io.Event = .unset, - }; + return values[block_idx]; + } + + pub fn getBlock(self: Self, block_idx: u32) ![]T { + const offset: u64 = std.mem.readInt(u64, self.data[self.table_start + (block_idx * 8) ..][0..8], .little); + + const block = try self.table.alloc(T, if (block_idx == (self.table_num - 1 / VALUES_PER_BLOCK)) + self.table_num % VALUES_PER_BLOCK + else + VALUES_PER_BLOCK); + + var meta: MetadataReader = .init(self.table.alloc, self.data, self.decomp, offset); + try meta.interface.readSliceEndian(T, block, .little); + + return block; + } }; } diff --git a/src/util/protected_map.zig b/src/util/protected_map.zig new file mode 100644 index 0000000..1cadf73 --- /dev/null +++ b/src/util/protected_map.zig @@ -0,0 +1,84 @@ +const std = @import("std"); +const Io = std.Io; + +pub fn ProtectedMap(comptime K: anytype, comptime T: anytype, comptime create_fn: anytype) type { + std.debug.assert(std.meta.activeTag(@typeInfo(create_fn)) == .@"fn"); + + const ret_info = @typeInfo(create_fn).@"fn".return_type; + + std.debug.assert(ret_info == T or + (std.meta.activeTag(@typeInfo(ret_info)) == .error_union and @typeInfo(ret_info).error_union.payload == T)); + + return struct { + const Map = @This(); + + alloc: std.mem.Allocator, + + map: std.AutoHashMap(K, ProtectedValue), + mut: Io.RwLock = .init, + + pub fn init(alloc: std.mem.Allocator) Map { + return .{ + .alloc = alloc, + + .map = .init(alloc), + }; + } + pub fn deinit(self: *Map) void { + self.map.deinit(); + } + + pub fn getOrPut(self: *Map, io: Io, key: K, create_fn_args: std.meta.ArgsTuple(create_fn)) !*T { + { + try self.mut.lockShared(io); + defer self.mut.unlockShared(io); + + const value = self.map.getPtr(key); + if (value != null) { + if (!value.?.filled.isSet()) + value.?.filled.wait(io); + if (value.?.err != null) return value.?.err != null; + return value.?.value; + } + } + try self.mut.lock(io); + + const res = self.map.getOrPut(key) catch |err| { + self.mut.unlock(io); + return err; + }; + if (res.found_existing) { + self.mut.unlock(io); + return self.getOrPut(io, key, create_fn_args); + } + res.value_ptr.* = .{}; + defer res.value_ptr.filled.set(io); + + res.mut.unlock(io); + + self.mut.lockSharedUncancelable(io); + defer self.mut.unlockShared(io); + + if (@TypeOf(CreateError) == void) { + res.value_ptr.value = @call(.auto, create_fn, create_fn_args); + } else { + res.value_ptr.value = @call(.auto, create_fn, create_fn_args) catch |err| { + res.value_ptr.err = err; + }; + } + } + + // Map Types + + const CreateError: type = switch (@typeInfo(@typeInfo(create_fn).@"fn".return_type)) { + .error_union => |e| e.error_set, + else => void, + }; + + const ProtectedValue = struct { + value: T = undefined, + err: ?CreateError = null, + filled: Io.Event = .unset, + }; + }; +} diff --git a/src/xattr.zig b/src/xattr.zig new file mode 100644 index 0000000..2034ac0 --- /dev/null +++ b/src/xattr.zig @@ -0,0 +1,3 @@ +const std = @import("std"); + +const XattrTable = @This();