diff --git a/README.md b/README.md index a633f95..20d13d1 100644 --- a/README.md +++ b/README.md @@ -28,15 +28,14 @@ As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://g ## Issues -* Significantly slower then `unsquashfs` when nested images +* Noticably slower then `unsquashfs` for extraction, especially on larger images. * This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries. - * Not to mention it's written in C * Times seem to be largely dependent on file tree size and compression type. - * My main testing image (~100MB) using Zstd takes about 5x longer. - * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 30x longer. - * A Tensorflow docker image (~3.3GB) using Zstd takes about 12x longer. + * My main testing image (~100MB) using Zstd takes ~2x longer. + * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes ~28x longer. + * A Tensorflow docker image (~3.3GB) using Zstd takes ~3x longer. -Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes about 2x longer. +Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes ~2x longer. ## Recommendations on Usage diff --git a/file.go b/file.go index c3ebf6f..8c99b27 100644 --- a/file.go +++ b/file.go @@ -19,49 +19,47 @@ import ( // File represents a file inside a squashfs archive. type File struct { - full *data.FullReader - rdr *data.Reader - parent *FS + full data.FullReader + rdr data.Reader + rdrInit bool + parent FS r *Reader b squashfslow.FileBase dirsRead int } // Creates a new *File from the given *squashfs.Base -func (r *Reader) FileFromBase(b squashfslow.FileBase, parent *FS) *File { - return &File{ +func (r *Reader) FileFromBase(b squashfslow.FileBase, parent FS) File { + return File{ b: b, parent: parent, r: r, } } -func (f *File) FS() (*FS, error) { +func (f File) FS() (FS, error) { if !f.IsDir() { - return nil, errors.New("not a directory") + return FS{}, errors.New("not a directory") } - d, err := f.b.ToDir(&f.r.Low) + d, err := f.b.ToDir(f.r.Low) if err != nil { - return nil, err + return FS{}, err } - return &FS{d: d, parent: f.parent, r: f.r}, nil + return FS{d: d, parent: &f.parent, r: f.r}, nil } // Closes the underlying readers. // Further calls to Read and WriteTo will re-create the readers. // Never returns an error. func (f *File) Close() error { - if f.rdr != nil { - return f.rdr.Close() - } - f.rdr = nil - f.full = nil + f.rdr.Close() + f.full.Close() return nil } // Returns the file the symlink points to. // If the file isn't a symlink, or points to a file outside the archive, returns nil. -func (f *File) GetSymlinkFile() fs.File { +func (f File) GetSymlinkFile() fs.File { if !f.IsSymlink() { return nil } @@ -76,21 +74,21 @@ func (f *File) GetSymlinkFile() fs.File { } // Returns whether the file is a directory. -func (f *File) IsDir() bool { +func (f File) IsDir() bool { return f.b.IsDir() } // Returns whether the file is a regular file. -func (f *File) IsRegular() bool { +func (f File) IsRegular() bool { return f.b.IsRegular() } // Returns whether the file is a symlink. -func (f *File) IsSymlink() bool { +func (f File) IsSymlink() bool { return f.b.Inode.Type == inode.Sym || f.b.Inode.Type == inode.ESym } -func (f *File) Mode() fs.FileMode { +func (f File) Mode() fs.FileMode { return f.b.Inode.Mode() } @@ -99,7 +97,7 @@ func (f *File) Read(b []byte) (int, error) { if !f.IsRegular() { return 0, errors.New("file is not a regular file") } - if f.rdr == nil { + if !f.rdrInit { err := f.initializeReaders() if err != nil { return 0, err @@ -114,7 +112,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { if !f.IsDir() { return nil, errors.New("file is not a directory") } - d, err := f.b.ToDir(&f.r.Low) + d, err := f.b.ToDir(f.r.Low) if err != nil { return nil, err } @@ -141,7 +139,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { } // Returns the file's fs.FileInfo -func (f *File) Stat() (fs.FileInfo, error) { +func (f File) Stat() (fs.FileInfo, error) { uid, err := f.b.Uid(&f.r.Low) if err != nil { return nil, err @@ -154,7 +152,7 @@ func (f *File) Stat() (fs.FileInfo, error) { } // SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string. -func (f *File) SymlinkPath() string { +func (f File) SymlinkPath() string { switch f.b.Inode.Type { case inode.Sym: return string(f.b.Inode.Data.(inode.Symlink).Target) @@ -170,7 +168,7 @@ func (f *File) WriteTo(w io.Writer) (int64, error) { if !f.IsRegular() { return 0, errors.New("file is not a regular file") } - if f.full == nil { + if !f.rdrInit { err := f.initializeReaders() if err != nil { return 0, err @@ -181,11 +179,17 @@ func (f *File) WriteTo(w io.Writer) (int64, error) { func (f *File) initializeReaders() error { var err error - f.rdr, f.full, err = f.b.GetRegFileReaders(&f.r.Low) + f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.Low) + if err == nil { + f.rdrInit = true + } else { + f.rdr.Close() + f.full.Close() + } return err } -func (f *File) deviceDevices() (maj uint32, min uint32) { +func (f File) deviceDevices() (maj uint32, min uint32) { var dev uint32 switch f.b.Inode.Type { case inode.Char, inode.Block: @@ -196,8 +200,8 @@ func (f *File) deviceDevices() (maj uint32, min uint32) { return dev >> 8, dev & 0x000FF } -func (f *File) path() string { - if f.parent == nil { +func (f File) path() string { + if f.parent.d.Name == "" { return f.b.Name } return filepath.Join(f.parent.path(), f.b.Name) @@ -205,13 +209,13 @@ func (f *File) path() string { // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. // Uses default extraction options. -func (f *File) Extract(folder string) error { +func (f File) Extract(folder string) error { return f.ExtractWithOptions(folder, DefaultOptions()) } // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. // Allows setting various extraction options via ExtractionOptions. -func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { +func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error { if op.manager == nil { op.manager = routinemanager.NewManager(op.SimultaneousFiles) if op.LogOutput != nil { @@ -227,7 +231,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { } switch f.b.Inode.Type { case inode.Dir, inode.EDir: - d, err := f.b.ToDir(&f.r.Low) + d, err := f.b.ToDir(f.r.Low) if err != nil { if op.Verbose { log.Println("Failed to create squashfs.Directory for", path) diff --git a/fs.go b/fs.go index e1a282a..5971916 100644 --- a/fs.go +++ b/fs.go @@ -21,11 +21,11 @@ type FS struct { } // Creates a new *FS from the given squashfs.directory -func (r *Reader) FSFromDirectory(d squashfslow.Directory, parent *FS) *FS { - return &FS{ +func (r *Reader) FSFromDirectory(d squashfslow.Directory, parent FS) FS { + return FS{ d: d, r: r, - parent: parent, + parent: &parent, } } @@ -90,7 +90,7 @@ func (f *FS) Glob(pattern string) (out []string, err error) { } // Opens the file at name. Returns a *File as an fs.File. -func (f *FS) Open(name string) (fs.File, error) { +func (f FS) Open(name string) (fs.File, error) { name = filepath.Clean(name) if !fs.ValidPath(name) { return nil, &fs.PathError{ @@ -142,7 +142,7 @@ func (f *FS) Open(name string) (fs.File, error) { Err: fs.ErrNotExist, } } - d, err := b.ToDir(&f.r.Low) + d, err := b.ToDir(f.r.Low) if err != nil { return nil, err } @@ -151,7 +151,7 @@ func (f *FS) Open(name string) (fs.File, error) { // Returns all DirEntry's for the directory at name. // If name is not a directory, returns an error. -func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) { +func (f FS) ReadDir(name string) ([]fs.DirEntry, error) { name = filepath.Clean(name) if !fs.ValidPath(name) { return nil, &fs.PathError{ @@ -171,7 +171,7 @@ func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) { } // Returns the contents of the file at name. -func (f *FS) ReadFile(name string) (out []byte, err error) { +func (f FS) ReadFile(name string) (out []byte, err error) { name = filepath.Clean(name) if !fs.ValidPath(name) { return nil, &fs.PathError{ @@ -194,7 +194,7 @@ func (f *FS) ReadFile(name string) (out []byte, err error) { } // Returns the fs.FileInfo for the file at name. -func (f *FS) Stat(name string) (fs.FileInfo, error) { +func (f FS) Stat(name string) (fs.FileInfo, error) { name = filepath.Clean(name) if !fs.ValidPath(name) { return nil, &fs.PathError{ @@ -214,7 +214,7 @@ func (f *FS) Stat(name string) (fs.FileInfo, error) { } // Returns the FS at dir -func (f *FS) Sub(dir string) (fs.FS, error) { +func (f FS) Sub(dir string) (fs.FS, error) { dir = filepath.Clean(dir) if !fs.ValidPath(dir) { return nil, &fs.PathError{ @@ -242,26 +242,32 @@ func (f *FS) Sub(dir string) (fs.FS, error) { // Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. // Uses default extraction options. -func (f *FS) Extract(folder string) error { +func (f FS) Extract(folder string) error { return f.File().Extract(folder) } // Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. // Allows setting various extraction options via ExtractionOptions. -func (f *FS) ExtractWithOptions(folder string, op *ExtractionOptions) error { +func (f FS) ExtractWithOptions(folder string, op *ExtractionOptions) error { return f.File().ExtractWithOptions(folder, op) } // Returns the FS as a *File -func (f *FS) File() *File { +func (f FS) File() *File { + if f.parent != nil { + return &File{ + b: f.d.FileBase, + parent: *f.parent, + r: f.r, + } + } return &File{ - b: f.d.FileBase, - parent: f.parent, - r: f.r, + b: f.d.FileBase, + r: f.r, } } -func (f *FS) path() string { +func (f FS) path() string { if f.parent == nil { return f.d.Name } diff --git a/internal/decompress/lz4.go b/internal/decompress/lz4.go index e2f3bdb..eb2af9e 100644 --- a/internal/decompress/lz4.go +++ b/internal/decompress/lz4.go @@ -3,13 +3,28 @@ package decompress import ( "bytes" "io" + "sync" "github.com/pierrec/lz4/v4" ) -type Lz4 struct{} +type Lz4 struct { + pool sync.Pool +} -func (l Lz4) Decompress(data []byte) ([]byte, error) { - rdr := lz4.NewReader(bytes.NewReader(data)) +func NewLz4() *Lz4 { + return &Lz4{ + pool: sync.Pool{ + New: func() any { + return lz4.NewReader(nil) + }, + }, + } +} + +func (l *Lz4) Decompress(data []byte) ([]byte, error) { + rdr := l.pool.Get().(*lz4.Reader) + defer l.pool.Put(rdr) + rdr.Reset(bytes.NewReader(data)) return io.ReadAll(rdr) } diff --git a/internal/decompress/xz.go b/internal/decompress/xz.go index 02191db..4d1881d 100644 --- a/internal/decompress/xz.go +++ b/internal/decompress/xz.go @@ -3,14 +3,30 @@ package decompress import ( "bytes" "io" + "sync" "github.com/therootcompany/xz" ) -type Xz struct{} +type Xz struct { + pool sync.Pool +} -func (x Xz) Decompress(data []byte) ([]byte, error) { - rdr, err := xz.NewReader(bytes.NewReader(data), 0) +func NewXz() *Xz { + return &Xz{ + pool: sync.Pool{ + New: func() any { + rdr, _ := xz.NewReader(nil, 0) + return rdr + }, + }, + } +} + +func (x *Xz) Decompress(data []byte) ([]byte, error) { + rdr := x.pool.Get().(*xz.Reader) + defer x.pool.Put(rdr) + err := rdr.Reset(bytes.NewReader(data)) if err != nil { return nil, err } diff --git a/internal/decompress/zstd.go b/internal/decompress/zstd.go index ff24013..d65afac 100644 --- a/internal/decompress/zstd.go +++ b/internal/decompress/zstd.go @@ -1,19 +1,16 @@ package decompress import ( - "bytes" - "io" - "github.com/klauspost/compress/zstd" ) type Zstd struct{} func (z Zstd) Decompress(data []byte) ([]byte, error) { - rdr, err := zstd.NewReader(bytes.NewReader(data)) + rdr, err := zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderConcurrency(1)) if err != nil { return nil, err } defer rdr.Close() - return io.ReadAll(rdr) + return rdr.DecodeAll(data, nil) } diff --git a/internal/metadata/reader.go b/internal/metadata/reader.go index 5a810d1..ca0bac4 100644 --- a/internal/metadata/reader.go +++ b/internal/metadata/reader.go @@ -14,8 +14,8 @@ type Reader struct { curOffset uint16 } -func NewReader(r io.Reader, d decompress.Decompressor) *Reader { - return &Reader{ +func NewReader(r io.Reader, d decompress.Decompressor) Reader { + return Reader{ r: r, d: d, } @@ -23,14 +23,15 @@ func NewReader(r io.Reader, d decompress.Decompressor) *Reader { func (r *Reader) advance() error { r.curOffset = 0 - var size uint16 - err := binary.Read(r.r, binary.LittleEndian, &size) + dat := make([]byte, 2) + _, err := r.r.Read(dat) if err != nil { return err } + size := binary.LittleEndian.Uint16(dat) realSize := size &^ 0x8000 r.dat = make([]byte, realSize) - err = binary.Read(r.r, binary.LittleEndian, &r.dat) + _, err = r.r.Read(r.dat) if err != nil { return err } diff --git a/low/data/fullreader.go b/low/data/fullreader.go index 1e41eba..8ab50d6 100644 --- a/low/data/fullreader.go +++ b/low/data/fullreader.go @@ -1,15 +1,14 @@ package data import ( - "encoding/binary" "errors" "io" + "io/fs" "math" "runtime" "sync" "github.com/CalebQ42/squashfs/internal/decompress" - "github.com/CalebQ42/squashfs/internal/toreader" ) type FragReaderConstructor func() (io.Reader, error) @@ -23,10 +22,11 @@ type FullReader struct { finalBlockSize uint64 blockSize uint32 goroutineLimit uint16 + closed bool } -func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) *FullReader { - return &FullReader{ +func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) FullReader { + return FullReader{ r: r, d: d, sizes: sizes, @@ -37,6 +37,15 @@ func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor } } +func (r *FullReader) Close() error { + r.closed = true + r.r = nil + r.d = nil + r.frag = nil + r.sizes = nil + return nil +} + func (r *FullReader) AddFrag(frag FragReaderConstructor) { r.frag = frag } @@ -69,7 +78,7 @@ func (r FullReader) process(index uint64, fileOffset uint64, pool *sync.Pool, re return } ret.data = make([]byte, realSize) - ret.err = binary.Read(toreader.NewReader(r.r, int64(r.initialOffset)+int64(fileOffset)), binary.LittleEndian, &ret.data) + _, ret.err = r.r.ReadAt(ret.data, r.initialOffset+int64(fileOffset)) if r.sizes[index] == realSize { ret.data, ret.err = r.d.Decompress(ret.data) } @@ -77,6 +86,9 @@ func (r FullReader) process(index uint64, fileOffset uint64, pool *sync.Pool, re } func (r FullReader) WriteTo(w io.Writer) (int64, error) { + if r.closed { + return 0, fs.ErrClosed + } // if wa, is := w.(io.WriterAt); is { // return r.writeToWriteAt(wa) // } diff --git a/low/data/reader.go b/low/data/reader.go index e556369..898ab09 100644 --- a/low/data/reader.go +++ b/low/data/reader.go @@ -1,8 +1,8 @@ package data import ( - "encoding/binary" "io" + "io/fs" "github.com/CalebQ42/squashfs/internal/decompress" ) @@ -17,10 +17,11 @@ type Reader struct { curIndex uint64 finalBlockSize uint64 blockSize uint32 + closed bool } -func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) *Reader { - return &Reader{ +func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) Reader { + return Reader{ r: r, d: d, sizes: sizes, @@ -54,7 +55,7 @@ func (r *Reader) advance() error { return nil } r.dat = make([]byte, realSize) - err = binary.Read(r.r, binary.LittleEndian, &r.dat) + _, err = r.r.Read(r.dat) if err != nil { return err } @@ -66,6 +67,9 @@ func (r *Reader) advance() error { } func (r *Reader) Read(b []byte) (int, error) { + if r.closed { + return 0, fs.ErrClosed + } curRead := 0 var toRead int for curRead < len(b) { @@ -83,6 +87,9 @@ func (r *Reader) Read(b []byte) (int, error) { } func (r *Reader) Close() error { + r.closed = true + r.r = nil + r.d = nil if r.frag != nil { if l, ok := r.frag.(*io.LimitedReader); ok { if cl, ok := l.R.(io.Closer); ok { @@ -90,6 +97,8 @@ func (r *Reader) Close() error { } } } + r.frag = nil + r.sizes = nil r.dat = nil return nil } diff --git a/low/directory.go b/low/directory.go index 097078b..a83cc16 100644 --- a/low/directory.go +++ b/low/directory.go @@ -18,7 +18,7 @@ type Directory struct { Entries []directory.Entry } -func (r *Reader) directoryFromRef(ref uint64, name string) (Directory, error) { +func (r Reader) directoryFromRef(ref uint64, name string) (Directory, error) { i, err := r.InodeFromRef(ref) if err != nil { return Directory{}, err @@ -44,7 +44,7 @@ func (r *Reader) directoryFromRef(ref uint64, name string) (Directory, error) { if err != nil { return Directory{}, err } - entries, err := directory.ReadDirectory(dirRdr, size) + entries, err := directory.ReadDirectory(&dirRdr, size) if err != nil { return Directory{}, err } @@ -54,7 +54,7 @@ func (r *Reader) directoryFromRef(ref uint64, name string) (Directory, error) { }, nil } -func (d *Directory) Open(r *Reader, path string) (FileBase, error) { +func (d Directory) Open(r Reader, path string) (FileBase, error) { path = filepath.Clean(path) if path == "." || path == "" { return d.FileBase, nil diff --git a/low/file_base.go b/low/file_base.go index a095276..f4e57e3 100644 --- a/low/file_base.go +++ b/low/file_base.go @@ -16,11 +16,11 @@ type FileBase struct { Name string } -func (r *Reader) BaseFromInode(i inode.Inode, name string) FileBase { +func (r Reader) BaseFromInode(i inode.Inode, name string) FileBase { return FileBase{Inode: i, Name: name} } -func (r *Reader) BaseFromEntry(e directory.Entry) (FileBase, error) { +func (r Reader) BaseFromEntry(e directory.Entry) (FileBase, error) { in, err := r.InodeFromEntry(e) if err != nil { return FileBase{}, err @@ -28,7 +28,7 @@ func (r *Reader) BaseFromEntry(e directory.Entry) (FileBase, error) { return FileBase{Inode: in, Name: e.Name}, nil } -func (r *Reader) BaseFromRef(ref uint64, name string) (FileBase, error) { +func (r Reader) BaseFromRef(ref uint64, name string) (FileBase, error) { in, err := r.InodeFromRef(ref) if err != nil { return FileBase{}, err @@ -36,19 +36,19 @@ func (r *Reader) BaseFromRef(ref uint64, name string) (FileBase, error) { return FileBase{Inode: in, Name: name}, nil } -func (b *FileBase) Uid(r *Reader) (uint32, error) { +func (b FileBase) Uid(r *Reader) (uint32, error) { return r.Id(b.Inode.UidInd) } -func (b *FileBase) Gid(r *Reader) (uint32, error) { +func (b FileBase) Gid(r *Reader) (uint32, error) { return r.Id(b.Inode.GidInd) } -func (b *FileBase) IsDir() bool { +func (b FileBase) IsDir() bool { return b.Inode.Type == inode.Dir || b.Inode.Type == inode.EDir } -func (b *FileBase) ToDir(r *Reader) (Directory, error) { +func (b FileBase) ToDir(r Reader) (Directory, error) { var blockStart uint32 var size uint32 var offset uint16 @@ -70,23 +70,23 @@ func (b *FileBase) ToDir(r *Reader) (Directory, error) { if err != nil { return Directory{}, err } - entries, err := directory.ReadDirectory(dirRdr, size) + entries, err := directory.ReadDirectory(&dirRdr, size) if err != nil { return Directory{}, err } return Directory{ - FileBase: *b, + FileBase: b, Entries: entries, }, nil } -func (b *FileBase) IsRegular() bool { +func (b FileBase) IsRegular() bool { return b.Inode.Type == inode.Fil || b.Inode.Type == inode.EFil } -func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, error) { +func (b FileBase) GetRegFileReaders(r Reader) (data.Reader, data.FullReader, error) { if !b.IsRegular() { - return nil, nil, errors.New("not a regular file") + return data.Reader{}, data.FullReader{}, errors.New("not a regular file") } var blockStart uint64 var fragIndex uint32 @@ -113,13 +113,13 @@ func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, } frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize) frag.Read(make([]byte, fragOffset)) - return io.LimitReader(frag, int64(fragSize)), nil + return io.LimitReader(&frag, int64(fragSize)), nil } outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize) if fragIndex != 0xffffffff { f, err := frag() if err != nil { - return nil, nil, err + return data.Reader{}, data.FullReader{}, err } outRdr.AddFrag(f) } @@ -130,9 +130,9 @@ func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, return outRdr, outFull, nil } -func (b *FileBase) GetFullReader(r *Reader) (*data.FullReader, error) { +func (b FileBase) GetFullReader(r *Reader) (data.FullReader, error) { if !b.IsRegular() { - return nil, errors.New("not a regular file") + return data.FullReader{}, errors.New("not a regular file") } var blockStart uint64 var fragIndex uint32 @@ -161,15 +161,15 @@ func (b *FileBase) GetFullReader(r *Reader) (*data.FullReader, error) { } frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize) frag.Read(make([]byte, fragOffset)) - return io.LimitReader(frag, int64(fragSize)), nil + return io.LimitReader(&frag, int64(fragSize)), nil }) } return outFull, nil } -func (b *FileBase) GetReader(r *Reader) (*data.Reader, error) { +func (b FileBase) GetReader(r *Reader) (data.Reader, error) { if !b.IsRegular() { - return nil, errors.New("not a regular file") + return data.Reader{}, errors.New("not a regular file") } var blockStart uint64 var fragIndex uint32 @@ -193,11 +193,11 @@ func (b *FileBase) GetReader(r *Reader) (*data.Reader, error) { if fragIndex != 0xffffffff { ent, err := r.fragEntry(fragIndex) if err != nil { - return nil, err + return data.Reader{}, err } frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize) frag.Read(make([]byte, fragOffset)) - outRdr.AddFrag(io.LimitReader(frag, int64(fragSize))) + outRdr.AddFrag(io.LimitReader(&frag, int64(fragSize))) } return outRdr, nil } diff --git a/low/inode.go b/low/inode.go index 494e428..de52d1d 100644 --- a/low/inode.go +++ b/low/inode.go @@ -7,7 +7,7 @@ import ( "github.com/CalebQ42/squashfs/low/inode" ) -func (r *Reader) InodeFromRef(ref uint64) (inode.Inode, error) { +func (r Reader) InodeFromRef(ref uint64) (inode.Inode, error) { offset, meta := (ref>>16)+r.Superblock.InodeTableStart, ref&0xFFFF rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) defer rdr.Close() @@ -15,12 +15,12 @@ func (r *Reader) InodeFromRef(ref uint64) (inode.Inode, error) { if err != nil { return inode.Inode{}, err } - return inode.Read(rdr, r.Superblock.BlockSize) + return inode.Read(&rdr, r.Superblock.BlockSize) } -func (r *Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) { +func (r Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) { rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d) defer rdr.Close() rdr.Read(make([]byte, e.Offset)) - return inode.Read(rdr, r.Superblock.BlockSize) + return inode.Read(&rdr, r.Superblock.BlockSize) } diff --git a/low/inode/dir.go b/low/inode/dir.go index bffe973..ec5ff7e 100644 --- a/low/inode/dir.go +++ b/low/inode/dir.go @@ -13,7 +13,7 @@ type Directory struct { ParentNum uint32 } -type eDirectoryInit struct { +type EDirectory struct { LinkCount uint32 Size uint32 BlockStart uint32 @@ -21,42 +21,55 @@ type eDirectoryInit struct { IndCount uint16 Offset uint16 XattrInd uint32 -} - -type EDirectory struct { - eDirectoryInit - Indexes []DirectoryIndex -} - -type directoryIndexInit struct { - Ind uint32 - Start uint32 - NameSize uint32 + Indexes []DirectoryIndex } type DirectoryIndex struct { - directoryIndexInit - Name []byte + Ind uint32 + Start uint32 + NameSize uint32 + Name []byte } func ReadDir(r io.Reader) (d Directory, err error) { - err = binary.Read(r, binary.LittleEndian, &d) + dat := make([]byte, 16) + _, err = r.Read(dat) + if err != nil { + return + } + d.BlockStart = binary.LittleEndian.Uint32(dat) + d.LinkCount = binary.LittleEndian.Uint32(dat[4:]) + d.Size = binary.LittleEndian.Uint16(dat[8:]) + d.Offset = binary.LittleEndian.Uint16(dat[10:]) + d.ParentNum = binary.LittleEndian.Uint32(dat[12:]) return } func ReadEDir(r io.Reader) (d EDirectory, err error) { - err = binary.Read(r, binary.LittleEndian, &d.eDirectoryInit) + dat := make([]byte, 24) + _, err = r.Read(dat) if err != nil { return } + d.LinkCount = binary.LittleEndian.Uint32(dat) + d.Size = binary.LittleEndian.Uint32(dat[4:]) + d.BlockStart = binary.LittleEndian.Uint32(dat[8:]) + d.ParentNum = binary.LittleEndian.Uint32(dat[12:]) + d.IndCount = binary.LittleEndian.Uint16(dat[16:]) + d.Offset = binary.LittleEndian.Uint16(dat[18:]) + d.XattrInd = binary.LittleEndian.Uint32(dat[20:]) d.Indexes = make([]DirectoryIndex, d.IndCount) - for i := range d.Indexes { - err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].directoryIndexInit) + for i := range d.IndCount { + dat = make([]byte, 12) + _, err = r.Read(dat) if err != nil { return } + d.Indexes[i].Ind = binary.LittleEndian.Uint32(dat) + d.Indexes[i].Start = binary.LittleEndian.Uint32(dat[4:]) + d.Indexes[i].NameSize = binary.LittleEndian.Uint32(dat[8:]) d.Indexes[i].Name = make([]byte, d.Indexes[i].NameSize+1) - err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].Name) + _, err = r.Read(d.Indexes[i].Name) if err != nil { return } diff --git a/low/inode/file.go b/low/inode/file.go index 1b4d461..2b81014 100644 --- a/low/inode/file.go +++ b/low/inode/file.go @@ -6,15 +6,11 @@ import ( "math" ) -type fileInit struct { +type File struct { BlockStart uint32 FragInd uint32 FragOffset uint32 Size uint32 -} - -type File struct { - fileInit BlockSizes []uint32 } @@ -34,16 +30,28 @@ type EFile struct { } func ReadFile(r io.Reader, blockSize uint32) (f File, err error) { - err = binary.Read(r, binary.LittleEndian, &f.fileInit) + dat := make([]byte, 16) + _, err = r.Read(dat) if err != nil { return } + f.BlockStart = binary.LittleEndian.Uint32(dat) + f.FragInd = binary.LittleEndian.Uint32(dat[4:]) + f.FragOffset = binary.LittleEndian.Uint32(dat[8:]) + f.Size = binary.LittleEndian.Uint32(dat[12:]) toRead := int(math.Floor(float64(f.Size) / float64(blockSize))) if f.FragInd == 0xFFFFFFFF && f.Size%blockSize > 0 { toRead++ } + dat = make([]byte, toRead*4) + _, err = r.Read(dat) + if err != nil { + return + } f.BlockSizes = make([]uint32, toRead) - err = binary.Read(r, binary.LittleEndian, &f.BlockSizes) + for i := range toRead { + f.BlockSizes[i] = binary.LittleEndian.Uint32(dat[i*4:]) + } return } diff --git a/low/reader.go b/low/reader.go index 1f415ea..3cf24b2 100644 --- a/low/reader.go +++ b/low/reader.go @@ -39,21 +39,20 @@ type Reader struct { Superblock superblock } -func NewReader(r io.ReaderAt) (rdr *Reader, err error) { - rdr = new(Reader) +func NewReader(r io.ReaderAt) (rdr Reader, err error) { rdr.r = r err = binary.Read(toreader.NewReader(r, 0), binary.LittleEndian, &rdr.Superblock) if err != nil { - return nil, errors.Join(errors.New("failed to read superblock"), err) + return rdr, errors.Join(errors.New("failed to read superblock"), err) } if !rdr.Superblock.ValidMagic() { - return nil, ErrorMagic + return rdr, ErrorMagic } if !rdr.Superblock.ValidBlockLog() { - return nil, ErrorLog + return rdr, ErrorLog } if !rdr.Superblock.ValidVersion() { - return nil, ErrorVersion + return rdr, ErrorVersion } switch rdr.Superblock.CompType { case ZlibCompression: @@ -61,25 +60,25 @@ func NewReader(r io.ReaderAt) (rdr *Reader, err error) { case LZMACompression: rdr.d, err = decompress.NewLzma() if err != nil { - return nil, err + return rdr, err } case LZOCompression: rdr.d, err = decompress.NewLzo() if err != nil { - return nil, err + return rdr, err } case XZCompression: - rdr.d = decompress.Xz{} + rdr.d = decompress.NewXz() case LZ4Compression: - rdr.d = decompress.Lz4{} + rdr.d = decompress.NewLz4() case ZSTDCompression: rdr.d = decompress.Zstd{} default: - return nil, errors.New("invalid compression type. possible corrupted archive") + return rdr, errors.New("invalid compression type. possible corrupted archive") } rdr.Root, err = rdr.directoryFromRef(rdr.Superblock.RootInodeRef, "") if err != nil { - return nil, errors.Join(errors.New("failed to read root directory"), err) + return rdr, errors.Join(errors.New("failed to read root directory"), err) } return } @@ -105,7 +104,8 @@ func (r *Reader) Id(i uint16) (uint32, error) { var idsToRead uint16 var idsTmp []uint32 var err error - var rdr *metadata.Reader + var rdr metadata.Reader + // We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset) if err != nil { @@ -114,7 +114,7 @@ func (r *Reader) Id(i uint16) (uint32, error) { idsToRead = min(r.Superblock.IdCount-uint16(len(r.idTable)), 2048) idsTmp = make([]uint32, idsToRead) rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) - err = binary.Read(rdr, binary.LittleEndian, &idsTmp) + err = binary.Read(&rdr, binary.LittleEndian, &idsTmp) rdr.Close() if err != nil { return 0, err @@ -145,7 +145,8 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) { var fragsToRead uint32 var fragsTmp []fragEntry var err error - var rdr *metadata.Reader + var rdr metadata.Reader + // We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset) if err != nil { @@ -154,7 +155,7 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) { fragsToRead = min(r.Superblock.FragCount-uint32(len(r.fragTable)), 512) fragsTmp = make([]fragEntry, fragsToRead) rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) - err = binary.Read(rdr, binary.LittleEndian, &fragsTmp) + err = binary.Read(&rdr, binary.LittleEndian, &fragsTmp) rdr.Close() if err != nil { return fragEntry{}, err @@ -188,7 +189,8 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) { var refsToRead uint32 var refsTmp []uint64 var err error - var rdr *metadata.Reader + var rdr metadata.Reader + // We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset) if err != nil { @@ -197,7 +199,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) { refsToRead = min(r.Superblock.InodeCount-uint32(len(r.exportTable)), 1024) refsTmp = make([]uint64, refsToRead) rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) - err = binary.Read(rdr, binary.LittleEndian, &refsTmp) + err = binary.Read(&rdr, binary.LittleEndian, &refsTmp) rdr.Close() if err != nil { return 0, err @@ -207,7 +209,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) { return r.exportTable[i], nil } -func (r *Reader) Inode(i uint32) (inode.Inode, error) { +func (r Reader) Inode(i uint32) (inode.Inode, error) { ref, err := r.inodeRef(i) if err != nil { return inode.Inode{}, err diff --git a/low/reader_test.go b/low/reader_test.go index 6a01c09..403a7ba 100644 --- a/low/reader_test.go +++ b/low/reader_test.go @@ -77,7 +77,7 @@ func TestReader(t *testing.T) { path := filepath.Join(tmpDir, "extractTest") os.RemoveAll(path) os.MkdirAll(path, 0777) - err = extractToDir(rdr, &rdr.Root.FileBase, path) + err = extractToDir(rdr, rdr.Root.FileBase, path) t.Fatal(err) } @@ -101,11 +101,11 @@ func TestSingleFile(t *testing.T) { if err != nil { t.Fatal(err) } - err = extractToDir(rdr, &b, path) + err = extractToDir(rdr, b, path) t.Fatal(err) } -func extractToDir(rdr *Reader, b *FileBase, folder string) error { +func extractToDir(rdr Reader, b FileBase, folder string) error { path := filepath.Join(folder, b.Name) if b.IsDir() { d, err := b.ToDir(rdr) @@ -122,7 +122,7 @@ func extractToDir(rdr *Reader, b *FileBase, folder string) error { if err != nil { return err } - err = extractToDir(rdr, &nestBast, path) + err = extractToDir(rdr, nestBast, path) if err != nil { return err } diff --git a/reader.go b/reader.go index a31a6be..94361f0 100644 --- a/reader.go +++ b/reader.go @@ -9,26 +9,26 @@ import ( ) type Reader struct { - *FS + FS Low squashfslow.Reader } -func NewReader(r io.ReaderAt) (*Reader, error) { +func NewReader(r io.ReaderAt) (Reader, error) { rdr, err := squashfslow.NewReader(r) if err != nil { - return nil, err + return Reader{}, err } - out := &Reader{ - Low: *rdr, + out := Reader{ + Low: rdr, } - out.FS = &FS{ + out.FS = FS{ d: rdr.Root, - r: out, + r: &out, } return out, nil } -func NewReaderAtOffset(r io.ReaderAt, offset int64) (*Reader, error) { +func NewReaderAtOffset(r io.ReaderAt, offset int64) (Reader, error) { return NewReader(toreader.NewOffsetReader(r, offset)) } diff --git a/squashfs.test b/squashfs.test new file mode 100755 index 0000000..6854392 Binary files /dev/null and b/squashfs.test differ diff --git a/squashfs_test.go b/squashfs_test.go index ff3f074..1d086e6 100644 --- a/squashfs_test.go +++ b/squashfs_test.go @@ -17,7 +17,7 @@ import ( const ( squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs" - squashfsName = "LinuxPATest.sfs" + squashfsName = "tensorflow.sqfs" ) func preTest(dir string) (fil *os.File, err error) { @@ -68,6 +68,26 @@ func TestMisc(t *testing.T) { // t.Fatal("UM") } +func BenchmarkExtract(b *testing.B) { + tmpDir := "testing" + fil, err := preTest(tmpDir) + if err != nil { + b.Fatal(err) + } + libPath := filepath.Join(tmpDir, "ExtractLib") + os.RemoveAll(libPath) + op := FastOptions() + op.IgnorePerm = true + rdr, err := NewReader(fil) + if err != nil { + b.Fatal(err) + } + err = rdr.ExtractWithOptions(libPath, op) + if err != nil { + b.Fatal(err) + } +} + func BenchmarkRace(b *testing.B) { tmpDir := "testing" fil, err := preTest(tmpDir) @@ -91,7 +111,7 @@ func BenchmarkRace(b *testing.B) { b.Fatal(err) } libTime = time.Since(start) - cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name()) + cmd := exec.Command("unsquashfs", "-q", "-d", unsquashPath, fil.Name()) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr start = time.Now() @@ -167,7 +187,7 @@ func TestExtractQuick(t *testing.T) { } } -var filePath = "Start.exe" +var filePath = "usr/sbin/add-shell" func TestSingleFile(t *testing.T) { tmpDir := "testing" @@ -175,7 +195,7 @@ func TestSingleFile(t *testing.T) { if err != nil { t.Fatal(err) } - os.Remove(filepath.Join(tmpDir, filePath)) + os.RemoveAll("testing/stuff") rdr, err := NewReader(fil) if err != nil { t.Fatal(err) @@ -186,7 +206,7 @@ func TestSingleFile(t *testing.T) { } op := DefaultOptions() op.Verbose = true - err = f.(*File).ExtractWithOptions("testing", op) + err = f.(*File).ExtractWithOptions("testing/stuff", op) if err != nil { t.Fatal(err) }