diff --git a/README.md b/README.md index da5e660..73e7767 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,12 @@ As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://g ## Issues -* Significantly slower then `unsquashfs` when extracting folders +* Significantly slower then `unsquashfs` when nested images * This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries. + * Not to mention it's written in C * Times seem to be largely dependent on file tree size and compression type. - * My main testing image (~100MB) using Zstd takes about 6x longer. - * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 32x longer. + * My main testing image (~100MB) using Zstd takes about 5x longer. + * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 30x longer. * A Tensorflow docker image (~3.3GB) using Zstd takes about 12x longer. Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes about 2x longer. diff --git a/file.go b/file.go index 7b2fcdf..4c4f39a 100644 --- a/file.go +++ b/file.go @@ -19,16 +19,16 @@ import ( // File represents a file inside a squashfs archive. type File struct { - b *squashfslow.FileBase full *data.FullReader rdr *data.Reader parent *FS r *Reader + b squashfslow.FileBase dirsRead int } // Creates a new *File from the given *squashfs.Base -func (r *Reader) FileFromBase(b *squashfslow.FileBase, parent *FS) *File { +func (r *Reader) FileFromBase(b squashfslow.FileBase, parent *FS) *File { return &File{ b: b, parent: parent, @@ -40,7 +40,7 @@ func (f *File) FS() (*FS, error) { if !f.IsDir() { return nil, errors.New("not a directory") } - d, err := f.b.ToDir(f.r.Low) + d, err := f.b.ToDir(&f.r.Low) if err != nil { return nil, err } @@ -114,7 +114,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { if !f.IsDir() { return nil, errors.New("file is not a directory") } - d, err := f.b.ToDir(f.r.Low) + d, err := f.b.ToDir(&f.r.Low) if err != nil { return nil, err } @@ -142,7 +142,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { // Returns the file's fs.FileInfo func (f *File) Stat() (fs.FileInfo, error) { - return newFileInfo(f.b.Name, f.b.Inode), nil + return newFileInfo(f.b.Name, &f.b.Inode), nil } // SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string. @@ -173,7 +173,7 @@ func (f *File) WriteTo(w io.Writer) (int64, error) { func (f *File) initializeReaders() error { var err error - f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.Low) + f.rdr, f.full, err = f.b.GetRegFileReaders(&f.r.Low) return err } @@ -218,7 +218,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { } switch f.b.Inode.Type { case inode.Dir, inode.EDir: - d, err := f.b.ToDir(f.r.Low) + d, err := f.b.ToDir(&f.r.Low) if err != nil { if op.Verbose { log.Println("Failed to create squashfs.Directory for", path) @@ -234,7 +234,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { } return errors.Join(errors.New("failed to get base from entry: "+path), err) } - go func(b *squashfslow.FileBase, path string) { + go func(b squashfslow.FileBase, path string) { i := op.manager.Lock() if b.IsDir() { extDir := filepath.Join(path, b.Name) @@ -285,7 +285,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { return errors.Join(errors.New("failed to create file: "+path), err) } defer outFil.Close() - full, err := f.b.GetFullReader(f.r.Low) + full, err := f.b.GetFullReader(&f.r.Low) if err != nil { if op.Verbose { log.Println("Failed to create full reader for", path) @@ -406,7 +406,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { if op.IgnorePerm { return nil } - uid, err := f.b.Uid(f.r.Low) + uid, err := f.b.Uid(&f.r.Low) if err != nil { if op.Verbose { log.Println("Failed to get uid for", path) @@ -414,7 +414,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { } return nil } - gid, err := f.b.Gid(f.r.Low) + gid, err := f.b.Gid(&f.r.Low) if err != nil { if op.Verbose { log.Println("Failed to get gid for", path) diff --git a/file_info.go b/file_info.go index d7390a6..28af45a 100644 --- a/file_info.go +++ b/file_info.go @@ -21,7 +21,7 @@ func (r Reader) newFileInfo(e directory.Entry) (fileInfo, error) { if err != nil { return fileInfo{}, err } - return newFileInfo(e.Name, i), nil + return newFileInfo(e.Name, &i), nil } func newFileInfo(name string, i *inode.Inode) fileInfo { diff --git a/fs.go b/fs.go index 43d9e1e..0048d0b 100644 --- a/fs.go +++ b/fs.go @@ -15,13 +15,13 @@ import ( // FS is a fs.FS representation of a squashfs directory. // Implements fs.GlobFS, fs.ReadDirFS, fs.ReadFileFS, fs.StatFS, and fs.SubFS type FS struct { - d *squashfslow.Directory r *Reader parent *FS + d squashfslow.Directory } // Creates a new *FS from the given squashfs.directory -func (r *Reader) FSFromDirectory(d *squashfslow.Directory, parent *FS) *FS { +func (r *Reader) FSFromDirectory(d squashfslow.Directory, parent *FS) *FS { return &FS{ d: d, r: r, @@ -142,7 +142,7 @@ func (f *FS) Open(name string) (fs.File, error) { Err: fs.ErrNotExist, } } - d, err := b.ToDir(f.r.Low) + d, err := b.ToDir(&f.r.Low) if err != nil { return nil, err } @@ -255,7 +255,7 @@ func (f *FS) ExtractWithOptions(folder string, op *ExtractionOptions) error { // Returns the FS as a *File func (f *FS) File() *File { return &File{ - b: &f.d.FileBase, + b: f.d.FileBase, parent: f.parent, r: f.r, } diff --git a/go.mod b/go.mod index 39a24a7..e6c72d8 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ module github.com/CalebQ42/squashfs -go 1.21.5 +go 1.22.5 require ( - github.com/pierrec/lz4/v4 v4.1.19 - github.com/ulikunitz/xz v0.5.11 - github.com/klauspost/compress v1.17.4 + github.com/klauspost/compress v1.17.9 + github.com/pierrec/lz4/v4 v4.1.21 github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e github.com/therootcompany/xz v1.0.1 + github.com/ulikunitz/xz v0.5.12 ) diff --git a/go.sum b/go.sum index 894206e..164a13f 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,10 @@ -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4= -github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs= github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= -github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= diff --git a/low/directory.go b/low/directory.go index b219d77..097078b 100644 --- a/low/directory.go +++ b/low/directory.go @@ -18,10 +18,10 @@ type Directory struct { Entries []directory.Entry } -func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) { +func (r *Reader) directoryFromRef(ref uint64, name string) (Directory, error) { i, err := r.InodeFromRef(ref) if err != nil { - return nil, err + return Directory{}, err } var blockStart uint32 var size uint32 @@ -36,48 +36,48 @@ func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) { size = i.Data.(inode.EDirectory).Size offset = i.Data.(inode.EDirectory).Offset default: - return nil, errors.New("not a directory") + return Directory{}, errors.New("not a directory") } dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d) defer dirRdr.Close() _, err = dirRdr.Read(make([]byte, offset)) if err != nil { - return nil, err + return Directory{}, err } entries, err := directory.ReadDirectory(dirRdr, size) if err != nil { - return nil, err + return Directory{}, err } - return &Directory{ - FileBase: *r.BaseFromInode(i, name), + return Directory{ + FileBase: r.BaseFromInode(i, name), Entries: entries, }, nil } -func (d *Directory) Open(r *Reader, path string) (*FileBase, error) { +func (d *Directory) Open(r *Reader, path string) (FileBase, error) { path = filepath.Clean(path) if path == "." || path == "" { - return &d.FileBase, nil + return d.FileBase, nil } split := strings.Split(path, "/") i, found := slices.BinarySearchFunc(d.Entries, split[0], func(e directory.Entry, name string) int { return strings.Compare(e.Name, name) }) if !found { - return nil, fs.ErrNotExist + return FileBase{}, fs.ErrNotExist } b, err := r.BaseFromEntry(d.Entries[i]) if err != nil { - return nil, err + return FileBase{}, err } if len(split) == 1 { return b, nil } else if !b.IsDir() { - return nil, fs.ErrNotExist + return FileBase{}, fs.ErrNotExist } dir, err := b.ToDir(r) if err != nil { - return nil, err + return FileBase{}, err } return dir.Open(r, strings.Join(split[1:], "/")) } diff --git a/low/file_base.go b/low/file_base.go index e87bc9e..a095276 100644 --- a/low/file_base.go +++ b/low/file_base.go @@ -12,28 +12,28 @@ import ( ) type FileBase struct { - Inode *inode.Inode + Inode inode.Inode Name string } -func (r *Reader) BaseFromInode(i *inode.Inode, name string) *FileBase { - return &FileBase{Inode: i, Name: name} +func (r *Reader) BaseFromInode(i inode.Inode, name string) FileBase { + return FileBase{Inode: i, Name: name} } -func (r *Reader) BaseFromEntry(e directory.Entry) (*FileBase, error) { +func (r *Reader) BaseFromEntry(e directory.Entry) (FileBase, error) { in, err := r.InodeFromEntry(e) if err != nil { - return nil, err + return FileBase{}, err } - return &FileBase{Inode: in, Name: e.Name}, nil + return FileBase{Inode: in, Name: e.Name}, nil } -func (r *Reader) BaseFromRef(ref uint64, name string) (*FileBase, error) { +func (r *Reader) BaseFromRef(ref uint64, name string) (FileBase, error) { in, err := r.InodeFromRef(ref) if err != nil { - return nil, err + return FileBase{}, err } - return &FileBase{Inode: in, Name: name}, nil + return FileBase{Inode: in, Name: name}, nil } func (b *FileBase) Uid(r *Reader) (uint32, error) { @@ -48,7 +48,7 @@ func (b *FileBase) IsDir() bool { return b.Inode.Type == inode.Dir || b.Inode.Type == inode.EDir } -func (b *FileBase) ToDir(r *Reader) (*Directory, error) { +func (b *FileBase) ToDir(r *Reader) (Directory, error) { var blockStart uint32 var size uint32 var offset uint16 @@ -62,19 +62,19 @@ func (b *FileBase) ToDir(r *Reader) (*Directory, error) { size = b.Inode.Data.(inode.EDirectory).Size offset = b.Inode.Data.(inode.EDirectory).Offset default: - return nil, errors.New("not a directory") + return Directory{}, errors.New("not a directory") } dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d) defer dirRdr.Close() _, err := dirRdr.Read(make([]byte, offset)) if err != nil { - return nil, err + return Directory{}, err } entries, err := directory.ReadDirectory(dirRdr, size) if err != nil { - return nil, err + return Directory{}, err } - return &Directory{ + return Directory{ FileBase: *b, Entries: entries, }, nil diff --git a/low/inode.go b/low/inode.go index a10a2d1..494e428 100644 --- a/low/inode.go +++ b/low/inode.go @@ -7,18 +7,18 @@ import ( "github.com/CalebQ42/squashfs/low/inode" ) -func (r *Reader) InodeFromRef(ref uint64) (*inode.Inode, error) { +func (r *Reader) InodeFromRef(ref uint64) (inode.Inode, error) { offset, meta := (ref>>16)+r.Superblock.InodeTableStart, ref&0xFFFF rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) defer rdr.Close() _, err := rdr.Read(make([]byte, meta)) if err != nil { - return nil, err + return inode.Inode{}, err } return inode.Read(rdr, r.Superblock.BlockSize) } -func (r *Reader) InodeFromEntry(e directory.Entry) (*inode.Inode, error) { +func (r *Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) { rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d) defer rdr.Close() rdr.Read(make([]byte, e.Offset)) diff --git a/low/inode/inode.go b/low/inode/inode.go index 21f0a2e..11976f5 100644 --- a/low/inode/inode.go +++ b/low/inode/inode.go @@ -39,8 +39,7 @@ type Inode struct { Data any } -func Read(r io.Reader, blockSize uint32) (i *Inode, err error) { - i = new(Inode) +func Read(r io.Reader, blockSize uint32) (i Inode, err error) { err = binary.Read(r, binary.LittleEndian, &i.Header) if err != nil { return diff --git a/low/reader.go b/low/reader.go index d6d6bef..ea71895 100644 --- a/low/reader.go +++ b/low/reader.go @@ -32,7 +32,7 @@ var ( type Reader struct { r io.ReaderAt d decompress.Decompressor - Root *Directory + Root Directory fragTable []fragEntry idTable []uint32 exportTable []uint64 @@ -210,10 +210,10 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) { return r.exportTable[i], nil } -func (r *Reader) Inode(i uint32) (*inode.Inode, error) { +func (r *Reader) Inode(i uint32) (inode.Inode, error) { ref, err := r.inodeRef(i) if err != nil { - return nil, err + return inode.Inode{}, err } return r.InodeFromRef(ref) } diff --git a/low/reader_test.go b/low/reader_test.go index 7fe237f..6a4a4d0 100644 --- a/low/reader_test.go +++ b/low/reader_test.go @@ -88,7 +88,7 @@ func TestSingleFile(t *testing.T) { if err != nil { t.Fatal(err) } - err = extractToDir(rdr, b, path) + err = extractToDir(rdr, &b, path) t.Fatal(err) } @@ -103,13 +103,13 @@ func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder strin if err != nil { return err } - var nestBast *squashfslow.FileBase + var nestBast squashfslow.FileBase for _, e := range d.Entries { nestBast, err = rdr.BaseFromEntry(e) if err != nil { return err } - err = extractToDir(rdr, nestBast, path) + err = extractToDir(rdr, &nestBast, path) if err != nil { return err } diff --git a/reader.go b/reader.go index ce499f8..647cd58 100644 --- a/reader.go +++ b/reader.go @@ -9,7 +9,7 @@ import ( type Reader struct { *FS - Low *squashfslow.Reader + Low squashfslow.Reader } func NewReader(r io.ReaderAt) (*Reader, error) { @@ -18,7 +18,7 @@ func NewReader(r io.ReaderAt) (*Reader, error) { return nil, err } out := &Reader{ - Low: rdr, + Low: *rdr, } out.FS = &FS{ d: rdr.Root, diff --git a/squashfs_test.go b/squashfs_test.go index 0ae1658..1479774 100644 --- a/squashfs_test.go +++ b/squashfs_test.go @@ -67,7 +67,7 @@ func TestMisc(t *testing.T) { } _ = rdr // Put testing here - t.Fatal("UM") + // t.Fatal("UM") } func BenchmarkRace(b *testing.B) { @@ -82,6 +82,7 @@ func BenchmarkRace(b *testing.B) { os.RemoveAll(unsquashPath) var libTime, unsquashTime time.Duration op := squashfs.FastOptions() + op.IgnorePerm = true start := time.Now() rdr, err := squashfs.NewReader(fil) if err != nil { @@ -101,9 +102,9 @@ func BenchmarkRace(b *testing.B) { b.Log("Unsquashfs error:", err) } unsquashTime = time.Since(start) - b.Log("Library took:", libTime.Round(time.Millisecond)) - b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond)) - b.Log("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster") + // b.Log("Library took:", libTime.Round(time.Millisecond)) + // b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond)) + b.Fatal("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster") } func TestExtractQuick(t *testing.T) {