diff --git a/README.md b/README.md index a633f95..20d13d1 100644 --- a/README.md +++ b/README.md @@ -28,15 +28,14 @@ As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://g ## Issues -* Significantly slower then `unsquashfs` when nested images +* Noticably slower then `unsquashfs` for extraction, especially on larger images. * This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries. - * Not to mention it's written in C * Times seem to be largely dependent on file tree size and compression type. - * My main testing image (~100MB) using Zstd takes about 5x longer. - * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 30x longer. - * A Tensorflow docker image (~3.3GB) using Zstd takes about 12x longer. + * My main testing image (~100MB) using Zstd takes ~2x longer. + * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes ~28x longer. + * A Tensorflow docker image (~3.3GB) using Zstd takes ~3x longer. -Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes about 2x longer. +Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes ~2x longer. ## Recommendations on Usage diff --git a/internal/decompress/lz4.go b/internal/decompress/lz4.go index e2f3bdb..eb2af9e 100644 --- a/internal/decompress/lz4.go +++ b/internal/decompress/lz4.go @@ -3,13 +3,28 @@ package decompress import ( "bytes" "io" + "sync" "github.com/pierrec/lz4/v4" ) -type Lz4 struct{} +type Lz4 struct { + pool sync.Pool +} -func (l Lz4) Decompress(data []byte) ([]byte, error) { - rdr := lz4.NewReader(bytes.NewReader(data)) +func NewLz4() *Lz4 { + return &Lz4{ + pool: sync.Pool{ + New: func() any { + return lz4.NewReader(nil) + }, + }, + } +} + +func (l *Lz4) Decompress(data []byte) ([]byte, error) { + rdr := l.pool.Get().(*lz4.Reader) + defer l.pool.Put(rdr) + rdr.Reset(bytes.NewReader(data)) return io.ReadAll(rdr) } diff --git a/internal/decompress/xz.go b/internal/decompress/xz.go index 02191db..4d1881d 100644 --- a/internal/decompress/xz.go +++ b/internal/decompress/xz.go @@ -3,14 +3,30 @@ package decompress import ( "bytes" "io" + "sync" "github.com/therootcompany/xz" ) -type Xz struct{} +type Xz struct { + pool sync.Pool +} -func (x Xz) Decompress(data []byte) ([]byte, error) { - rdr, err := xz.NewReader(bytes.NewReader(data), 0) +func NewXz() *Xz { + return &Xz{ + pool: sync.Pool{ + New: func() any { + rdr, _ := xz.NewReader(nil, 0) + return rdr + }, + }, + } +} + +func (x *Xz) Decompress(data []byte) ([]byte, error) { + rdr := x.pool.Get().(*xz.Reader) + defer x.pool.Put(rdr) + err := rdr.Reset(bytes.NewReader(data)) if err != nil { return nil, err } diff --git a/internal/decompress/zstd.go b/internal/decompress/zstd.go index ff24013..d65afac 100644 --- a/internal/decompress/zstd.go +++ b/internal/decompress/zstd.go @@ -1,19 +1,16 @@ package decompress import ( - "bytes" - "io" - "github.com/klauspost/compress/zstd" ) type Zstd struct{} func (z Zstd) Decompress(data []byte) ([]byte, error) { - rdr, err := zstd.NewReader(bytes.NewReader(data)) + rdr, err := zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderConcurrency(1)) if err != nil { return nil, err } defer rdr.Close() - return io.ReadAll(rdr) + return rdr.DecodeAll(data, nil) } diff --git a/internal/metadata/reader.go b/internal/metadata/reader.go index e91c0a6..ca0bac4 100644 --- a/internal/metadata/reader.go +++ b/internal/metadata/reader.go @@ -14,8 +14,8 @@ type Reader struct { curOffset uint16 } -func NewReader(r io.Reader, d decompress.Decompressor) *Reader { - return &Reader{ +func NewReader(r io.Reader, d decompress.Decompressor) Reader { + return Reader{ r: r, d: d, } diff --git a/low/directory.go b/low/directory.go index 07f0c80..a83cc16 100644 --- a/low/directory.go +++ b/low/directory.go @@ -44,7 +44,7 @@ func (r Reader) directoryFromRef(ref uint64, name string) (Directory, error) { if err != nil { return Directory{}, err } - entries, err := directory.ReadDirectory(dirRdr, size) + entries, err := directory.ReadDirectory(&dirRdr, size) if err != nil { return Directory{}, err } diff --git a/low/file_base.go b/low/file_base.go index 96aa5ef..f4e57e3 100644 --- a/low/file_base.go +++ b/low/file_base.go @@ -70,7 +70,7 @@ func (b FileBase) ToDir(r Reader) (Directory, error) { if err != nil { return Directory{}, err } - entries, err := directory.ReadDirectory(dirRdr, size) + entries, err := directory.ReadDirectory(&dirRdr, size) if err != nil { return Directory{}, err } diff --git a/low/inode.go b/low/inode.go index 8780feb..de52d1d 100644 --- a/low/inode.go +++ b/low/inode.go @@ -15,12 +15,12 @@ func (r Reader) InodeFromRef(ref uint64) (inode.Inode, error) { if err != nil { return inode.Inode{}, err } - return inode.Read(rdr, r.Superblock.BlockSize) + return inode.Read(&rdr, r.Superblock.BlockSize) } func (r Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) { rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d) defer rdr.Close() rdr.Read(make([]byte, e.Offset)) - return inode.Read(rdr, r.Superblock.BlockSize) + return inode.Read(&rdr, r.Superblock.BlockSize) } diff --git a/low/reader.go b/low/reader.go index c40d277..3cf24b2 100644 --- a/low/reader.go +++ b/low/reader.go @@ -68,9 +68,9 @@ func NewReader(r io.ReaderAt) (rdr Reader, err error) { return rdr, err } case XZCompression: - rdr.d = decompress.Xz{} + rdr.d = decompress.NewXz() case LZ4Compression: - rdr.d = decompress.Lz4{} + rdr.d = decompress.NewLz4() case ZSTDCompression: rdr.d = decompress.Zstd{} default: @@ -104,7 +104,8 @@ func (r *Reader) Id(i uint16) (uint32, error) { var idsToRead uint16 var idsTmp []uint32 var err error - var rdr *metadata.Reader + var rdr metadata.Reader + // We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset) if err != nil { @@ -113,7 +114,7 @@ func (r *Reader) Id(i uint16) (uint32, error) { idsToRead = min(r.Superblock.IdCount-uint16(len(r.idTable)), 2048) idsTmp = make([]uint32, idsToRead) rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) - err = binary.Read(rdr, binary.LittleEndian, &idsTmp) + err = binary.Read(&rdr, binary.LittleEndian, &idsTmp) rdr.Close() if err != nil { return 0, err @@ -144,7 +145,8 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) { var fragsToRead uint32 var fragsTmp []fragEntry var err error - var rdr *metadata.Reader + var rdr metadata.Reader + // We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset) if err != nil { @@ -153,7 +155,7 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) { fragsToRead = min(r.Superblock.FragCount-uint32(len(r.fragTable)), 512) fragsTmp = make([]fragEntry, fragsToRead) rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) - err = binary.Read(rdr, binary.LittleEndian, &fragsTmp) + err = binary.Read(&rdr, binary.LittleEndian, &fragsTmp) rdr.Close() if err != nil { return fragEntry{}, err @@ -187,7 +189,8 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) { var refsToRead uint32 var refsTmp []uint64 var err error - var rdr *metadata.Reader + var rdr metadata.Reader + // We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset) if err != nil { @@ -196,7 +199,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) { refsToRead = min(r.Superblock.InodeCount-uint32(len(r.exportTable)), 1024) refsTmp = make([]uint64, refsToRead) rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) - err = binary.Read(rdr, binary.LittleEndian, &refsTmp) + err = binary.Read(&rdr, binary.LittleEndian, &refsTmp) rdr.Close() if err != nil { return 0, err diff --git a/squashfs.test b/squashfs.test index 22a7559..6854392 100755 Binary files a/squashfs.test and b/squashfs.test differ diff --git a/squashfs_test.go b/squashfs_test.go index 4e7a7a5..1d086e6 100644 --- a/squashfs_test.go +++ b/squashfs_test.go @@ -17,7 +17,7 @@ import ( const ( squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs" - squashfsName = "LinuxPATest.sfs" + squashfsName = "tensorflow.sqfs" ) func preTest(dir string) (fil *os.File, err error) { @@ -111,7 +111,7 @@ func BenchmarkRace(b *testing.B) { b.Fatal(err) } libTime = time.Since(start) - cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name()) + cmd := exec.Command("unsquashfs", "-q", "-d", unsquashPath, fil.Name()) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr start = time.Now() @@ -187,7 +187,7 @@ func TestExtractQuick(t *testing.T) { } } -var filePath = "Start.exe" +var filePath = "usr/sbin/add-shell" func TestSingleFile(t *testing.T) { tmpDir := "testing" @@ -195,7 +195,7 @@ func TestSingleFile(t *testing.T) { if err != nil { t.Fatal(err) } - os.Remove(filepath.Join(tmpDir, filePath)) + os.RemoveAll("testing/stuff") rdr, err := NewReader(fil) if err != nil { t.Fatal(err) @@ -206,7 +206,7 @@ func TestSingleFile(t *testing.T) { } op := DefaultOptions() op.Verbose = true - err = f.(*File).ExtractWithOptions("testing", op) + err = f.(*File).ExtractWithOptions("testing/stuff", op) if err != nil { t.Fatal(err) }