Further performance improvements

Further removed multiple pointer instances
Re-use decompression readers (except zstd due to bugs)

README.md

@@ -28,15 +28,14 @@ As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://g
 
 ## Issues
 
-* Significantly slower then `unsquashfs` when nested images
+* Noticably slower then `unsquashfs` for extraction, especially on larger images.
   * This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries.
-    * Not to mention it's written in C
   * Times seem to be largely dependent on file tree size and compression type.
-    * My main testing image (~100MB) using Zstd takes about 5x longer.
+    * My main testing image (~100MB) using Zstd takes ~2x longer.
-    * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 30x longer.
+    * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes ~28x longer.
-    * A Tensorflow docker image (~3.3GB) using Zstd takes about 12x longer.
+    * A Tensorflow docker image (~3.3GB) using Zstd takes ~3x longer.
 
-Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes about 2x longer.
+Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes ~2x longer.
 
 ## Recommendations on Usage
 

internal/decompress/lz4.go

@@ -3,13 +3,28 @@ package decompress
 import (
 	"bytes"
 	"io"
+	"sync"
 
 	"github.com/pierrec/lz4/v4"
 )
 
-type Lz4 struct{}
+type Lz4 struct {
+	pool sync.Pool
+}
 
-func (l Lz4) Decompress(data []byte) ([]byte, error) {
+func NewLz4() *Lz4 {
-	rdr := lz4.NewReader(bytes.NewReader(data))
+	return &Lz4{
+		pool: sync.Pool{
+			New: func() any {
+				return lz4.NewReader(nil)
+			},
+		},
+	}
+}
+
+func (l *Lz4) Decompress(data []byte) ([]byte, error) {
+	rdr := l.pool.Get().(*lz4.Reader)
+	defer l.pool.Put(rdr)
+	rdr.Reset(bytes.NewReader(data))
 	return io.ReadAll(rdr)
 }

internal/decompress/xz.go

@@ -3,14 +3,30 @@ package decompress
 import (
 	"bytes"
 	"io"
+	"sync"
 
 	"github.com/therootcompany/xz"
 )
 
-type Xz struct{}
+type Xz struct {
+	pool sync.Pool
+}
 
-func (x Xz) Decompress(data []byte) ([]byte, error) {
+func NewXz() *Xz {
-	rdr, err := xz.NewReader(bytes.NewReader(data), 0)
+	return &Xz{
+		pool: sync.Pool{
+			New: func() any {
+				rdr, _ := xz.NewReader(nil, 0)
+				return rdr
+			},
+		},
+	}
+}
+
+func (x *Xz) Decompress(data []byte) ([]byte, error) {
+	rdr := x.pool.Get().(*xz.Reader)
+	defer x.pool.Put(rdr)
+	err := rdr.Reset(bytes.NewReader(data))
 	if err != nil {
 		return nil, err
 	}

internal/decompress/zstd.go

@@ -1,19 +1,16 @@
 package decompress
 
 import (
-	"bytes"
-	"io"
-
 	"github.com/klauspost/compress/zstd"
 )
 
 type Zstd struct{}
 
 func (z Zstd) Decompress(data []byte) ([]byte, error) {
-	rdr, err := zstd.NewReader(bytes.NewReader(data))
+	rdr, err := zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderConcurrency(1))
 	if err != nil {
 		return nil, err
 	}
 	defer rdr.Close()
-	return io.ReadAll(rdr)
+	return rdr.DecodeAll(data, nil)
 }

internal/metadata/reader.go

@@ -14,8 +14,8 @@ type Reader struct {
 	curOffset uint16
 }
 
-func NewReader(r io.Reader, d decompress.Decompressor) *Reader {
+func NewReader(r io.Reader, d decompress.Decompressor) Reader {
-	return &Reader{
+	return Reader{
 		r: r,
 		d: d,
 	}

low/directory.go

@@ -44,7 +44,7 @@ func (r Reader) directoryFromRef(ref uint64, name string) (Directory, error) {
 	if err != nil {
 		return Directory{}, err
 	}
-	entries, err := directory.ReadDirectory(dirRdr, size)
+	entries, err := directory.ReadDirectory(&dirRdr, size)
 	if err != nil {
 		return Directory{}, err
 	}

low/file_base.go

@@ -70,7 +70,7 @@ func (b FileBase) ToDir(r Reader) (Directory, error) {
 	if err != nil {
 		return Directory{}, err
 	}
-	entries, err := directory.ReadDirectory(dirRdr, size)
+	entries, err := directory.ReadDirectory(&dirRdr, size)
 	if err != nil {
 		return Directory{}, err
 	}

low/inode.go

@@ -15,12 +15,12 @@ func (r Reader) InodeFromRef(ref uint64) (inode.Inode, error) {
 	if err != nil {
 		return inode.Inode{}, err
 	}
-	return inode.Read(rdr, r.Superblock.BlockSize)
+	return inode.Read(&rdr, r.Superblock.BlockSize)
 }
 
 func (r Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) {
 	rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d)
 	defer rdr.Close()
 	rdr.Read(make([]byte, e.Offset))
-	return inode.Read(rdr, r.Superblock.BlockSize)
+	return inode.Read(&rdr, r.Superblock.BlockSize)
 }

low/reader.go

@@ -68,9 +68,9 @@ func NewReader(r io.ReaderAt) (rdr Reader, err error) {
 			return rdr, err
 		}
 	case XZCompression:
-		rdr.d = decompress.Xz{}
+		rdr.d = decompress.NewXz()
 	case LZ4Compression:
-		rdr.d = decompress.Lz4{}
+		rdr.d = decompress.NewLz4()
 	case ZSTDCompression:
 		rdr.d = decompress.Zstd{}
 	default:
@@ -104,7 +104,8 @@ func (r *Reader) Id(i uint16) (uint32, error) {
 	var idsToRead uint16
 	var idsTmp []uint32
 	var err error
-	var rdr *metadata.Reader
+	var rdr metadata.Reader
+	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
 	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
 		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset)
 		if err != nil {
@@ -113,7 +114,7 @@ func (r *Reader) Id(i uint16) (uint32, error) {
 		idsToRead = min(r.Superblock.IdCount-uint16(len(r.idTable)), 2048)
 		idsTmp = make([]uint32, idsToRead)
 		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
-		err = binary.Read(rdr, binary.LittleEndian, &idsTmp)
+		err = binary.Read(&rdr, binary.LittleEndian, &idsTmp)
 		rdr.Close()
 		if err != nil {
 			return 0, err
@@ -144,7 +145,8 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
 	var fragsToRead uint32
 	var fragsTmp []fragEntry
 	var err error
-	var rdr *metadata.Reader
+	var rdr metadata.Reader
+	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
 	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
 		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset)
 		if err != nil {
@@ -153,7 +155,7 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
 		fragsToRead = min(r.Superblock.FragCount-uint32(len(r.fragTable)), 512)
 		fragsTmp = make([]fragEntry, fragsToRead)
 		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
-		err = binary.Read(rdr, binary.LittleEndian, &fragsTmp)
+		err = binary.Read(&rdr, binary.LittleEndian, &fragsTmp)
 		rdr.Close()
 		if err != nil {
 			return fragEntry{}, err
@@ -187,7 +189,8 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
 	var refsToRead uint32
 	var refsTmp []uint64
 	var err error
-	var rdr *metadata.Reader
+	var rdr metadata.Reader
+	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
 	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
 		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset)
 		if err != nil {
@@ -196,7 +199,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
 		refsToRead = min(r.Superblock.InodeCount-uint32(len(r.exportTable)), 1024)
 		refsTmp = make([]uint64, refsToRead)
 		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
-		err = binary.Read(rdr, binary.LittleEndian, &refsTmp)
+		err = binary.Read(&rdr, binary.LittleEndian, &refsTmp)
 		rdr.Close()
 		if err != nil {
 			return 0, err

squashfs_test.go

@@ -17,7 +17,7 @@ import (
 
 const (
 	squashfsURL  = "https://darkstorm.tech/files/LinuxPATest.sfs"
-	squashfsName = "LinuxPATest.sfs"
+	squashfsName = "tensorflow.sqfs"
 )
 
 func preTest(dir string) (fil *os.File, err error) {
@@ -111,7 +111,7 @@ func BenchmarkRace(b *testing.B) {
 		b.Fatal(err)
 	}
 	libTime = time.Since(start)
-	cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name())
+	cmd := exec.Command("unsquashfs", "-q", "-d", unsquashPath, fil.Name())
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	start = time.Now()
@@ -187,7 +187,7 @@ func TestExtractQuick(t *testing.T) {
 	}
 }
 
-var filePath = "Start.exe"
+var filePath = "usr/sbin/add-shell"
 
 func TestSingleFile(t *testing.T) {
 	tmpDir := "testing"
@@ -195,7 +195,7 @@ func TestSingleFile(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	os.Remove(filepath.Join(tmpDir, filePath))
+	os.RemoveAll("testing/stuff")
 	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
@@ -206,7 +206,7 @@ func TestSingleFile(t *testing.T) {
 	}
 	op := DefaultOptions()
 	op.Verbose = true
-	err = f.(*File).ExtractWithOptions("testing", op)
+	err = f.(*File).ExtractWithOptions("testing/stuff", op)
 	if err != nil {
 		t.Fatal(err)
 	}