Belac/squashfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 55 Wiki Activity

Further performance improvements

Browse Source 
Further removed multiple pointer instances
Re-use decompression readers (except zstd due to bugs)
This commit is contained in:
Caleb Gardner
2025-04-10 11:20:55 -05:00
parent 6b0e9ef2c6
commit 6224c4be41
11 changed files with 66 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+5 -6
View File
@@ -28,15 +28,14 @@ As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://g
## Issues
* Significantly slower then `unsquashfs` when nested images
* Noticably slower then `unsquashfs` for extraction, especially on larger images.
* This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries.
* Not to mention it's written in C
* Times seem to be largely dependent on file tree size and compression type.
* My main testing image (~100MB) using Zstd takes about 5x longer.
* An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 30x longer.
* A Tensorflow docker image (~3.3GB) using Zstd takes about 12x longer.
* My main testing image (~100MB) using Zstd takes ~2x longer.
* An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes ~28x longer.
* A Tensorflow docker image (~3.3GB) using Zstd takes ~3x longer.
Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes about 2x longer.
Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes ~2x longer.
## Recommendations on Usage
internal/decompress/lz4.go
+18 -3
View File
@@ -3,13 +3,28 @@ package decompress
import (
"bytes"
"io"
"sync"
"github.com/pierrec/lz4/v4"
)
type Lz4 struct{}
type Lz4 struct {
pool sync.Pool
}
func (l Lz4) Decompress(data []byte) ([]byte, error) {
rdr := lz4.NewReader(bytes.NewReader(data))
func NewLz4() *Lz4 {
return &Lz4{
pool: sync.Pool{
New: func() any {
return lz4.NewReader(nil)
},
},
}
}
func (l *Lz4) Decompress(data []byte) ([]byte, error) {
rdr := l.pool.Get().(*lz4.Reader)
defer l.pool.Put(rdr)
rdr.Reset(bytes.NewReader(data))
return io.ReadAll(rdr)
}
internal/decompress/xz.go
+19 -3
View File
@@ -3,14 +3,30 @@ package decompress
import (
"bytes"
"io"
"sync"
"github.com/therootcompany/xz"
)
type Xz struct{}
type Xz struct {
pool sync.Pool
}
func (x Xz) Decompress(data []byte) ([]byte, error) {
rdr, err := xz.NewReader(bytes.NewReader(data), 0)
func NewXz() *Xz {
return &Xz{
pool: sync.Pool{
New: func() any {
rdr, _ := xz.NewReader(nil, 0)
return rdr
},
},
}
}
func (x *Xz) Decompress(data []byte) ([]byte, error) {
rdr := x.pool.Get().(*xz.Reader)
defer x.pool.Put(rdr)
err := rdr.Reset(bytes.NewReader(data))
if err != nil {
return nil, err
}
internal/decompress/zstd.go
+2 -5
View File
@@ -1,19 +1,16 @@
package decompress
import (
"bytes"
"io"
"github.com/klauspost/compress/zstd"
)
type Zstd struct{}
func (z Zstd) Decompress(data []byte) ([]byte, error) {
rdr, err := zstd.NewReader(bytes.NewReader(data))
rdr, err := zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderConcurrency(1))
if err != nil {
return nil, err
}
defer rdr.Close()
return io.ReadAll(rdr)
return rdr.DecodeAll(data, nil)
}
internal/metadata/reader.go
+2 -2
View File
@@ -14,8 +14,8 @@ type Reader struct {
curOffset uint16
}
func NewReader(r io.Reader, d decompress.Decompressor) *Reader {
return &Reader{
func NewReader(r io.Reader, d decompress.Decompressor) Reader {
return Reader{
r: r,
d: d,
}
low/directory.go
+1 -1
View File
@@ -44,7 +44,7 @@ func (r Reader) directoryFromRef(ref uint64, name string) (Directory, error) {
if err != nil {
return Directory{}, err
}
entries, err := directory.ReadDirectory(dirRdr, size)
entries, err := directory.ReadDirectory(&dirRdr, size)
if err != nil {
return Directory{}, err
}
low/file_base.go
+1 -1
View File
@@ -70,7 +70,7 @@ func (b FileBase) ToDir(r Reader) (Directory, error) {
if err != nil {
return Directory{}, err
}
entries, err := directory.ReadDirectory(dirRdr, size)
entries, err := directory.ReadDirectory(&dirRdr, size)
if err != nil {
return Directory{}, err
}
low/inode.go
+2 -2
View File
@@ -15,12 +15,12 @@ func (r Reader) InodeFromRef(ref uint64) (inode.Inode, error) {
if err != nil {
return inode.Inode{}, err
}
return inode.Read(rdr, r.Superblock.BlockSize)
return inode.Read(&rdr, r.Superblock.BlockSize)
}
func (r Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) {
rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d)
defer rdr.Close()
rdr.Read(make([]byte, e.Offset))
return inode.Read(rdr, r.Superblock.BlockSize)
return inode.Read(&rdr, r.Superblock.BlockSize)
}
low/reader.go
+11 -8
View File
@@ -68,9 +68,9 @@ func NewReader(r io.ReaderAt) (rdr Reader, err error) {
return rdr, err
}
case XZCompression:
rdr.d = decompress.Xz{}
rdr.d = decompress.NewXz()
case LZ4Compression:
rdr.d = decompress.Lz4{}
rdr.d = decompress.NewLz4()
case ZSTDCompression:
rdr.d = decompress.Zstd{}
default:
@@ -104,7 +104,8 @@ func (r *Reader) Id(i uint16) (uint32, error) {
var idsToRead uint16
var idsTmp []uint32
var err error
var rdr *metadata.Reader
var rdr metadata.Reader
// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset)
if err != nil {
@@ -113,7 +114,7 @@ func (r *Reader) Id(i uint16) (uint32, error) {
idsToRead = min(r.Superblock.IdCount-uint16(len(r.idTable)), 2048)
idsTmp = make([]uint32, idsToRead)
rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
err = binary.Read(rdr, binary.LittleEndian, &idsTmp)
err = binary.Read(&rdr, binary.LittleEndian, &idsTmp)
rdr.Close()
if err != nil {
return 0, err
@@ -144,7 +145,8 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
var fragsToRead uint32
var fragsTmp []fragEntry
var err error
var rdr *metadata.Reader
var rdr metadata.Reader
// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset)
if err != nil {
@@ -153,7 +155,7 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
fragsToRead = min(r.Superblock.FragCount-uint32(len(r.fragTable)), 512)
fragsTmp = make([]fragEntry, fragsToRead)
rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
err = binary.Read(rdr, binary.LittleEndian, &fragsTmp)
err = binary.Read(&rdr, binary.LittleEndian, &fragsTmp)
rdr.Close()
if err != nil {
return fragEntry{}, err
@@ -187,7 +189,8 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
var refsToRead uint32
var refsTmp []uint64
var err error
var rdr *metadata.Reader
var rdr metadata.Reader
// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset)
if err != nil {
@@ -196,7 +199,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
refsToRead = min(r.Superblock.InodeCount-uint32(len(r.exportTable)), 1024)
refsTmp = make([]uint64, refsToRead)
rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
err = binary.Read(rdr, binary.LittleEndian, &refsTmp)
err = binary.Read(&rdr, binary.LittleEndian, &refsTmp)
rdr.Close()
if err != nil {
return 0, err
squashfs.test
BIN
View File
Binary file not shown.
squashfs_test.go
+5 -5
View File
@@ -17,7 +17,7 @@ import (
const (
squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs"
squashfsName = "LinuxPATest.sfs"
squashfsName = "tensorflow.sqfs"
)
func preTest(dir string) (fil *os.File, err error) {
@@ -111,7 +111,7 @@ func BenchmarkRace(b *testing.B) {
b.Fatal(err)
}
libTime = time.Since(start)
cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name())
cmd := exec.Command("unsquashfs", "-q", "-d", unsquashPath, fil.Name())
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
start = time.Now()
@@ -187,7 +187,7 @@ func TestExtractQuick(t *testing.T) {
}
}
var filePath = "Start.exe"
var filePath = "usr/sbin/add-shell"
func TestSingleFile(t *testing.T) {
tmpDir := "testing"
@@ -195,7 +195,7 @@ func TestSingleFile(t *testing.T) {
if err != nil {
t.Fatal(err)
}
os.Remove(filepath.Join(tmpDir, filePath))
os.RemoveAll("testing/stuff")
rdr, err := NewReader(fil)
if err != nil {
t.Fatal(err)
@@ -206,7 +206,7 @@ func TestSingleFile(t *testing.T) {
}
op := DefaultOptions()
op.Verbose = true
err = f.(*File).ExtractWithOptions("testing", op)
err = f.(*File).ExtractWithOptions("testing/stuff", op)
if err != nil {
t.Fatal(err)
}