Empty io.Reader buffer on EOF

Didn't seem to have any performance advantage

file.go

@@ -179,9 +179,10 @@ func (f *File) initializeReaders() error {
 
 func (f *File) deviceDevices() (maj uint32, min uint32) {
 	var dev uint32
-	if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.Block {
+	switch f.b.Inode.Type {
+	case inode.Char, inode.Block:
 		dev = f.b.Inode.Data.(inode.Device).Dev
-	} else if f.b.Inode.Type == inode.EChar || f.b.Inode.Type == inode.EBlock {
+	case inode.EChar, inode.EBlock:
 		dev = f.b.Inode.Data.(inode.EDevice).Dev
 	}
 	return dev >> 8, dev & 0x000FF
@@ -266,7 +267,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			}(b, path)
 		}
 		var errCache []error
-		for i := 0; i < len(d.Entries); i++ {
+		for range d.Entries {
 			err := <-errChan
 			if err != nil {
 				errCache = append(errCache, err)
@@ -363,11 +364,12 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 		}
 		path = filepath.Join(path, f.b.Name)
 		var typ string
-		if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.EChar {
+		switch f.b.Inode.Type {
+		case inode.Char, inode.EChar:
 			typ = "c"
-		} else if f.b.Inode.Type == inode.Block || f.b.Inode.Type == inode.EBlock {
+		case inode.Block, inode.EBlock:
 			typ = "b"
-		} else { //Fifo IPC
+		default: //Fifo IPC
 			if runtime.GOOS == "darwin" {
 				if op.Verbose {
 					log.Println(f.path(), "ignored. A Fifo file and can't be created on Darwin.")

file_info.go

@@ -26,9 +26,10 @@ func (r Reader) newFileInfo(e directory.Entry) (fileInfo, error) {
 
 func newFileInfo(name string, i *inode.Inode) fileInfo {
 	var size int64
-	if i.Type == inode.Fil {
+	switch i.Type {
+	case inode.Fil:
 		size = int64(i.Data.(inode.File).Size)
-	} else if i.Type == inode.EFil {
+	case inode.EFil:
 		size = int64(i.Data.(inode.EFile).Size)
 	}
 	return fileInfo{

fs.go

@@ -42,7 +42,7 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 		}
 	}
 	split := strings.Split(pattern, "/")
-	for i := 0; i < len(f.d.Entries); i++ {
+	for i := range f.d.Entries {
 		if match, _ := path.Match(split[0], f.d.Entries[i].Name); match {
 			if len(split) == 1 {
 				out = append(out, f.d.Entries[i].Name)
@@ -80,7 +80,7 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 					Err:  err,
 				}
 			}
-			for i := 0; i < len(subGlob); i++ {
+			for i := range subGlob {
 				subGlob[i] = f.d.Name + "/" + subGlob[i]
 			}
 			out = append(out, subGlob...)

go.mod

@@ -1,10 +1,10 @@
 module github.com/CalebQ42/squashfs
 
-go 1.22.5
+go 1.24.0
 
 require (
-	github.com/klauspost/compress v1.17.9
-	github.com/pierrec/lz4/v4 v4.1.21
+	github.com/klauspost/compress v1.18.0
+	github.com/pierrec/lz4/v4 v4.1.22
 	github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e
 	github.com/therootcompany/xz v1.0.1
 	github.com/ulikunitz/xz v0.5.12

go.sum

@@ -1,7 +1,11 @@
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
 github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs=
 github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=

internal/metadata/reader.go

@@ -50,10 +50,7 @@ func (r *Reader) Read(b []byte) (int, error) {
 				return curRead, err
 			}
 		}
-		toRead = len(b) - curRead
-		if toRead > len(r.dat)-int(r.curOffset) {
-			toRead = len(r.dat) - int(r.curOffset)
-		}
+		toRead = min(len(b)-curRead, len(r.dat)-int(r.curOffset))
 		copy(b[curRead:], r.dat[r.curOffset:int(r.curOffset)+toRead])
 		r.curOffset += uint16(toRead)
 		curRead += toRead

low/data/fullreader.go

@@ -18,7 +18,6 @@ type FullReader struct {
 	r              io.ReaderAt
 	d              decompress.Decompressor
 	frag           FragReaderConstructor
-	retPool        *sync.Pool
 	sizes          []uint32
 	initialOffset  int64
 	finalBlockSize uint64
@@ -35,11 +34,6 @@ func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor
 		goroutineLimit: uint16(runtime.NumCPU()),
 		finalBlockSize: finalBlockSize,
 		blockSize:      blockSize,
-		retPool: &sync.Pool{
-			New: func() any {
-				return &retValue{}
-			},
-		},
 	}
 }
 
@@ -48,6 +42,9 @@ func (r *FullReader) AddFrag(frag FragReaderConstructor) {
 }
 
 func (r *FullReader) SetGoroutineLimit(limit uint16) {
+	if limit <= 0 {
+		r.goroutineLimit = 1
+	}
 	r.goroutineLimit = limit
 }
 
@@ -57,8 +54,8 @@ type retValue struct {
 	index uint64
 }
 
-func (r *FullReader) process(index uint64, fileOffset uint64, retChan chan *retValue) {
-	ret := r.retPool.Get().(*retValue)
+func (r FullReader) process(index uint64, fileOffset uint64, pool *sync.Pool, retChan chan *retValue) {
+	ret := pool.Get().(*retValue)
 	ret.index = index
 	realSize := r.sizes[index] &^ (1 << 24)
 	if realSize == 0 {
@@ -79,7 +76,10 @@ func (r *FullReader) process(index uint64, fileOffset uint64, retChan chan *retV
 	retChan <- ret
 }
 
-func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
+func (r FullReader) WriteTo(w io.Writer) (int64, error) {
+	// if wa, is := w.(io.WriterAt); is {
+	// 	return r.writeToWriteAt(wa)
+	// }
 	var curIndex uint64
 	var curOffset uint64
 	var toProcess uint16
@@ -87,14 +87,16 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
 	cache := make(map[uint64]*retValue)
 	var errCache []error
 	retChan := make(chan *retValue, r.goroutineLimit)
+	pool := &sync.Pool{
+		New: func() any {
+			return &retValue{}
+		},
+	}
 	for i := uint64(0); i < uint64(math.Ceil(float64(len(r.sizes))/float64(r.goroutineLimit))); i++ {
-		toProcess = uint16(len(r.sizes)) - (uint16(i) * r.goroutineLimit)
-		if toProcess > r.goroutineLimit {
-			toProcess = r.goroutineLimit
-		}
+		toProcess = min(uint16(len(r.sizes))-(uint16(i)*r.goroutineLimit), r.goroutineLimit)
 		// Start all the goroutines
 		for j := uint16(0); j < toProcess; j++ {
-			go r.process((i*uint64(r.goroutineLimit))+uint64(j), curOffset, retChan)
+			go r.process((i*uint64(r.goroutineLimit))+uint64(j), curOffset, pool, retChan)
 			curOffset += uint64(r.sizes[(i*uint64(r.goroutineLimit))+uint64(j)]) &^ (1 << 24)
 		}
 		// Then consume the results on retChan
@@ -128,7 +130,7 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
 				}
 				continue
 			}
-			r.retPool.Put(res)
+			pool.Put(res)
 			curIndex++
 			// Now we recursively try to clear the cache
 			for len(cache) > 0 {
@@ -146,7 +148,7 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
 					break
 				}
 				delete(cache, curIndex)
-				r.retPool.Put(res)
+				pool.Put(res)
 				curIndex++
 			}
 		}
@@ -172,3 +174,75 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
 	}
 	return wrote, nil
 }
+
+// func (r FullReader) writeToWriteAt(w io.WriterAt) (out int64, outErr error) {
+// 	wait := &sync.WaitGroup{}
+// 	wait.Add(len(r.sizes))
+// 	mgr := routinemanager.NewManager(r.goroutineLimit)
+// 	curOffset := r.initialOffset
+// 	for i := uint64(0); i < uint64(len(r.sizes)); i++ {
+// 		go func(index uint64, fileOffset int64) {
+// 			lckNum := mgr.Lock()
+// 			defer mgr.Unlock(lckNum)
+// 			defer wait.Done()
+// 			realSize := r.sizes[index] &^ (1 << 24)
+// 			if realSize == 0 {
+// 				if index == uint64(len(r.sizes))-1 && r.frag == nil {
+// 					_, err := w.WriteAt([]byte{0}, int64((uint64(r.blockSize)*index)+r.finalBlockSize)-1)
+// 					if err != nil {
+// 						outErr = errors.Join(outErr, err)
+// 						return
+// 					}
+// 					out = max(out, int64((uint64(r.blockSize)*index)+r.finalBlockSize))
+// 				}
+// 				return
+// 			}
+// 			data := make([]byte, realSize)
+// 			err := binary.Read(toreader.NewReader(r.r, int64(fileOffset)), binary.LittleEndian, &data)
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			if r.sizes[index] == realSize {
+// 				data, err = r.d.Decompress(data)
+// 			}
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			_, err = w.WriteAt(data, int64(uint64(r.blockSize)*index))
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			out = max(out, int64(uint64(r.blockSize)*(index+1)))
+// 		}(i, curOffset)
+// 		curOffset += int64(r.sizes[i]) &^ (1 << 24)
+// 	}
+// 	if r.frag != nil {
+// 		wait.Add(1)
+// 		go func() {
+// 			lckNum := mgr.Lock()
+// 			defer mgr.Unlock(lckNum)
+// 			defer wait.Done()
+// 			rdr, err := r.frag()
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			dat, err := io.ReadAll(rdr)
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			_, err = w.WriteAt(dat, int64(int(r.blockSize)*len(r.sizes)))
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			out = int64(int(r.blockSize)*len(r.sizes)) + int64(r.finalBlockSize)
+// 		}()
+// 	}
+// 	wait.Wait()
+// 	return
+// }

low/data/reader.go

@@ -41,6 +41,7 @@ func (r *Reader) advance() error {
 		r.dat, err = io.ReadAll(r.frag)
 		return err
 	} else if r.curIndex >= uint64(len(r.sizes)) {
+		r.dat = []byte{}
 		return io.EOF
 	}
 	realSize := r.sizes[r.curIndex] &^ (1 << 24)
@@ -73,10 +74,7 @@ func (r *Reader) Read(b []byte) (int, error) {
 				return curRead, err
 			}
 		}
-		toRead = len(b) - curRead
-		if toRead > len(r.dat)-r.curOffset {
-			toRead = len(r.dat) - r.curOffset
-		}
+		toRead = min(len(b)-curRead, len(r.dat)-r.curOffset)
 		toRead = copy(b[curRead:], r.dat[r.curOffset:r.curOffset+toRead])
 		r.curOffset += toRead
 		curRead += toRead

low/reader.go

@@ -88,7 +88,7 @@ func (r *Reader) Id(i uint16) (uint32, error) {
 	// Populate the id table as needed
 	var blockNum uint32
 	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
-		blockNum = uint32(math.Ceil(float64(i)/2048)) - 1
+		blockNum = uint32(math.Ceil(float64(i+1)/2048)) - 1
 	} else {
 		blockNum = 0
 	}
@@ -105,10 +105,7 @@ func (r *Reader) Id(i uint16) (uint32, error) {
 		if err != nil {
 			return 0, err
 		}
-		idsToRead = r.Superblock.IdCount - uint16(len(r.idTable))
-		if idsToRead > 2048 {
-			idsToRead = 2048
-		}
+		idsToRead = min(r.Superblock.IdCount-uint16(len(r.idTable)), 2048)
 		idsTmp = make([]uint32, idsToRead)
 		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
 		err = binary.Read(rdr, binary.LittleEndian, &idsTmp)
@@ -131,7 +128,7 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
 	// Populate the fragment table as needed
 	var blockNum uint32
 	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
-		blockNum = uint32(math.Ceil(float64(i)/512)) - 1
+		blockNum = uint32(math.Ceil(float64(i+1)/512)) - 1
 	} else {
 		blockNum = 0
 	}
@@ -148,10 +145,7 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
 		if err != nil {
 			return fragEntry{}, err
 		}
-		fragsToRead = r.Superblock.FragCount - uint32(len(r.fragTable))
-		if fragsToRead > 512 {
-			fragsToRead = 512
-		}
+		fragsToRead = min(r.Superblock.FragCount-uint32(len(r.fragTable)), 512)
 		fragsTmp = make([]fragEntry, fragsToRead)
 		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
 		err = binary.Read(rdr, binary.LittleEndian, &fragsTmp)
@@ -177,7 +171,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
 	// Populate the export table as needed
 	var blockNum uint32
 	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
-		blockNum = uint32(math.Ceil(float64(i)/1024)) - 1
+		blockNum = uint32(math.Ceil(float64(i+1)/1024)) - 1
 	} else {
 		blockNum = 0
 	}
@@ -194,10 +188,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
 		if err != nil {
 			return 0, err
 		}
-		refsToRead = r.Superblock.InodeCount - uint32(len(r.exportTable))
-		if refsToRead > 1024 {
-			refsToRead = 1024
-		}
+		refsToRead = min(r.Superblock.InodeCount-uint32(len(r.exportTable)), 1024)
 		refsTmp = make([]uint64, refsToRead)
 		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
 		err = binary.Read(rdr, binary.LittleEndian, &refsTmp)

low/reader_test.go

@@ -1,4 +1,4 @@
-package squashfslow_test
+package squashfslow
 
 import (
 	"fmt"
@@ -8,13 +8,11 @@ import (
 	"os/exec"
 	"path/filepath"
 	"testing"
-
-	squashfslow "github.com/CalebQ42/squashfs/low"
 )
 
 const (
 	squashfsURL  = "https://darkstorm.tech/files/LinuxPATest.sfs"
-	squashfsName = "LinuxPATest.sfs"
+	squashfsName = "airootfs.sfs"
 )
 
 func preTest(dir string) (fil *os.File, err error) {
@@ -50,6 +48,21 @@ func preTest(dir string) (fil *os.File, err error) {
 	return
 }
 
+func TestMisc(t *testing.T) {
+	tmpDir := "../testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fil.Close()
+	rdr, err := NewReader(fil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(rdr.Superblock.FragCount)
+	t.Fatal(rdr.fragEntry(1233))
+}
+
 func TestReader(t *testing.T) {
 	tmpDir := "../testing"
 	fil, err := preTest(tmpDir)
@@ -57,7 +70,7 @@ func TestReader(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer fil.Close()
-	rdr, err := squashfslow.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -77,7 +90,7 @@ func TestSingleFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer fil.Close()
-	rdr, err := squashfslow.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -92,7 +105,7 @@ func TestSingleFile(t *testing.T) {
 	t.Fatal(err)
 }
 
-func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder string) error {
+func extractToDir(rdr *Reader, b *FileBase, folder string) error {
 	path := filepath.Join(folder, b.Name)
 	if b.IsDir() {
 		d, err := b.ToDir(rdr)
@@ -103,7 +116,7 @@ func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder strin
 		if err != nil {
 			return err
 		}
-		var nestBast squashfslow.FileBase
+		var nestBast FileBase
 		for _, e := range d.Entries {
 			nestBast, err = rdr.BaseFromEntry(e)
 			if err != nil {

squashfs_test.go

@@ -1,4 +1,4 @@
-package squashfs_test
+package squashfs
 
 //Actually proper tests go here.
 
@@ -13,13 +13,11 @@ import (
 	"strconv"
 	"testing"
 	"time"
-
-	"github.com/CalebQ42/squashfs"
 )
 
 const (
 	squashfsURL  = "https://darkstorm.tech/files/LinuxPATest.sfs"
-	squashfsName = "airootfs.sfs"
+	squashfsName = "LinuxPATest.sfs"
 )
 
 func preTest(dir string) (fil *os.File, err error) {
@@ -61,7 +59,7 @@ func TestMisc(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	rdr, err := squashfs.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -81,10 +79,10 @@ func BenchmarkRace(b *testing.B) {
 	os.RemoveAll(libPath)
 	os.RemoveAll(unsquashPath)
 	var libTime, unsquashTime time.Duration
-	op := squashfs.FastOptions()
+	op := FastOptions()
 	op.IgnorePerm = true
 	start := time.Now()
-	rdr, err := squashfs.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -102,15 +100,15 @@ func BenchmarkRace(b *testing.B) {
 		b.Log("Unsquashfs error:", err)
 	}
 	unsquashTime = time.Since(start)
-	// b.Log("Library took:", libTime.Round(time.Millisecond))
-	// b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond))
-	b.Fatal("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster")
+	b.Log("Library took:", libTime.Round(time.Millisecond))
+	b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond))
+	b.Log("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster")
 }
 
 func TestExtractQuick(t *testing.T) {
 	//First, setup everything and extract the archive using the library and unsquashfs
 
-	// tmpDir := b.TempDir()
+	// tmpDir := bTempDir()
 	tmpDir := "testing"
 	fil, err := preTest(tmpDir)
 	if err != nil {
@@ -120,13 +118,13 @@ func TestExtractQuick(t *testing.T) {
 	unsquashPath := filepath.Join(tmpDir, "ExtractSquashfs")
 	os.RemoveAll(libPath)
 	os.RemoveAll(unsquashPath)
-	rdr, err := squashfs.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	os.RemoveAll(filepath.Join(tmpDir, "testLog.txt"))
 	logFil, _ := os.Create(filepath.Join(tmpDir, "testLog.txt"))
-	op := squashfs.DefaultOptions()
+	op := FastOptions()
 	op.Verbose = true
 	op.IgnorePerm = true
 	op.LogOutput = logFil
@@ -169,7 +167,7 @@ func TestExtractQuick(t *testing.T) {
 	}
 }
 
-var filePath = "bin"
+var filePath = "Start.exe"
 
 func TestSingleFile(t *testing.T) {
 	tmpDir := "testing"
@@ -178,7 +176,7 @@ func TestSingleFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	os.Remove(filepath.Join(tmpDir, filePath))
-	rdr, err := squashfs.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -186,7 +184,9 @@ func TestSingleFile(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = f.(*squashfs.File).ExtractWithOptions("testing", &squashfs.ExtractionOptions{Verbose: true})
+	op := DefaultOptions()
+	op.Verbose = true
+	err = f.(*File).ExtractWithOptions("testing", op)
 	if err != nil {
 		t.Fatal(err)
 	}