Fix #44

[PATCH] Change xz module due to licensing concerns

.gitignore

@@ -1,2 +1,7 @@
 testing
 /go-unsquashfs
+squashfs.test
+
+# Memory and CPU pprof profiles
+mem.out
+cpu.out

cmd/go-unsquashfs/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"io/fs"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -12,6 +11,7 @@ import (
 	"time"
 
 	"github.com/CalebQ42/squashfs"
+	squashfslow "github.com/CalebQ42/squashfs/low"
 )
 
 func userName(uid int, numeric bool) string {
@@ -36,32 +36,84 @@ func groupName(gid int, numeric bool) string {
 	return gs
 }
 
-func printEntry(root, path string, d fs.DirEntry, numeric bool) {
+var hardLinks = make(map[uint32]string)
-	fi, _ := d.Info()
+
+func printFile(rdr *squashfs.Reader, path string, f *squashfs.File) {
+	path = filepath.Join(path, f.Low.Name)
+	fi, _ := f.Stat()
 	sfi := fi.(squashfs.FileInfo)
 	owner := fmt.Sprintf("%s/%s",
-		userName(sfi.Uid(), numeric),
+		userName(sfi.Uid(), *numeric),
-		groupName(sfi.Gid(), numeric))
+		groupName(sfi.Gid(), *numeric))
-	link := ""
+	var link string
+	var isHardLink bool
+	if *showHardLinks {
+		link, isHardLink = hardLinks[f.Low.Inode.Num]
+		if !isHardLink {
+			hardLinks[f.Low.Inode.Num] = path
+		}
+	}
+	var size int64
+	if isHardLink {
+		size = 0
+	} else {
+		size = fi.Size()
+	}
 	if sfi.IsSymlink() {
 		link = " -> " + sfi.SymlinkPath()
+	} else if isHardLink {
+		link = " link to " + link
 	}
 	fmt.Printf("%s %s %*d %s %s%s\n",
 		strings.ToLower(fi.Mode().String()),
-		owner, 26-len(owner), fi.Size(),
+		owner, 26-len(owner), size,
 		fi.ModTime().Format("2006-01-02 15:04"),
-		filepath.Join(root, path), link)
+		path, link)
+	if f.IsDir() {
+		fs, _ := f.FS()
+		printDir(rdr, path, fs)
+	}
 }
 
+func printDir(rdr *squashfs.Reader, path string, f squashfs.FS) {
+	var base squashfslow.FileBase
+	var fil squashfs.File
+	var err error
+	for _, e := range f.LowDir.Entries {
+		base, err = rdr.Low.BaseFromEntry(e)
+		if err != nil {
+			panic(err)
+		}
+		fil = rdr.FileFromBase(base, f)
+		printFile(rdr, path, &fil)
+	}
+}
+
+var (
+	verbose       *bool
+	list          *bool
+	long          *bool
+	numeric       *bool
+	offset        *int64
+	ignore        *bool
+	file          *string
+	showHardLinks *bool
+)
+
 func main() {
-	verbose := flag.Bool("v", false, "Verbose")
+	verbose = flag.Bool("v", false, "Verbose")
-	list := flag.Bool("l", false, "List")
+	list = flag.Bool("l", false, "List")
-	long := flag.Bool("ll", false, "List with attributes")
+	long = flag.Bool("ll", false, "List with attributes")
-	numeric := flag.Bool("lln", false, "List with attributes and numeric ids")
+	numeric = flag.Bool("lln", false, "List with attributes and numeric ids")
-	offset := flag.Int64("o", 0, "Offset")
+	showHardLinks = flag.Bool("show-hard-links", false, "When used with ll or lln, shows hard links")
-	ignore := flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755")
+	offset = flag.Int64("o", 0, "Offset")
+	ignore = flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755")
+	file = flag.String("e", "", "File or folder to extract")
 	flag.Parse()
-	if len(flag.Args()) < 2 {
+	if (*list || *long || *numeric) && flag.NArg() < 1 {
+		fmt.Println("Please provide a file name")
+		os.Exit(0)
+	} else if (!*list && !*long && !*numeric) && flag.NArg() < 2 {
 		fmt.Println("Please provide a file name and extraction path")
 		os.Exit(0)
 	}
@@ -73,26 +125,22 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
+	extractFil := r.File()
+	if *file != "" {
+		extractFil, err = r.OpenFile(*file)
+		if err != nil {
+			panic(err)
+		}
+	}
 	if *list || *long || *numeric {
-		root := flag.Arg(1)
+		printFile(&r, "", extractFil)
-		fs.WalkDir(r, ".", func(path string, d fs.DirEntry, err error) error {
-			if err != nil {
-				panic(err)
-			}
-			if *long || *numeric {
-				printEntry(root, path, d, *numeric)
-			} else {
-				fmt.Println(filepath.Join(root, path))
-			}
-			return nil
-		})
 		return
 	}
 	op := squashfs.DefaultOptions()
 	op.Verbose = *verbose
 	op.IgnorePerm = *ignore
 	n := time.Now()
-	err = r.ExtractWithOptions(flag.Arg(1), op)
+	err = extractFil.ExtractWithOptions(flag.Arg(1), op)
 	if err != nil {
 		panic(err)
 	}

extraction_options.go

@@ -4,46 +4,34 @@ import (
 	"io"
 	"io/fs"
 	"runtime"
-
+	"sync"
-	"github.com/CalebQ42/squashfs/internal/routinemanager"
 )
 
 type ExtractionOptions struct {
-	manager            *routinemanager.Manager
+	dispatcher         chan struct{} // Limits the amount of work being done simultaneously.
-	LogOutput          io.Writer   //Where the verbose log should write.
+	fullRdrPool        sync.Pool     // Pool for data.FullReader results.
-	DereferenceSymlink bool        //Replace symlinks with the target file.
+	LogOutput          io.Writer     //Where the verbose log should write.
-	UnbreakSymlink     bool        //Try to make sure symlinks remain unbroken when extracted, without changing the symlink.
+	DereferenceSymlink bool          //Replace symlinks with the target file.
-	Verbose            bool        //Prints extra info to log on an error.
+	UnbreakSymlink     bool          //Try to make sure symlinks remain unbroken when extracted, without changing the symlink.
-	IgnorePerm         bool        //Ignore file's permissions and instead use Perm.
+	Verbose            bool          //Prints extra info to log on an error.
-	Perm               fs.FileMode //Permission to use when IgnorePerm. Defaults to 0777.
+	IgnorePerm         bool          //Ignore file's permissions and instead use Perm.
-	SimultaneousFiles  uint16      //Number of files to process in parallel. Default set based on runtime.NumCPU().
+	Perm               fs.FileMode   //Permission to use when IgnorePerm. Defaults to 0777.
-	ExtractionRoutines uint16      //Number of goroutines to use for each file's extraction. Only applies to regular files. Default set based on runtime.NumCPU().
+	ExtractionRoutines uint16        //The number of threads to use during extraction. Defaults to a number based on runtime.NumCPU().
+	SimultaneousFiles  uint16        //Depreciated: Only use ExtractionRoutines
 }
 
-// The default extraction options.
+// The default extraction options. Uses half of your CPU cores.
 func DefaultOptions() *ExtractionOptions {
-	cores := uint16(runtime.NumCPU() / 2)
-	var files, routines uint16
-	if cores <= 4 {
-		files = 1
-		routines = cores
-	} else {
-		files = cores - 4
-		routines = 4
-	}
 	return &ExtractionOptions{
 		Perm:               0777,
-		SimultaneousFiles:  files,
+		ExtractionRoutines: uint16(runtime.NumCPU() / 2),
-		ExtractionRoutines: routines,
 	}
 }
 
-// Less limited default options. Can run up 2x faster than DefaultOptions.
+// Faster extraction option. Uses all CPU cores.
-// Tends to use all available CPU resources.
 func FastOptions() *ExtractionOptions {
 	return &ExtractionOptions{
 		Perm:               0777,
-		SimultaneousFiles:  uint16(runtime.NumCPU()),
 		ExtractionRoutines: uint16(runtime.NumCPU()),
 	}
 }

file.go

@@ -10,8 +10,8 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
+	"sync"
 
-	"github.com/CalebQ42/squashfs/internal/routinemanager"
 	squashfslow "github.com/CalebQ42/squashfs/low"
 	"github.com/CalebQ42/squashfs/low/data"
 	"github.com/CalebQ42/squashfs/low/inode"
@@ -24,14 +24,14 @@ type File struct {
 	rdrInit  bool
 	parent   FS
 	r        *Reader
-	b        squashfslow.FileBase
+	Low      squashfslow.FileBase
 	dirsRead int
 }
 
 // Creates a new *File from the given *squashfs.Base
 func (r *Reader) FileFromBase(b squashfslow.FileBase, parent FS) File {
 	return File{
-		b:      b,
+		Low:    b,
 		parent: parent,
 		r:      r,
 	}
@@ -41,11 +41,11 @@ func (f File) FS() (FS, error) {
 	if !f.IsDir() {
 		return FS{}, errors.New("not a directory")
 	}
-	d, err := f.b.ToDir(f.r.Low)
+	d, err := f.Low.ToDir(f.r.Low)
 	if err != nil {
 		return FS{}, err
 	}
-	return FS{d: d, parent: &f.parent, r: f.r}, nil
+	return FS{LowDir: d, parent: &f.parent, r: f.r}, nil
 }
 
 // Closes the underlying readers.
@@ -54,6 +54,7 @@ func (f File) FS() (FS, error) {
 func (f *File) Close() error {
 	f.rdr.Close()
 	f.full.Close()
+	f.rdrInit = false
 	return nil
 }
 
@@ -75,21 +76,21 @@ func (f File) GetSymlinkFile() fs.File {
 
 // Returns whether the file is a directory.
 func (f File) IsDir() bool {
-	return f.b.IsDir()
+	return f.Low.IsDir()
 }
 
 // Returns whether the file is a regular file.
 func (f File) IsRegular() bool {
-	return f.b.IsRegular()
+	return f.Low.IsRegular()
 }
 
 // Returns whether the file is a symlink.
 func (f File) IsSymlink() bool {
-	return f.b.Inode.Type == inode.Sym || f.b.Inode.Type == inode.ESym
+	return f.Low.Inode.Type == inode.Sym || f.Low.Inode.Type == inode.ESym
 }
 
 func (f File) Mode() fs.FileMode {
-	return f.b.Inode.Mode()
+	return f.Low.Inode.Mode()
 }
 
 // Read reads the data from the file. Only works if file is a normal file.
@@ -112,7 +113,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
 	if !f.IsDir() {
 		return nil, errors.New("file is not a directory")
 	}
-	d, err := f.b.ToDir(f.r.Low)
+	d, err := f.Low.ToDir(f.r.Low)
 	if err != nil {
 		return nil, err
 	}
@@ -140,24 +141,24 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
 
 // Returns the file's fs.FileInfo
 func (f File) Stat() (fs.FileInfo, error) {
-	uid, err := f.b.Uid(&f.r.Low)
+	uid, err := f.Low.Uid(&f.r.Low)
 	if err != nil {
 		return nil, err
 	}
-	gid, err := f.b.Gid(&f.r.Low)
+	gid, err := f.Low.Gid(&f.r.Low)
 	if err != nil {
 		return nil, err
 	}
-	return newFileInfo(f.b.Name, uid, gid, &f.b.Inode), nil
+	return newFileInfo(f.Low.Name, uid, gid, &f.Low.Inode), nil
 }
 
 // SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string.
 func (f File) SymlinkPath() string {
-	switch f.b.Inode.Type {
+	switch f.Low.Inode.Type {
 	case inode.Sym:
-		return string(f.b.Inode.Data.(inode.Symlink).Target)
+		return string(f.Low.Inode.Data.(inode.Symlink).Target)
 	case inode.ESym:
-		return string(f.b.Inode.Data.(inode.ESymlink).Target)
+		return string(f.Low.Inode.Data.(inode.ESymlink).Target)
 	}
 	return ""
 }
@@ -179,7 +180,7 @@ func (f *File) WriteTo(w io.Writer) (int64, error) {
 
 func (f *File) initializeReaders() error {
 	var err error
-	f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.Low)
+	f.rdr, f.full, err = f.Low.GetRegFileReaders(f.r.Low)
 	if err == nil {
 		f.rdrInit = true
 	} else {
@@ -191,20 +192,20 @@ func (f *File) initializeReaders() error {
 
 func (f File) deviceDevices() (maj uint32, min uint32) {
 	var dev uint32
-	switch f.b.Inode.Type {
+	switch f.Low.Inode.Type {
 	case inode.Char, inode.Block:
-		dev = f.b.Inode.Data.(inode.Device).Dev
+		dev = f.Low.Inode.Data.(inode.Device).Dev
 	case inode.EChar, inode.EBlock:
-		dev = f.b.Inode.Data.(inode.EDevice).Dev
+		dev = f.Low.Inode.Data.(inode.EDevice).Dev
 	}
 	return dev >> 8, dev & 0x000FF
 }
 
 func (f File) path() string {
-	if f.parent.d.Name == "" {
+	if f.parent.LowDir.Name == "" {
-		return f.b.Name
+		return f.Low.Name
 	}
-	return filepath.Join(f.parent.path(), f.b.Name)
+	return filepath.Join(f.parent.path(), f.Low.Name)
 }
 
 // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
@@ -216,8 +217,16 @@ func (f File) Extract(folder string) error {
 // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
 // Allows setting various extraction options via ExtractionOptions.
 func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
-	if op.manager == nil {
+	if op.dispatcher == nil {
-		op.manager = routinemanager.NewManager(op.SimultaneousFiles)
+		op.fullRdrPool = sync.Pool{
+			New: func() any {
+				return &data.BlockResults{}
+			},
+		}
+		op.dispatcher = make(chan struct{}, op.ExtractionRoutines)
+		for range op.ExtractionRoutines {
+			op.dispatcher <- struct{}{}
+		}
 		if op.LogOutput != nil {
 			log.SetOutput(op.LogOutput)
 		}
@@ -229,13 +238,15 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			return err
 		}
 	}
-	switch f.b.Inode.Type {
+	switch f.Low.Inode.Type {
 	case inode.Dir, inode.EDir:
-		d, err := f.b.ToDir(f.r.Low)
+		<-op.dispatcher
+		d, err := f.Low.ToDir(f.r.Low)
 		if err != nil {
 			if op.Verbose {
 				log.Println("Failed to create squashfs.Directory for", path)
 			}
+			op.dispatcher <- struct{}{}
 			return errors.Join(errors.New("failed to create squashfs.Directory: "+path), err)
 		}
 		errChan := make(chan error, len(d.Entries))
@@ -248,19 +259,21 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 				return errors.Join(errors.New("failed to get base from entry: "+path), err)
 			}
 			go func(b squashfslow.FileBase, path string) {
-				i := op.manager.Lock()
 				if b.IsDir() {
+					<-op.dispatcher
 					extDir := filepath.Join(path, b.Name)
 					err = os.Mkdir(extDir, 0777)
-					op.manager.Unlock(i)
 					if err != nil {
 						if op.Verbose {
 							log.Println("Failed to create directory", path)
 						}
+						op.dispatcher <- struct{}{}
 						errChan <- errors.Join(errors.New("failed to create directory: "+path), err)
 						return
 					}
-					err = f.r.FileFromBase(b, f.r.FSFromDirectory(d, f.parent)).ExtractWithOptions(extDir, op)
+					fil := f.r.FileFromBase(b, f.r.FSFromDirectory(d, f.parent))
+					op.dispatcher <- struct{}{}
+					err = fil.ExtractWithOptions(extDir, op)
 					if err != nil {
 						if op.Verbose {
 							log.Println("Failed to extract directory", path)
@@ -272,12 +285,12 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 				} else {
 					fil := f.r.FileFromBase(b, f.r.FSFromDirectory(d, f.parent))
 					err = fil.ExtractWithOptions(path, op)
-					op.manager.Unlock(i)
 					fil.Close()
 					errChan <- err
 				}
 			}(b, path)
 		}
+		op.dispatcher <- struct{}{}
 		var errCache []error
 		for range d.Entries {
 			err := <-errChan
@@ -289,23 +302,28 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			return errors.Join(errors.New("failed to extract folder: "+path), errors.Join(errCache...))
 		}
 	case inode.Fil, inode.EFil:
-		path = filepath.Join(path, f.b.Name)
+		<-op.dispatcher
+		path = filepath.Join(path, f.Low.Name)
 		outFil, err := os.Create(path)
 		if err != nil {
 			if op.Verbose {
 				log.Println("Failed to create file", path)
 			}
+			op.dispatcher <- struct{}{}
 			return errors.Join(errors.New("failed to create file: "+path), err)
 		}
 		defer outFil.Close()
-		full, err := f.b.GetFullReader(&f.r.Low)
+		full, err := f.Low.GetFullReader(&f.r.Low)
+		defer full.Close()
 		if err != nil {
 			if op.Verbose {
 				log.Println("Failed to create full reader for", path)
 			}
+			op.dispatcher <- struct{}{}
 			return errors.Join(errors.New("failed to create full reader: "+path), err)
 		}
-		full.SetGoroutineLimit(op.ExtractionRoutines)
+		full.SetDispatcherPool(op.dispatcher, &op.fullRdrPool)
+		op.dispatcher <- struct{}{}
 		_, err = full.WriteTo(outFil)
 		if err != nil {
 			if op.Verbose {
@@ -314,6 +332,8 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			return errors.Join(errors.New("failed to write file: "+path), err)
 		}
 	case inode.Sym, inode.ESym:
+		<-op.dispatcher
+		defer func() { op.dispatcher <- struct{}{} }()
 		symPath := f.SymlinkPath()
 		if op.DereferenceSymlink {
 			filTmp := f.GetSymlinkFile()
@@ -324,11 +344,11 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 				return errors.New("failed to get symlink's file")
 			}
 			fil := filTmp.(*File)
-			fil.b.Name = f.b.Name
+			fil.Low.Name = f.Low.Name
 			err := fil.ExtractWithOptions(path, op)
 			if err != nil {
 				if op.Verbose {
-					log.Println("Failed to extract symlink's file:", filepath.Join(path, f.b.Name))
+					log.Println("Failed to extract symlink's file:", filepath.Join(path, f.Low.Name))
 				}
 				return errors.Join(errors.New("failed to extract symlink's file: "+path), err)
 			}
@@ -351,7 +371,7 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 					return errors.Join(errors.New("failed to extract symlink's file: "+extractLoc), err)
 				}
 			}
-			path = filepath.Join(path, f.b.Name)
+			path = filepath.Join(path, f.Low.Name)
 			err := os.Symlink(f.SymlinkPath(), path)
 			if err != nil {
 				if op.Verbose {
@@ -361,6 +381,8 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			}
 		}
 	case inode.Char, inode.EChar, inode.Block, inode.EBlock, inode.Fifo, inode.EFifo:
+		<-op.dispatcher
+		defer func() { op.dispatcher <- struct{}{} }()
 		if runtime.GOOS == "windows" {
 			if op.Verbose {
 				log.Println(f.path(), "ignored. A device link and can't be created on Windows.")
@@ -374,9 +396,9 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			}
 			return errors.Join(errors.New("mknot command not found"), err)
 		}
-		path = filepath.Join(path, f.b.Name)
+		path = filepath.Join(path, f.Low.Name)
 		var typ string
-		switch f.b.Inode.Type {
+		switch f.Low.Inode.Type {
 		case inode.Char, inode.EChar:
 			typ = "c"
 		case inode.Block, inode.EBlock:
@@ -412,7 +434,7 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 		}
 		return nil
 	default:
-		return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.b.Inode.Type)))
+		return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.Low.Inode.Type)))
 	}
 	if op.Verbose {
 		log.Println(f.path(), "extracted to", path)
@@ -420,7 +442,7 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 	if op.IgnorePerm {
 		return nil
 	}
-	uid, err := f.b.Uid(&f.r.Low)
+	uid, err := f.Low.Uid(&f.r.Low)
 	if err != nil {
 		if op.Verbose {
 			log.Println("Failed to get uid for", path)
@@ -428,7 +450,7 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 		}
 		return nil
 	}
-	gid, err := f.b.Gid(&f.r.Low)
+	gid, err := f.Low.Gid(&f.r.Low)
 	if err != nil {
 		if op.Verbose {
 			log.Println("Failed to get gid for", path)

fs.go

@@ -17,13 +17,13 @@ import (
 type FS struct {
 	r      *Reader
 	parent *FS
-	d      squashfslow.Directory
+	LowDir squashfslow.Directory
 }
 
 // Creates a new *FS from the given squashfs.directory
 func (r *Reader) FSFromDirectory(d squashfslow.Directory, parent FS) FS {
 	return FS{
-		d:      d,
+		LowDir: d,
 		r:      r,
 		parent: &parent,
 	}
@@ -42,10 +42,10 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 		}
 	}
 	split := strings.Split(pattern, "/")
-	for i := range f.d.Entries {
+	for i := range f.LowDir.Entries {
-		if match, _ := path.Match(split[0], f.d.Entries[i].Name); match {
+		if match, _ := path.Match(split[0], f.LowDir.Entries[i].Name); match {
 			if len(split) == 1 {
-				out = append(out, f.d.Entries[i].Name)
+				out = append(out, f.LowDir.Entries[i].Name)
 				continue
 			}
 			sub, err := f.Sub(split[0])
@@ -81,7 +81,7 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 				}
 			}
 			for i := range subGlob {
-				subGlob[i] = f.d.Name + "/" + subGlob[i]
+				subGlob[i] = f.LowDir.Name + "/" + subGlob[i]
 			}
 			out = append(out, subGlob...)
 		}
@@ -91,6 +91,10 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 
 // Opens the file at name. Returns a *File as an fs.File.
 func (f FS) Open(name string) (fs.File, error) {
+	return f.OpenFile(name)
+}
+
+func (f FS) OpenFile(name string) (*File, error) {
 	name = filepath.Clean(name)
 	if !fs.ValidPath(name) {
 		return nil, &fs.PathError{
@@ -111,10 +115,10 @@ func (f FS) Open(name string) (fs.File, error) {
 				Err:  fs.ErrNotExist,
 			}
 		} else {
-			return f.parent.Open(strings.Join(split[1:], "/"))
+			return f.parent.OpenFile(strings.Join(split[1:], "/"))
 		}
 	}
-	i, found := slices.BinarySearchFunc(f.d.Entries, split[0], func(e directory.Entry, name string) int {
+	i, found := slices.BinarySearchFunc(f.LowDir.Entries, split[0], func(e directory.Entry, name string) int {
 		return strings.Compare(e.Name, name)
 	})
 	if !found {
@@ -124,13 +128,13 @@ func (f FS) Open(name string) (fs.File, error) {
 			Err:  fs.ErrNotExist,
 		}
 	}
-	b, err := f.r.Low.BaseFromEntry(f.d.Entries[i])
+	b, err := f.r.Low.BaseFromEntry(f.LowDir.Entries[i])
 	if err != nil {
 		return nil, err
 	}
 	if len(split) == 1 {
 		return &File{
-			b:      b,
+			Low:    b,
 			r:      f.r,
 			parent: f,
 		}, nil
@@ -146,7 +150,7 @@ func (f FS) Open(name string) (fs.File, error) {
 	if err != nil {
 		return nil, err
 	}
-	return f.r.FSFromDirectory(d, f).Open(strings.Join(split[1:], "/"))
+	return f.r.FSFromDirectory(d, f).OpenFile(strings.Join(split[1:], "/"))
 }
 
 // Returns all DirEntry's for the directory at name.
@@ -256,20 +260,20 @@ func (f FS) ExtractWithOptions(folder string, op *ExtractionOptions) error {
 func (f FS) File() *File {
 	if f.parent != nil {
 		return &File{
-			b:      f.d.FileBase,
+			Low:    f.LowDir.FileBase,
 			parent: *f.parent,
 			r:      f.r,
 		}
 	}
 	return &File{
-		b: f.d.FileBase,
+		Low: f.LowDir.FileBase,
-		r: f.r,
+		r:   f.r,
 	}
 }
 
 func (f FS) path() string {
 	if f.parent == nil {
-		return f.d.Name
+		return f.LowDir.Name
 	}
-	return filepath.Join(f.parent.path(), f.d.Name)
+	return filepath.Join(f.parent.path(), f.LowDir.Name)
 }

go.mod

@@ -4,8 +4,8 @@ go 1.24.0
 
 require (
 	github.com/klauspost/compress v1.18.0
+	github.com/mikelolasagasti/xz v1.0.1
 	github.com/pierrec/lz4/v4 v4.1.22
 	github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e
-	github.com/therootcompany/xz v1.0.1
 	github.com/ulikunitz/xz v0.5.12
 )

go.sum

@@ -1,10 +1,10 @@
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0=
+github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc=
 github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
 github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs=
 github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M=
-github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
-github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
 github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
 github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=

internal/decompress/xz.go

@@ -5,7 +5,7 @@ import (
 	"io"
 	"sync"
 
-	"github.com/therootcompany/xz"
+	"github.com/mikelolasagasti/xz"
 )
 
 type Xz struct {

internal/decompress/zlib.go

@@ -2,17 +2,31 @@ package decompress
 
 import (
 	"bytes"
-	"compress/zlib"
 	"io"
+	"sync"
+
+	"github.com/klauspost/compress/zlib"
 )
 
-type Zlib struct{}
+type Zlib struct {
+	pool sync.Pool
+}
 
-func (z Zlib) Decompress(data []byte) ([]byte, error) {
+func NewZlib() *Zlib {
-	rdr, err := zlib.NewReader(bytes.NewReader(data))
+	return &Zlib{}
+}
+
+func (z *Zlib) Decompress(data []byte) ([]byte, error) {
+	rdr := z.pool.Get()
+	defer z.pool.Put(rdr)
+	var err error
+	if rdr == nil {
+		rdr, err = zlib.NewReader(bytes.NewReader(data))
+	} else {
+		err = rdr.(zlib.Resetter).Reset(bytes.NewReader(data), nil)
+	}
 	if err != nil {
 		return nil, err
 	}
-	defer rdr.Close()
+	return io.ReadAll(rdr.(io.ReadCloser))
-	return io.ReadAll(rdr)
 }

internal/decompress/zstd.go

@@ -4,13 +4,21 @@ import (
 	"github.com/klauspost/compress/zstd"
 )
 
-type Zstd struct{}
+type Zstd struct {
+	rdr *zstd.Decoder
+}
+
+func NewZstd() Zstd {
+	rdr, _ := zstd.NewReader(nil, zstd.WithDecoderLowmem(true))
+	return Zstd{
+		rdr: rdr,
+	}
+}
 
 func (z Zstd) Decompress(data []byte) ([]byte, error) {
-	rdr, err := zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderConcurrency(1))
+	dat, err := z.rdr.DecodeAll(data, nil)
 	if err != nil {
 		return nil, err
 	}
-	defer rdr.Close()
+	return dat, err
-	return rdr.DecodeAll(data, nil)
 }

low/README.md

@@ -1,3 +1,5 @@
 # Lower-Level Squashfs
 
 This library is a lower level version of the main [squashfs](https://github.com/CalebQ42/squashfs) library that doesn't try to be easy to use and exposes a lot of information that is not necesary for must use cases.
+
+I will try to keep the API stable, but it is not guarenteed.

low/caching_paged_reader_test.go

@@ -0,0 +1,127 @@
+package squashfslow
+
+// TODO: Make work
+// func requireNoError(t *testing.T, err error) {
+// 	t.Helper()
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// }
+
+// func assertEqual(t *testing.T, want int, got int) {
+// 	t.Helper()
+// 	if want != got {
+// 		t.Errorf("want %d, got %d", want, got)
+// 	}
+// }
+
+// func assertLength(t *testing.T, want int, slice []int) {
+// 	t.Helper()
+// 	if len(slice) != want {
+// 		t.Errorf("want len %d, got %d", want, len(slice))
+// 	}
+// }
+
+// func assertErrorIs(t *testing.T, err error, wantErr error) {
+// 	t.Helper()
+// 	if err == nil {
+// 		t.Errorf("want %s, got nil", wantErr)
+// 		return
+// 	}
+// 	if !errors.Is(err, wantErr) {
+// 		t.Errorf("want %s, got %v", wantErr, err)
+// 	}
+// }
+
+// func TestCachingPagedReader(t *testing.T) {
+// 	// Mock readBlocks function
+// 	mockReadNMore := func(startBlock, numItems int) ([]int, error) {
+// 		if startBlock < 0 {
+// 			return nil, errors.New("invalid block start")
+// 		}
+// 		var result []int
+// 		for i := 0; i < numItems; i++ {
+// 			result = append(result, startBlock*512+i)
+// 		}
+// 		return result, nil
+// 	}
+
+// 	t.Run("ValidRequestWithinFirstBlock", func(t *testing.T) {
+// 		tab := NewTable[int]()
+// 		currentItems := make([]int, 0)
+// 		item, err := readPagedItems(300, 512, &currentItems, 2048, mockReadNMore)
+// 		requireNoError(t, err)
+// 		assertEqual(t, 300, item)
+// 		assertLength(t, 512, currentItems) // Ensure one block is read
+// 	})
+
+// 	t.Run("ValidRequestAcrossMultipleBlocks", func(t *testing.T) {
+// 		currentItems := make([]int, 0)
+// 		item, err := readPagedItems(600, 512, &currentItems, 2048, mockReadNMore)
+// 		requireNoError(t, err)
+// 		assertEqual(t, 600, item)
+// 		assertLength(t, 1024, currentItems)
+// 	})
+
+// 	t.Run("SequentialRequestsWithinBlocks", func(t *testing.T) {
+// 		currentItems := make([]int, 0)
+// 		// First request
+// 		item, err := readPagedItems(300, 512, &currentItems, 2048, mockReadNMore)
+// 		requireNoError(t, err)
+// 		assertEqual(t, 300, item)
+
+// 		// Second request in the same block
+// 		item, err = readPagedItems(400, 512, &currentItems, 2048, mockReadNMore)
+// 		requireNoError(t, err)
+// 		assertEqual(t, 400, item)
+// 		assertLength(t, 512, currentItems)
+// 	})
+
+// 	t.Run("RequestExactBlockBoundary", func(t *testing.T) {
+// 		currentItems := make([]int, 0)
+// 		item, err := readPagedItems(511, 512, &currentItems, 2048, mockReadNMore)
+// 		requireNoError(t, err)
+// 		assertEqual(t, 511, item)
+// 		assertLength(t, 512, currentItems)
+
+// 		// Request the next block's first item
+// 		item, err = readPagedItems(512, 512, &currentItems, 2048, mockReadNMore)
+// 		requireNoError(t, err)
+// 		assertEqual(t, 512, item)
+// 		assertLength(t, 1024, currentItems)
+// 	})
+
+// 	t.Run("OutOfBoundsRequest", func(t *testing.T) {
+// 		currentItems := make([]int, 0)
+// 		_, err := readPagedItems(2048, 512, &currentItems, 2048, mockReadNMore)
+// 		assertErrorIs(t, err, errOutOfBounds)
+// 	})
+
+// 	t.Run("RequestBeyondReadBlocks", func(t *testing.T) {
+// 		readFail := errors.New("failed to read block")
+// 		failingReadBlocks := func(startBlock, numBlocks int) ([]int, error) {
+// 			if startBlock > 1 {
+// 				return nil, readFail
+// 			}
+// 			var result []int
+// 			for i := 0; i < numBlocks*512; i++ {
+// 				result = append(result, startBlock*512+i)
+// 			}
+// 			return result, nil
+// 		}
+
+// 		currentItems := make([]int, 0)
+// 		_, err := readPagedItems(1024, 512, &currentItems, 2048, failingReadBlocks)
+// 		assertErrorIs(t, err, readFail)
+// 	})
+
+// 	t.Run("partial last page", func(t *testing.T) {
+// 		currentItems := make([]int, 0)
+
+// 		// Request the next block's first item
+// 		item, err := readPagedItems(512, 512, &currentItems, 612, mockReadNMore)
+// 		requireNoError(t, err)
+// 		assertEqual(t, 512, item)
+// 		assertLength(t, 612, currentItems)
+// 	})
+// }

low/data/fullreader.go

@@ -3,258 +3,230 @@ package data
 import (
 	"errors"
 	"io"
-	"io/fs"
-	"math"
 	"runtime"
 	"sync"
 
 	"github.com/CalebQ42/squashfs/internal/decompress"
 )
 
-type FragReaderConstructor func() (io.Reader, error)
-
 type FullReader struct {
-	r              io.ReaderAt
+	fileSize     uint64
-	d              decompress.Decompressor
+	blockSize    uint32
-	frag           FragReaderConstructor
+	dispatcher   chan struct{}
-	sizes          []uint32
+	pool         *sync.Pool
-	initialOffset  int64
+	rdr          io.ReaderAt
-	finalBlockSize uint64
+	decomp       decompress.Decompressor
-	blockSize      uint32
+	sizes        []uint32
-	goroutineLimit uint16
+	blockOffsets []uint64
-	closed         bool
+	fragDat      []byte
 }
 
-func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) FullReader {
+func NewFullReader(rdr io.ReaderAt, decomp decompress.Decompressor, blockSize uint32, size uint64, start uint64, sizes []uint32) FullReader {
-	return FullReader{
+	out := FullReader{
-		r:              r,
+		fileSize:  size,
-		d:              d,
+		blockSize: blockSize,
-		sizes:          sizes,
+		rdr:       rdr,
-		initialOffset:  initialOffset,
+		decomp:    decomp,
-		goroutineLimit: uint16(runtime.NumCPU()),
+		sizes:     sizes,
-		finalBlockSize: finalBlockSize,
-		blockSize:      blockSize,
 	}
+	out.blockOffsets = make([]uint64, len(sizes))
+	curOffset := start
+	for i := range sizes {
+		out.blockOffsets[i] = curOffset
+		curOffset += uint64(sizes[i]) &^ (1 << 24)
+	}
+	return out
 }
 
-func (r *FullReader) Close() error {
+func (f *FullReader) Close() error {
-	r.closed = true
+	f.fragDat = nil
-	r.r = nil
+	f.sizes = nil
-	r.d = nil
+	f.blockOffsets = nil
-	r.frag = nil
-	r.sizes = nil
 	return nil
 }
 
-func (r *FullReader) AddFrag(frag FragReaderConstructor) {
+func (f *FullReader) AddFragData(blockStart uint64, blockSize uint32, offset uint32) error {
-	r.frag = frag
+	realSize := blockSize &^ (1 << 24)
-}
+	dat := make([]byte, realSize)
-
+	_, err := f.rdr.ReadAt(dat, int64(blockStart))
-func (r *FullReader) SetGoroutineLimit(limit uint16) {
+	if err != nil {
-	if limit <= 0 {
+		return err
-		r.goroutineLimit = 1
 	}
-	r.goroutineLimit = limit
+	if blockSize == realSize {
+		dat, err = f.decomp.Decompress(dat)
+		if err != nil {
+			return err
+		}
+	}
+	f.fragDat = make([]byte, f.fileSize%uint64(f.blockSize))
+	copy(f.fragDat, dat[offset:])
+	dat = nil
+	return nil
 }
 
-type retValue struct {
+func (f *FullReader) SetDispatcherPool(dispatcher chan struct{}, pool *sync.Pool) {
-	err   error
+	f.dispatcher = dispatcher
-	data  []byte
+	f.pool = pool
-	index uint64
 }
 
-func (r FullReader) process(index uint64, fileOffset uint64, pool *sync.Pool, retChan chan *retValue) {
+// The number of blocks, including the fragment block if present
-	ret := pool.Get().(*retValue)
+func (f FullReader) BlockNum() uint32 {
-	ret.index = index
+	out := len(f.sizes)
-	realSize := r.sizes[index] &^ (1 << 24)
+	if f.fragDat != nil {
+		out++
+	}
+	return uint32(out)
+}
+
+// Returns the data block at the given index
+func (f FullReader) Block(i uint32) ([]byte, error) {
+	if i == uint32(len(f.sizes)) && f.fragDat != nil {
+		return f.fragDat, nil
+	}
+	if i >= uint32(len(f.sizes)) {
+		return nil, errors.New("invalid block index")
+	}
+	realSize := f.sizes[i] &^ (1 << 24)
 	if realSize == 0 {
-		if index == uint64(len(r.sizes))-1 && r.frag == nil {
+		if i == uint32(len(f.sizes)-1) && f.fragDat == nil {
-			ret.data = make([]byte, r.finalBlockSize)
+			return make([]byte, f.fileSize%uint64(f.blockSize)), nil
-		} else {
-			ret.data = make([]byte, r.blockSize)
 		}
-		ret.err = nil
+		return make([]byte, f.blockSize), nil
-		retChan <- ret
-		return
 	}
-	ret.data = make([]byte, realSize)
+	dat := make([]byte, realSize)
-	_, ret.err = r.r.ReadAt(ret.data, r.initialOffset+int64(fileOffset))
+	_, err := f.rdr.ReadAt(dat, int64(f.blockOffsets[i]))
-	if r.sizes[index] == realSize {
+	if err != nil {
-		ret.data, ret.err = r.d.Decompress(ret.data)
+		return nil, err
 	}
-	retChan <- ret
+	if realSize == f.sizes[i] {
+		dat, err = f.decomp.Decompress(dat)
+	}
+	return dat, err
 }
 
-func (r FullReader) WriteTo(w io.Writer) (int64, error) {
+func (f FullReader) blockFromPool(i uint32) *BlockResults {
-	if r.closed {
+	out := f.pool.Get().(*BlockResults)
-		return 0, fs.ErrClosed
+	out.idx = i
+	out.err = nil
+	if i == uint32(len(f.sizes)) && f.fragDat != nil {
+		out.block = f.fragDat
+		return out
 	}
-	// if wa, is := w.(io.WriterAt); is {
+	if i >= uint32(len(f.sizes)) {
-	// 	return r.writeToWriteAt(wa)
+		out.err = errors.New("invalid block index")
-	// }
+		return out
-	var curIndex uint64
-	var curOffset uint64
-	var toProcess uint16
-	var wrote int64
-	cache := make(map[uint64]*retValue)
-	var errCache []error
-	retChan := make(chan *retValue, r.goroutineLimit)
-	pool := &sync.Pool{
-		New: func() any {
-			return &retValue{}
-		},
 	}
-	for i := uint64(0); i < uint64(math.Ceil(float64(len(r.sizes))/float64(r.goroutineLimit))); i++ {
+	realSize := f.sizes[i] &^ (1 << 24)
-		toProcess = min(uint16(len(r.sizes))-(uint16(i)*r.goroutineLimit), r.goroutineLimit)
+	if realSize == 0 {
-		// Start all the goroutines
+		if i == uint32(len(f.sizes)-1) && f.fragDat == nil {
-		for j := uint16(0); j < toProcess; j++ {
+			out.block = make([]byte, f.fileSize%uint64(f.blockSize))
-			go r.process((i*uint64(r.goroutineLimit))+uint64(j), curOffset, pool, retChan)
+			return out
-			curOffset += uint64(r.sizes[(i*uint64(r.goroutineLimit))+uint64(j)]) &^ (1 << 24)
 		}
-		// Then consume the results on retChan
+		out.block = make([]byte, f.blockSize)
-		for j := uint16(0); j < toProcess; j++ {
+	}
-			res := <-retChan
+	out.block = make([]byte, realSize)
-			// If there's an error, we don't care about the results.
+	_, out.err = f.rdr.ReadAt(out.block, int64(f.blockOffsets[i]))
-			if res.err != nil {
+	if out.err != nil {
-				errCache = append(errCache, res.err)
+		return out
-				if len(cache) > 0 {
+	}
-					clear(cache)
+	if realSize == f.sizes[i] {
-				}
+		out.block, out.err = f.decomp.Decompress(out.block)
-				continue
+	}
+	return out
+}
+
+type BlockResults struct {
+	idx   uint32
+	block []byte
+	err   error
+}
+
+func (f FullReader) WriteTo(w io.Writer) (wrote int64, err error) {
+	if f.dispatcher == nil {
+		f.dispatcher = make(chan struct{}, runtime.NumCPU())
+		for range runtime.NumCPU() {
+			f.dispatcher <- struct{}{}
+		}
+	}
+	if f.pool == nil {
+		f.pool = &sync.Pool{
+			New: func() any {
+				return &BlockResults{}
+			},
+		}
+	}
+	open := true
+	resChan := make(chan *BlockResults, len(f.dispatcher))
+	var results map[uint32]*BlockResults
+	if _, is := w.(io.WriterAt); !is {
+		results = make(map[uint32]*BlockResults)
+	}
+	for i := range f.BlockNum() {
+		go func(idx uint32) {
+			<-f.dispatcher
+			defer func() { f.dispatcher <- struct{}{} }()
+			if !open {
+				resChan <- f.pool.Get().(*BlockResults)
+				return
 			}
-			// If there has been an error previously, we don't care about the results.
+			resChan <- f.blockFromPool(idx)
-			// We still want to wait for all the goroutines to prevent resources being wasted.
+		}(i)
-			if len(errCache) > 0 {
+	}
-				continue
+	out := int64(0)
-			}
+	errOut := make([]error, 0)
-			// If we don't need the data yet, we cache it and move on
+	for i := uint32(0); i < f.BlockNum(); {
-			if res.index != curIndex {
+		res := <-resChan
-				cache[res.index] = res
+		defer f.pool.Put(res)
-				continue
+		if res.err != nil {
-			}
+			open = false
-			// If we do need the data, we write it
+			errOut = append(errOut, res.err)
-			wr, err := w.Write(res.data)
+		}
-			wrote += int64(wr)
+		if len(errOut) > 0 {
+			i++
+			continue
+		}
+		if wa, is := w.(io.WriterAt); is {
+			_, err := wa.WriteAt(res.block, int64(res.idx)*int64(f.blockSize))
 			if err != nil {
-				errCache = append(errCache, err)
+				errOut = append(errOut, err)
-				if len(cache) > 0 {
+			} else {
-					clear(cache)
+				out = max(out, int64(res.idx)*int64(f.blockSize)+int64(len(res.block)))
-				}
-				continue
 			}
-			pool.Put(res)
+			i++
-			curIndex++
+			continue
-			// Now we recursively try to clear the cache
+		}
-			for len(cache) > 0 {
+		var err error
-				res, ok := cache[curIndex]
+		if res.idx == i {
-				if !ok {
+			_, err = w.Write(res.block)
-					break
+			if err != nil {
-				}
+				errOut = append(errOut, err)
-				wr, err := w.Write(res.data)
+			} else {
-				wrote += int64(wr)
+				out = max(out, int64(res.idx)*int64(f.blockSize)+int64(len(res.block)))
+			}
+			i++
+		} else {
+			results[res.idx] = res
+		}
+		var has bool
+		for {
+			res, has = results[i]
+			if has {
+				_, err = w.Write(res.block)
 				if err != nil {
-					errCache = append(errCache, err)
+					errOut = append(errOut, err)
-					if len(cache) > 0 {
+				} else {
-						clear(cache)
+					out = max(out, int64(res.idx)*int64(f.blockSize)+int64(len(res.block)))
-					}
-					break
 				}
-				delete(cache, curIndex)
+				i++
-				pool.Put(res)
+				delete(results, i)
-				curIndex++
+				f.pool.Put(res)
+			} else {
+				break
 			}
 		}
-		if len(errCache) > 0 {
-			return wrote, errors.Join(errCache...)
-		}
 	}
-	if r.frag != nil {
+	if len(errOut) > 0 {
-		rdr, err := r.frag()
+		return out, errors.Join(errOut...)
-		if err != nil {
-			return wrote, err
-		}
-		wr, err := io.Copy(w, rdr)
-		wrote += wr
-		if l, ok := rdr.(*io.LimitedReader); ok {
-			if cl, ok := l.R.(io.Closer); ok {
-				cl.Close()
-			}
-		}
-		if err != nil {
-			return wrote, err
-		}
 	}
-	return wrote, nil
+	return out, nil
 }
-
-// func (r FullReader) writeToWriteAt(w io.WriterAt) (out int64, outErr error) {
-// 	wait := &sync.WaitGroup{}
-// 	wait.Add(len(r.sizes))
-// 	mgr := routinemanager.NewManager(r.goroutineLimit)
-// 	curOffset := r.initialOffset
-// 	for i := uint64(0); i < uint64(len(r.sizes)); i++ {
-// 		go func(index uint64, fileOffset int64) {
-// 			lckNum := mgr.Lock()
-// 			defer mgr.Unlock(lckNum)
-// 			defer wait.Done()
-// 			realSize := r.sizes[index] &^ (1 << 24)
-// 			if realSize == 0 {
-// 				if index == uint64(len(r.sizes))-1 && r.frag == nil {
-// 					_, err := w.WriteAt([]byte{0}, int64((uint64(r.blockSize)*index)+r.finalBlockSize)-1)
-// 					if err != nil {
-// 						outErr = errors.Join(outErr, err)
-// 						return
-// 					}
-// 					out = max(out, int64((uint64(r.blockSize)*index)+r.finalBlockSize))
-// 				}
-// 				return
-// 			}
-// 			data := make([]byte, realSize)
-// 			err := binary.Read(toreader.NewReader(r.r, int64(fileOffset)), binary.LittleEndian, &data)
-// 			if err != nil {
-// 				outErr = errors.Join(outErr, err)
-// 				return
-// 			}
-// 			if r.sizes[index] == realSize {
-// 				data, err = r.d.Decompress(data)
-// 			}
-// 			if err != nil {
-// 				outErr = errors.Join(outErr, err)
-// 				return
-// 			}
-// 			_, err = w.WriteAt(data, int64(uint64(r.blockSize)*index))
-// 			if err != nil {
-// 				outErr = errors.Join(outErr, err)
-// 				return
-// 			}
-// 			out = max(out, int64(uint64(r.blockSize)*(index+1)))
-// 		}(i, curOffset)
-// 		curOffset += int64(r.sizes[i]) &^ (1 << 24)
-// 	}
-// 	if r.frag != nil {
-// 		wait.Add(1)
-// 		go func() {
-// 			lckNum := mgr.Lock()
-// 			defer mgr.Unlock(lckNum)
-// 			defer wait.Done()
-// 			rdr, err := r.frag()
-// 			if err != nil {
-// 				outErr = errors.Join(outErr, err)
-// 				return
-// 			}
-// 			dat, err := io.ReadAll(rdr)
-// 			if err != nil {
-// 				outErr = errors.Join(outErr, err)
-// 				return
-// 			}
-// 			_, err = w.WriteAt(dat, int64(int(r.blockSize)*len(r.sizes)))
-// 			if err != nil {
-// 				outErr = errors.Join(outErr, err)
-// 				return
-// 			}
-// 			out = int64(int(r.blockSize)*len(r.sizes)) + int64(r.finalBlockSize)
-// 		}()
-// 	}
-// 	wait.Wait()
-// 	return
-// }

low/data/reader.go

@@ -1,104 +1,62 @@
 package data
 
-import (
+import "io"
-	"io"
-	"io/fs"
-
-	"github.com/CalebQ42/squashfs/internal/decompress"
-)
 
 type Reader struct {
-	r              io.Reader
+	f         *FullReader
-	d              decompress.Decompressor
+	curBlock  []byte
-	frag           io.Reader
+	nextIdx   uint32
-	sizes          []uint32
+	curOffset uint32
-	dat            []byte
-	curOffset      int
-	curIndex       uint64
-	finalBlockSize uint64
-	blockSize      uint32
-	closed         bool
 }
 
-func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) Reader {
+func NewReader(f *FullReader) (Reader, error) {
-	return Reader{
+	dat, err := f.Block(0)
-		r:              r,
+	if err != nil {
-		d:              d,
+		return Reader{}, err
-		sizes:          sizes,
-		finalBlockSize: finalBlockSize,
-		blockSize:      blockSize,
 	}
+	return Reader{
+		f:         f,
+		curBlock:  dat,
+		nextIdx:   1,
+		curOffset: 0,
+	}, nil
 }
 
-func (r *Reader) AddFrag(fragRdr io.Reader) {
+func (d *Reader) Close() error {
-	r.frag = fragRdr
+	d.curBlock = nil
+	return nil
 }
 
-func (r *Reader) advance() error {
+func (d *Reader) advanceBlock() error {
-	r.curOffset = 0
+	if d.nextIdx >= d.f.BlockNum() {
-	defer func() { r.curIndex++ }()
+		d.curBlock = nil
-	var err error
-	if r.curIndex == uint64(len(r.sizes)) && r.frag != nil {
-		r.dat, err = io.ReadAll(r.frag)
-		return err
-	} else if r.curIndex >= uint64(len(r.sizes)) {
-		r.dat = []byte{}
 		return io.EOF
 	}
-	realSize := r.sizes[r.curIndex] &^ (1 << 24)
+	var err error
-	if realSize == 0 {
+	d.curBlock, err = d.f.Block(d.nextIdx)
-		if r.curIndex == uint64(len(r.sizes))-1 && r.frag == nil {
-			r.dat = make([]byte, r.finalBlockSize)
-		} else {
-			r.dat = make([]byte, r.blockSize)
-		}
-		return nil
-	}
-	r.dat = make([]byte, realSize)
-	_, err = r.r.Read(r.dat)
 	if err != nil {
 		return err
 	}
-	if r.sizes[r.curIndex] != realSize {
+	d.nextIdx++
-		return nil
+	d.curOffset = 0
-	}
-	r.dat, err = r.d.Decompress(r.dat)
-	return err
-}
-
-func (r *Reader) Read(b []byte) (int, error) {
-	if r.closed {
-		return 0, fs.ErrClosed
-	}
-	curRead := 0
-	var toRead int
-	for curRead < len(b) {
-		if r.curOffset >= len(r.dat) {
-			if err := r.advance(); err != nil {
-				return curRead, err
-			}
-		}
-		toRead = min(len(b)-curRead, len(r.dat)-r.curOffset)
-		toRead = copy(b[curRead:], r.dat[r.curOffset:r.curOffset+toRead])
-		r.curOffset += toRead
-		curRead += toRead
-	}
-	return curRead, nil
-}
-
-func (r *Reader) Close() error {
-	r.closed = true
-	r.r = nil
-	r.d = nil
-	if r.frag != nil {
-		if l, ok := r.frag.(*io.LimitedReader); ok {
-			if cl, ok := l.R.(io.Closer); ok {
-				cl.Close()
-			}
-		}
-	}
-	r.frag = nil
-	r.sizes = nil
-	r.dat = nil
 	return nil
 }
+
+func (d *Reader) Read(buf []byte) (int, error) {
+	totRed := 0
+	toRead := 0
+	var err error
+	for totRed < len(buf) {
+		if int(d.curOffset) >= len(d.curBlock) {
+			err = d.advanceBlock()
+			if err != nil {
+				return totRed, err
+			}
+		}
+		toRead = min(len(d.curBlock)-int(d.curOffset), len(buf)-totRed)
+		copy(buf[totRed:], d.curBlock[d.curOffset:d.curOffset+uint32(toRead)])
+		totRed += toRead
+		d.curOffset += uint32(toRead)
+	}
+	return totRed, nil
+}

low/directory/directory.go

@@ -11,12 +11,45 @@ type header struct {
 	Num        uint32
 }
 
-type decEntry struct {
+func readHeader(r io.Reader) (h header, err error) {
+	dat := make([]byte, 12)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	h.Count = binary.LittleEndian.Uint32(dat)
+	h.BlockStart = binary.LittleEndian.Uint32(dat[4:])
+	h.Num = binary.LittleEndian.Uint32(dat[8:])
+	return
+}
+
+type dirEntry struct {
 	Offset    uint16
 	NumOffset int16
 	InodeType uint16
 	NameSize  uint16
-	// Name []byte (not decoded along with decEntry)
+	Name      []byte
+}
+
+func readEntry(r io.Reader) (e dirEntry, err error) {
+	dat := make([]byte, 8)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	e.Offset = binary.LittleEndian.Uint16(dat)
+	_, err = binary.Decode(dat[2:], binary.LittleEndian, &e.NumOffset)
+	if err != nil {
+		return
+	}
+	e.InodeType = binary.LittleEndian.Uint16(dat[4:])
+	e.NameSize = binary.LittleEndian.Uint16(dat[6:])
+	e.Name = make([]byte, e.NameSize+1)
+	_, err = r.Read(e.Name)
+	if err != nil {
+		return
+	}
+	return
 }
 
 type Entry struct {
@@ -31,20 +64,15 @@ func ReadDirectory(r io.Reader, size uint32) (out []Entry, err error) {
 	size -= 3
 	var curRead uint32
 	var h header
-	var de decEntry
+	var de dirEntry
 	for curRead < size {
-		err = binary.Read(r, binary.LittleEndian, &h)
+		h, err = readHeader(r)
 		if err != nil {
 			return
 		}
 		curRead += 12
 		for i := uint32(0); i < h.Count+1 && curRead < size; i++ {
-			err = binary.Read(r, binary.LittleEndian, &de)
+			de, err = readEntry(r)
-			if err != nil {
-				return
-			}
-			nameTmp := make([]byte, de.NameSize+1)
-			err = binary.Read(r, binary.LittleEndian, &nameTmp)
 			if err != nil {
 				return
 			}
@@ -52,7 +80,7 @@ func ReadDirectory(r io.Reader, size uint32) (out []Entry, err error) {
 			out = append(out, Entry{
 				BlockStart: h.BlockStart,
 				Offset:     de.Offset,
-				Name:       string(nameTmp),
+				Name:       string(de.Name),
 				InodeType:  de.InodeType,
 				Num:        h.Num + uint32(de.NumOffset),
 			})

low/file_base.go

@@ -2,7 +2,6 @@ package squashfslow
 
 import (
 	"errors"
-	"io"
 
 	"github.com/CalebQ42/squashfs/internal/metadata"
 	"github.com/CalebQ42/squashfs/internal/toreader"
@@ -84,6 +83,8 @@ func (b FileBase) IsRegular() bool {
 	return b.Inode.Type == inode.Fil || b.Inode.Type == inode.EFil
 }
 
+// Returns a regular file's readers. They are linked, so the data.Reader calls to the data.FullReader.
+// Aka: closing the FullReader breaks the Reader
 func (b FileBase) GetRegFileReaders(r Reader) (data.Reader, data.FullReader, error) {
 	if !b.IsRegular() {
 		return data.Reader{}, data.FullReader{}, errors.New("not a regular file")
@@ -91,41 +92,32 @@ func (b FileBase) GetRegFileReaders(r Reader) (data.Reader, data.FullReader, err
 	var blockStart uint64
 	var fragIndex uint32
 	var fragOffset uint32
-	var fragSize uint64
 	var sizes []uint32
+	var fileSize uint64
 	if b.Inode.Type == inode.Fil {
 		blockStart = uint64(b.Inode.Data.(inode.File).BlockStart)
 		fragIndex = b.Inode.Data.(inode.File).FragInd
 		fragOffset = b.Inode.Data.(inode.File).FragOffset
 		sizes = b.Inode.Data.(inode.File).BlockSizes
-		fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize)
+		fileSize = uint64(b.Inode.Data.(inode.File).Size)
 	} else {
 		blockStart = b.Inode.Data.(inode.EFile).BlockStart
 		fragIndex = b.Inode.Data.(inode.EFile).FragInd
 		fragOffset = b.Inode.Data.(inode.EFile).FragOffset
 		sizes = b.Inode.Data.(inode.EFile).BlockSizes
-		fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize)
+		fileSize = b.Inode.Data.(inode.EFile).Size
 	}
-	frag := func() (io.Reader, error) {
+	outFull := data.NewFullReader(r.r, r.d, r.Superblock.BlockSize, fileSize, blockStart, sizes)
+	if fragIndex != 0xFFFFFFFF {
 		ent, err := r.fragEntry(fragIndex)
-		if err != nil {
-			return nil, err
-		}
-		frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
-		frag.Read(make([]byte, fragOffset))
-		return io.LimitReader(&frag, int64(fragSize)), nil
-	}
-	outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize)
-	if fragIndex != 0xffffffff {
-		f, err := frag()
 		if err != nil {
 			return data.Reader{}, data.FullReader{}, err
 		}
-		outRdr.AddFrag(f)
+		outFull.AddFragData(ent.Start, ent.Size, fragOffset)
 	}
-	outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes, fragSize, r.Superblock.BlockSize)
+	outRdr, err := data.NewReader(&outFull)
-	if fragIndex != 0xffffffff {
+	if err != nil {
-		outFull.AddFrag(frag)
+		return data.Reader{}, data.FullReader{}, err
 	}
 	return outRdr, outFull, nil
 }
@@ -137,67 +129,28 @@ func (b FileBase) GetFullReader(r *Reader) (data.FullReader, error) {
 	var blockStart uint64
 	var fragIndex uint32
 	var fragOffset uint32
-	var fragSize uint64
 	var sizes []uint32
+	var fileSize uint64
 	if b.Inode.Type == inode.Fil {
 		blockStart = uint64(b.Inode.Data.(inode.File).BlockStart)
 		fragIndex = b.Inode.Data.(inode.File).FragInd
 		fragOffset = b.Inode.Data.(inode.File).FragOffset
 		sizes = b.Inode.Data.(inode.File).BlockSizes
-		fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize)
+		fileSize = uint64(b.Inode.Data.(inode.File).Size)
 	} else {
 		blockStart = b.Inode.Data.(inode.EFile).BlockStart
 		fragIndex = b.Inode.Data.(inode.EFile).FragInd
 		fragOffset = b.Inode.Data.(inode.EFile).FragOffset
 		sizes = b.Inode.Data.(inode.EFile).BlockSizes
-		fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize)
+		fileSize = b.Inode.Data.(inode.EFile).Size
 	}
-	outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes, fragSize, r.Superblock.BlockSize)
+	outFull := data.NewFullReader(r.r, r.d, r.Superblock.BlockSize, fileSize, blockStart, sizes)
-	if fragIndex != 0xffffffff {
+	if fragIndex != 0xFFFFFFFF {
-		outFull.AddFrag(func() (io.Reader, error) {
+		ent, err := r.fragEntry(fragIndex)
-			ent, err := r.fragEntry(fragIndex)
+		if err != nil {
-			if err != nil {
+			return data.FullReader{}, err
-				return nil, err
+		}
-			}
+		outFull.AddFragData(ent.Start, ent.Size, fragOffset)
-			frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
-			frag.Read(make([]byte, fragOffset))
-			return io.LimitReader(&frag, int64(fragSize)), nil
-		})
 	}
 	return outFull, nil
 }
-
-func (b FileBase) GetReader(r *Reader) (data.Reader, error) {
-	if !b.IsRegular() {
-		return data.Reader{}, errors.New("not a regular file")
-	}
-	var blockStart uint64
-	var fragIndex uint32
-	var fragOffset uint32
-	var fragSize uint64
-	var sizes []uint32
-	if b.Inode.Type == inode.Fil {
-		blockStart = uint64(b.Inode.Data.(inode.File).BlockStart)
-		fragIndex = b.Inode.Data.(inode.File).FragInd
-		fragOffset = b.Inode.Data.(inode.File).FragOffset
-		sizes = b.Inode.Data.(inode.File).BlockSizes
-		fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize)
-	} else {
-		blockStart = b.Inode.Data.(inode.EFile).BlockStart
-		fragIndex = b.Inode.Data.(inode.EFile).FragInd
-		fragOffset = b.Inode.Data.(inode.EFile).FragOffset
-		sizes = b.Inode.Data.(inode.EFile).BlockSizes
-		fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize)
-	}
-	outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize)
-	if fragIndex != 0xffffffff {
-		ent, err := r.fragEntry(fragIndex)
-		if err != nil {
-			return data.Reader{}, err
-		}
-		frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
-		frag.Read(make([]byte, fragOffset))
-		outRdr.AddFrag(io.LimitReader(&frag, int64(fragSize)))
-	}
-	return outRdr, nil
-}

low/inode.go

@@ -7,7 +7,9 @@ import (
 	"github.com/CalebQ42/squashfs/low/inode"
 )
 
-func (r Reader) InodeFromRef(ref uint64) (inode.Inode, error) {
+type InodeRef = uint64
+
+func (r Reader) InodeFromRef(ref InodeRef) (inode.Inode, error) {
 	offset, meta := (ref>>16)+r.Superblock.InodeTableStart, ref&0xFFFF
 	rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
 	defer rdr.Close()

low/inode/dir.go

@@ -13,6 +13,20 @@ type Directory struct {
 	ParentNum  uint32
 }
 
+func ReadDir(r io.Reader) (d Directory, err error) {
+	dat := make([]byte, 16)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	d.BlockStart = binary.LittleEndian.Uint32(dat)
+	d.LinkCount = binary.LittleEndian.Uint32(dat[4:])
+	d.Size = binary.LittleEndian.Uint16(dat[8:])
+	d.Offset = binary.LittleEndian.Uint16(dat[10:])
+	d.ParentNum = binary.LittleEndian.Uint32(dat[12:])
+	return
+}
+
 type EDirectory struct {
 	LinkCount  uint32
 	Size       uint32
@@ -31,20 +45,6 @@ type DirectoryIndex struct {
 	Name     []byte
 }
 
-func ReadDir(r io.Reader) (d Directory, err error) {
-	dat := make([]byte, 16)
-	_, err = r.Read(dat)
-	if err != nil {
-		return
-	}
-	d.BlockStart = binary.LittleEndian.Uint32(dat)
-	d.LinkCount = binary.LittleEndian.Uint32(dat[4:])
-	d.Size = binary.LittleEndian.Uint16(dat[8:])
-	d.Offset = binary.LittleEndian.Uint16(dat[10:])
-	d.ParentNum = binary.LittleEndian.Uint32(dat[12:])
-	return
-}
-
 func ReadEDir(r io.Reader) (d EDirectory, err error) {
 	dat := make([]byte, 24)
 	_, err = r.Read(dat)

low/inode/file.go

@@ -14,31 +14,16 @@ type File struct {
 	BlockSizes []uint32
 }
 
-type eFileInit struct {
-	BlockStart uint64
-	Size       uint64
-	Sparse     uint64
-	LinkCount  uint32
-	FragInd    uint32
-	FragOffset uint32
-	XattrInd   uint32
-}
-
-type EFile struct {
-	eFileInit
-	BlockSizes []uint32
-}
-
 func ReadFile(r io.Reader, blockSize uint32) (f File, err error) {
 	dat := make([]byte, 16)
 	_, err = r.Read(dat)
 	if err != nil {
 		return
 	}
-	f.BlockStart = binary.LittleEndian.Uint32(dat)
+	f.BlockStart = binary.LittleEndian.Uint32(dat[0:4])
-	f.FragInd = binary.LittleEndian.Uint32(dat[4:])
+	f.FragInd = binary.LittleEndian.Uint32(dat[4:8])
-	f.FragOffset = binary.LittleEndian.Uint32(dat[8:])
+	f.FragOffset = binary.LittleEndian.Uint32(dat[8:12])
-	f.Size = binary.LittleEndian.Uint32(dat[12:])
+	f.Size = binary.LittleEndian.Uint32(dat[12:16])
 	toRead := int(math.Floor(float64(f.Size) / float64(blockSize)))
 	if f.FragInd == 0xFFFFFFFF && f.Size%blockSize > 0 {
 		toRead++
@@ -55,16 +40,42 @@ func ReadFile(r io.Reader, blockSize uint32) (f File, err error) {
 	return
 }
 
+type EFile struct {
+	BlockStart uint64
+	Size       uint64
+	Sparse     uint64
+	LinkCount  uint32
+	FragInd    uint32
+	FragOffset uint32
+	XattrInd   uint32
+	BlockSizes []uint32
+}
+
 func ReadEFile(r io.Reader, blockSize uint32) (f EFile, err error) {
-	err = binary.Read(r, binary.LittleEndian, &f.eFileInit)
+	dat := make([]byte, 40)
+	_, err = r.Read(dat)
 	if err != nil {
 		return
 	}
-	toRead := int(math.Floor(float64(f.Size) / float64(blockSize)))
+	f.BlockStart = binary.LittleEndian.Uint64(dat[0:8])
+	f.Size = binary.LittleEndian.Uint64(dat[8:16])
+	f.Sparse = binary.LittleEndian.Uint64(dat[16:24])
+	f.LinkCount = binary.LittleEndian.Uint32(dat[24:28])
+	f.FragInd = binary.LittleEndian.Uint32(dat[28:32])
+	f.FragOffset = binary.LittleEndian.Uint32(dat[32:36])
+	f.XattrInd = binary.LittleEndian.Uint32(dat[36:40])
+	toRead := f.Size / uint64(blockSize)
 	if f.FragInd == 0xFFFFFFFF && f.Size%uint64(blockSize) > 0 {
 		toRead++
 	}
+	dat = make([]byte, toRead*4)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
 	f.BlockSizes = make([]uint32, toRead)
-	err = binary.Read(r, binary.LittleEndian, &f.BlockSizes)
+	for i := range toRead {
+		f.BlockSizes[i] = binary.LittleEndian.Uint32(dat[i*4:])
+	}
 	return
 }

low/inode/inode.go

@@ -40,10 +40,17 @@ type Inode struct {
 }
 
 func Read(r io.Reader, blockSize uint32) (i Inode, err error) {
-	err = binary.Read(r, binary.LittleEndian, &i.Header)
+	dat := make([]byte, 16)
+	_, err = r.Read(dat)
 	if err != nil {
 		return
 	}
+	i.Type = binary.LittleEndian.Uint16(dat[0:2])
+	i.Perm = binary.LittleEndian.Uint16(dat[2:4])
+	i.UidInd = binary.LittleEndian.Uint16(dat[4:6])
+	i.GidInd = binary.LittleEndian.Uint16(dat[6:8])
+	i.ModTime = binary.LittleEndian.Uint32(dat[8:12])
+	i.Num = binary.LittleEndian.Uint32(dat[12:16])
 	switch i.Type {
 	case Dir:
 		i.Data, err = ReadDir(r)

low/inode/misc.go

@@ -10,18 +10,31 @@ type Device struct {
 	Dev       uint32
 }
 
+func ReadDevice(r io.Reader) (d Device, err error) {
+	dat := make([]byte, 8)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	d.LinkCount = binary.LittleEndian.Uint32(dat)
+	d.Dev = binary.LittleEndian.Uint32(dat[4:])
+	return
+}
+
 type EDevice struct {
 	Device
 	XattrInd uint32
 }
 
-func ReadDevice(r io.Reader) (d Device, err error) {
-	err = binary.Read(r, binary.LittleEndian, &d)
-	return
-}
-
 func ReadEDevice(r io.Reader) (d EDevice, err error) {
-	err = binary.Read(r, binary.LittleEndian, &d)
+	dat := make([]byte, 12)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	d.LinkCount = binary.LittleEndian.Uint32(dat)
+	d.Dev = binary.LittleEndian.Uint32(dat[4:])
+	d.XattrInd = binary.LittleEndian.Uint32(dat[8:])
 	return
 }
 
@@ -29,17 +42,28 @@ type IPC struct {
 	LinkCount uint32
 }
 
+func ReadIPC(r io.Reader) (i IPC, err error) {
+	dat := make([]byte, 4)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	i.LinkCount = binary.LittleEndian.Uint32(dat)
+	return
+}
+
 type EIPC struct {
 	IPC
 	XattrInd uint32
 }
 
-func ReadIPC(r io.Reader) (i IPC, err error) {
-	err = binary.Read(r, binary.LittleEndian, &i)
-	return
-}
-
 func ReadEIPC(r io.Reader) (i EIPC, err error) {
-	err = binary.Read(r, binary.LittleEndian, &i)
+	dat := make([]byte, 8)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	i.LinkCount = binary.LittleEndian.Uint32(dat)
+	i.XattrInd = binary.LittleEndian.Uint32(dat[4:])
 	return
 }

low/inode/sym.go

@@ -5,42 +5,50 @@ import (
 	"io"
 )
 
-type symlinkInit struct {
+type Symlink struct {
 	LinkCount  uint32
 	TargetSize uint32
-}
+	Target     []byte
-
-type Symlink struct {
-	symlinkInit
-	Target []byte
-}
-
-type ESymlink struct {
-	symlinkInit
-	Target   []byte
-	XattrInd uint32
 }
 
 func ReadSym(r io.Reader) (s Symlink, err error) {
-	err = binary.Read(r, binary.LittleEndian, &s.symlinkInit)
+	dat := make([]byte, 8)
+	_, err = r.Read(dat)
 	if err != nil {
 		return
 	}
+	s.LinkCount = binary.LittleEndian.Uint32(dat)
+	s.TargetSize = binary.LittleEndian.Uint32(dat[4:])
 	s.Target = make([]byte, s.TargetSize)
-	err = binary.Read(r, binary.LittleEndian, &s.Target)
+	_, err = r.Read(s.Target)
 	return
 }
 
+type ESymlink struct {
+	LinkCount  uint32
+	TargetSize uint32
+	Target     []byte
+	XattrInd   uint32
+}
+
 func ReadESym(r io.Reader) (s ESymlink, err error) {
-	err = binary.Read(r, binary.LittleEndian, &s.symlinkInit)
+	dat := make([]byte, 8)
+	_, err = r.Read(dat)
 	if err != nil {
 		return
 	}
+	s.LinkCount = binary.LittleEndian.Uint32(dat)
+	s.TargetSize = binary.LittleEndian.Uint32(dat[4:])
 	s.Target = make([]byte, s.TargetSize)
-	err = binary.Read(r, binary.LittleEndian, &s.Target)
+	_, err = r.Read(s.Target)
 	if err != nil {
 		return
 	}
-	err = binary.Read(r, binary.LittleEndian, &s.XattrInd)
+	dat = make([]byte, 4)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	s.XattrInd = binary.LittleEndian.Uint32(dat)
 	return
 }

low/reader.go

@@ -4,10 +4,8 @@ import (
 	"encoding/binary"
 	"errors"
 	"io"
-	"math"
 
 	"github.com/CalebQ42/squashfs/internal/decompress"
-	"github.com/CalebQ42/squashfs/internal/metadata"
 	"github.com/CalebQ42/squashfs/internal/toreader"
 	"github.com/CalebQ42/squashfs/low/inode"
 )
@@ -30,13 +28,13 @@ var (
 )
 
 type Reader struct {
+	Root        Directory
+	Superblock  superblock
 	r           io.ReaderAt
 	d           decompress.Decompressor
-	Root        Directory
+	fragTable   *Table[fragEntry]
-	fragTable   []fragEntry
+	idTable     *Table[uint32]
-	idTable     []uint32
+	exportTable *Table[InodeRef]
-	exportTable []uint64
-	Superblock  superblock
 }
 
 func NewReader(r io.ReaderAt) (rdr Reader, err error) {
@@ -56,7 +54,7 @@ func NewReader(r io.ReaderAt) (rdr Reader, err error) {
 	}
 	switch rdr.Superblock.CompType {
 	case ZlibCompression:
-		rdr.d = decompress.Zlib{}
+		rdr.d = decompress.NewZlib()
 	case LZMACompression:
 		rdr.d, err = decompress.NewLzma()
 		if err != nil {
@@ -72,7 +70,7 @@ func NewReader(r io.ReaderAt) (rdr Reader, err error) {
 	case LZ4Compression:
 		rdr.d = decompress.NewLz4()
 	case ZSTDCompression:
-		rdr.d = decompress.Zstd{}
+		rdr.d = decompress.NewZstd()
 	default:
 		return rdr, errors.New("invalid compression type. possible corrupted archive")
 	}
@@ -80,137 +78,59 @@ func NewReader(r io.ReaderAt) (rdr Reader, err error) {
 	if err != nil {
 		return rdr, errors.Join(errors.New("failed to read root directory"), err)
 	}
+	rdr.fragTable = NewTable(&rdr, rdr.Superblock.FragTableStart, rdr.Superblock.FragCount, readFrag)
+	rdr.idTable = NewTable(&rdr, rdr.Superblock.IdTableStart, uint32(rdr.Superblock.IdCount), readId)
+	rdr.exportTable = NewTable(&rdr, rdr.Superblock.ExportTableStart, rdr.Superblock.InodeCount, readRef)
 	return
 }
 
+func readFrag(r io.Reader) (fragEntry, error) {
+	dat := make([]byte, 16)
+	_, err := r.Read(dat)
+	if err != nil {
+		return fragEntry{}, err
+	}
+	return fragEntry{
+		Start: binary.LittleEndian.Uint64(dat[0:8]),
+		Size:  binary.LittleEndian.Uint32(dat[8:12]),
+	}, nil
+}
+
+func readId(r io.Reader) (uint32, error) {
+	dat := make([]byte, 4)
+	_, err := r.Read(dat)
+	if err != nil {
+		return 0, err
+	}
+	return binary.LittleEndian.Uint32(dat), nil
+}
+
+func readRef(r io.Reader) (InodeRef, error) {
+	dat := make([]byte, 8)
+	_, err := r.Read(dat)
+	if err != nil {
+		return 0, err
+	}
+	return binary.LittleEndian.Uint64(dat), nil
+}
+
 // Get a uid/gid at the given index. Lazily populates the reader's Id table as necessary.
 func (r *Reader) Id(i uint16) (uint32, error) {
-	if len(r.idTable) > int(i) {
+	return r.idTable.Get(uint32(i))
-		return r.idTable[i], nil
-	} else if i >= r.Superblock.IdCount {
-		return 0, errors.New("id out of bounds")
-	}
-	// Populate the id table as needed
-	var blockNum uint32
-	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
-		blockNum = uint32(math.Ceil(float64(i+1)/2048)) - 1
-	} else {
-		blockNum = 0
-	}
-	blocksRead := len(r.idTable) / 2048
-	blocksToRead := int(blockNum) - blocksRead + 1
-
-	var offset uint64
-	var idsToRead uint16
-	var idsTmp []uint32
-	var err error
-	var rdr metadata.Reader
-	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
-	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
-		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset)
-		if err != nil {
-			return 0, err
-		}
-		idsToRead = min(r.Superblock.IdCount-uint16(len(r.idTable)), 2048)
-		idsTmp = make([]uint32, idsToRead)
-		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
-		err = binary.Read(&rdr, binary.LittleEndian, &idsTmp)
-		rdr.Close()
-		if err != nil {
-			return 0, err
-		}
-		r.idTable = append(r.idTable, idsTmp...)
-	}
-	return r.idTable[i], nil
 }
 
 // Get a fragment entry at the given index. Lazily populates the reader's fragment table as necessary.
 func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
-	if len(r.fragTable) > int(i) {
+	return r.fragTable.Get(i)
-		return r.fragTable[i], nil
-	} else if i >= r.Superblock.FragCount {
-		return fragEntry{}, errors.New("fragment out of bounds")
-	}
-	// Populate the fragment table as needed
-	var blockNum uint32
-	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
-		blockNum = uint32(math.Ceil(float64(i+1)/512)) - 1
-	} else {
-		blockNum = 0
-	}
-	blocksRead := len(r.fragTable) / 512
-	blocksToRead := int(blockNum) - blocksRead + 1
-
-	var offset uint64
-	var fragsToRead uint32
-	var fragsTmp []fragEntry
-	var err error
-	var rdr metadata.Reader
-	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
-	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
-		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset)
-		if err != nil {
-			return fragEntry{}, err
-		}
-		fragsToRead = min(r.Superblock.FragCount-uint32(len(r.fragTable)), 512)
-		fragsTmp = make([]fragEntry, fragsToRead)
-		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
-		err = binary.Read(&rdr, binary.LittleEndian, &fragsTmp)
-		rdr.Close()
-		if err != nil {
-			return fragEntry{}, err
-		}
-		r.fragTable = append(r.fragTable, fragsTmp...)
-	}
-	return r.fragTable[i], nil
 }
 
 // Get an inode reference at the given index. Lazily populates the reader's export table as necessary.
-func (r *Reader) inodeRef(i uint32) (uint64, error) {
+func (r *Reader) inodeRef(i uint32) (InodeRef, error) {
-	if !r.Superblock.Exportable() {
+	return r.exportTable.Get(i)
-		return 0, ErrorNotExportable
-	}
-	if len(r.exportTable) > int(i) {
-		return r.exportTable[i], nil
-	} else if i >= r.Superblock.InodeCount {
-		return 0, errors.New("inode out of bounds")
-	}
-	// Populate the export table as needed
-	var blockNum uint32
-	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
-		blockNum = uint32(math.Ceil(float64(i+1)/1024)) - 1
-	} else {
-		blockNum = 0
-	}
-	blocksRead := len(r.exportTable) / 1024
-	blocksToRead := int(blockNum) - blocksRead + 1
-
-	var offset uint64
-	var refsToRead uint32
-	var refsTmp []uint64
-	var err error
-	var rdr metadata.Reader
-	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
-	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
-		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset)
-		if err != nil {
-			return 0, err
-		}
-		refsToRead = min(r.Superblock.InodeCount-uint32(len(r.exportTable)), 1024)
-		refsTmp = make([]uint64, refsToRead)
-		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
-		err = binary.Read(&rdr, binary.LittleEndian, &refsTmp)
-		rdr.Close()
-		if err != nil {
-			return 0, err
-		}
-		r.exportTable = append(r.exportTable, refsTmp...)
-	}
-	return r.exportTable[i], nil
 }
 
 func (r Reader) Inode(i uint32) (inode.Inode, error) {
-	ref, err := r.inodeRef(i)
+	ref, err := r.inodeRef(i - 1) // Inode table is 1 indexed
 	if err != nil {
 		return inode.Inode{}, err
 	}

low/reader_test.go

@@ -78,7 +78,9 @@ func TestReader(t *testing.T) {
 	os.RemoveAll(path)
 	os.MkdirAll(path, 0777)
 	err = extractToDir(rdr, rdr.Root.FileBase, path)
-	t.Fatal(err)
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 var singleFile = "PortableApps/CPU-X/CPU-X-v4.2.0-x86_64.AppImage"
@@ -102,7 +104,9 @@ func TestSingleFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	err = extractToDir(rdr, b, path)
-	t.Fatal(err)
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func extractToDir(rdr Reader, b FileBase, folder string) error {

low/superblock.go

@@ -14,7 +14,7 @@ type superblock struct {
 	IdCount          uint16
 	VerMaj           uint16
 	VerMin           uint16
-	RootInodeRef     uint64
+	RootInodeRef     InodeRef
 	Size             uint64
 	IdTableStart     uint64
 	XattrTableStart  uint64

low/table.go

@@ -0,0 +1,85 @@
+package squashfslow
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+	"sync"
+
+	"github.com/CalebQ42/squashfs/internal/metadata"
+	"github.com/CalebQ42/squashfs/internal/toreader"
+)
+
+var errOutOfBounds = errors.New("out of bounds")
+var errUnexpectedOutOfBounds = errors.New("unexpected out of bounds")
+var errNilCollection = errors.New("nil collection")
+
+type CreateFunction[T any] = func(io.Reader) (T, error)
+
+type Table[T any] struct {
+	totalItems    uint32
+	itemsPerBlock uint32
+	offset        uint64
+	mut           sync.RWMutex
+	currentItems  []T
+	rdr           *Reader
+	createFunc    CreateFunction[T]
+}
+
+func NewTable[T any](rdr *Reader, start uint64, totalItems uint32, createFunc CreateFunction[T]) *Table[T] {
+	var zero T
+	return &Table[T]{
+		totalItems:    totalItems,
+		itemsPerBlock: 8192 / uint32(binary.Size(zero)),
+		offset:        start,
+		mut:           sync.RWMutex{},
+		rdr:           rdr,
+		createFunc:    createFunc,
+	}
+}
+
+func (t *Table[T]) Get(requestedItemIndex uint32) (T, error) {
+	t.mut.RLock()
+	if requestedItemIndex >= t.totalItems {
+		t.mut.RUnlock()
+		var zero T
+		return zero, errOutOfBounds
+	}
+	if uint32(len(t.currentItems)) > requestedItemIndex {
+		t.mut.RUnlock()
+		return t.currentItems[requestedItemIndex], nil
+	}
+	t.mut.RUnlock()
+	return t.fillAndGet(requestedItemIndex)
+}
+
+func (t *Table[T]) fillAndGet(requestedItemIndex uint32) (T, error) {
+	t.mut.Lock()
+	defer t.mut.Unlock()
+	var offset uint64
+	var toRead uint32
+	var rdr *toreader.Reader
+	var metaRdr metadata.Reader
+	var err error
+	for uint32(len(t.currentItems)) <= requestedItemIndex {
+		rdr = toreader.NewReader(t.rdr.r, int64(t.offset))
+		err = binary.Read(rdr, binary.LittleEndian, &offset)
+		if err != nil {
+			var zero T
+			return zero, err
+		}
+		t.offset += 8
+		toRead = min(t.itemsPerBlock, t.totalItems-uint32(len(t.currentItems)))
+		oldLen := uint32(len(t.currentItems))
+		t.currentItems = append(t.currentItems, make([]T, toRead)...)
+		metaRdr = metadata.NewReader(toreader.NewReader(t.rdr.r, int64(offset)), t.rdr.d)
+		for i := range toRead {
+			t.currentItems[oldLen+i], err = t.createFunc(&metaRdr)
+			if err != nil {
+				var zero T
+				return zero, err
+			}
+		}
+	}
+	return t.currentItems[requestedItemIndex], nil
+}

reader.go

@@ -22,8 +22,8 @@ func NewReader(r io.ReaderAt) (Reader, error) {
 		Low: rdr,
 	}
 	out.FS = FS{
-		d: rdr.Root,
+		LowDir: rdr.Root,
-		r: &out,
+		r:   &out,
 	}
 	return out, nil
 }

squashfs_test.go

@@ -76,13 +76,11 @@ func BenchmarkExtract(b *testing.B) {
 	}
 	libPath := filepath.Join(tmpDir, "ExtractLib")
 	os.RemoveAll(libPath)
-	op := FastOptions()
-	op.IgnorePerm = true
 	rdr, err := NewReader(fil)
 	if err != nil {
 		b.Fatal(err)
 	}
-	err = rdr.ExtractWithOptions(libPath, op)
+	err = rdr.ExtractWithOptions(libPath, FastOptions())
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -99,14 +97,12 @@ func BenchmarkRace(b *testing.B) {
 	os.RemoveAll(libPath)
 	os.RemoveAll(unsquashPath)
 	var libTime, unsquashTime time.Duration
-	op := FastOptions()
-	op.IgnorePerm = true
 	start := time.Now()
 	rdr, err := NewReader(fil)
 	if err != nil {
 		b.Fatal(err)
 	}
-	err = rdr.ExtractWithOptions(libPath, op)
+	err = rdr.ExtractWithOptions(libPath, FastOptions())
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -120,8 +116,8 @@ func BenchmarkRace(b *testing.B) {
 		b.Log("Unsquashfs error:", err)
 	}
 	unsquashTime = time.Since(start)
-	b.Log("Library took:", libTime.Round(time.Millisecond))
+	// b.Log("Library took:", libTime.Round(time.Millisecond))
-	b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond))
+	// b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond))
 	b.Log("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster")
 }