Merge pull request #40 from CalebQ42/go-unsquashfs-hardlinks

go-unsquashfs hardlinks & squashfslow access

README.md

@@ -11,6 +11,10 @@ Currently has support for reading squashfs files and extracting files and folder
 Special thanks to <https://dr-emann.github.io/squashfs/> for some VERY important information in an easy to understand format.
 Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree/master/internal/squashfs) as I referenced it to figure some things out (and double check others).
 
+## Build tags
+
+As of `v1.1.0` this library has two optional build tags: `no_gpl` and `no_obsolete`. `no_gpl` disables the ability to read archives with lzo compression due to the library's gpl license. `no_obsolete` removes "obsolete" compression types for a reduced compilation size; currently this only disable lzma compression since it's superseded by xz.
+
 ## FUSE
 
 As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://github.com/CalebQ42/squashfuse).
@@ -24,14 +28,14 @@ As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://g
 
 ## Issues
 
-* Significantly slower then `unsquashfs` when extracting folders
+* Noticably slower then `unsquashfs` for extraction, especially on larger images.
   * This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries.
   * Times seem to be largely dependent on file tree size and compression type.
-    * My main testing image (~100MB) using Zstd takes about 6x longer.
+    * My main testing image (~100MB) using Zstd takes ~2x longer.
-    * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 32x longer.
+    * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes ~28x longer.
-    * A Tensorflow docker image (~3.3GB) using Zstd takes about 12x longer.
+    * A Tensorflow docker image (~3.3GB) using Zstd takes ~3x longer.
 
-Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes about 2x longer.
+Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes ~2x longer.
 
 ## Recommendations on Usage
 

cmd/go-unsquashfs/main.go

@@ -4,16 +4,116 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/CalebQ42/squashfs"
+	squashfslow "github.com/CalebQ42/squashfs/low"
+)
+
+func userName(uid int, numeric bool) string {
+	us := strconv.Itoa(uid)
+	if numeric {
+		return us
+	}
+	if u, err := user.LookupId(us); err == nil {
+		return u.Username
+	}
+	return us
+}
+
+func groupName(gid int, numeric bool) string {
+	gs := strconv.Itoa(gid)
+	if numeric {
+		return gs
+	}
+	if g, err := user.LookupGroupId(gs); err == nil {
+		return g.Name
+	}
+	return gs
+}
+
+var hardLinks = make(map[uint32]string)
+
+func printFile(rdr *squashfs.Reader, path string, f *squashfs.File) {
+	path = filepath.Join(path, f.Low.Name)
+	fi, _ := f.Stat()
+	sfi := fi.(squashfs.FileInfo)
+	owner := fmt.Sprintf("%s/%s",
+		userName(sfi.Uid(), *numeric),
+		groupName(sfi.Gid(), *numeric))
+	var link string
+	var isHardLink bool
+	if *showHardLinks {
+		link, isHardLink = hardLinks[f.Low.Inode.Num]
+		if !isHardLink {
+			hardLinks[f.Low.Inode.Num] = path
+		}
+	}
+	var size int64
+	if isHardLink {
+		size = 0
+	} else {
+		size = fi.Size()
+	}
+	if sfi.IsSymlink() {
+		link = " -> " + sfi.SymlinkPath()
+	} else if isHardLink {
+		link = " link to " + link
+	}
+	fmt.Printf("%s %s %*d %s %s%s\n",
+		strings.ToLower(fi.Mode().String()),
+		owner, 26-len(owner), size,
+		fi.ModTime().Format("2006-01-02 15:04"),
+		path, link)
+	if f.IsDir() {
+		fs, _ := f.FS()
+		printDir(rdr, path, fs)
+	}
+}
+
+func printDir(rdr *squashfs.Reader, path string, f squashfs.FS) {
+	var base squashfslow.FileBase
+	var fil squashfs.File
+	var err error
+	for _, e := range f.LowDir.Entries {
+		base, err = rdr.Low.BaseFromEntry(e)
+		if err != nil {
+			panic(err)
+		}
+		fil = rdr.FileFromBase(base, f)
+		printFile(rdr, path, &fil)
+	}
+}
+
+var (
+	verbose       *bool
+	list          *bool
+	long          *bool
+	numeric       *bool
+	offset        *int64
+	ignore        *bool
+	file          *string
+	showHardLinks *bool
 )
 
 func main() {
-	verbose := flag.Bool("v", false, "Verbose")
+	verbose = flag.Bool("v", false, "Verbose")
-	ignore := flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755")
+	list = flag.Bool("l", false, "List")
+	long = flag.Bool("ll", false, "List with attributes")
+	numeric = flag.Bool("lln", false, "List with attributes and numeric ids")
+	showHardLinks = flag.Bool("show-hard-links", false, "When used with ll or lln, shows hard links")
+	offset = flag.Int64("o", 0, "Offset")
+	ignore = flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755")
+	file = flag.String("e", "", "File or folder to extract")
 	flag.Parse()
-	if len(flag.Args()) < 2 {
+	if (*list || *long || *numeric) && flag.NArg() < 1 {
+		fmt.Println("Please provide a file name")
+		os.Exit(0)
+	} else if (!*list && !*long && !*numeric) && flag.NArg() < 2 {
 		fmt.Println("Please provide a file name and extraction path")
 		os.Exit(0)
 	}
@@ -21,15 +121,26 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	r, err := squashfs.NewReader(f)
+	r, err := squashfs.NewReaderAtOffset(f, *offset)
 	if err != nil {
 		panic(err)
 	}
+	extractFil := r.File()
+	if *file != "" {
+		extractFil, err = r.OpenFile(*file)
+		if err != nil {
+			panic(err)
+		}
+	}
+	if *list || *long || *numeric {
+		printFile(&r, "", extractFil)
+		return
+	}
 	op := squashfs.DefaultOptions()
 	op.Verbose = *verbose
 	op.IgnorePerm = *ignore
 	n := time.Now()
-	err = r.ExtractWithOptions(flag.Arg(1), op)
+	err = extractFil.ExtractWithOptions(flag.Arg(1), op)
 	if err != nil {
 		panic(err)
 	}

file.go

@@ -19,49 +19,47 @@ import (
 
 // File represents a file inside a squashfs archive.
 type File struct {
-	b        *squashfslow.FileBase
+	full     data.FullReader
-	full     *data.FullReader
+	rdr      data.Reader
-	rdr      *data.Reader
+	rdrInit  bool
-	parent   *FS
+	parent   FS
 	r        *Reader
+	Low      squashfslow.FileBase
 	dirsRead int
 }
 
 // Creates a new *File from the given *squashfs.Base
-func (r *Reader) FileFromBase(b *squashfslow.FileBase, parent *FS) *File {
+func (r *Reader) FileFromBase(b squashfslow.FileBase, parent FS) File {
-	return &File{
+	return File{
-		b:      b,
+		Low:    b,
 		parent: parent,
 		r:      r,
 	}
 }
 
-func (f *File) FS() (*FS, error) {
+func (f File) FS() (FS, error) {
 	if !f.IsDir() {
-		return nil, errors.New("not a directory")
+		return FS{}, errors.New("not a directory")
 	}
-	d, err := f.b.ToDir(f.r.Low)
+	d, err := f.Low.ToDir(f.r.Low)
 	if err != nil {
-		return nil, err
+		return FS{}, err
 	}
-	return &FS{d: d, parent: f.parent, r: f.r}, nil
+	return FS{LowDir: d, parent: &f.parent, r: f.r}, nil
 }
 
 // Closes the underlying readers.
 // Further calls to Read and WriteTo will re-create the readers.
 // Never returns an error.
 func (f *File) Close() error {
-	if f.rdr != nil {
+	f.rdr.Close()
-		return f.rdr.Close()
+	f.full.Close()
-	}
-	f.rdr = nil
-	f.full = nil
 	return nil
 }
 
 // Returns the file the symlink points to.
 // If the file isn't a symlink, or points to a file outside the archive, returns nil.
-func (f *File) GetSymlinkFile() fs.File {
+func (f File) GetSymlinkFile() fs.File {
 	if !f.IsSymlink() {
 		return nil
 	}
@@ -76,22 +74,22 @@ func (f *File) GetSymlinkFile() fs.File {
 }
 
 // Returns whether the file is a directory.
-func (f *File) IsDir() bool {
+func (f File) IsDir() bool {
-	return f.b.IsDir()
+	return f.Low.IsDir()
 }
 
 // Returns whether the file is a regular file.
-func (f *File) IsRegular() bool {
+func (f File) IsRegular() bool {
-	return f.b.IsRegular()
+	return f.Low.IsRegular()
 }
 
 // Returns whether the file is a symlink.
-func (f *File) IsSymlink() bool {
+func (f File) IsSymlink() bool {
-	return f.b.Inode.Type == inode.Sym || f.b.Inode.Type == inode.ESym
+	return f.Low.Inode.Type == inode.Sym || f.Low.Inode.Type == inode.ESym
 }
 
-func (f *File) Mode() fs.FileMode {
+func (f File) Mode() fs.FileMode {
-	return f.b.Inode.Mode()
+	return f.Low.Inode.Mode()
 }
 
 // Read reads the data from the file. Only works if file is a normal file.
@@ -99,7 +97,7 @@ func (f *File) Read(b []byte) (int, error) {
 	if !f.IsRegular() {
 		return 0, errors.New("file is not a regular file")
 	}
-	if f.rdr == nil {
+	if !f.rdrInit {
 		err := f.initializeReaders()
 		if err != nil {
 			return 0, err
@@ -114,7 +112,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
 	if !f.IsDir() {
 		return nil, errors.New("file is not a directory")
 	}
-	d, err := f.b.ToDir(f.r.Low)
+	d, err := f.Low.ToDir(f.r.Low)
 	if err != nil {
 		return nil, err
 	}
@@ -127,7 +125,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
 		}
 	}
 	var out []fs.DirEntry
-	var fi fileInfo
+	var fi FileInfo
 	for _, e := range d.Entries[start:end] {
 		fi, err = f.r.newFileInfo(e)
 		if err != nil {
@@ -141,17 +139,25 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
 }
 
 // Returns the file's fs.FileInfo
-func (f *File) Stat() (fs.FileInfo, error) {
+func (f File) Stat() (fs.FileInfo, error) {
-	return newFileInfo(f.b.Name, f.b.Inode), nil
+	uid, err := f.Low.Uid(&f.r.Low)
+	if err != nil {
+		return nil, err
+	}
+	gid, err := f.Low.Gid(&f.r.Low)
+	if err != nil {
+		return nil, err
+	}
+	return newFileInfo(f.Low.Name, uid, gid, &f.Low.Inode), nil
 }
 
 // SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string.
-func (f *File) SymlinkPath() string {
+func (f File) SymlinkPath() string {
-	switch f.b.Inode.Type {
+	switch f.Low.Inode.Type {
 	case inode.Sym:
-		return string(f.b.Inode.Data.(inode.Symlink).Target)
+		return string(f.Low.Inode.Data.(inode.Symlink).Target)
 	case inode.ESym:
-		return string(f.b.Inode.Data.(inode.ESymlink).Target)
+		return string(f.Low.Inode.Data.(inode.ESymlink).Target)
 	}
 	return ""
 }
@@ -162,7 +168,7 @@ func (f *File) WriteTo(w io.Writer) (int64, error) {
 	if !f.IsRegular() {
 		return 0, errors.New("file is not a regular file")
 	}
-	if f.full == nil {
+	if !f.rdrInit {
 		err := f.initializeReaders()
 		if err != nil {
 			return 0, err
@@ -173,36 +179,43 @@ func (f *File) WriteTo(w io.Writer) (int64, error) {
 
 func (f *File) initializeReaders() error {
 	var err error
-	f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.Low)
+	f.rdr, f.full, err = f.Low.GetRegFileReaders(f.r.Low)
+	if err == nil {
+		f.rdrInit = true
+	} else {
+		f.rdr.Close()
+		f.full.Close()
+	}
 	return err
 }
 
-func (f *File) deviceDevices() (maj uint32, min uint32) {
+func (f File) deviceDevices() (maj uint32, min uint32) {
 	var dev uint32
-	if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.Block {
+	switch f.Low.Inode.Type {
-		dev = f.b.Inode.Data.(inode.Device).Dev
+	case inode.Char, inode.Block:
-	} else if f.b.Inode.Type == inode.EChar || f.b.Inode.Type == inode.EBlock {
+		dev = f.Low.Inode.Data.(inode.Device).Dev
-		dev = f.b.Inode.Data.(inode.EDevice).Dev
+	case inode.EChar, inode.EBlock:
+		dev = f.Low.Inode.Data.(inode.EDevice).Dev
 	}
 	return dev >> 8, dev & 0x000FF
 }
 
-func (f *File) path() string {
+func (f File) path() string {
-	if f.parent == nil {
+	if f.parent.LowDir.Name == "" {
-		return f.b.Name
+		return f.Low.Name
 	}
-	return filepath.Join(f.parent.path(), f.b.Name)
+	return filepath.Join(f.parent.path(), f.Low.Name)
 }
 
 // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
 // Uses default extraction options.
-func (f *File) Extract(folder string) error {
+func (f File) Extract(folder string) error {
 	return f.ExtractWithOptions(folder, DefaultOptions())
 }
 
 // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
 // Allows setting various extraction options via ExtractionOptions.
-func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
+func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 	if op.manager == nil {
 		op.manager = routinemanager.NewManager(op.SimultaneousFiles)
 		if op.LogOutput != nil {
@@ -216,9 +229,9 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			return err
 		}
 	}
-	switch f.b.Inode.Type {
+	switch f.Low.Inode.Type {
 	case inode.Dir, inode.EDir:
-		d, err := f.b.ToDir(f.r.Low)
+		d, err := f.Low.ToDir(f.r.Low)
 		if err != nil {
 			if op.Verbose {
 				log.Println("Failed to create squashfs.Directory for", path)
@@ -234,7 +247,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 				}
 				return errors.Join(errors.New("failed to get base from entry: "+path), err)
 			}
-			go func(b *squashfslow.FileBase, path string) {
+			go func(b squashfslow.FileBase, path string) {
 				i := op.manager.Lock()
 				if b.IsDir() {
 					extDir := filepath.Join(path, b.Name)
@@ -266,7 +279,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			}(b, path)
 		}
 		var errCache []error
-		for i := 0; i < len(d.Entries); i++ {
+		for range d.Entries {
 			err := <-errChan
 			if err != nil {
 				errCache = append(errCache, err)
@@ -276,7 +289,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			return errors.Join(errors.New("failed to extract folder: "+path), errors.Join(errCache...))
 		}
 	case inode.Fil, inode.EFil:
-		path = filepath.Join(path, f.b.Name)
+		path = filepath.Join(path, f.Low.Name)
 		outFil, err := os.Create(path)
 		if err != nil {
 			if op.Verbose {
@@ -285,7 +298,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			return errors.Join(errors.New("failed to create file: "+path), err)
 		}
 		defer outFil.Close()
-		full, err := f.b.GetFullReader(f.r.Low)
+		full, err := f.Low.GetFullReader(&f.r.Low)
 		if err != nil {
 			if op.Verbose {
 				log.Println("Failed to create full reader for", path)
@@ -311,11 +324,11 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 				return errors.New("failed to get symlink's file")
 			}
 			fil := filTmp.(*File)
-			fil.b.Name = f.b.Name
+			fil.Low.Name = f.Low.Name
 			err := fil.ExtractWithOptions(path, op)
 			if err != nil {
 				if op.Verbose {
-					log.Println("Failed to extract symlink's file:", filepath.Join(path, f.b.Name))
+					log.Println("Failed to extract symlink's file:", filepath.Join(path, f.Low.Name))
 				}
 				return errors.Join(errors.New("failed to extract symlink's file: "+path), err)
 			}
@@ -338,7 +351,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 					return errors.Join(errors.New("failed to extract symlink's file: "+extractLoc), err)
 				}
 			}
-			path = filepath.Join(path, f.b.Name)
+			path = filepath.Join(path, f.Low.Name)
 			err := os.Symlink(f.SymlinkPath(), path)
 			if err != nil {
 				if op.Verbose {
@@ -361,13 +374,14 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			}
 			return errors.Join(errors.New("mknot command not found"), err)
 		}
-		path = filepath.Join(path, f.b.Name)
+		path = filepath.Join(path, f.Low.Name)
 		var typ string
-		if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.EChar {
+		switch f.Low.Inode.Type {
+		case inode.Char, inode.EChar:
 			typ = "c"
-		} else if f.b.Inode.Type == inode.Block || f.b.Inode.Type == inode.EBlock {
+		case inode.Block, inode.EBlock:
 			typ = "b"
-		} else { //Fifo IPC
+		default: //Fifo IPC
 			if runtime.GOOS == "darwin" {
 				if op.Verbose {
 					log.Println(f.path(), "ignored. A Fifo file and can't be created on Darwin.")
@@ -398,7 +412,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 		}
 		return nil
 	default:
-		return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.b.Inode.Type)))
+		return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.Low.Inode.Type)))
 	}
 	if op.Verbose {
 		log.Println(f.path(), "extracted to", path)
@@ -406,7 +420,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 	if op.IgnorePerm {
 		return nil
 	}
-	uid, err := f.b.Uid(f.r.Low)
+	uid, err := f.Low.Uid(&f.r.Low)
 	if err != nil {
 		if op.Verbose {
 			log.Println("Failed to get uid for", path)
@@ -414,7 +428,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 		}
 		return nil
 	}
-	gid, err := f.b.Gid(f.r.Low)
+	gid, err := f.Low.Gid(&f.r.Low)
 	if err != nil {
 		if op.Verbose {
 			log.Println("Failed to get gid for", path)

file_info.go

@@ -8,61 +8,119 @@ import (
 	"github.com/CalebQ42/squashfs/low/inode"
 )
 
-type fileInfo struct {
+type FileInfo struct {
 	name     string
+	uid      uint32
+	gid      uint32
 	size     int64
+	target   string
 	perm     uint32
 	modTime  uint32
 	fileType uint16
 }
 
-func (r Reader) newFileInfo(e directory.Entry) (fileInfo, error) {
+func (r Reader) newFileInfo(e directory.Entry) (FileInfo, error) {
-	i, err := r.Low.InodeFromEntry(e)
+	b, err := r.Low.BaseFromEntry(e)
 	if err != nil {
-		return fileInfo{}, err
+		return FileInfo{}, err
 	}
-	return newFileInfo(e.Name, i), nil
+	uid, err := b.Uid(&r.Low)
+	if err != nil {
+		return FileInfo{}, err
+	}
+	gid, err := b.Gid(&r.Low)
+	if err != nil {
+		return FileInfo{}, err
+	}
+	return newFileInfo(e.Name, uid, gid, &b.Inode), nil
 }
 
-func newFileInfo(name string, i *inode.Inode) fileInfo {
+func newFileInfo(name string, uid, gid uint32, i *inode.Inode) FileInfo {
 	var size int64
-	if i.Type == inode.Fil {
+	var target string
+	switch i.Type {
+	case inode.Fil:
 		size = int64(i.Data.(inode.File).Size)
-	} else if i.Type == inode.EFil {
+	case inode.EFil:
 		size = int64(i.Data.(inode.EFile).Size)
+	case inode.Sym:
+		target = string(i.Data.(inode.Symlink).Target)
+	case inode.ESym:
+		target = string(i.Data.(inode.ESymlink).Target)
 	}
-	return fileInfo{
+	return FileInfo{
 		name:     name,
+		uid:      uid,
+		gid:      gid,
 		size:     size,
+		target:   target,
 		perm:     uint32(i.Perm),
 		modTime:  i.ModTime,
 		fileType: i.Type,
 	}
 }
 
-func (f fileInfo) Name() string {
+func (f FileInfo) Name() string {
 	return f.name
 }
 
-func (f fileInfo) Size() int64 {
+func (f FileInfo) Uid() int {
+	return int(f.uid)
+}
+
+func (f FileInfo) Gid() int {
+	return int(f.gid)
+}
+
+func (f FileInfo) Size() int64 {
 	return f.size
 }
 
-func (f fileInfo) Mode() fs.FileMode {
+func (f FileInfo) SymlinkPath() string {
-	if f.IsDir() {
+	return f.target
+}
+
+func (f FileInfo) Mode() fs.FileMode {
+	switch f.fileType {
+	case inode.Dir, inode.EDir:
 		return fs.FileMode(f.perm | uint32(fs.ModeDir))
+	case inode.Sym, inode.ESym:
+		return fs.FileMode(f.perm | uint32(fs.ModeSymlink))
+	case inode.Char, inode.EChar, inode.Block, inode.EBlock:
+		return fs.FileMode(f.perm | uint32(fs.ModeDevice))
+	case inode.Fifo, inode.EFifo:
+		return fs.FileMode(f.perm | uint32(fs.ModeNamedPipe))
+	case inode.Sock, inode.ESock:
+		return fs.FileMode(f.perm | uint32(fs.ModeSocket))
 	}
 	return fs.FileMode(f.perm)
 }
 
-func (f fileInfo) ModTime() time.Time {
+func (f FileInfo) ModTime() time.Time {
 	return time.Unix(int64(f.modTime), 0)
 }
 
-func (f fileInfo) IsDir() bool {
+func (f FileInfo) IsDir() bool {
 	return f.fileType == inode.Dir || f.fileType == inode.EDir
 }
 
-func (f fileInfo) Sys() any {
+func (f FileInfo) IsSymlink() bool {
+	return f.fileType == inode.Sym || f.fileType == inode.ESym
+}
+
+func (f FileInfo) IsDevice() bool {
+	return f.fileType == inode.Block || f.fileType == inode.EBlock ||
+		f.fileType == inode.Char || f.fileType == inode.EChar
+}
+
+func (f FileInfo) IsFifo() bool {
+	return f.fileType == inode.Fifo || f.fileType == inode.EFifo
+}
+
+func (f FileInfo) IsSocket() bool {
+	return f.fileType == inode.Sock || f.fileType == inode.ESock
+}
+
+func (f FileInfo) Sys() any {
 	return nil
 }

fs.go

@@ -15,17 +15,17 @@ import (
 // FS is a fs.FS representation of a squashfs directory.
 // Implements fs.GlobFS, fs.ReadDirFS, fs.ReadFileFS, fs.StatFS, and fs.SubFS
 type FS struct {
-	d      *squashfslow.Directory
 	r      *Reader
 	parent *FS
+	LowDir squashfslow.Directory
 }
 
 // Creates a new *FS from the given squashfs.directory
-func (r *Reader) FSFromDirectory(d *squashfslow.Directory, parent *FS) *FS {
+func (r *Reader) FSFromDirectory(d squashfslow.Directory, parent FS) FS {
-	return &FS{
+	return FS{
-		d:      d,
+		LowDir: d,
 		r:      r,
-		parent: parent,
+		parent: &parent,
 	}
 }
 
@@ -42,10 +42,10 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 		}
 	}
 	split := strings.Split(pattern, "/")
-	for i := 0; i < len(f.d.Entries); i++ {
+	for i := range f.LowDir.Entries {
-		if match, _ := path.Match(split[0], f.d.Entries[i].Name); match {
+		if match, _ := path.Match(split[0], f.LowDir.Entries[i].Name); match {
 			if len(split) == 1 {
-				out = append(out, f.d.Entries[i].Name)
+				out = append(out, f.LowDir.Entries[i].Name)
 				continue
 			}
 			sub, err := f.Sub(split[0])
@@ -80,8 +80,8 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 					Err:  err,
 				}
 			}
-			for i := 0; i < len(subGlob); i++ {
+			for i := range subGlob {
-				subGlob[i] = f.d.Name + "/" + subGlob[i]
+				subGlob[i] = f.LowDir.Name + "/" + subGlob[i]
 			}
 			out = append(out, subGlob...)
 		}
@@ -90,7 +90,11 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 }
 
 // Opens the file at name. Returns a *File as an fs.File.
-func (f *FS) Open(name string) (fs.File, error) {
+func (f FS) Open(name string) (fs.File, error) {
+	return f.OpenFile(name)
+}
+
+func (f FS) OpenFile(name string) (*File, error) {
 	name = filepath.Clean(name)
 	if !fs.ValidPath(name) {
 		return nil, &fs.PathError{
@@ -111,10 +115,10 @@ func (f *FS) Open(name string) (fs.File, error) {
 				Err:  fs.ErrNotExist,
 			}
 		} else {
-			return f.parent.Open(strings.Join(split[1:], "/"))
+			return f.parent.OpenFile(strings.Join(split[1:], "/"))
 		}
 	}
-	i, found := slices.BinarySearchFunc(f.d.Entries, split[0], func(e directory.Entry, name string) int {
+	i, found := slices.BinarySearchFunc(f.LowDir.Entries, split[0], func(e directory.Entry, name string) int {
 		return strings.Compare(e.Name, name)
 	})
 	if !found {
@@ -124,13 +128,13 @@ func (f *FS) Open(name string) (fs.File, error) {
 			Err:  fs.ErrNotExist,
 		}
 	}
-	b, err := f.r.Low.BaseFromEntry(f.d.Entries[i])
+	b, err := f.r.Low.BaseFromEntry(f.LowDir.Entries[i])
 	if err != nil {
 		return nil, err
 	}
 	if len(split) == 1 {
 		return &File{
-			b:      b,
+			Low:    b,
 			r:      f.r,
 			parent: f,
 		}, nil
@@ -146,12 +150,12 @@ func (f *FS) Open(name string) (fs.File, error) {
 	if err != nil {
 		return nil, err
 	}
-	return f.r.FSFromDirectory(d, f).Open(strings.Join(split[1:], "/"))
+	return f.r.FSFromDirectory(d, f).OpenFile(strings.Join(split[1:], "/"))
 }
 
 // Returns all DirEntry's for the directory at name.
 // If name is not a directory, returns an error.
-func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) {
+func (f FS) ReadDir(name string) ([]fs.DirEntry, error) {
 	name = filepath.Clean(name)
 	if !fs.ValidPath(name) {
 		return nil, &fs.PathError{
@@ -171,7 +175,7 @@ func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) {
 }
 
 // Returns the contents of the file at name.
-func (f *FS) ReadFile(name string) (out []byte, err error) {
+func (f FS) ReadFile(name string) (out []byte, err error) {
 	name = filepath.Clean(name)
 	if !fs.ValidPath(name) {
 		return nil, &fs.PathError{
@@ -194,7 +198,7 @@ func (f *FS) ReadFile(name string) (out []byte, err error) {
 }
 
 // Returns the fs.FileInfo for the file at name.
-func (f *FS) Stat(name string) (fs.FileInfo, error) {
+func (f FS) Stat(name string) (fs.FileInfo, error) {
 	name = filepath.Clean(name)
 	if !fs.ValidPath(name) {
 		return nil, &fs.PathError{
@@ -214,7 +218,7 @@ func (f *FS) Stat(name string) (fs.FileInfo, error) {
 }
 
 // Returns the FS at dir
-func (f *FS) Sub(dir string) (fs.FS, error) {
+func (f FS) Sub(dir string) (fs.FS, error) {
 	dir = filepath.Clean(dir)
 	if !fs.ValidPath(dir) {
 		return nil, &fs.PathError{
@@ -242,28 +246,34 @@ func (f *FS) Sub(dir string) (fs.FS, error) {
 
 // Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
 // Uses default extraction options.
-func (f *FS) Extract(folder string) error {
+func (f FS) Extract(folder string) error {
 	return f.File().Extract(folder)
 }
 
 // Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
 // Allows setting various extraction options via ExtractionOptions.
-func (f *FS) ExtractWithOptions(folder string, op *ExtractionOptions) error {
+func (f FS) ExtractWithOptions(folder string, op *ExtractionOptions) error {
 	return f.File().ExtractWithOptions(folder, op)
 }
 
 // Returns the FS as a *File
-func (f *FS) File() *File {
+func (f FS) File() *File {
+	if f.parent != nil {
 		return &File{
-		b:      &f.d.FileBase,
+			Low:    f.LowDir.FileBase,
-		parent: f.parent,
+			parent: *f.parent,
+			r:      f.r,
+		}
+	}
+	return &File{
+		Low: f.LowDir.FileBase,
 		r:   f.r,
 	}
 }
 
-func (f *FS) path() string {
+func (f FS) path() string {
 	if f.parent == nil {
-		return f.d.Name
+		return f.LowDir.Name
 	}
-	return filepath.Join(f.parent.path(), f.d.Name)
+	return filepath.Join(f.parent.path(), f.LowDir.Name)
 }

go.mod

@@ -1,11 +1,11 @@
 module github.com/CalebQ42/squashfs
 
-go 1.21.5
+go 1.24.0
 
 require (
-	github.com/pierrec/lz4/v4 v4.1.19
+	github.com/klauspost/compress v1.18.0
-	github.com/ulikunitz/xz v0.5.11
+	github.com/pierrec/lz4/v4 v4.1.22
-	github.com/klauspost/compress v1.17.4
 	github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e
 	github.com/therootcompany/xz v1.0.1
+	github.com/ulikunitz/xz v0.5.12
 )

go.sum

@@ -1,10 +1,10 @@
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
-github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
-github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs=
 github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
 github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
-github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
+github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
-github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=

internal/decompress/lz4.go

@@ -3,13 +3,28 @@ package decompress
 import (
 	"bytes"
 	"io"
+	"sync"
 
 	"github.com/pierrec/lz4/v4"
 )
 
-type Lz4 struct{}
+type Lz4 struct {
+	pool sync.Pool
+}
 
-func (l Lz4) Decompress(data []byte) ([]byte, error) {
+func NewLz4() *Lz4 {
-	rdr := lz4.NewReader(bytes.NewReader(data))
+	return &Lz4{
+		pool: sync.Pool{
+			New: func() any {
+				return lz4.NewReader(nil)
+			},
+		},
+	}
+}
+
+func (l *Lz4) Decompress(data []byte) ([]byte, error) {
+	rdr := l.pool.Get().(*lz4.Reader)
+	defer l.pool.Put(rdr)
+	rdr.Reset(bytes.NewReader(data))
 	return io.ReadAll(rdr)
 }

internal/decompress/lzma.go

@@ -1,3 +1,5 @@
+//go:build !no_obsolete
+
 package decompress
 
 import (
@@ -9,6 +11,10 @@ import (
 
 type Lzma struct{}
 
+func NewLzma() (Lzma, error) {
+	return Lzma{}, nil
+}
+
 func (l Lzma) Decompress(data []byte) ([]byte, error) {
 	rdr, err := lzma.NewReader(bytes.NewReader(data))
 	if err != nil {

internal/decompress/lzma_disabled.go

@@ -0,0 +1,17 @@
+//go:build no_obsolete
+
+package decompress
+
+import (
+	"errors"
+)
+
+type Lzma struct{}
+
+func NewLzma() (Lzma, error) {
+	return Lzma{}, errors.New("lzma compression is disable in this build with no_obsolete")
+}
+
+func (l Lzma) Decompress(data []byte) ([]byte, error) {
+	return nil, errors.New("lzma compression is disable in this build with no_obsolete")
+}

internal/decompress/lzo.go

@@ -1,3 +1,5 @@
+//go:build !no_gpl
+
 package decompress
 
 import (
@@ -8,6 +10,10 @@ import (
 
 type Lzo struct{}
 
+func NewLzo() (Lzo, error) {
+	return Lzo{}, nil
+}
+
 func (l Lzo) Decompress(data []byte) ([]byte, error) {
 	return lzo.Decompress1X(bytes.NewReader(data), len(data), 0)
 }

internal/decompress/lzo_disabled.go

@@ -0,0 +1,15 @@
+//go:build no_gpl
+
+package decompress
+
+import "errors"
+
+type Lzo struct{}
+
+func NewLzo() (Lzo, error) {
+	return Lzo{}, errors.New("lzo compression is disable in this build with no_gpl")
+}
+
+func (l Lzo) Decompress(data []byte) ([]byte, error) {
+	return nil, errors.New("lzo compression is disable in this build with no_gpl")
+}

internal/decompress/xz.go

@@ -3,14 +3,30 @@ package decompress
 import (
 	"bytes"
 	"io"
+	"sync"
 
 	"github.com/therootcompany/xz"
 )
 
-type Xz struct{}
+type Xz struct {
+	pool sync.Pool
+}
 
-func (x Xz) Decompress(data []byte) ([]byte, error) {
+func NewXz() *Xz {
-	rdr, err := xz.NewReader(bytes.NewReader(data), 0)
+	return &Xz{
+		pool: sync.Pool{
+			New: func() any {
+				rdr, _ := xz.NewReader(nil, 0)
+				return rdr
+			},
+		},
+	}
+}
+
+func (x *Xz) Decompress(data []byte) ([]byte, error) {
+	rdr := x.pool.Get().(*xz.Reader)
+	defer x.pool.Put(rdr)
+	err := rdr.Reset(bytes.NewReader(data))
 	if err != nil {
 		return nil, err
 	}

internal/decompress/zlib.go

@@ -2,17 +2,31 @@ package decompress
 
 import (
 	"bytes"
-	"compress/zlib"
 	"io"
+	"sync"
+
+	"github.com/klauspost/compress/zlib"
 )
 
-type Zlib struct{}
+type Zlib struct {
+	pool sync.Pool
+}
 
-func (z Zlib) Decompress(data []byte) ([]byte, error) {
+func NewZlib() *Zlib {
-	rdr, err := zlib.NewReader(bytes.NewReader(data))
+	return &Zlib{}
+}
+
+func (z *Zlib) Decompress(data []byte) ([]byte, error) {
+	rdr := z.pool.Get()
+	defer z.pool.Put(rdr)
+	var err error
+	if rdr == nil {
+		rdr, err = zlib.NewReader(bytes.NewReader(data))
+	} else {
+		err = rdr.(zlib.Resetter).Reset(bytes.NewReader(data), nil)
+	}
 	if err != nil {
 		return nil, err
 	}
-	defer rdr.Close()
+	return io.ReadAll(rdr.(io.ReadCloser))
-	return io.ReadAll(rdr)
 }

internal/decompress/zstd.go

@@ -1,19 +1,20 @@
 package decompress
 
 import (
-	"bytes"
-	"io"
-
 	"github.com/klauspost/compress/zstd"
 )
 
-type Zstd struct{}
+type Zstd struct {
+	rdr *zstd.Decoder
+}
+
+func NewZstd() Zstd {
+	rdr, _ := zstd.NewReader(nil, zstd.WithDecoderLowmem(true))
+	return Zstd{
+		rdr: rdr,
+	}
+}
 
 func (z Zstd) Decompress(data []byte) ([]byte, error) {
-	rdr, err := zstd.NewReader(bytes.NewReader(data))
+	return z.rdr.DecodeAll(data, nil)
-	if err != nil {
-		return nil, err
-	}
-	defer rdr.Close()
-	return io.ReadAll(rdr)
 }

internal/metadata/reader.go

@@ -14,8 +14,8 @@ type Reader struct {
 	curOffset uint16
 }
 
-func NewReader(r io.Reader, d decompress.Decompressor) *Reader {
+func NewReader(r io.Reader, d decompress.Decompressor) Reader {
-	return &Reader{
+	return Reader{
 		r: r,
 		d: d,
 	}
@@ -23,14 +23,15 @@ func NewReader(r io.Reader, d decompress.Decompressor) *Reader {
 
 func (r *Reader) advance() error {
 	r.curOffset = 0
-	var size uint16
+	dat := make([]byte, 2)
-	err := binary.Read(r.r, binary.LittleEndian, &size)
+	_, err := r.r.Read(dat)
 	if err != nil {
 		return err
 	}
+	size := binary.LittleEndian.Uint16(dat)
 	realSize := size &^ 0x8000
 	r.dat = make([]byte, realSize)
-	err = binary.Read(r.r, binary.LittleEndian, &r.dat)
+	_, err = r.r.Read(r.dat)
 	if err != nil {
 		return err
 	}
@@ -50,10 +51,7 @@ func (r *Reader) Read(b []byte) (int, error) {
 				return curRead, err
 			}
 		}
-		toRead = len(b) - curRead
+		toRead = min(len(b)-curRead, len(r.dat)-int(r.curOffset))
-		if toRead > len(r.dat)-int(r.curOffset) {
-			toRead = len(r.dat) - int(r.curOffset)
-		}
 		copy(b[curRead:], r.dat[r.curOffset:int(r.curOffset)+toRead])
 		r.curOffset += uint16(toRead)
 		curRead += toRead

internal/toreader/offsetreader.go

@@ -0,0 +1,19 @@
+package toreader
+
+import "io"
+
+type OffsetReader struct {
+	r   io.ReaderAt
+	off int64
+}
+
+func NewOffsetReader(r io.ReaderAt, off int64) *OffsetReader {
+	return &OffsetReader{
+		r:   r,
+		off: off,
+	}
+}
+
+func (r OffsetReader) ReadAt(p []byte, off int64) (n int, e error) {
+	return r.r.ReadAt(p, off+r.off)
+}

low/data/fullreader.go

@@ -1,15 +1,14 @@
 package data
 
 import (
-	"encoding/binary"
 	"errors"
 	"io"
+	"io/fs"
 	"math"
 	"runtime"
 	"sync"
 
 	"github.com/CalebQ42/squashfs/internal/decompress"
-	"github.com/CalebQ42/squashfs/internal/toreader"
 )
 
 type FragReaderConstructor func() (io.Reader, error)
@@ -18,16 +17,16 @@ type FullReader struct {
 	r              io.ReaderAt
 	d              decompress.Decompressor
 	frag           FragReaderConstructor
-	retPool        *sync.Pool
 	sizes          []uint32
 	initialOffset  int64
 	finalBlockSize uint64
 	blockSize      uint32
 	goroutineLimit uint16
+	closed         bool
 }
 
-func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) *FullReader {
+func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) FullReader {
-	return &FullReader{
+	return FullReader{
 		r:              r,
 		d:              d,
 		sizes:          sizes,
@@ -35,19 +34,26 @@ func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor
 		goroutineLimit: uint16(runtime.NumCPU()),
 		finalBlockSize: finalBlockSize,
 		blockSize:      blockSize,
-		retPool: &sync.Pool{
-			New: func() any {
-				return &retValue{}
-			},
-		},
 	}
 }
 
+func (r *FullReader) Close() error {
+	r.closed = true
+	r.r = nil
+	r.d = nil
+	r.frag = nil
+	r.sizes = nil
+	return nil
+}
+
 func (r *FullReader) AddFrag(frag FragReaderConstructor) {
 	r.frag = frag
 }
 
 func (r *FullReader) SetGoroutineLimit(limit uint16) {
+	if limit <= 0 {
+		r.goroutineLimit = 1
+	}
 	r.goroutineLimit = limit
 }
 
@@ -57,8 +63,8 @@ type retValue struct {
 	index uint64
 }
 
-func (r *FullReader) process(index uint64, fileOffset uint64, retChan chan *retValue) {
+func (r FullReader) process(index uint64, fileOffset uint64, pool *sync.Pool, retChan chan *retValue) {
-	ret := r.retPool.Get().(*retValue)
+	ret := pool.Get().(*retValue)
 	ret.index = index
 	realSize := r.sizes[index] &^ (1 << 24)
 	if realSize == 0 {
@@ -72,14 +78,20 @@ func (r *FullReader) process(index uint64, fileOffset uint64, retChan chan *retV
 		return
 	}
 	ret.data = make([]byte, realSize)
-	ret.err = binary.Read(toreader.NewReader(r.r, int64(r.initialOffset)+int64(fileOffset)), binary.LittleEndian, &ret.data)
+	_, ret.err = r.r.ReadAt(ret.data, r.initialOffset+int64(fileOffset))
 	if r.sizes[index] == realSize {
 		ret.data, ret.err = r.d.Decompress(ret.data)
 	}
 	retChan <- ret
 }
 
-func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
+func (r FullReader) WriteTo(w io.Writer) (int64, error) {
+	if r.closed {
+		return 0, fs.ErrClosed
+	}
+	// if wa, is := w.(io.WriterAt); is {
+	// 	return r.writeToWriteAt(wa)
+	// }
 	var curIndex uint64
 	var curOffset uint64
 	var toProcess uint16
@@ -87,14 +99,16 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
 	cache := make(map[uint64]*retValue)
 	var errCache []error
 	retChan := make(chan *retValue, r.goroutineLimit)
-	for i := uint64(0); i < uint64(math.Ceil(float64(len(r.sizes))/float64(r.goroutineLimit))); i++ {
+	pool := &sync.Pool{
-		toProcess = uint16(len(r.sizes)) - (uint16(i) * r.goroutineLimit)
+		New: func() any {
-		if toProcess > r.goroutineLimit {
+			return &retValue{}
-			toProcess = r.goroutineLimit
+		},
 	}
+	for i := uint64(0); i < uint64(math.Ceil(float64(len(r.sizes))/float64(r.goroutineLimit))); i++ {
+		toProcess = min(uint16(len(r.sizes))-(uint16(i)*r.goroutineLimit), r.goroutineLimit)
 		// Start all the goroutines
 		for j := uint16(0); j < toProcess; j++ {
-			go r.process((i*uint64(r.goroutineLimit))+uint64(j), curOffset, retChan)
+			go r.process((i*uint64(r.goroutineLimit))+uint64(j), curOffset, pool, retChan)
 			curOffset += uint64(r.sizes[(i*uint64(r.goroutineLimit))+uint64(j)]) &^ (1 << 24)
 		}
 		// Then consume the results on retChan
@@ -128,7 +142,7 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
 				}
 				continue
 			}
-			r.retPool.Put(res)
+			pool.Put(res)
 			curIndex++
 			// Now we recursively try to clear the cache
 			for len(cache) > 0 {
@@ -146,7 +160,7 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
 					break
 				}
 				delete(cache, curIndex)
-				r.retPool.Put(res)
+				pool.Put(res)
 				curIndex++
 			}
 		}
@@ -172,3 +186,75 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
 	}
 	return wrote, nil
 }
+
+// func (r FullReader) writeToWriteAt(w io.WriterAt) (out int64, outErr error) {
+// 	wait := &sync.WaitGroup{}
+// 	wait.Add(len(r.sizes))
+// 	mgr := routinemanager.NewManager(r.goroutineLimit)
+// 	curOffset := r.initialOffset
+// 	for i := uint64(0); i < uint64(len(r.sizes)); i++ {
+// 		go func(index uint64, fileOffset int64) {
+// 			lckNum := mgr.Lock()
+// 			defer mgr.Unlock(lckNum)
+// 			defer wait.Done()
+// 			realSize := r.sizes[index] &^ (1 << 24)
+// 			if realSize == 0 {
+// 				if index == uint64(len(r.sizes))-1 && r.frag == nil {
+// 					_, err := w.WriteAt([]byte{0}, int64((uint64(r.blockSize)*index)+r.finalBlockSize)-1)
+// 					if err != nil {
+// 						outErr = errors.Join(outErr, err)
+// 						return
+// 					}
+// 					out = max(out, int64((uint64(r.blockSize)*index)+r.finalBlockSize))
+// 				}
+// 				return
+// 			}
+// 			data := make([]byte, realSize)
+// 			err := binary.Read(toreader.NewReader(r.r, int64(fileOffset)), binary.LittleEndian, &data)
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			if r.sizes[index] == realSize {
+// 				data, err = r.d.Decompress(data)
+// 			}
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			_, err = w.WriteAt(data, int64(uint64(r.blockSize)*index))
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			out = max(out, int64(uint64(r.blockSize)*(index+1)))
+// 		}(i, curOffset)
+// 		curOffset += int64(r.sizes[i]) &^ (1 << 24)
+// 	}
+// 	if r.frag != nil {
+// 		wait.Add(1)
+// 		go func() {
+// 			lckNum := mgr.Lock()
+// 			defer mgr.Unlock(lckNum)
+// 			defer wait.Done()
+// 			rdr, err := r.frag()
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			dat, err := io.ReadAll(rdr)
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			_, err = w.WriteAt(dat, int64(int(r.blockSize)*len(r.sizes)))
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			out = int64(int(r.blockSize)*len(r.sizes)) + int64(r.finalBlockSize)
+// 		}()
+// 	}
+// 	wait.Wait()
+// 	return
+// }

low/data/reader.go

@@ -1,8 +1,8 @@
 package data
 
 import (
-	"encoding/binary"
 	"io"
+	"io/fs"
 
 	"github.com/CalebQ42/squashfs/internal/decompress"
 )
@@ -17,10 +17,11 @@ type Reader struct {
 	curIndex       uint64
 	finalBlockSize uint64
 	blockSize      uint32
+	closed         bool
 }
 
-func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) *Reader {
+func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) Reader {
-	return &Reader{
+	return Reader{
 		r:              r,
 		d:              d,
 		sizes:          sizes,
@@ -41,6 +42,7 @@ func (r *Reader) advance() error {
 		r.dat, err = io.ReadAll(r.frag)
 		return err
 	} else if r.curIndex >= uint64(len(r.sizes)) {
+		r.dat = []byte{}
 		return io.EOF
 	}
 	realSize := r.sizes[r.curIndex] &^ (1 << 24)
@@ -53,7 +55,7 @@ func (r *Reader) advance() error {
 		return nil
 	}
 	r.dat = make([]byte, realSize)
-	err = binary.Read(r.r, binary.LittleEndian, &r.dat)
+	_, err = r.r.Read(r.dat)
 	if err != nil {
 		return err
 	}
@@ -65,6 +67,9 @@ func (r *Reader) advance() error {
 }
 
 func (r *Reader) Read(b []byte) (int, error) {
+	if r.closed {
+		return 0, fs.ErrClosed
+	}
 	curRead := 0
 	var toRead int
 	for curRead < len(b) {
@@ -73,10 +78,7 @@ func (r *Reader) Read(b []byte) (int, error) {
 				return curRead, err
 			}
 		}
-		toRead = len(b) - curRead
+		toRead = min(len(b)-curRead, len(r.dat)-r.curOffset)
-		if toRead > len(r.dat)-r.curOffset {
-			toRead = len(r.dat) - r.curOffset
-		}
 		toRead = copy(b[curRead:], r.dat[r.curOffset:r.curOffset+toRead])
 		r.curOffset += toRead
 		curRead += toRead
@@ -85,6 +87,9 @@ func (r *Reader) Read(b []byte) (int, error) {
 }
 
 func (r *Reader) Close() error {
+	r.closed = true
+	r.r = nil
+	r.d = nil
 	if r.frag != nil {
 		if l, ok := r.frag.(*io.LimitedReader); ok {
 			if cl, ok := l.R.(io.Closer); ok {
@@ -92,6 +97,8 @@ func (r *Reader) Close() error {
 			}
 		}
 	}
+	r.frag = nil
+	r.sizes = nil
 	r.dat = nil
 	return nil
 }

low/directory.go

@@ -18,10 +18,10 @@ type Directory struct {
 	Entries []directory.Entry
 }
 
-func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) {
+func (r Reader) directoryFromRef(ref uint64, name string) (Directory, error) {
 	i, err := r.InodeFromRef(ref)
 	if err != nil {
-		return nil, err
+		return Directory{}, err
 	}
 	var blockStart uint32
 	var size uint32
@@ -36,48 +36,48 @@ func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) {
 		size = i.Data.(inode.EDirectory).Size
 		offset = i.Data.(inode.EDirectory).Offset
 	default:
-		return nil, errors.New("not a directory")
+		return Directory{}, errors.New("not a directory")
 	}
 	dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d)
 	defer dirRdr.Close()
 	_, err = dirRdr.Read(make([]byte, offset))
 	if err != nil {
-		return nil, err
+		return Directory{}, err
 	}
-	entries, err := directory.ReadDirectory(dirRdr, size)
+	entries, err := directory.ReadDirectory(&dirRdr, size)
 	if err != nil {
-		return nil, err
+		return Directory{}, err
 	}
-	return &Directory{
+	return Directory{
-		FileBase: *r.BaseFromInode(i, name),
+		FileBase: r.BaseFromInode(i, name),
 		Entries:  entries,
 	}, nil
 }
 
-func (d *Directory) Open(r *Reader, path string) (*FileBase, error) {
+func (d Directory) Open(r Reader, path string) (FileBase, error) {
 	path = filepath.Clean(path)
 	if path == "." || path == "" {
-		return &d.FileBase, nil
+		return d.FileBase, nil
 	}
 	split := strings.Split(path, "/")
 	i, found := slices.BinarySearchFunc(d.Entries, split[0], func(e directory.Entry, name string) int {
 		return strings.Compare(e.Name, name)
 	})
 	if !found {
-		return nil, fs.ErrNotExist
+		return FileBase{}, fs.ErrNotExist
 	}
 	b, err := r.BaseFromEntry(d.Entries[i])
 	if err != nil {
-		return nil, err
+		return FileBase{}, err
 	}
 	if len(split) == 1 {
 		return b, nil
 	} else if !b.IsDir() {
-		return nil, fs.ErrNotExist
+		return FileBase{}, fs.ErrNotExist
 	}
 	dir, err := b.ToDir(r)
 	if err != nil {
-		return nil, err
+		return FileBase{}, err
 	}
 	return dir.Open(r, strings.Join(split[1:], "/"))
 }

low/file_base.go

@@ -12,43 +12,43 @@ import (
 )
 
 type FileBase struct {
-	Inode *inode.Inode
+	Inode inode.Inode
 	Name  string
 }
 
-func (r *Reader) BaseFromInode(i *inode.Inode, name string) *FileBase {
+func (r Reader) BaseFromInode(i inode.Inode, name string) FileBase {
-	return &FileBase{Inode: i, Name: name}
+	return FileBase{Inode: i, Name: name}
 }
 
-func (r *Reader) BaseFromEntry(e directory.Entry) (*FileBase, error) {
+func (r Reader) BaseFromEntry(e directory.Entry) (FileBase, error) {
 	in, err := r.InodeFromEntry(e)
 	if err != nil {
-		return nil, err
+		return FileBase{}, err
 	}
-	return &FileBase{Inode: in, Name: e.Name}, nil
+	return FileBase{Inode: in, Name: e.Name}, nil
 }
 
-func (r *Reader) BaseFromRef(ref uint64, name string) (*FileBase, error) {
+func (r Reader) BaseFromRef(ref uint64, name string) (FileBase, error) {
 	in, err := r.InodeFromRef(ref)
 	if err != nil {
-		return nil, err
+		return FileBase{}, err
 	}
-	return &FileBase{Inode: in, Name: name}, nil
+	return FileBase{Inode: in, Name: name}, nil
 }
 
-func (b *FileBase) Uid(r *Reader) (uint32, error) {
+func (b FileBase) Uid(r *Reader) (uint32, error) {
 	return r.Id(b.Inode.UidInd)
 }
 
-func (b *FileBase) Gid(r *Reader) (uint32, error) {
+func (b FileBase) Gid(r *Reader) (uint32, error) {
 	return r.Id(b.Inode.GidInd)
 }
 
-func (b *FileBase) IsDir() bool {
+func (b FileBase) IsDir() bool {
 	return b.Inode.Type == inode.Dir || b.Inode.Type == inode.EDir
 }
 
-func (b *FileBase) ToDir(r *Reader) (*Directory, error) {
+func (b FileBase) ToDir(r Reader) (Directory, error) {
 	var blockStart uint32
 	var size uint32
 	var offset uint16
@@ -62,31 +62,31 @@ func (b *FileBase) ToDir(r *Reader) (*Directory, error) {
 		size = b.Inode.Data.(inode.EDirectory).Size
 		offset = b.Inode.Data.(inode.EDirectory).Offset
 	default:
-		return nil, errors.New("not a directory")
+		return Directory{}, errors.New("not a directory")
 	}
 	dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d)
 	defer dirRdr.Close()
 	_, err := dirRdr.Read(make([]byte, offset))
 	if err != nil {
-		return nil, err
+		return Directory{}, err
 	}
-	entries, err := directory.ReadDirectory(dirRdr, size)
+	entries, err := directory.ReadDirectory(&dirRdr, size)
 	if err != nil {
-		return nil, err
+		return Directory{}, err
 	}
-	return &Directory{
+	return Directory{
-		FileBase: *b,
+		FileBase: b,
 		Entries:  entries,
 	}, nil
 }
 
-func (b *FileBase) IsRegular() bool {
+func (b FileBase) IsRegular() bool {
 	return b.Inode.Type == inode.Fil || b.Inode.Type == inode.EFil
 }
 
-func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, error) {
+func (b FileBase) GetRegFileReaders(r Reader) (data.Reader, data.FullReader, error) {
 	if !b.IsRegular() {
-		return nil, nil, errors.New("not a regular file")
+		return data.Reader{}, data.FullReader{}, errors.New("not a regular file")
 	}
 	var blockStart uint64
 	var fragIndex uint32
@@ -113,13 +113,13 @@ func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader,
 		}
 		frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
 		frag.Read(make([]byte, fragOffset))
-		return io.LimitReader(frag, int64(fragSize)), nil
+		return io.LimitReader(&frag, int64(fragSize)), nil
 	}
 	outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize)
 	if fragIndex != 0xffffffff {
 		f, err := frag()
 		if err != nil {
-			return nil, nil, err
+			return data.Reader{}, data.FullReader{}, err
 		}
 		outRdr.AddFrag(f)
 	}
@@ -130,9 +130,9 @@ func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader,
 	return outRdr, outFull, nil
 }
 
-func (b *FileBase) GetFullReader(r *Reader) (*data.FullReader, error) {
+func (b FileBase) GetFullReader(r *Reader) (data.FullReader, error) {
 	if !b.IsRegular() {
-		return nil, errors.New("not a regular file")
+		return data.FullReader{}, errors.New("not a regular file")
 	}
 	var blockStart uint64
 	var fragIndex uint32
@@ -161,15 +161,15 @@ func (b *FileBase) GetFullReader(r *Reader) (*data.FullReader, error) {
 			}
 			frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
 			frag.Read(make([]byte, fragOffset))
-			return io.LimitReader(frag, int64(fragSize)), nil
+			return io.LimitReader(&frag, int64(fragSize)), nil
 		})
 	}
 	return outFull, nil
 }
 
-func (b *FileBase) GetReader(r *Reader) (*data.Reader, error) {
+func (b FileBase) GetReader(r *Reader) (data.Reader, error) {
 	if !b.IsRegular() {
-		return nil, errors.New("not a regular file")
+		return data.Reader{}, errors.New("not a regular file")
 	}
 	var blockStart uint64
 	var fragIndex uint32
@@ -193,11 +193,11 @@ func (b *FileBase) GetReader(r *Reader) (*data.Reader, error) {
 	if fragIndex != 0xffffffff {
 		ent, err := r.fragEntry(fragIndex)
 		if err != nil {
-			return nil, err
+			return data.Reader{}, err
 		}
 		frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
 		frag.Read(make([]byte, fragOffset))
-		outRdr.AddFrag(io.LimitReader(frag, int64(fragSize)))
+		outRdr.AddFrag(io.LimitReader(&frag, int64(fragSize)))
 	}
 	return outRdr, nil
 }

low/inode.go

@@ -7,20 +7,20 @@ import (
 	"github.com/CalebQ42/squashfs/low/inode"
 )
 
-func (r *Reader) InodeFromRef(ref uint64) (*inode.Inode, error) {
+func (r Reader) InodeFromRef(ref uint64) (inode.Inode, error) {
 	offset, meta := (ref>>16)+r.Superblock.InodeTableStart, ref&0xFFFF
 	rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
 	defer rdr.Close()
 	_, err := rdr.Read(make([]byte, meta))
 	if err != nil {
-		return nil, err
+		return inode.Inode{}, err
 	}
-	return inode.Read(rdr, r.Superblock.BlockSize)
+	return inode.Read(&rdr, r.Superblock.BlockSize)
 }
 
-func (r *Reader) InodeFromEntry(e directory.Entry) (*inode.Inode, error) {
+func (r Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) {
 	rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d)
 	defer rdr.Close()
 	rdr.Read(make([]byte, e.Offset))
-	return inode.Read(rdr, r.Superblock.BlockSize)
+	return inode.Read(&rdr, r.Superblock.BlockSize)
 }

low/inode/dir.go

@@ -13,7 +13,7 @@ type Directory struct {
 	ParentNum  uint32
 }
 
-type eDirectoryInit struct {
+type EDirectory struct {
 	LinkCount  uint32
 	Size       uint32
 	BlockStart uint32
@@ -21,42 +21,55 @@ type eDirectoryInit struct {
 	IndCount   uint16
 	Offset     uint16
 	XattrInd   uint32
-}
-
-type EDirectory struct {
-	eDirectoryInit
 	Indexes    []DirectoryIndex
 }
 
-type directoryIndexInit struct {
+type DirectoryIndex struct {
 	Ind      uint32
 	Start    uint32
 	NameSize uint32
-}
-
-type DirectoryIndex struct {
-	directoryIndexInit
 	Name     []byte
 }
 
 func ReadDir(r io.Reader) (d Directory, err error) {
-	err = binary.Read(r, binary.LittleEndian, &d)
+	dat := make([]byte, 16)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	d.BlockStart = binary.LittleEndian.Uint32(dat)
+	d.LinkCount = binary.LittleEndian.Uint32(dat[4:])
+	d.Size = binary.LittleEndian.Uint16(dat[8:])
+	d.Offset = binary.LittleEndian.Uint16(dat[10:])
+	d.ParentNum = binary.LittleEndian.Uint32(dat[12:])
 	return
 }
 
 func ReadEDir(r io.Reader) (d EDirectory, err error) {
-	err = binary.Read(r, binary.LittleEndian, &d.eDirectoryInit)
+	dat := make([]byte, 24)
+	_, err = r.Read(dat)
 	if err != nil {
 		return
 	}
+	d.LinkCount = binary.LittleEndian.Uint32(dat)
+	d.Size = binary.LittleEndian.Uint32(dat[4:])
+	d.BlockStart = binary.LittleEndian.Uint32(dat[8:])
+	d.ParentNum = binary.LittleEndian.Uint32(dat[12:])
+	d.IndCount = binary.LittleEndian.Uint16(dat[16:])
+	d.Offset = binary.LittleEndian.Uint16(dat[18:])
+	d.XattrInd = binary.LittleEndian.Uint32(dat[20:])
 	d.Indexes = make([]DirectoryIndex, d.IndCount)
-	for i := range d.Indexes {
+	for i := range d.IndCount {
-		err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].directoryIndexInit)
+		dat = make([]byte, 12)
+		_, err = r.Read(dat)
 		if err != nil {
 			return
 		}
+		d.Indexes[i].Ind = binary.LittleEndian.Uint32(dat)
+		d.Indexes[i].Start = binary.LittleEndian.Uint32(dat[4:])
+		d.Indexes[i].NameSize = binary.LittleEndian.Uint32(dat[8:])
 		d.Indexes[i].Name = make([]byte, d.Indexes[i].NameSize+1)
-		err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].Name)
+		_, err = r.Read(d.Indexes[i].Name)
 		if err != nil {
 			return
 		}

low/inode/file.go

@@ -6,15 +6,11 @@ import (
 	"math"
 )
 
-type fileInit struct {
+type File struct {
 	BlockStart uint32
 	FragInd    uint32
 	FragOffset uint32
 	Size       uint32
-}
-
-type File struct {
-	fileInit
 	BlockSizes []uint32
 }
 
@@ -34,16 +30,28 @@ type EFile struct {
 }
 
 func ReadFile(r io.Reader, blockSize uint32) (f File, err error) {
-	err = binary.Read(r, binary.LittleEndian, &f.fileInit)
+	dat := make([]byte, 16)
+	_, err = r.Read(dat)
 	if err != nil {
 		return
 	}
+	f.BlockStart = binary.LittleEndian.Uint32(dat)
+	f.FragInd = binary.LittleEndian.Uint32(dat[4:])
+	f.FragOffset = binary.LittleEndian.Uint32(dat[8:])
+	f.Size = binary.LittleEndian.Uint32(dat[12:])
 	toRead := int(math.Floor(float64(f.Size) / float64(blockSize)))
 	if f.FragInd == 0xFFFFFFFF && f.Size%blockSize > 0 {
 		toRead++
 	}
+	dat = make([]byte, toRead*4)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
 	f.BlockSizes = make([]uint32, toRead)
-	err = binary.Read(r, binary.LittleEndian, &f.BlockSizes)
+	for i := range toRead {
+		f.BlockSizes[i] = binary.LittleEndian.Uint32(dat[i*4:])
+	}
 	return
 }
 

low/inode/inode.go

@@ -39,8 +39,7 @@ type Inode struct {
 	Data any
 }
 
-func Read(r io.Reader, blockSize uint32) (i *Inode, err error) {
+func Read(r io.Reader, blockSize uint32) (i Inode, err error) {
-	i = new(Inode)
 	err = binary.Read(r, binary.LittleEndian, &i.Header)
 	if err != nil {
 		return
@@ -82,23 +81,17 @@ func Read(r io.Reader, blockSize uint32) (i *Inode, err error) {
 
 func (i Inode) Mode() (out fs.FileMode) {
 	out = fs.FileMode(i.Perm)
-	switch i.Data.(type) {
+	switch i.Type {
-	case Directory:
+	case Dir, EDir:
 		out |= fs.ModeDir
-	case EDirectory:
+	case Sym, ESym:
-		out |= fs.ModeDir
-	case Symlink:
 		out |= fs.ModeSymlink
-	case ESymlink:
+	case Char, EChar, Block, EBlock:
-		out |= fs.ModeSymlink
-	case Device:
 		out |= fs.ModeDevice
-	case EDevice:
+	case Fifo, EFifo:
-		out |= fs.ModeDevice
-	case IPC:
-		out |= fs.ModeNamedPipe
-	case EIPC:
 		out |= fs.ModeNamedPipe
+	case Sock, ESock:
+		out |= fs.ModeSocket
 	}
 	return
 }

low/reader.go

@@ -32,48 +32,53 @@ var (
 type Reader struct {
 	r           io.ReaderAt
 	d           decompress.Decompressor
-	Root        *Directory
+	Root        Directory
 	fragTable   []fragEntry
 	idTable     []uint32
 	exportTable []uint64
 	Superblock  superblock
 }
 
-func NewReader(r io.ReaderAt) (rdr *Reader, err error) {
+func NewReader(r io.ReaderAt) (rdr Reader, err error) {
-	rdr = new(Reader)
 	rdr.r = r
 	err = binary.Read(toreader.NewReader(r, 0), binary.LittleEndian, &rdr.Superblock)
 	if err != nil {
-		return nil, errors.Join(errors.New("failed to read superblock"), err)
+		return rdr, errors.Join(errors.New("failed to read superblock"), err)
 	}
 	if !rdr.Superblock.ValidMagic() {
-		return nil, ErrorMagic
+		return rdr, ErrorMagic
 	}
 	if !rdr.Superblock.ValidBlockLog() {
-		return nil, ErrorLog
+		return rdr, ErrorLog
 	}
 	if !rdr.Superblock.ValidVersion() {
-		return nil, ErrorVersion
+		return rdr, ErrorVersion
 	}
 	switch rdr.Superblock.CompType {
 	case ZlibCompression:
-		rdr.d = decompress.Zlib{}
+		rdr.d = decompress.NewZlib()
 	case LZMACompression:
-		rdr.d = decompress.Lzma{}
+		rdr.d, err = decompress.NewLzma()
+		if err != nil {
+			return rdr, err
+		}
 	case LZOCompression:
-		rdr.d = decompress.Lzo{}
+		rdr.d, err = decompress.NewLzo()
+		if err != nil {
+			return rdr, err
+		}
 	case XZCompression:
-		rdr.d = decompress.Xz{}
+		rdr.d = decompress.NewXz()
 	case LZ4Compression:
-		rdr.d = decompress.Lz4{}
+		rdr.d = decompress.NewLz4()
 	case ZSTDCompression:
-		rdr.d = &decompress.Zstd{}
+		rdr.d = decompress.NewZstd()
 	default:
-		return nil, errors.New("invalid compression type. possible corrupted archive")
+		return rdr, errors.New("invalid compression type. possible corrupted archive")
 	}
 	rdr.Root, err = rdr.directoryFromRef(rdr.Superblock.RootInodeRef, "")
 	if err != nil {
-		return nil, errors.Join(errors.New("failed to read root directory"), err)
+		return rdr, errors.Join(errors.New("failed to read root directory"), err)
 	}
 	return
 }
@@ -88,7 +93,7 @@ func (r *Reader) Id(i uint16) (uint32, error) {
 	// Populate the id table as needed
 	var blockNum uint32
 	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
-		blockNum = uint32(math.Ceil(float64(i)/2048)) - 1
+		blockNum = uint32(math.Ceil(float64(i+1)/2048)) - 1
 	} else {
 		blockNum = 0
 	}
@@ -99,19 +104,17 @@ func (r *Reader) Id(i uint16) (uint32, error) {
 	var idsToRead uint16
 	var idsTmp []uint32
 	var err error
-	var rdr *metadata.Reader
+	var rdr metadata.Reader
+	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
 	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
 		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset)
 		if err != nil {
 			return 0, err
 		}
-		idsToRead = r.Superblock.IdCount - uint16(len(r.idTable))
+		idsToRead = min(r.Superblock.IdCount-uint16(len(r.idTable)), 2048)
-		if idsToRead > 2048 {
-			idsToRead = 2048
-		}
 		idsTmp = make([]uint32, idsToRead)
 		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
-		err = binary.Read(rdr, binary.LittleEndian, &idsTmp)
+		err = binary.Read(&rdr, binary.LittleEndian, &idsTmp)
 		rdr.Close()
 		if err != nil {
 			return 0, err
@@ -131,7 +134,7 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
 	// Populate the fragment table as needed
 	var blockNum uint32
 	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
-		blockNum = uint32(math.Ceil(float64(i)/512)) - 1
+		blockNum = uint32(math.Ceil(float64(i+1)/512)) - 1
 	} else {
 		blockNum = 0
 	}
@@ -142,19 +145,17 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
 	var fragsToRead uint32
 	var fragsTmp []fragEntry
 	var err error
-	var rdr *metadata.Reader
+	var rdr metadata.Reader
+	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
 	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
 		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset)
 		if err != nil {
 			return fragEntry{}, err
 		}
-		fragsToRead = r.Superblock.FragCount - uint32(len(r.fragTable))
+		fragsToRead = min(r.Superblock.FragCount-uint32(len(r.fragTable)), 512)
-		if fragsToRead > 512 {
-			fragsToRead = 512
-		}
 		fragsTmp = make([]fragEntry, fragsToRead)
 		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
-		err = binary.Read(rdr, binary.LittleEndian, &fragsTmp)
+		err = binary.Read(&rdr, binary.LittleEndian, &fragsTmp)
 		rdr.Close()
 		if err != nil {
 			return fragEntry{}, err
@@ -177,7 +178,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
 	// Populate the export table as needed
 	var blockNum uint32
 	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
-		blockNum = uint32(math.Ceil(float64(i)/1024)) - 1
+		blockNum = uint32(math.Ceil(float64(i+1)/1024)) - 1
 	} else {
 		blockNum = 0
 	}
@@ -188,19 +189,17 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
 	var refsToRead uint32
 	var refsTmp []uint64
 	var err error
-	var rdr *metadata.Reader
+	var rdr metadata.Reader
+	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
 	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
 		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset)
 		if err != nil {
 			return 0, err
 		}
-		refsToRead = r.Superblock.InodeCount - uint32(len(r.exportTable))
+		refsToRead = min(r.Superblock.InodeCount-uint32(len(r.exportTable)), 1024)
-		if refsToRead > 1024 {
-			refsToRead = 1024
-		}
 		refsTmp = make([]uint64, refsToRead)
 		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
-		err = binary.Read(rdr, binary.LittleEndian, &refsTmp)
+		err = binary.Read(&rdr, binary.LittleEndian, &refsTmp)
 		rdr.Close()
 		if err != nil {
 			return 0, err
@@ -210,10 +209,10 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
 	return r.exportTable[i], nil
 }
 
-func (r *Reader) Inode(i uint32) (*inode.Inode, error) {
+func (r Reader) Inode(i uint32) (inode.Inode, error) {
 	ref, err := r.inodeRef(i)
 	if err != nil {
-		return nil, err
+		return inode.Inode{}, err
 	}
 	return r.InodeFromRef(ref)
 }

low/reader_test.go

@@ -1,4 +1,4 @@
-package squashfslow_test
+package squashfslow
 
 import (
 	"fmt"
@@ -8,13 +8,11 @@ import (
 	"os/exec"
 	"path/filepath"
 	"testing"
-
-	squashfslow "github.com/CalebQ42/squashfs/low"
 )
 
 const (
 	squashfsURL  = "https://darkstorm.tech/files/LinuxPATest.sfs"
-	squashfsName = "LinuxPATest.sfs"
+	squashfsName = "airootfs.sfs"
 )
 
 func preTest(dir string) (fil *os.File, err error) {
@@ -50,6 +48,21 @@ func preTest(dir string) (fil *os.File, err error) {
 	return
 }
 
+func TestMisc(t *testing.T) {
+	tmpDir := "../testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fil.Close()
+	rdr, err := NewReader(fil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(rdr.Superblock.FragCount)
+	t.Fatal(rdr.fragEntry(1233))
+}
+
 func TestReader(t *testing.T) {
 	tmpDir := "../testing"
 	fil, err := preTest(tmpDir)
@@ -57,14 +70,14 @@ func TestReader(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer fil.Close()
-	rdr, err := squashfslow.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	path := filepath.Join(tmpDir, "extractTest")
 	os.RemoveAll(path)
 	os.MkdirAll(path, 0777)
-	err = extractToDir(rdr, &rdr.Root.FileBase, path)
+	err = extractToDir(rdr, rdr.Root.FileBase, path)
 	t.Fatal(err)
 }
 
@@ -77,7 +90,7 @@ func TestSingleFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer fil.Close()
-	rdr, err := squashfslow.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -92,7 +105,7 @@ func TestSingleFile(t *testing.T) {
 	t.Fatal(err)
 }
 
-func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder string) error {
+func extractToDir(rdr Reader, b FileBase, folder string) error {
 	path := filepath.Join(folder, b.Name)
 	if b.IsDir() {
 		d, err := b.ToDir(rdr)
@@ -103,7 +116,7 @@ func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder strin
 		if err != nil {
 			return err
 		}
-		var nestBast *squashfslow.FileBase
+		var nestBast FileBase
 		for _, e := range d.Entries {
 			nestBast, err = rdr.BaseFromEntry(e)
 			if err != nil {

reader.go

@@ -4,29 +4,34 @@ import (
 	"io"
 	"time"
 
+	"github.com/CalebQ42/squashfs/internal/toreader"
 	squashfslow "github.com/CalebQ42/squashfs/low"
 )
 
 type Reader struct {
-	*FS
+	FS
-	Low *squashfslow.Reader
+	Low squashfslow.Reader
 }
 
-func NewReader(r io.ReaderAt) (*Reader, error) {
+func NewReader(r io.ReaderAt) (Reader, error) {
 	rdr, err := squashfslow.NewReader(r)
 	if err != nil {
-		return nil, err
+		return Reader{}, err
 	}
-	out := &Reader{
+	out := Reader{
 		Low: rdr,
 	}
-	out.FS = &FS{
+	out.FS = FS{
-		d: rdr.Root,
+		LowDir: rdr.Root,
-		r: out,
+		r:   &out,
 	}
 	return out, nil
 }
 
+func NewReaderAtOffset(r io.ReaderAt, offset int64) (Reader, error) {
+	return NewReader(toreader.NewOffsetReader(r, offset))
+}
+
 func (r *Reader) ModTime() time.Time {
 	return time.Unix(int64(r.Low.Superblock.ModTime), 0)
 }

squashfs_test.go

@@ -1,4 +1,4 @@
-package squashfs_test
+package squashfs
 
 //Actually proper tests go here.
 
@@ -13,13 +13,11 @@ import (
 	"strconv"
 	"testing"
 	"time"
-
-	"github.com/CalebQ42/squashfs"
 )
 
 const (
 	squashfsURL  = "https://darkstorm.tech/files/LinuxPATest.sfs"
-	squashfsName = "airootfs.sfs"
+	squashfsName = "tensorflow.sqfs"
 )
 
 func preTest(dir string) (fil *os.File, err error) {
@@ -61,13 +59,31 @@ func TestMisc(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	rdr, err := squashfs.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	_ = rdr
 	// Put testing here
-	t.Fatal("UM")
+	// t.Fatal("UM")
+}
+
+func BenchmarkExtract(b *testing.B) {
+	tmpDir := "testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		b.Fatal(err)
+	}
+	libPath := filepath.Join(tmpDir, "ExtractLib")
+	os.RemoveAll(libPath)
+	rdr, err := NewReader(fil)
+	if err != nil {
+		b.Fatal(err)
+	}
+	err = rdr.ExtractWithOptions(libPath, FastOptions())
+	if err != nil {
+		b.Fatal(err)
+	}
 }
 
 func BenchmarkRace(b *testing.B) {
@@ -81,18 +97,17 @@ func BenchmarkRace(b *testing.B) {
 	os.RemoveAll(libPath)
 	os.RemoveAll(unsquashPath)
 	var libTime, unsquashTime time.Duration
-	op := squashfs.FastOptions()
 	start := time.Now()
-	rdr, err := squashfs.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		b.Fatal(err)
 	}
-	err = rdr.ExtractWithOptions(libPath, op)
+	err = rdr.ExtractWithOptions(libPath, FastOptions())
 	if err != nil {
 		b.Fatal(err)
 	}
 	libTime = time.Since(start)
-	cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name())
+	cmd := exec.Command("unsquashfs", "-q", "-d", unsquashPath, fil.Name())
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	start = time.Now()
@@ -109,7 +124,7 @@ func BenchmarkRace(b *testing.B) {
 func TestExtractQuick(t *testing.T) {
 	//First, setup everything and extract the archive using the library and unsquashfs
 
-	// tmpDir := b.TempDir()
+	// tmpDir := bTempDir()
 	tmpDir := "testing"
 	fil, err := preTest(tmpDir)
 	if err != nil {
@@ -119,13 +134,13 @@ func TestExtractQuick(t *testing.T) {
 	unsquashPath := filepath.Join(tmpDir, "ExtractSquashfs")
 	os.RemoveAll(libPath)
 	os.RemoveAll(unsquashPath)
-	rdr, err := squashfs.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	os.RemoveAll(filepath.Join(tmpDir, "testLog.txt"))
 	logFil, _ := os.Create(filepath.Join(tmpDir, "testLog.txt"))
-	op := squashfs.DefaultOptions()
+	op := FastOptions()
 	op.Verbose = true
 	op.IgnorePerm = true
 	op.LogOutput = logFil
@@ -168,7 +183,7 @@ func TestExtractQuick(t *testing.T) {
 	}
 }
 
-var filePath = "bin"
+var filePath = "usr/sbin/add-shell"
 
 func TestSingleFile(t *testing.T) {
 	tmpDir := "testing"
@@ -176,8 +191,8 @@ func TestSingleFile(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	os.Remove(filepath.Join(tmpDir, filePath))
+	os.RemoveAll("testing/stuff")
-	rdr, err := squashfs.NewReader(fil)
+	rdr, err := NewReader(fil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -185,9 +200,10 @@ func TestSingleFile(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = f.(*squashfs.File).ExtractWithOptions("testing", &squashfs.ExtractionOptions{Verbose: true})
+	op := DefaultOptions()
+	op.Verbose = true
+	err = f.(*File).ExtractWithOptions("testing/stuff", op)
 	if err != nil {
 		t.Fatal(err)
 	}
-	t.Fatal("HI")
 }