Merge pull request #39 from CalebQ42/perfExp

Performance

.gitignore

@@ -1 +1,2 @@
-testing
+testing
+/go-unsquashfs

README.md

@@ -1,28 +1,44 @@
-# squashfs (WIP)
+# squashfs
 
 [![PkgGoDev](https://pkg.go.dev/badge/github.com/CalebQ42/squashfs)](https://pkg.go.dev/github.com/CalebQ42/squashfs) [![Go Report Card](https://goreportcard.com/badge/github.com/CalebQ42/squashfs)](https://goreportcard.com/report/github.com/CalebQ42/squashfs)
 
-A PURE Go library to read and write squashfs.
+A PURE Go library to read squashfs. There is currently no plans to add archive creation support as it will almost always be better to just call `mksquashfs`. I could see some possible use cases, but probably won't spend time on it unless it's requested (open a discussion if you want this feature).
+
+The library has two parts with this `github.com/CalebQ42/squashfs` being easy to use as it implements `io/fs` interfaces and doesn't expose unnecessary information. 95% this is the library you want. If you need lower level access to the information, use `github.com/CalebQ42/squashfs/low` where far more information is exposed.
 
 Currently has support for reading squashfs files and extracting files and folders.
 
 Special thanks to <https://dr-emann.github.io/squashfs/> for some VERY important information in an easy to understand format.
 Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree/master/internal/squashfs) as I referenced it to figure some things out (and double check others).
 
-## [TODO](https://github.com/CalebQ42/squashfs/projects/1?fullscreen=true)
+## Build tags
+
+As of `v1.1.0` this library has two optional build tags: `no_gpl` and `no_obsolete`. `no_gpl` disables the ability to read archives with lzo compression due to the library's gpl license. `no_obsolete` removes "obsolete" compression types for a reduced compilation size; currently this only disable lzma compression since it's superseded by xz.
+
+## FUSE
+
+As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://github.com/CalebQ42/squashfuse).
 
 ## Limitations
 
-This library is pure Go (including external libraries) which can cause some issues, which are listed below. Right now this library is also not feature complete, so check out the TODO list above for what I'm still planning on adding.
+* No Xattr parsing.
+* Socket files are not extracted.
+  * From my research, it seems like a socket file would be useless if it could be created.
+* Fifo files are ignored on `darwin`
 
-* No LZO compression. This is purely due to a lack of a good LZO pure golang library. If one is made, it would be a simple job to add it in.
-* GZIP Compression
-  * Strategies might or might not work.
-  * Custom window sizes might or might not work.
-* XZ Compression
-  * LZMA executable filters are NOT supported.
-* No Xattr parsing. This is simply because I haven't done any research on it and how to apply these in a pure go way.
+## Issues
 
-## Performane
+* Noticably slower then `unsquashfs` for extraction, especially on larger images.
+  * This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries.
+  * Times seem to be largely dependent on file tree size and compression type.
+    * My main testing image (~100MB) using Zstd takes ~2x longer.
+    * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes ~28x longer.
+    * A Tensorflow docker image (~3.3GB) using Zstd takes ~3x longer.
 
-This library, decompressing the Firefox AppImage and using go tests, takes about twice as long as `unsquashfs` on my quad core laptop. (~1 second with the library and about half a second with `unsquashfs`)
+Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes ~2x longer.
+
+## Recommendations on Usage
+
+Due to the above performance consideration, this library should only be used to access files within the archive without extraction, or to mount it via Fuse.
+
+* Neither of these use cases are largely effected by the issue above.

cmd/go-unsquashfs/main.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/CalebQ42/squashfs"
+)
+
+func userName(uid int, numeric bool) string {
+	us := strconv.Itoa(uid)
+	if numeric {
+		return us
+	}
+	if u, err := user.LookupId(us); err == nil {
+		return u.Username
+	}
+	return us
+}
+
+func groupName(gid int, numeric bool) string {
+	gs := strconv.Itoa(gid)
+	if numeric {
+		return gs
+	}
+	if g, err := user.LookupGroupId(gs); err == nil {
+		return g.Name
+	}
+	return gs
+}
+
+func printEntry(root, path string, d fs.DirEntry, numeric bool) {
+	fi, _ := d.Info()
+	sfi := fi.(squashfs.FileInfo)
+	owner := fmt.Sprintf("%s/%s",
+		userName(sfi.Uid(), numeric),
+		groupName(sfi.Gid(), numeric))
+	link := ""
+	if sfi.IsSymlink() {
+		link = " -> " + sfi.SymlinkPath()
+	}
+	fmt.Printf("%s %s %*d %s %s%s\n",
+		strings.ToLower(fi.Mode().String()),
+		owner, 26-len(owner), fi.Size(),
+		fi.ModTime().Format("2006-01-02 15:04"),
+		filepath.Join(root, path), link)
+}
+
+func main() {
+	verbose := flag.Bool("v", false, "Verbose")
+	list := flag.Bool("l", false, "List")
+	long := flag.Bool("ll", false, "List with attributes")
+	numeric := flag.Bool("lln", false, "List with attributes and numeric ids")
+	offset := flag.Int64("o", 0, "Offset")
+	ignore := flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755")
+	flag.Parse()
+	if len(flag.Args()) < 2 {
+		fmt.Println("Please provide a file name and extraction path")
+		os.Exit(0)
+	}
+	f, err := os.Open(flag.Arg(0))
+	if err != nil {
+		panic(err)
+	}
+	r, err := squashfs.NewReaderAtOffset(f, *offset)
+	if err != nil {
+		panic(err)
+	}
+	if *list || *long || *numeric {
+		root := flag.Arg(1)
+		fs.WalkDir(r, ".", func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				panic(err)
+			}
+			if *long || *numeric {
+				printEntry(root, path, d, *numeric)
+			} else {
+				fmt.Println(filepath.Join(root, path))
+			}
+			return nil
+		})
+		return
+	}
+	op := squashfs.DefaultOptions()
+	op.Verbose = *verbose
+	op.IgnorePerm = *ignore
+	n := time.Now()
+	err = r.ExtractWithOptions(flag.Arg(1), op)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println("Took:", time.Since(n))
+}

direntry_fileinfo.go

@@ -1,137 +0,0 @@
-package squashfs
-
-import (
-	"io"
-	"io/fs"
-	"time"
-
-	"github.com/CalebQ42/squashfs/internal/directory"
-	"github.com/CalebQ42/squashfs/internal/inode"
-)
-
-//DirEntry is a child of a directory.
-type DirEntry struct {
-	en     *directory.Entry
-	parent *FS
-	r      *Reader
-}
-
-func (r *Reader) newDirEntry(en *directory.Entry, parent *FS) *DirEntry {
-	return &DirEntry{
-		en:     en,
-		parent: parent,
-		r:      r,
-	}
-}
-
-//Name returns the DirEntry's name
-func (d DirEntry) Name() string {
-	return d.en.Name
-}
-
-//IsDir Yep.
-func (d DirEntry) IsDir() bool {
-	return d.en.Type == inode.DirType
-}
-
-//Type returns the type bits of fs.FileMode of the DirEntry.
-func (d DirEntry) Type() fs.FileMode {
-	switch d.en.Type {
-	case inode.DirType:
-		return fs.ModeDir
-	case inode.SymType:
-		return fs.ModeSymlink
-	default:
-		return 0
-	}
-}
-
-//Info returns the fs.FileInfo for the given DirEntry.
-func (d DirEntry) Info() (fs.FileInfo, error) {
-	in, err := d.r.getInodeFromEntry(d.en)
-	if err != nil {
-		return nil, err
-	}
-	return &FileInfo{
-		name:   d.en.Name,
-		i:      in,
-		parent: d.parent,
-		r:      d.r,
-	}, nil
-}
-
-//GetInodeFromEntry returns the inode associated with a given directory.Entry
-func (r *Reader) getInodeFromEntry(en *directory.Entry) (*inode.Inode, error) {
-	br, err := r.newMetadataReader(int64(r.super.InodeTableStart + uint64(en.InodeOffset)))
-	if err != nil {
-		return nil, err
-	}
-	_, err = br.Seek(int64(en.InodeBlockOffset), io.SeekStart)
-	if err != nil {
-		return nil, err
-	}
-	i, err := inode.ProcessInode(br, r.super.BlockSize)
-	if err != nil {
-		return nil, err
-	}
-	return i, nil
-}
-
-//FileInfo is a fs.FileInfo for a file.
-type FileInfo struct {
-	i      *inode.Inode
-	parent *FS
-	r      *Reader
-	name   string
-}
-
-//Name is the file's name.
-func (f FileInfo) Name() string {
-	return f.name
-}
-
-//Size is the file's size if it's a regular file. Otherwise, returns 0.
-func (f FileInfo) Size() int64 {
-	switch f.i.Type {
-	case inode.FileType:
-		return int64(f.i.Info.(inode.File).Size)
-	case inode.ExtFileType:
-		return int64(f.i.Info.(inode.ExtFile).Size)
-	}
-	return 0
-}
-
-//Mode returns the fs.FileMode bits of the file.
-func (f FileInfo) Mode() fs.FileMode {
-	mode := fs.FileMode(f.i.Permissions)
-	switch f.i.Type {
-	case inode.DirType | inode.ExtDirType:
-		return mode | fs.ModeDir
-	case inode.ExtDirType:
-		return mode | fs.ModeDir
-	case inode.SymType:
-		return mode | fs.ModeSymlink
-	case inode.ExtSymType:
-		return mode | fs.ModeSymlink
-	}
-	return mode
-}
-
-//ModTime is the last time the file was modified.
-func (f FileInfo) ModTime() time.Time {
-	return time.Unix(int64(f.i.ModifiedTime), 0)
-}
-
-//IsDir yep.
-func (f FileInfo) IsDir() bool {
-	return f.i.Type == inode.DirType || f.i.Type == inode.ExtDirType
-}
-
-//Sys returns the File for the FileInfo. If something goes wrong, nil is returned.
-func (f FileInfo) Sys() interface{} {
-	fil, err := f.File()
-	if err != nil {
-		return nil
-	}
-	return fil
-}

extraction_options.go

@@ -0,0 +1,49 @@
+package squashfs
+
+import (
+	"io"
+	"io/fs"
+	"runtime"
+
+	"github.com/CalebQ42/squashfs/internal/routinemanager"
+)
+
+type ExtractionOptions struct {
+	manager            *routinemanager.Manager
+	LogOutput          io.Writer   //Where the verbose log should write.
+	DereferenceSymlink bool        //Replace symlinks with the target file.
+	UnbreakSymlink     bool        //Try to make sure symlinks remain unbroken when extracted, without changing the symlink.
+	Verbose            bool        //Prints extra info to log on an error.
+	IgnorePerm         bool        //Ignore file's permissions and instead use Perm.
+	Perm               fs.FileMode //Permission to use when IgnorePerm. Defaults to 0777.
+	SimultaneousFiles  uint16      //Number of files to process in parallel. Default set based on runtime.NumCPU().
+	ExtractionRoutines uint16      //Number of goroutines to use for each file's extraction. Only applies to regular files. Default set based on runtime.NumCPU().
+}
+
+// The default extraction options.
+func DefaultOptions() *ExtractionOptions {
+	cores := uint16(runtime.NumCPU() / 2)
+	var files, routines uint16
+	if cores <= 4 {
+		files = 1
+		routines = cores
+	} else {
+		files = cores - 4
+		routines = 4
+	}
+	return &ExtractionOptions{
+		Perm:               0777,
+		SimultaneousFiles:  files,
+		ExtractionRoutines: routines,
+	}
+}
+
+// Less limited default options. Can run up 2x faster than DefaultOptions.
+// Tends to use all available CPU resources.
+func FastOptions() *ExtractionOptions {
+	return &ExtractionOptions{
+		Perm:               0777,
+		SimultaneousFiles:  uint16(runtime.NumCPU()),
+		ExtractionRoutines: uint16(runtime.NumCPU()),
+	}
+}

file.go

@@ -0,0 +1,442 @@
+package squashfs
+
+import (
+	"errors"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+
+	"github.com/CalebQ42/squashfs/internal/routinemanager"
+	squashfslow "github.com/CalebQ42/squashfs/low"
+	"github.com/CalebQ42/squashfs/low/data"
+	"github.com/CalebQ42/squashfs/low/inode"
+)
+
+// File represents a file inside a squashfs archive.
+type File struct {
+	full     data.FullReader
+	rdr      data.Reader
+	rdrInit  bool
+	parent   FS
+	r        *Reader
+	b        squashfslow.FileBase
+	dirsRead int
+}
+
+// Creates a new *File from the given *squashfs.Base
+func (r *Reader) FileFromBase(b squashfslow.FileBase, parent FS) File {
+	return File{
+		b:      b,
+		parent: parent,
+		r:      r,
+	}
+}
+
+func (f File) FS() (FS, error) {
+	if !f.IsDir() {
+		return FS{}, errors.New("not a directory")
+	}
+	d, err := f.b.ToDir(f.r.Low)
+	if err != nil {
+		return FS{}, err
+	}
+	return FS{d: d, parent: &f.parent, r: f.r}, nil
+}
+
+// Closes the underlying readers.
+// Further calls to Read and WriteTo will re-create the readers.
+// Never returns an error.
+func (f *File) Close() error {
+	f.rdr.Close()
+	f.full.Close()
+	return nil
+}
+
+// Returns the file the symlink points to.
+// If the file isn't a symlink, or points to a file outside the archive, returns nil.
+func (f File) GetSymlinkFile() fs.File {
+	if !f.IsSymlink() {
+		return nil
+	}
+	if filepath.IsAbs(f.SymlinkPath()) {
+		return nil
+	}
+	fil, err := f.parent.Open(f.SymlinkPath())
+	if err != nil {
+		return nil
+	}
+	return fil
+}
+
+// Returns whether the file is a directory.
+func (f File) IsDir() bool {
+	return f.b.IsDir()
+}
+
+// Returns whether the file is a regular file.
+func (f File) IsRegular() bool {
+	return f.b.IsRegular()
+}
+
+// Returns whether the file is a symlink.
+func (f File) IsSymlink() bool {
+	return f.b.Inode.Type == inode.Sym || f.b.Inode.Type == inode.ESym
+}
+
+func (f File) Mode() fs.FileMode {
+	return f.b.Inode.Mode()
+}
+
+// Read reads the data from the file. Only works if file is a normal file.
+func (f *File) Read(b []byte) (int, error) {
+	if !f.IsRegular() {
+		return 0, errors.New("file is not a regular file")
+	}
+	if !f.rdrInit {
+		err := f.initializeReaders()
+		if err != nil {
+			return 0, err
+		}
+	}
+	return f.rdr.Read(b)
+}
+
+// ReadDir returns n fs.DirEntry's that's contained in the File (if it's a directory).
+// If n <= 0 all fs.DirEntry's are returned.
+func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
+	if !f.IsDir() {
+		return nil, errors.New("file is not a directory")
+	}
+	d, err := f.b.ToDir(f.r.Low)
+	if err != nil {
+		return nil, err
+	}
+	start, end := 0, len(d.Entries)
+	if n > 0 {
+		start, end = f.dirsRead, f.dirsRead+n
+		if end > len(d.Entries) {
+			end = len(d.Entries)
+			err = io.EOF
+		}
+	}
+	var out []fs.DirEntry
+	var fi FileInfo
+	for _, e := range d.Entries[start:end] {
+		fi, err = f.r.newFileInfo(e)
+		if err != nil {
+			f.dirsRead += len(out)
+			return out, err
+		}
+		out = append(out, fs.FileInfoToDirEntry(fi))
+	}
+	f.dirsRead += len(out)
+	return out, err
+}
+
+// Returns the file's fs.FileInfo
+func (f File) Stat() (fs.FileInfo, error) {
+	uid, err := f.b.Uid(&f.r.Low)
+	if err != nil {
+		return nil, err
+	}
+	gid, err := f.b.Gid(&f.r.Low)
+	if err != nil {
+		return nil, err
+	}
+	return newFileInfo(f.b.Name, uid, gid, &f.b.Inode), nil
+}
+
+// SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string.
+func (f File) SymlinkPath() string {
+	switch f.b.Inode.Type {
+	case inode.Sym:
+		return string(f.b.Inode.Data.(inode.Symlink).Target)
+	case inode.ESym:
+		return string(f.b.Inode.Data.(inode.ESymlink).Target)
+	}
+	return ""
+}
+
+// Writes all data from the file to the given writer in a multi-threaded manner.
+// The underlying reader is separate
+func (f *File) WriteTo(w io.Writer) (int64, error) {
+	if !f.IsRegular() {
+		return 0, errors.New("file is not a regular file")
+	}
+	if !f.rdrInit {
+		err := f.initializeReaders()
+		if err != nil {
+			return 0, err
+		}
+	}
+	return f.full.WriteTo(w)
+}
+
+func (f *File) initializeReaders() error {
+	var err error
+	f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.Low)
+	if err == nil {
+		f.rdrInit = true
+	} else {
+		f.rdr.Close()
+		f.full.Close()
+	}
+	return err
+}
+
+func (f File) deviceDevices() (maj uint32, min uint32) {
+	var dev uint32
+	switch f.b.Inode.Type {
+	case inode.Char, inode.Block:
+		dev = f.b.Inode.Data.(inode.Device).Dev
+	case inode.EChar, inode.EBlock:
+		dev = f.b.Inode.Data.(inode.EDevice).Dev
+	}
+	return dev >> 8, dev & 0x000FF
+}
+
+func (f File) path() string {
+	if f.parent.d.Name == "" {
+		return f.b.Name
+	}
+	return filepath.Join(f.parent.path(), f.b.Name)
+}
+
+// Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
+// Uses default extraction options.
+func (f File) Extract(folder string) error {
+	return f.ExtractWithOptions(folder, DefaultOptions())
+}
+
+// Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
+// Allows setting various extraction options via ExtractionOptions.
+func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
+	if op.manager == nil {
+		op.manager = routinemanager.NewManager(op.SimultaneousFiles)
+		if op.LogOutput != nil {
+			log.SetOutput(op.LogOutput)
+		}
+		err := os.MkdirAll(path, 0777)
+		if err != nil {
+			if op.Verbose {
+				log.Println("Failed to create initial directory", path)
+			}
+			return err
+		}
+	}
+	switch f.b.Inode.Type {
+	case inode.Dir, inode.EDir:
+		d, err := f.b.ToDir(f.r.Low)
+		if err != nil {
+			if op.Verbose {
+				log.Println("Failed to create squashfs.Directory for", path)
+			}
+			return errors.Join(errors.New("failed to create squashfs.Directory: "+path), err)
+		}
+		errChan := make(chan error, len(d.Entries))
+		for i := range d.Entries {
+			b, err := f.r.Low.BaseFromEntry(d.Entries[i])
+			if err != nil {
+				if op.Verbose {
+					log.Println("Failed to get squashfs.Base from entry for", path)
+				}
+				return errors.Join(errors.New("failed to get base from entry: "+path), err)
+			}
+			go func(b squashfslow.FileBase, path string) {
+				i := op.manager.Lock()
+				if b.IsDir() {
+					extDir := filepath.Join(path, b.Name)
+					err = os.Mkdir(extDir, 0777)
+					op.manager.Unlock(i)
+					if err != nil {
+						if op.Verbose {
+							log.Println("Failed to create directory", path)
+						}
+						errChan <- errors.Join(errors.New("failed to create directory: "+path), err)
+						return
+					}
+					err = f.r.FileFromBase(b, f.r.FSFromDirectory(d, f.parent)).ExtractWithOptions(extDir, op)
+					if err != nil {
+						if op.Verbose {
+							log.Println("Failed to extract directory", path)
+						}
+						errChan <- errors.Join(errors.New("failed to extract directory: "+path), err)
+						return
+					}
+					errChan <- nil
+				} else {
+					fil := f.r.FileFromBase(b, f.r.FSFromDirectory(d, f.parent))
+					err = fil.ExtractWithOptions(path, op)
+					op.manager.Unlock(i)
+					fil.Close()
+					errChan <- err
+				}
+			}(b, path)
+		}
+		var errCache []error
+		for range d.Entries {
+			err := <-errChan
+			if err != nil {
+				errCache = append(errCache, err)
+			}
+		}
+		if len(errCache) > 0 {
+			return errors.Join(errors.New("failed to extract folder: "+path), errors.Join(errCache...))
+		}
+	case inode.Fil, inode.EFil:
+		path = filepath.Join(path, f.b.Name)
+		outFil, err := os.Create(path)
+		if err != nil {
+			if op.Verbose {
+				log.Println("Failed to create file", path)
+			}
+			return errors.Join(errors.New("failed to create file: "+path), err)
+		}
+		defer outFil.Close()
+		full, err := f.b.GetFullReader(&f.r.Low)
+		if err != nil {
+			if op.Verbose {
+				log.Println("Failed to create full reader for", path)
+			}
+			return errors.Join(errors.New("failed to create full reader: "+path), err)
+		}
+		full.SetGoroutineLimit(op.ExtractionRoutines)
+		_, err = full.WriteTo(outFil)
+		if err != nil {
+			if op.Verbose {
+				log.Println("Failed to write file", path)
+			}
+			return errors.Join(errors.New("failed to write file: "+path), err)
+		}
+	case inode.Sym, inode.ESym:
+		symPath := f.SymlinkPath()
+		if op.DereferenceSymlink {
+			filTmp := f.GetSymlinkFile()
+			if filTmp == nil {
+				if op.Verbose {
+					log.Println("Failed to get symlink's file:", f.path())
+				}
+				return errors.New("failed to get symlink's file")
+			}
+			fil := filTmp.(*File)
+			fil.b.Name = f.b.Name
+			err := fil.ExtractWithOptions(path, op)
+			if err != nil {
+				if op.Verbose {
+					log.Println("Failed to extract symlink's file:", filepath.Join(path, f.b.Name))
+				}
+				return errors.Join(errors.New("failed to extract symlink's file: "+path), err)
+			}
+		} else {
+			if op.UnbreakSymlink {
+				filTmp := f.GetSymlinkFile()
+				if filTmp == nil {
+					if op.Verbose {
+						log.Println("Failed to get symlink's file:", f.path())
+					}
+					return errors.New("failed to get symlink's file")
+				}
+				extractLoc := filepath.Join(path, filepath.Dir(symPath))
+				fil := filTmp.(*File)
+				err := fil.ExtractWithOptions(extractLoc, op)
+				if err != nil {
+					if op.Verbose {
+						log.Println("Error while extracting", fil.path(), "to make sure symlink at", f.path(), "is unbroken")
+					}
+					return errors.Join(errors.New("failed to extract symlink's file: "+extractLoc), err)
+				}
+			}
+			path = filepath.Join(path, f.b.Name)
+			err := os.Symlink(f.SymlinkPath(), path)
+			if err != nil {
+				if op.Verbose {
+					log.Println("Failed to create symlink:", path)
+				}
+				return errors.Join(errors.New("failed to create symlink: "+path), err)
+			}
+		}
+	case inode.Char, inode.EChar, inode.Block, inode.EBlock, inode.Fifo, inode.EFifo:
+		if runtime.GOOS == "windows" {
+			if op.Verbose {
+				log.Println(f.path(), "ignored. A device link and can't be created on Windows.")
+			}
+			return nil
+		}
+		_, err := exec.LookPath("mknod")
+		if err != nil {
+			if op.Verbose {
+				log.Println("mknot command not found, cannot create device link for", f.path())
+			}
+			return errors.Join(errors.New("mknot command not found"), err)
+		}
+		path = filepath.Join(path, f.b.Name)
+		var typ string
+		switch f.b.Inode.Type {
+		case inode.Char, inode.EChar:
+			typ = "c"
+		case inode.Block, inode.EBlock:
+			typ = "b"
+		default: //Fifo IPC
+			if runtime.GOOS == "darwin" {
+				if op.Verbose {
+					log.Println(f.path(), "ignored. A Fifo file and can't be created on Darwin.")
+				}
+				return nil
+			}
+			typ = "p"
+		}
+		cmd := exec.Command("mknod", path, typ)
+		if typ != "p" {
+			maj, min := f.deviceDevices()
+			cmd.Args = append(cmd.Args, strconv.Itoa(int(maj)), strconv.Itoa(int(min)))
+		}
+		if op.Verbose {
+			cmd.Stdout = op.LogOutput
+			cmd.Stderr = op.LogOutput
+		}
+		err = cmd.Run()
+		if err != nil {
+			if op.Verbose {
+				log.Println("Error while running mknod for", path)
+			}
+			return errors.Join(errors.New("error while running mknod for "+path), err)
+		}
+	case inode.Sock, inode.ESock:
+		if op.Verbose {
+			log.Println(f.path(), "ignored since it's a socket file.")
+		}
+		return nil
+	default:
+		return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.b.Inode.Type)))
+	}
+	if op.Verbose {
+		log.Println(f.path(), "extracted to", path)
+	}
+	if op.IgnorePerm {
+		return nil
+	}
+	uid, err := f.b.Uid(&f.r.Low)
+	if err != nil {
+		if op.Verbose {
+			log.Println("Failed to get uid for", path)
+			log.Println(err)
+		}
+		return nil
+	}
+	gid, err := f.b.Gid(&f.r.Low)
+	if err != nil {
+		if op.Verbose {
+			log.Println("Failed to get gid for", path)
+			log.Println(err)
+		}
+		return nil
+	}
+	os.Chmod(path, f.Mode())
+	os.Chown(path, int(uid), int(gid))
+	return nil
+}

file_info.go

@@ -0,0 +1,126 @@
+package squashfs
+
+import (
+	"io/fs"
+	"time"
+
+	"github.com/CalebQ42/squashfs/low/directory"
+	"github.com/CalebQ42/squashfs/low/inode"
+)
+
+type FileInfo struct {
+	name     string
+	uid      uint32
+	gid      uint32
+	size     int64
+	target   string
+	perm     uint32
+	modTime  uint32
+	fileType uint16
+}
+
+func (r Reader) newFileInfo(e directory.Entry) (FileInfo, error) {
+	b, err := r.Low.BaseFromEntry(e)
+	if err != nil {
+		return FileInfo{}, err
+	}
+	uid, err := b.Uid(&r.Low)
+	if err != nil {
+		return FileInfo{}, err
+	}
+	gid, err := b.Gid(&r.Low)
+	if err != nil {
+		return FileInfo{}, err
+	}
+	return newFileInfo(e.Name, uid, gid, &b.Inode), nil
+}
+
+func newFileInfo(name string, uid, gid uint32, i *inode.Inode) FileInfo {
+	var size int64
+	var target string
+	switch i.Type {
+	case inode.Fil:
+		size = int64(i.Data.(inode.File).Size)
+	case inode.EFil:
+		size = int64(i.Data.(inode.EFile).Size)
+	case inode.Sym:
+		target = string(i.Data.(inode.Symlink).Target)
+	case inode.ESym:
+		target = string(i.Data.(inode.ESymlink).Target)
+	}
+	return FileInfo{
+		name:     name,
+		uid:      uid,
+		gid:      gid,
+		size:     size,
+		target:   target,
+		perm:     uint32(i.Perm),
+		modTime:  i.ModTime,
+		fileType: i.Type,
+	}
+}
+
+func (f FileInfo) Name() string {
+	return f.name
+}
+
+func (f FileInfo) Uid() int {
+	return int(f.uid)
+}
+
+func (f FileInfo) Gid() int {
+	return int(f.gid)
+}
+
+func (f FileInfo) Size() int64 {
+	return f.size
+}
+
+func (f FileInfo) SymlinkPath() string {
+	return f.target
+}
+
+func (f FileInfo) Mode() fs.FileMode {
+	switch f.fileType {
+	case inode.Dir, inode.EDir:
+		return fs.FileMode(f.perm | uint32(fs.ModeDir))
+	case inode.Sym, inode.ESym:
+		return fs.FileMode(f.perm | uint32(fs.ModeSymlink))
+	case inode.Char, inode.EChar, inode.Block, inode.EBlock:
+		return fs.FileMode(f.perm | uint32(fs.ModeDevice))
+	case inode.Fifo, inode.EFifo:
+		return fs.FileMode(f.perm | uint32(fs.ModeNamedPipe))
+	case inode.Sock, inode.ESock:
+		return fs.FileMode(f.perm | uint32(fs.ModeSocket))
+	}
+	return fs.FileMode(f.perm)
+}
+
+func (f FileInfo) ModTime() time.Time {
+	return time.Unix(int64(f.modTime), 0)
+}
+
+func (f FileInfo) IsDir() bool {
+	return f.fileType == inode.Dir || f.fileType == inode.EDir
+}
+
+func (f FileInfo) IsSymlink() bool {
+	return f.fileType == inode.Sym || f.fileType == inode.ESym
+}
+
+func (f FileInfo) IsDevice() bool {
+	return f.fileType == inode.Block || f.fileType == inode.EBlock ||
+		f.fileType == inode.Char || f.fileType == inode.EChar
+}
+
+func (f FileInfo) IsFifo() bool {
+	return f.fileType == inode.Fifo || f.fileType == inode.EFifo
+}
+
+func (f FileInfo) IsSocket() bool {
+	return f.fileType == inode.Sock || f.fileType == inode.ESock
+}
+
+func (f FileInfo) Sys() any {
+	return nil
+}

fragment.go

@@ -1,80 +0,0 @@
-package squashfs
-
-import (
-	"encoding/binary"
-	"errors"
-	"io"
-
-	"github.com/CalebQ42/squashfs/internal/inode"
-)
-
-//FragmentEntry is an entry in the fragment table
-type fragmentEntry struct {
-	Start uint64
-	Size  uint32
-	_     uint32 //unused
-}
-
-//GetFragmentDataFromInode returns the fragment data for a given inode.
-//If the inode does not have a fragment, harmlessly returns an empty slice without an error.
-func (r *Reader) getFragmentDataFromInode(in *inode.Inode) ([]byte, error) {
-	var size uint64
-	var fragIndex uint32
-	var fragOffset uint32
-	if in.Type == inode.FileType {
-		bf := in.Info.(inode.File)
-		if !bf.Fragmented {
-			return make([]byte, 0), nil
-		}
-		if bf.BlockStart == 0 {
-			size = uint64(bf.Size)
-		} else {
-			size = uint64(bf.BlockSizes[len(bf.BlockSizes)-1])
-		}
-		fragIndex = bf.FragmentIndex
-		fragOffset = bf.FragmentOffset
-	} else if in.Type == inode.ExtFileType {
-		bf := in.Info.(inode.ExtFile)
-		if !bf.Fragmented {
-			return make([]byte, 0), nil
-		}
-		if bf.BlockStart == 0 {
-			size = bf.Size
-		} else {
-			size = uint64(bf.BlockSizes[len(bf.BlockSizes)-1])
-		}
-		fragIndex = bf.FragmentIndex
-		fragOffset = bf.FragmentOffset
-	} else {
-		return nil, errors.New("inode type not supported")
-	}
-	//reading the fragment entry first
-	fragEntryRdr, err := r.newMetadataReader(int64(r.fragOffsets[int(fragIndex/512)]))
-	if err != nil {
-		return nil, err
-	}
-	_, err = fragEntryRdr.Seek(int64(16*fragIndex), io.SeekStart)
-	if err != nil {
-		return nil, err
-	}
-	var entry fragmentEntry
-	err = binary.Read(fragEntryRdr, binary.LittleEndian, &entry)
-	if err != nil {
-		return nil, err
-	}
-	//now reading the actual fragment
-	dr, err := r.newDataReader(int64(entry.Start), []uint32{entry.Size})
-	if err != nil {
-		return nil, err
-	}
-	_, err = dr.Read(make([]byte, fragOffset))
-	if err != nil {
-		return nil, err
-	}
-	tmp := make([]byte, size)
-	err = binary.Read(dr, binary.LittleEndian, &tmp)
-	if err != nil {
-		return nil, err
-	}
-	return tmp, nil
-}

fs.go

@@ -0,0 +1,275 @@
+package squashfs
+
+import (
+	"io"
+	"io/fs"
+	"path"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	squashfslow "github.com/CalebQ42/squashfs/low"
+	"github.com/CalebQ42/squashfs/low/directory"
+)
+
+// FS is a fs.FS representation of a squashfs directory.
+// Implements fs.GlobFS, fs.ReadDirFS, fs.ReadFileFS, fs.StatFS, and fs.SubFS
+type FS struct {
+	r      *Reader
+	parent *FS
+	d      squashfslow.Directory
+}
+
+// Creates a new *FS from the given squashfs.directory
+func (r *Reader) FSFromDirectory(d squashfslow.Directory, parent FS) FS {
+	return FS{
+		d:      d,
+		r:      r,
+		parent: &parent,
+	}
+}
+
+// Glob returns the name of the files at the given pattern.
+// All paths are relative to the FS.
+// Uses filepath.Match to compare names.
+func (f *FS) Glob(pattern string) (out []string, err error) {
+	pattern = filepath.Clean(pattern)
+	if !fs.ValidPath(pattern) {
+		return nil, &fs.PathError{
+			Op:   "glob",
+			Path: pattern,
+			Err:  fs.ErrInvalid,
+		}
+	}
+	split := strings.Split(pattern, "/")
+	for i := range f.d.Entries {
+		if match, _ := path.Match(split[0], f.d.Entries[i].Name); match {
+			if len(split) == 1 {
+				out = append(out, f.d.Entries[i].Name)
+				continue
+			}
+			sub, err := f.Sub(split[0])
+			if err != nil {
+				if pathErr, ok := err.(*fs.PathError); ok {
+					if pathErr.Err == fs.ErrNotExist {
+						continue
+					}
+					pathErr.Op = "glob"
+					pathErr.Path = pattern
+					return nil, pathErr
+				}
+				return nil, &fs.PathError{
+					Op:   "glob",
+					Path: pattern,
+					Err:  err,
+				}
+			}
+			subGlob, err := sub.(fs.GlobFS).Glob(strings.Join(split[1:], "/"))
+			if err != nil {
+				if pathErr, ok := err.(*fs.PathError); ok {
+					if pathErr.Err == fs.ErrNotExist {
+						continue
+					}
+					pathErr.Op = "glob"
+					pathErr.Path = pattern
+					return nil, pathErr
+				}
+				return nil, &fs.PathError{
+					Op:   "glob",
+					Path: pattern,
+					Err:  err,
+				}
+			}
+			for i := range subGlob {
+				subGlob[i] = f.d.Name + "/" + subGlob[i]
+			}
+			out = append(out, subGlob...)
+		}
+	}
+	return
+}
+
+// Opens the file at name. Returns a *File as an fs.File.
+func (f FS) Open(name string) (fs.File, error) {
+	name = filepath.Clean(name)
+	if !fs.ValidPath(name) {
+		return nil, &fs.PathError{
+			Op:   "open",
+			Path: name,
+			Err:  fs.ErrInvalid,
+		}
+	}
+	if name == "." || name == "" {
+		return f.File(), nil
+	}
+	split := strings.Split(name, "/")
+	if split[0] == ".." {
+		if f.parent == nil { // root directory
+			return nil, &fs.PathError{
+				Op:   "open",
+				Path: name,
+				Err:  fs.ErrNotExist,
+			}
+		} else {
+			return f.parent.Open(strings.Join(split[1:], "/"))
+		}
+	}
+	i, found := slices.BinarySearchFunc(f.d.Entries, split[0], func(e directory.Entry, name string) int {
+		return strings.Compare(e.Name, name)
+	})
+	if !found {
+		return nil, &fs.PathError{
+			Op:   "open",
+			Path: name,
+			Err:  fs.ErrNotExist,
+		}
+	}
+	b, err := f.r.Low.BaseFromEntry(f.d.Entries[i])
+	if err != nil {
+		return nil, err
+	}
+	if len(split) == 1 {
+		return &File{
+			b:      b,
+			r:      f.r,
+			parent: f,
+		}, nil
+	}
+	if !b.IsDir() {
+		return nil, &fs.PathError{
+			Op:   "open",
+			Path: name,
+			Err:  fs.ErrNotExist,
+		}
+	}
+	d, err := b.ToDir(f.r.Low)
+	if err != nil {
+		return nil, err
+	}
+	return f.r.FSFromDirectory(d, f).Open(strings.Join(split[1:], "/"))
+}
+
+// Returns all DirEntry's for the directory at name.
+// If name is not a directory, returns an error.
+func (f FS) ReadDir(name string) ([]fs.DirEntry, error) {
+	name = filepath.Clean(name)
+	if !fs.ValidPath(name) {
+		return nil, &fs.PathError{
+			Op:   "readdir",
+			Path: name,
+			Err:  fs.ErrInvalid,
+		}
+	}
+	if name == "." || name == "" {
+		return f.File().ReadDir(-1)
+	}
+	fil, err := f.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	return fil.(*File).ReadDir(-1)
+}
+
+// Returns the contents of the file at name.
+func (f FS) ReadFile(name string) (out []byte, err error) {
+	name = filepath.Clean(name)
+	if !fs.ValidPath(name) {
+		return nil, &fs.PathError{
+			Op:   "readfile",
+			Path: name,
+			Err:  fs.ErrInvalid,
+		}
+	}
+	if name == "." || name == "" {
+		return nil, fs.ErrInvalid
+	}
+	fil, err := f.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	if !fil.(*File).IsRegular() {
+		return nil, fs.ErrInvalid
+	}
+	return io.ReadAll(fil)
+}
+
+// Returns the fs.FileInfo for the file at name.
+func (f FS) Stat(name string) (fs.FileInfo, error) {
+	name = filepath.Clean(name)
+	if !fs.ValidPath(name) {
+		return nil, &fs.PathError{
+			Op:   "stat",
+			Path: name,
+			Err:  fs.ErrInvalid,
+		}
+	}
+	if name == "." || name == "" {
+		return f.File().Stat()
+	}
+	fil, err := f.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	return fil.(*File).Stat()
+}
+
+// Returns the FS at dir
+func (f FS) Sub(dir string) (fs.FS, error) {
+	dir = filepath.Clean(dir)
+	if !fs.ValidPath(dir) {
+		return nil, &fs.PathError{
+			Op:   "dir",
+			Path: dir,
+			Err:  fs.ErrInvalid,
+		}
+	}
+	if dir == "." || dir == "" {
+		return f, nil
+	}
+	fil, err := f.Open(dir)
+	if err != nil {
+		return nil, err
+	}
+	if !fil.(*File).IsDir() {
+		return nil, &fs.PathError{
+			Op:   "dir",
+			Path: dir,
+			Err:  fs.ErrInvalid,
+		}
+	}
+	return fil.(*File).FS()
+}
+
+// Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
+// Uses default extraction options.
+func (f FS) Extract(folder string) error {
+	return f.File().Extract(folder)
+}
+
+// Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
+// Allows setting various extraction options via ExtractionOptions.
+func (f FS) ExtractWithOptions(folder string, op *ExtractionOptions) error {
+	return f.File().ExtractWithOptions(folder, op)
+}
+
+// Returns the FS as a *File
+func (f FS) File() *File {
+	if f.parent != nil {
+		return &File{
+			b:      f.d.FileBase,
+			parent: *f.parent,
+			r:      f.r,
+		}
+	}
+	return &File{
+		b: f.d.FileBase,
+		r: f.r,
+	}
+}
+
+func (f FS) path() string {
+	if f.parent == nil {
+		return f.d.Name
+	}
+	return filepath.Join(f.parent.path(), f.d.Name)
+}

go.mod

@@ -1,23 +1,11 @@
 module github.com/CalebQ42/squashfs
 
-go 1.17
+go 1.24.0
 
 require (
-	github.com/CalebQ42/GoAppImage v0.5.0
-	github.com/adrg/xdg v0.3.3 // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20210420193930-a4630ec28c79 // indirect
-	github.com/klauspost/compress v1.12.2
-	github.com/pierrec/lz4/v4 v4.1.6
-	github.com/smartystreets/assertions v1.2.0 // indirect
-	github.com/ulikunitz/xz v0.5.10
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-)
-
-require (
-	github.com/golang/snappy v0.0.3 // indirect
-	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/klauspost/compress v1.18.0
+	github.com/pierrec/lz4/v4 v4.1.22
 	github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e
 	github.com/therootcompany/xz v1.0.1
-	go.lsp.dev/uri v0.3.0 // indirect
-	gopkg.in/ini.v1 v1.62.0 // indirect
+	github.com/ulikunitz/xz v0.5.12
 )

go.sum

@@ -1,348 +1,10 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/CalebQ42/GoAppImage v0.5.0 h1:znoKNXtliH754tS9sYwyOIg/0wFDjFN5Twc7PAh1rSM=
-github.com/CalebQ42/GoAppImage v0.5.0/go.mod h1:qHudJKAn/dlkNWNnH4h1YKXp29EZ7Bppsn7sNP2HuvU=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/adrg/xdg v0.2.2/go.mod h1:7I2hH/IT30IsupOpKZ5ue7/qNi3CoKzD6tL3HwpaRMQ=
-github.com/adrg/xdg v0.3.3 h1:s/tV7MdqQnzB1nKY8aqHvAMD+uCiuEDzVB5HLRY849U=
-github.com/adrg/xdg v0.3.3/go.mod h1:61xAR2VZcggl2St4O9ohF5qCKe08+JDmE4VNzPFQvOQ=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20210420193930-a4630ec28c79 h1:ATVz3rDvK4xX0nHx57zYSHRVIK/+lFwln9KJr8wvuk0=
-github.com/gopherjs/gopherjs v0.0.0-20210420193930-a4630ec28c79/go.mod h1:Opf9rtYVq0eTyX+aRVmRO9hE8ERAozcdrBxWG9Q6mkQ=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8=
-github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
-github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pierrec/lz4/v4 v4.1.6 h1:ueMTcBBFrbT8K4uGDNNZPa8Z7LtPV7Cl0TDjaeHxP44=
-github.com/pierrec/lz4/v4 v4.1.6/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs=
 github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
-github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
-github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
 github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
-github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.lsp.dev/uri v0.3.0 h1:KcZJmh6nFIBeJzTugn5JTU6OOyG0lDOo3R9KwTxTYbo=
-go.lsp.dev/uri v0.3.0/go.mod h1:P5sbO1IQR+qySTWOCnhnK7phBx+W3zbLqSMDJNTw88I=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
-gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
+github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=

internal/compression/gzip.go

@@ -1,76 +0,0 @@
-package compression
-
-import (
-	"bytes"
-	"encoding/binary"
-	"io"
-
-	"github.com/klauspost/compress/zlib"
-)
-
-type gzipInit struct {
-	CompressionLevel int32
-	WindowSize       int16
-	Strategies       int16
-}
-
-//Gzip is a decompressor for gzip type compression. Uses zlib for compression and decompression
-type Gzip struct {
-	wrt *zlib.Writer
-	gzipInit
-	HasCustomWindow bool
-	HasStrategies   bool
-}
-
-//NewGzipCompressorWithOptions creates a new gzip compressor/decompressor with options read from the given reader.
-func NewGzipCompressorWithOptions(r io.Reader) (*Gzip, error) {
-	var gzip Gzip
-	err := binary.Read(r, binary.LittleEndian, &gzip.gzipInit)
-	if err != nil {
-		return nil, err
-	}
-	//TODO: proper support for window size and strategies
-	gzip.HasCustomWindow = gzip.WindowSize != 15
-	gzip.HasStrategies = gzip.Strategies != 0 && gzip.Strategies != 1
-	return &gzip, nil
-}
-
-//Decompress reads the entirety of the given reader and returns it uncompressed as a byte slice.
-func (g *Gzip) Decompress(r io.Reader) ([]byte, error) {
-	rdr, err := zlib.NewReader(r)
-	if err != nil {
-		return nil, err
-	}
-	var data bytes.Buffer
-	_, err = io.Copy(&data, rdr)
-	if err != nil {
-		return nil, err
-	}
-	return data.Bytes(), nil
-}
-
-//Compress compresses the given data (as a byte array) and returns the compressed data.
-func (g *Gzip) Compress(data []byte) ([]byte, error) {
-	var buf bytes.Buffer
-	var err error
-	if g.wrt == nil {
-		if g.CompressionLevel == 0 {
-			g.wrt = zlib.NewWriter(&buf)
-		} else {
-			g.wrt, err = zlib.NewWriterLevel(&buf, int(g.CompressionLevel))
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-	wrt, err := zlib.NewWriterLevel(&buf, int(g.CompressionLevel))
-	if err != nil {
-		return nil, err
-	}
-	_, err = wrt.Write(data)
-	if err != nil {
-		return nil, err
-	}
-	wrt.Close()
-	return buf.Bytes(), nil
-}

internal/compression/lz4.go

@@ -1,55 +0,0 @@
-package compression
-
-import (
-	"bytes"
-	"encoding/binary"
-	"io"
-
-	"github.com/pierrec/lz4/v4"
-)
-
-//Lz4 is a Lz4 Compressor/Decompressor
-type Lz4 struct {
-	HC bool
-}
-
-//NewLz4CompressorWithOptions creates a new lz4 compressor/decompressor with options read from the given reader.
-func NewLz4CompressorWithOptions(r io.Reader) (*Lz4, error) {
-	var lz4 Lz4
-	var init struct {
-		Version int32
-		Flags   int32
-	}
-	err := binary.Read(r, binary.LittleEndian, &init)
-	if err != nil {
-		return nil, err
-	}
-	lz4.HC = init.Flags == 1
-	return &lz4, nil
-}
-
-//Decompress decompresses all data from r and returns the uncompressed bytes
-func (l *Lz4) Decompress(r io.Reader) ([]byte, error) {
-	rdr := lz4.NewReader(r)
-	var buf bytes.Buffer
-	_, err := io.Copy(&buf, rdr)
-	return buf.Bytes(), err
-}
-
-//Compress implements compression.Compress
-func (l *Lz4) Compress(data []byte) ([]byte, error) {
-	var buf bytes.Buffer
-	w := lz4.NewWriter(&buf)
-	if l.HC {
-		err := w.Apply(lz4.CompressionLevelOption(lz4.Level9))
-		if err != nil {
-			return nil, err
-		}
-	}
-	_, err := w.Write(data)
-	if err != nil {
-		return nil, err
-	}
-	w.Close()
-	return buf.Bytes(), nil
-}

internal/compression/lzma.go

@@ -1,40 +0,0 @@
-package compression
-
-import (
-	"bytes"
-	"io"
-
-	"github.com/ulikunitz/xz/lzma"
-)
-
-//Lzma is a lzma decompressor
-type Lzma struct{}
-
-//Decompress decompresses all the data in the given reader and returns the uncompressed bytes.
-func (l *Lzma) Decompress(rdr io.Reader) ([]byte, error) {
-	r, err := lzma.NewReader(rdr)
-	if err != nil {
-		return nil, err
-	}
-	var buf bytes.Buffer
-	_, err = io.Copy(&buf, r)
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-//Compress implements compression.Compress
-func (l *Lzma) Compress(data []byte) ([]byte, error) {
-	var buf bytes.Buffer
-	w, err := lzma.NewWriter(&buf)
-	if err != nil {
-		return nil, err
-	}
-	_, err = w.Write(data)
-	if err != nil {
-		return nil, err
-	}
-	w.Close()
-	return buf.Bytes(), nil
-}

internal/compression/lzo.go

@@ -1,30 +0,0 @@
-package compression
-
-import (
-	"encoding/binary"
-	"io"
-
-	lzo "github.com/rasky/go-lzo"
-)
-
-type Lzo struct {
-	Algorithm int32
-	Level     int32
-}
-
-func NewLzoCompressorWithOptions(rdr io.Reader) (*Lzo, error) {
-	var lz Lzo
-	err := binary.Read(rdr, binary.LittleEndian, &lz)
-	if err != nil {
-		return nil, err
-	}
-	return &lz, nil
-}
-
-func (l Lzo) Decompress(rdr io.Reader) ([]byte, error) {
-	byt, err := lzo.Decompress1X(rdr, 0, 0)
-	if err != nil {
-		return nil, err
-	}
-	return byt, nil
-}

internal/compression/types.go

@@ -1,13 +0,0 @@
-package compression
-
-import "io"
-
-//Compressor is a squashfs decompressor interface. Allows for easy compression.
-type Compressor interface {
-	Compress([]byte) ([]byte, error)
-}
-
-//Decompressor is a squashfs decompressor interface. Allows for easy decompression no matter the type of compression.
-type Decompressor interface {
-	Decompress(io.Reader) ([]byte, error)
-}

internal/compression/xz.go

@@ -1,60 +0,0 @@
-package compression
-
-import (
-	"bytes"
-	"encoding/binary"
-	"io"
-
-	"github.com/therootcompany/xz"
-
-	wrtXz "github.com/ulikunitz/xz"
-)
-
-type Xz struct {
-	DictionarySize int32
-	Filters        int32
-}
-
-//NewXzCompressorWithOptions creates a new Xz compressor/decompressor that reads the compressor options from the given reader.
-func NewXzCompressorWithOptions(rdr io.Reader) (*Xz, error) {
-	var x Xz
-	err := binary.Read(rdr, binary.LittleEndian, &x)
-	if err != nil {
-		return nil, err
-	}
-	return &x, nil
-}
-
-//Decompress decompresses all the data from the rdr and returns the uncompressed bytes.
-func (x *Xz) Decompress(rdr io.Reader) ([]byte, error) {
-	r, err := xz.NewReader(rdr, 0)
-	if err != nil {
-		return nil, err
-	}
-	var buf bytes.Buffer
-	_, err = io.Copy(&buf, r)
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-//Compress implements compression.Compress
-func (x *Xz) Compress(data []byte) ([]byte, error) {
-	var buf bytes.Buffer
-	w, err := wrtXz.NewWriter(&buf)
-	if err != nil {
-		return nil, err
-	}
-	w.DictCap = int(x.DictionarySize)
-	err = w.Verify()
-	if err != nil {
-		return nil, err
-	}
-	_, err = w.Write(data)
-	if err != nil {
-		return nil, err
-	}
-	w.Close()
-	return buf.Bytes(), nil
-}

internal/compression/zstd.go

@@ -1,51 +0,0 @@
-package compression
-
-import (
-	"bytes"
-	"encoding/binary"
-	"io"
-
-	"github.com/klauspost/compress/zstd"
-)
-
-//Zstd is a zstd compressor/decompressor
-type Zstd struct {
-	CompressionLevel int32
-}
-
-//NewZstdCompressorWithOptions creates a new Zstd with options read from the given reader
-func NewZstdCompressorWithOptions(r io.Reader) (*Zstd, error) {
-	var zstd Zstd
-	err := binary.Read(r, binary.LittleEndian, &zstd)
-	if err != nil {
-		return nil, err
-	}
-	return &zstd, nil
-}
-
-//Decompress decompresses all data from the reader and returns the uncompressed data
-func (z *Zstd) Decompress(r io.Reader) ([]byte, error) {
-	rdr, err := zstd.NewReader(r)
-	if err != nil {
-		return nil, err
-	}
-	defer rdr.Close()
-	var buf bytes.Buffer
-	_, err = io.Copy(&buf, rdr)
-	return buf.Bytes(), err
-}
-
-//Compress impelements compression.Compress
-func (z *Zstd) Compress(data []byte) ([]byte, error) {
-	var buf bytes.Buffer
-	w, err := zstd.NewWriter(&buf, zstd.WithEncoderLevel(zstd.EncoderLevel(z.CompressionLevel)))
-	if err != nil {
-		return nil, err
-	}
-	_, err = w.Write(data)
-	if err != nil {
-		return nil, err
-	}
-	w.Close()
-	return buf.Bytes(), nil
-}

internal/decompress/decompress.go

@@ -0,0 +1,5 @@
+package decompress
+
+type Decompressor interface {
+	Decompress([]byte) ([]byte, error)
+}

internal/decompress/lz4.go

@@ -0,0 +1,30 @@
+package decompress
+
+import (
+	"bytes"
+	"io"
+	"sync"
+
+	"github.com/pierrec/lz4/v4"
+)
+
+type Lz4 struct {
+	pool sync.Pool
+}
+
+func NewLz4() *Lz4 {
+	return &Lz4{
+		pool: sync.Pool{
+			New: func() any {
+				return lz4.NewReader(nil)
+			},
+		},
+	}
+}
+
+func (l *Lz4) Decompress(data []byte) ([]byte, error) {
+	rdr := l.pool.Get().(*lz4.Reader)
+	defer l.pool.Put(rdr)
+	rdr.Reset(bytes.NewReader(data))
+	return io.ReadAll(rdr)
+}

internal/decompress/lzma.go

@@ -0,0 +1,24 @@
+//go:build !no_obsolete
+
+package decompress
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/ulikunitz/xz/lzma"
+)
+
+type Lzma struct{}
+
+func NewLzma() (Lzma, error) {
+	return Lzma{}, nil
+}
+
+func (l Lzma) Decompress(data []byte) ([]byte, error) {
+	rdr, err := lzma.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+	return io.ReadAll(rdr)
+}

internal/decompress/lzma_disabled.go

@@ -0,0 +1,17 @@
+//go:build no_obsolete
+
+package decompress
+
+import (
+	"errors"
+)
+
+type Lzma struct{}
+
+func NewLzma() (Lzma, error) {
+	return Lzma{}, errors.New("lzma compression is disable in this build with no_obsolete")
+}
+
+func (l Lzma) Decompress(data []byte) ([]byte, error) {
+	return nil, errors.New("lzma compression is disable in this build with no_obsolete")
+}

internal/decompress/lzo.go

@@ -0,0 +1,19 @@
+//go:build !no_gpl
+
+package decompress
+
+import (
+	"bytes"
+
+	"github.com/rasky/go-lzo"
+)
+
+type Lzo struct{}
+
+func NewLzo() (Lzo, error) {
+	return Lzo{}, nil
+}
+
+func (l Lzo) Decompress(data []byte) ([]byte, error) {
+	return lzo.Decompress1X(bytes.NewReader(data), len(data), 0)
+}

internal/decompress/lzo_disabled.go

@@ -0,0 +1,15 @@
+//go:build no_gpl
+
+package decompress
+
+import "errors"
+
+type Lzo struct{}
+
+func NewLzo() (Lzo, error) {
+	return Lzo{}, errors.New("lzo compression is disable in this build with no_gpl")
+}
+
+func (l Lzo) Decompress(data []byte) ([]byte, error) {
+	return nil, errors.New("lzo compression is disable in this build with no_gpl")
+}

internal/decompress/xz.go

@@ -0,0 +1,34 @@
+package decompress
+
+import (
+	"bytes"
+	"io"
+	"sync"
+
+	"github.com/therootcompany/xz"
+)
+
+type Xz struct {
+	pool sync.Pool
+}
+
+func NewXz() *Xz {
+	return &Xz{
+		pool: sync.Pool{
+			New: func() any {
+				rdr, _ := xz.NewReader(nil, 0)
+				return rdr
+			},
+		},
+	}
+}
+
+func (x *Xz) Decompress(data []byte) ([]byte, error) {
+	rdr := x.pool.Get().(*xz.Reader)
+	defer x.pool.Put(rdr)
+	err := rdr.Reset(bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+	return io.ReadAll(rdr)
+}

internal/decompress/zlib.go

@@ -0,0 +1,18 @@
+package decompress
+
+import (
+	"bytes"
+	"compress/zlib"
+	"io"
+)
+
+type Zlib struct{}
+
+func (z Zlib) Decompress(data []byte) ([]byte, error) {
+	rdr, err := zlib.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+	defer rdr.Close()
+	return io.ReadAll(rdr)
+}

internal/decompress/zstd.go

@@ -0,0 +1,16 @@
+package decompress
+
+import (
+	"github.com/klauspost/compress/zstd"
+)
+
+type Zstd struct{}
+
+func (z Zstd) Decompress(data []byte) ([]byte, error) {
+	rdr, err := zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderConcurrency(1))
+	if err != nil {
+		return nil, err
+	}
+	defer rdr.Close()
+	return rdr.DecodeAll(data, nil)
+}

internal/directory/directory.go

@@ -1,87 +0,0 @@
-package directory
-
-import (
-	"bytes"
-	"encoding/binary"
-	"io"
-)
-
-//Header is the header for a directory in the directory table
-type Header struct {
-	Count       uint32
-	InodeOffset uint32
-	InodeNumber uint32
-}
-
-//EntryRaw is the values that can be easily decoded
-type EntryRaw struct {
-	Offset      uint16
-	InodeOffset int16
-	Type        uint16
-	NameSize    uint16
-}
-
-//Entry is an entry in a directory.
-type Entry struct {
-	Name             string
-	InodeOffset      uint32
-	InodeBlockOffset uint16
-	Type             uint16
-}
-
-//NewEntry creates a new directory entry
-func NewEntry(rdr io.Reader) (*Entry, error) {
-	var raw EntryRaw
-	err := binary.Read(rdr, binary.LittleEndian, &raw)
-	if err != nil {
-		return nil, err
-	}
-	tmp := make([]byte, raw.NameSize+1)
-	err = binary.Read(rdr, binary.LittleEndian, &tmp)
-	if err != nil {
-		return nil, err
-	}
-	return &Entry{
-		InodeBlockOffset: raw.Offset,
-		Type:             raw.Type,
-		Name:             string(tmp),
-	}, nil
-}
-
-//NewDirectory reads the directory from rdr
-func NewDirectory(base io.Reader, size uint32) (entries []*Entry, err error) {
-	tmp := make([]byte, size)
-	base.Read(tmp)
-	rdr := bytes.NewBuffer(tmp)
-	for {
-		var hdr Header
-		err = binary.Read(rdr, binary.LittleEndian, &hdr)
-		if err == io.ErrUnexpectedEOF {
-			err = nil
-			break
-		} else if err != nil {
-			return nil, err
-		}
-		hdr.Count++
-		headers := hdr.Count / 256
-		if hdr.Count%256 > 0 {
-			headers++
-		}
-		for i := uint32(0); i < hdr.Count; i++ {
-			if i != 0 && i%256 == 0 {
-				err = binary.Read(rdr, binary.LittleEndian, &hdr)
-				if err != nil {
-					return nil, err
-				}
-			}
-			var ent *Entry
-			ent, err = NewEntry(rdr)
-			if err != nil {
-				return nil, err
-			}
-			ent.InodeOffset = hdr.InodeOffset
-			entries = append(entries, ent)
-		}
-	}
-	return
-}

internal/inode/inodetypes.go

@@ -1,257 +0,0 @@
-package inode
-
-import (
-	"encoding/binary"
-	"io"
-)
-
-//The different types of inodes as defined by inodetype
-const (
-	DirType = iota + 1
-	FileType
-	SymType
-	BlockDevType
-	CharDevType
-	FifoType
-	SocketType
-	ExtDirType
-	ExtFileType
-	ExtSymType
-	ExtBlockDeviceType
-	ExtCharDeviceType
-	ExtFifoType
-	ExtSocketType
-)
-
-//Header is the common header for all inodes
-type Header struct {
-	Type         uint16
-	Permissions  uint16
-	UID          uint16
-	GID          uint16
-	ModifiedTime uint32
-	Number       uint32
-}
-
-//Dir is self explainatory
-type Dir struct {
-	DirectoryIndex    uint32
-	HardLinks         uint32
-	DirectorySize     uint16
-	DirectoryOffset   uint16
-	ParentInodeNumber uint32
-}
-
-//ExtDirInit is the information that can be directoy decoded
-type ExtDirInit struct {
-	HardLinks         uint32
-	DirectorySize     uint32
-	DirectoryIndex    uint32
-	ParentInodeNumber uint32
-	IndexCount        uint16 //one less then directory indexes following structure
-	DirectoryOffset   uint16
-	XattrIndex        uint32
-}
-
-//ExtDir is a directory with extra info
-type ExtDir struct {
-	Indexes []DirIndex
-	ExtDirInit
-}
-
-//NewExtendedDirectory creates a new ExtendedDirectory
-func NewExtendedDirectory(rdr io.Reader) (ExtDir, error) {
-	var inode ExtDir
-	err := binary.Read(rdr, binary.LittleEndian, &inode.ExtDirInit)
-	if err != nil {
-		return inode, err
-	}
-	for i := uint16(0); i < inode.IndexCount; i++ {
-		var tmp DirIndex
-		tmp, err = NewDirectoryIndex(rdr)
-		if err != nil {
-			return inode, err
-		}
-		inode.Indexes = append(inode.Indexes, tmp)
-	}
-	return inode, err
-}
-
-//DirIndexInit holds the values that can be easily decoded
-type DirIndexInit struct {
-	Offset         uint32
-	DirTableOffset uint32
-	NameSize       uint32
-}
-
-//DirIndex is a quick lookup provided by an ExtendedDirectory
-type DirIndex struct {
-	Name string
-	DirIndexInit
-}
-
-//NewDirectoryIndex return a new DirectoryIndex
-func NewDirectoryIndex(rdr io.Reader) (DirIndex, error) {
-	var index DirIndex
-	err := binary.Read(rdr, binary.LittleEndian, &index.DirIndexInit)
-	if err != nil {
-		return index, err
-	}
-	tmp := make([]byte, index.NameSize+1, index.NameSize+1)
-	err = binary.Read(rdr, binary.LittleEndian, &tmp)
-	if err != nil {
-		return index, err
-	}
-	index.Name = string(tmp)
-	return index, nil
-}
-
-//FileInit is the information that can be directly decoded
-type FileInit struct {
-	BlockStart     uint32
-	FragmentIndex  uint32
-	FragmentOffset uint32
-	Size           uint32
-}
-
-//File is self explainatory
-type File struct {
-	BlockSizes []uint32
-	Fragmented bool
-	FileInit
-}
-
-//NewFile creates a new File
-func NewFile(rdr io.Reader, blockSize uint32) (File, error) {
-	var inode File
-	err := binary.Read(rdr, binary.LittleEndian, &inode.FileInit)
-	if err != nil {
-		return inode, err
-	}
-	inode.Fragmented = inode.FragmentIndex != 0xFFFFFFFF
-	blocks := inode.Size / blockSize
-	if inode.Size%blockSize > 0 {
-		blocks++
-	}
-	inode.BlockSizes = make([]uint32, blocks, blocks)
-	err = binary.Read(rdr, binary.LittleEndian, &inode.BlockSizes)
-	return inode, err
-}
-
-//ExtFileInit is the information that can be directly decoded
-type ExtFileInit struct {
-	BlockStart     uint64
-	Size           uint64
-	Sparse         uint64
-	HardLinks      uint32
-	FragmentIndex  uint32
-	FragmentOffset uint32
-	XattrIndex     uint32
-}
-
-//ExtFile is a file with more information
-type ExtFile struct {
-	BlockSizes []uint32
-	Fragmented bool
-	ExtFileInit
-}
-
-//NewExtendedFile creates a new ExtendedFile
-func NewExtendedFile(rdr io.Reader, blockSize uint32) (ExtFile, error) {
-	var inode ExtFile
-	err := binary.Read(rdr, binary.LittleEndian, &inode.ExtFileInit)
-	if err != nil {
-		return inode, err
-	}
-	inode.Fragmented = inode.FragmentIndex != 0xFFFFFFFF
-	blocks := inode.Size / uint64(blockSize)
-	if inode.Size%uint64(blockSize) > 0 {
-		blocks++
-	}
-	inode.BlockSizes = make([]uint32, blocks, blocks)
-	err = binary.Read(rdr, binary.LittleEndian, &inode.BlockSizes)
-	return inode, err
-}
-
-//SymInit is all the values that can be directly decoded
-type SymInit struct {
-	HardLinks      uint32
-	TargetPathSize uint32
-}
-
-//Sym is a symlink
-type Sym struct {
-	Path       string
-	targetPath []byte //len is TargetPathSize
-	SymInit
-}
-
-//NewSymlink creates a new Symlink
-func NewSymlink(rdr io.Reader) (Sym, error) {
-	var inode Sym
-	err := binary.Read(rdr, binary.LittleEndian, &inode.SymInit)
-	if err != nil {
-		return inode, err
-	}
-	inode.targetPath = make([]byte, inode.TargetPathSize, inode.TargetPathSize)
-	err = binary.Read(rdr, binary.LittleEndian, &inode.targetPath)
-	if err != nil {
-		return inode, err
-	}
-	inode.Path = string(inode.targetPath)
-	return inode, err
-}
-
-//ExtSymInit is all the values that can be directly decoded
-type ExtSymInit struct {
-	HardLinks      uint32
-	TargetPathSize uint32
-}
-
-//ExtSym is a symlink with extra information
-type ExtSym struct {
-	Path       string
-	targetPath []uint8
-	ExtSymInit
-	XattrIndex uint32
-}
-
-//NewExtendedSymlink creates a new ExtendedSymlink
-func NewExtendedSymlink(rdr io.Reader) (ExtSym, error) {
-	var inode ExtSym
-	err := binary.Read(rdr, binary.LittleEndian, &inode.ExtSymInit)
-	if err != nil {
-		return inode, err
-	}
-	inode.targetPath = make([]uint8, inode.TargetPathSize, inode.TargetPathSize)
-	err = binary.Read(rdr, binary.LittleEndian, &inode.targetPath)
-	if err != nil {
-		return inode, err
-	}
-	inode.Path = string(inode.targetPath)
-	err = binary.Read(rdr, binary.LittleEndian, &inode.XattrIndex)
-	return inode, err
-}
-
-//Device is a device
-type Device struct {
-	HardLinks uint32
-	Device    uint32
-}
-
-//ExtDevice is a device with more info
-type ExtDevice struct {
-	Device
-	XattrIndex uint32
-}
-
-//IPC is a Fifo or Socket device
-type IPC struct {
-	HardLink uint32
-}
-
-//ExtIPC is a IPC device with extra info
-type ExtIPC struct {
-	IPC
-	XattrIndex uint32
-}

internal/inode/process.go

@@ -1,128 +0,0 @@
-package inode
-
-import (
-	"encoding/binary"
-	"errors"
-	"io"
-	"strconv"
-)
-
-//Inode holds an inode. Header is the header that's common for all inodes.
-//
-//Info holds the actual Inode. Due to each inode type being a different type, it's store as an interface{}
-type Inode struct {
-	Header
-	Info interface{} //Info is the parsed specific data. It's type is defined by Type.
-}
-
-//ProcessInode tries to read an inode from the BlockReader
-func ProcessInode(br io.Reader, blockSize uint32) (*Inode, error) {
-	var in Inode
-	err := binary.Read(br, binary.LittleEndian, &in.Header)
-	if err != nil {
-		return nil, err
-	}
-	switch in.Type {
-	case DirType:
-		var inode Dir
-		err = binary.Read(br, binary.LittleEndian, &inode)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case FileType:
-		var inode File
-		inode, err = NewFile(br, blockSize)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case SymType:
-		var inode Sym
-		inode, err = NewSymlink(br)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case BlockDevType:
-		var inode Device
-		err = binary.Read(br, binary.LittleEndian, &inode)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case CharDevType:
-		var inode Device
-		err = binary.Read(br, binary.LittleEndian, &inode)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case FifoType:
-		var inode IPC
-		err = binary.Read(br, binary.LittleEndian, &inode)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case SocketType:
-		var inode IPC
-		err = binary.Read(br, binary.LittleEndian, &inode)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case ExtDirType:
-		var inode ExtDir
-		inode, err = NewExtendedDirectory(br)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case ExtFileType:
-		var inode ExtFile
-		inode, err = NewExtendedFile(br, blockSize)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case ExtSymType:
-		var inode ExtSym
-		inode, err = NewExtendedSymlink(br)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case ExtBlockDeviceType:
-		var inode ExtDevice
-		err = binary.Read(br, binary.LittleEndian, &inode)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case ExtCharDeviceType:
-		var inode ExtDevice
-		err = binary.Read(br, binary.LittleEndian, &inode)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case ExtFifoType:
-		var inode ExtIPC
-		err = binary.Read(br, binary.LittleEndian, &inode)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	case ExtSocketType:
-		var inode ExtIPC
-		err = binary.Read(br, binary.LittleEndian, &inode)
-		if err != nil {
-			return nil, err
-		}
-		in.Info = inode
-	default:
-		return nil, errors.New("Unsupported inode type: " + strconv.Itoa(int(in.Type)))
-	}
-	return &in, nil
-}

internal/metadata/reader.go

@@ -0,0 +1,65 @@
+package metadata
+
+import (
+	"encoding/binary"
+	"io"
+
+	"github.com/CalebQ42/squashfs/internal/decompress"
+)
+
+type Reader struct {
+	r         io.Reader
+	d         decompress.Decompressor
+	dat       []byte
+	curOffset uint16
+}
+
+func NewReader(r io.Reader, d decompress.Decompressor) Reader {
+	return Reader{
+		r: r,
+		d: d,
+	}
+}
+
+func (r *Reader) advance() error {
+	r.curOffset = 0
+	dat := make([]byte, 2)
+	_, err := r.r.Read(dat)
+	if err != nil {
+		return err
+	}
+	size := binary.LittleEndian.Uint16(dat)
+	realSize := size &^ 0x8000
+	r.dat = make([]byte, realSize)
+	_, err = r.r.Read(r.dat)
+	if err != nil {
+		return err
+	}
+	if size != realSize {
+		return nil
+	}
+	r.dat, err = r.d.Decompress(r.dat)
+	return err
+}
+
+func (r *Reader) Read(b []byte) (int, error) {
+	curRead := 0
+	var toRead int
+	for curRead < len(b) {
+		if r.curOffset >= uint16(len(r.dat)) {
+			if err := r.advance(); err != nil {
+				return curRead, err
+			}
+		}
+		toRead = min(len(b)-curRead, len(r.dat)-int(r.curOffset))
+		copy(b[curRead:], r.dat[r.curOffset:int(r.curOffset)+toRead])
+		r.curOffset += uint16(toRead)
+		curRead += toRead
+	}
+	return curRead, nil
+}
+
+func (r *Reader) Close() error {
+	r.dat = nil
+	return nil
+}

internal/rawreader/rawreader.go

@@ -1,135 +0,0 @@
-package rawreader
-
-import (
-	"bytes"
-	"errors"
-	"io"
-)
-
-func ConvertReader(r io.Reader) (RawReader, error) {
-	if rr, ok := r.(RawReader); ok {
-		return rr, nil
-	}
-	if rs, is := r.(io.ReadSeeker); is {
-		return &fromReadSeeker{
-			ReadSeeker: rs,
-		}, nil
-	}
-	buf := new(bytes.Buffer)
-	_, err := io.Copy(buf, r)
-	if err != nil {
-		return nil, err
-	}
-	return &fromReader{
-		data: buf.Bytes(),
-	}, nil
-}
-
-func ConvertReaderAt(r io.ReaderAt) RawReader {
-	if rr, ok := r.(RawReader); ok {
-		return rr
-	}
-	return &fromReaderAt{
-		ReaderAt: r,
-	}
-}
-
-//TODO: Add way to discard data from fromReader
-//RawReader implements the needed interfaces for reading a squashfs archive.
-type RawReader interface {
-	io.ReadSeeker
-	io.ReaderAt
-}
-
-type fromReader struct {
-	data []byte
-	off  int
-}
-
-func (r *fromReader) ReadAt(p []byte, off int64) (n int, err error) {
-	n = len(p)
-	if int(off)+len(p) > len(r.data) {
-		n = len(r.data) - int(off)
-		err = io.EOF
-	}
-	if n < 0 {
-		n = 0
-	}
-	for i := 0; i < n; i++ {
-		p[i] = r.data[int(off)+i]
-	}
-	return
-}
-
-func (r *fromReader) Seek(off int64, whence int) (n int64, err error) {
-	switch whence {
-	case io.SeekEnd:
-		r.off = len(r.data) - int(off)
-		if r.off < 0 {
-			r.off = 0
-			err = io.EOF
-		}
-	case io.SeekCurrent:
-		r.off += int(off)
-	case io.SeekStart:
-		r.off = int(off)
-	}
-	if r.off > len(r.data) {
-		r.off = len(r.data)
-		return int64(r.off), io.EOF
-	}
-	return int64(r.off), err
-}
-
-func (r *fromReader) Read(p []byte) (n int, err error) {
-	n = len(p)
-	if r.off+len(p) > len(r.data) {
-		n = len(r.data) - r.off
-		err = io.EOF
-	}
-	if n < 0 {
-		n = 0
-	}
-	for i := 0; i < n; i++ {
-		p[i] = r.data[r.off+i]
-	}
-	return
-}
-
-type fromReadSeeker struct {
-	io.ReadSeeker
-}
-
-func (r *fromReadSeeker) ReadAt(p []byte, off int64) (n int, err error) {
-	tmp, _ := r.Seek(0, io.SeekCurrent)
-	defer r.Seek(tmp, io.SeekStart)
-	_, err = r.Seek(off, io.SeekStart)
-	if err != nil {
-		return
-	}
-	return r.Read(p)
-}
-
-type fromReaderAt struct {
-	io.ReaderAt
-
-	off int
-}
-
-func (r *fromReaderAt) Read(p []byte) (n int, err error) {
-	n, err = r.ReadAt(p, int64(r.off))
-	r.off += n
-	return
-}
-
-func (r *fromReaderAt) Seek(off int64, whence int) (n int64, err error) {
-	switch whence {
-	case io.SeekEnd:
-		return 0, errors.New("cannot SeekEnd RawReader")
-	case io.SeekCurrent:
-		r.off += int(off)
-	case io.SeekStart:
-		r.off = int(off)
-	}
-	return int64(r.off), nil
-}

internal/routinemanager/manager.go

@@ -0,0 +1,25 @@
+package routinemanager
+
+type Manager struct {
+	channel     chan uint16
+	maxRoutines uint16
+}
+
+func NewManager(maxRoutines uint16) *Manager {
+	m := &Manager{
+		maxRoutines: maxRoutines,
+		channel:     make(chan uint16, maxRoutines),
+	}
+	for i := uint16(0); i < maxRoutines; i++ {
+		m.channel <- i
+	}
+	return m
+}
+
+func (m *Manager) Lock() uint16 {
+	return <-m.channel
+}
+
+func (m *Manager) Unlock(i uint16) {
+	m.channel <- i
+}

internal/toreader/offsetreader.go

@@ -0,0 +1,19 @@
+package toreader
+
+import "io"
+
+type OffsetReader struct {
+	r   io.ReaderAt
+	off int64
+}
+
+func NewOffsetReader(r io.ReaderAt, off int64) *OffsetReader {
+	return &OffsetReader{
+		r:   r,
+		off: off,
+	}
+}
+
+func (r OffsetReader) ReadAt(p []byte, off int64) (n int, e error) {
+	return r.r.ReadAt(p, off+r.off)
+}

internal/toreader/toreader.go

@@ -0,0 +1,21 @@
+package toreader
+
+import "io"
+
+type Reader struct {
+	r      io.ReaderAt
+	offset int64
+}
+
+func NewReader(r io.ReaderAt, start int64) *Reader {
+	return &Reader{
+		r:      r,
+		offset: start,
+	}
+}
+
+func (r *Reader) Read(b []byte) (int, error) {
+	n, err := r.r.ReadAt(b, r.offset)
+	r.offset += int64(n)
+	return n, err
+}

low/README.md

@@ -0,0 +1,3 @@
+# Lower-Level Squashfs
+
+This library is a lower level version of the main [squashfs](https://github.com/CalebQ42/squashfs) library that doesn't try to be easy to use and exposes a lot of information that is not necesary for must use cases.

low/data/fullreader.go

@@ -0,0 +1,260 @@
+package data
+
+import (
+	"errors"
+	"io"
+	"io/fs"
+	"math"
+	"runtime"
+	"sync"
+
+	"github.com/CalebQ42/squashfs/internal/decompress"
+)
+
+type FragReaderConstructor func() (io.Reader, error)
+
+type FullReader struct {
+	r              io.ReaderAt
+	d              decompress.Decompressor
+	frag           FragReaderConstructor
+	sizes          []uint32
+	initialOffset  int64
+	finalBlockSize uint64
+	blockSize      uint32
+	goroutineLimit uint16
+	closed         bool
+}
+
+func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) FullReader {
+	return FullReader{
+		r:              r,
+		d:              d,
+		sizes:          sizes,
+		initialOffset:  initialOffset,
+		goroutineLimit: uint16(runtime.NumCPU()),
+		finalBlockSize: finalBlockSize,
+		blockSize:      blockSize,
+	}
+}
+
+func (r *FullReader) Close() error {
+	r.closed = true
+	r.r = nil
+	r.d = nil
+	r.frag = nil
+	r.sizes = nil
+	return nil
+}
+
+func (r *FullReader) AddFrag(frag FragReaderConstructor) {
+	r.frag = frag
+}
+
+func (r *FullReader) SetGoroutineLimit(limit uint16) {
+	if limit <= 0 {
+		r.goroutineLimit = 1
+	}
+	r.goroutineLimit = limit
+}
+
+type retValue struct {
+	err   error
+	data  []byte
+	index uint64
+}
+
+func (r FullReader) process(index uint64, fileOffset uint64, pool *sync.Pool, retChan chan *retValue) {
+	ret := pool.Get().(*retValue)
+	ret.index = index
+	realSize := r.sizes[index] &^ (1 << 24)
+	if realSize == 0 {
+		if index == uint64(len(r.sizes))-1 && r.frag == nil {
+			ret.data = make([]byte, r.finalBlockSize)
+		} else {
+			ret.data = make([]byte, r.blockSize)
+		}
+		ret.err = nil
+		retChan <- ret
+		return
+	}
+	ret.data = make([]byte, realSize)
+	_, ret.err = r.r.ReadAt(ret.data, r.initialOffset+int64(fileOffset))
+	if r.sizes[index] == realSize {
+		ret.data, ret.err = r.d.Decompress(ret.data)
+	}
+	retChan <- ret
+}
+
+func (r FullReader) WriteTo(w io.Writer) (int64, error) {
+	if r.closed {
+		return 0, fs.ErrClosed
+	}
+	// if wa, is := w.(io.WriterAt); is {
+	// 	return r.writeToWriteAt(wa)
+	// }
+	var curIndex uint64
+	var curOffset uint64
+	var toProcess uint16
+	var wrote int64
+	cache := make(map[uint64]*retValue)
+	var errCache []error
+	retChan := make(chan *retValue, r.goroutineLimit)
+	pool := &sync.Pool{
+		New: func() any {
+			return &retValue{}
+		},
+	}
+	for i := uint64(0); i < uint64(math.Ceil(float64(len(r.sizes))/float64(r.goroutineLimit))); i++ {
+		toProcess = min(uint16(len(r.sizes))-(uint16(i)*r.goroutineLimit), r.goroutineLimit)
+		// Start all the goroutines
+		for j := uint16(0); j < toProcess; j++ {
+			go r.process((i*uint64(r.goroutineLimit))+uint64(j), curOffset, pool, retChan)
+			curOffset += uint64(r.sizes[(i*uint64(r.goroutineLimit))+uint64(j)]) &^ (1 << 24)
+		}
+		// Then consume the results on retChan
+		for j := uint16(0); j < toProcess; j++ {
+			res := <-retChan
+			// If there's an error, we don't care about the results.
+			if res.err != nil {
+				errCache = append(errCache, res.err)
+				if len(cache) > 0 {
+					clear(cache)
+				}
+				continue
+			}
+			// If there has been an error previously, we don't care about the results.
+			// We still want to wait for all the goroutines to prevent resources being wasted.
+			if len(errCache) > 0 {
+				continue
+			}
+			// If we don't need the data yet, we cache it and move on
+			if res.index != curIndex {
+				cache[res.index] = res
+				continue
+			}
+			// If we do need the data, we write it
+			wr, err := w.Write(res.data)
+			wrote += int64(wr)
+			if err != nil {
+				errCache = append(errCache, err)
+				if len(cache) > 0 {
+					clear(cache)
+				}
+				continue
+			}
+			pool.Put(res)
+			curIndex++
+			// Now we recursively try to clear the cache
+			for len(cache) > 0 {
+				res, ok := cache[curIndex]
+				if !ok {
+					break
+				}
+				wr, err := w.Write(res.data)
+				wrote += int64(wr)
+				if err != nil {
+					errCache = append(errCache, err)
+					if len(cache) > 0 {
+						clear(cache)
+					}
+					break
+				}
+				delete(cache, curIndex)
+				pool.Put(res)
+				curIndex++
+			}
+		}
+		if len(errCache) > 0 {
+			return wrote, errors.Join(errCache...)
+		}
+	}
+	if r.frag != nil {
+		rdr, err := r.frag()
+		if err != nil {
+			return wrote, err
+		}
+		wr, err := io.Copy(w, rdr)
+		wrote += wr
+		if l, ok := rdr.(*io.LimitedReader); ok {
+			if cl, ok := l.R.(io.Closer); ok {
+				cl.Close()
+			}
+		}
+		if err != nil {
+			return wrote, err
+		}
+	}
+	return wrote, nil
+}
+
+// func (r FullReader) writeToWriteAt(w io.WriterAt) (out int64, outErr error) {
+// 	wait := &sync.WaitGroup{}
+// 	wait.Add(len(r.sizes))
+// 	mgr := routinemanager.NewManager(r.goroutineLimit)
+// 	curOffset := r.initialOffset
+// 	for i := uint64(0); i < uint64(len(r.sizes)); i++ {
+// 		go func(index uint64, fileOffset int64) {
+// 			lckNum := mgr.Lock()
+// 			defer mgr.Unlock(lckNum)
+// 			defer wait.Done()
+// 			realSize := r.sizes[index] &^ (1 << 24)
+// 			if realSize == 0 {
+// 				if index == uint64(len(r.sizes))-1 && r.frag == nil {
+// 					_, err := w.WriteAt([]byte{0}, int64((uint64(r.blockSize)*index)+r.finalBlockSize)-1)
+// 					if err != nil {
+// 						outErr = errors.Join(outErr, err)
+// 						return
+// 					}
+// 					out = max(out, int64((uint64(r.blockSize)*index)+r.finalBlockSize))
+// 				}
+// 				return
+// 			}
+// 			data := make([]byte, realSize)
+// 			err := binary.Read(toreader.NewReader(r.r, int64(fileOffset)), binary.LittleEndian, &data)
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			if r.sizes[index] == realSize {
+// 				data, err = r.d.Decompress(data)
+// 			}
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			_, err = w.WriteAt(data, int64(uint64(r.blockSize)*index))
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			out = max(out, int64(uint64(r.blockSize)*(index+1)))
+// 		}(i, curOffset)
+// 		curOffset += int64(r.sizes[i]) &^ (1 << 24)
+// 	}
+// 	if r.frag != nil {
+// 		wait.Add(1)
+// 		go func() {
+// 			lckNum := mgr.Lock()
+// 			defer mgr.Unlock(lckNum)
+// 			defer wait.Done()
+// 			rdr, err := r.frag()
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			dat, err := io.ReadAll(rdr)
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			_, err = w.WriteAt(dat, int64(int(r.blockSize)*len(r.sizes)))
+// 			if err != nil {
+// 				outErr = errors.Join(outErr, err)
+// 				return
+// 			}
+// 			out = int64(int(r.blockSize)*len(r.sizes)) + int64(r.finalBlockSize)
+// 		}()
+// 	}
+// 	wait.Wait()
+// 	return
+// }

low/data/reader.go

@@ -0,0 +1,104 @@
+package data
+
+import (
+	"io"
+	"io/fs"
+
+	"github.com/CalebQ42/squashfs/internal/decompress"
+)
+
+type Reader struct {
+	r              io.Reader
+	d              decompress.Decompressor
+	frag           io.Reader
+	sizes          []uint32
+	dat            []byte
+	curOffset      int
+	curIndex       uint64
+	finalBlockSize uint64
+	blockSize      uint32
+	closed         bool
+}
+
+func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) Reader {
+	return Reader{
+		r:              r,
+		d:              d,
+		sizes:          sizes,
+		finalBlockSize: finalBlockSize,
+		blockSize:      blockSize,
+	}
+}
+
+func (r *Reader) AddFrag(fragRdr io.Reader) {
+	r.frag = fragRdr
+}
+
+func (r *Reader) advance() error {
+	r.curOffset = 0
+	defer func() { r.curIndex++ }()
+	var err error
+	if r.curIndex == uint64(len(r.sizes)) && r.frag != nil {
+		r.dat, err = io.ReadAll(r.frag)
+		return err
+	} else if r.curIndex >= uint64(len(r.sizes)) {
+		r.dat = []byte{}
+		return io.EOF
+	}
+	realSize := r.sizes[r.curIndex] &^ (1 << 24)
+	if realSize == 0 {
+		if r.curIndex == uint64(len(r.sizes))-1 && r.frag == nil {
+			r.dat = make([]byte, r.finalBlockSize)
+		} else {
+			r.dat = make([]byte, r.blockSize)
+		}
+		return nil
+	}
+	r.dat = make([]byte, realSize)
+	_, err = r.r.Read(r.dat)
+	if err != nil {
+		return err
+	}
+	if r.sizes[r.curIndex] != realSize {
+		return nil
+	}
+	r.dat, err = r.d.Decompress(r.dat)
+	return err
+}
+
+func (r *Reader) Read(b []byte) (int, error) {
+	if r.closed {
+		return 0, fs.ErrClosed
+	}
+	curRead := 0
+	var toRead int
+	for curRead < len(b) {
+		if r.curOffset >= len(r.dat) {
+			if err := r.advance(); err != nil {
+				return curRead, err
+			}
+		}
+		toRead = min(len(b)-curRead, len(r.dat)-r.curOffset)
+		toRead = copy(b[curRead:], r.dat[r.curOffset:r.curOffset+toRead])
+		r.curOffset += toRead
+		curRead += toRead
+	}
+	return curRead, nil
+}
+
+func (r *Reader) Close() error {
+	r.closed = true
+	r.r = nil
+	r.d = nil
+	if r.frag != nil {
+		if l, ok := r.frag.(*io.LimitedReader); ok {
+			if cl, ok := l.R.(io.Closer); ok {
+				cl.Close()
+			}
+		}
+	}
+	r.frag = nil
+	r.sizes = nil
+	r.dat = nil
+	return nil
+}

low/directory.go

@@ -0,0 +1,83 @@
+package squashfslow
+
+import (
+	"errors"
+	"io/fs"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"github.com/CalebQ42/squashfs/internal/metadata"
+	"github.com/CalebQ42/squashfs/internal/toreader"
+	"github.com/CalebQ42/squashfs/low/directory"
+	"github.com/CalebQ42/squashfs/low/inode"
+)
+
+type Directory struct {
+	FileBase
+	Entries []directory.Entry
+}
+
+func (r Reader) directoryFromRef(ref uint64, name string) (Directory, error) {
+	i, err := r.InodeFromRef(ref)
+	if err != nil {
+		return Directory{}, err
+	}
+	var blockStart uint32
+	var size uint32
+	var offset uint16
+	switch i.Type {
+	case inode.Dir:
+		blockStart = i.Data.(inode.Directory).BlockStart
+		size = uint32(i.Data.(inode.Directory).Size)
+		offset = i.Data.(inode.Directory).Offset
+	case inode.EDir:
+		blockStart = i.Data.(inode.EDirectory).BlockStart
+		size = i.Data.(inode.EDirectory).Size
+		offset = i.Data.(inode.EDirectory).Offset
+	default:
+		return Directory{}, errors.New("not a directory")
+	}
+	dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d)
+	defer dirRdr.Close()
+	_, err = dirRdr.Read(make([]byte, offset))
+	if err != nil {
+		return Directory{}, err
+	}
+	entries, err := directory.ReadDirectory(&dirRdr, size)
+	if err != nil {
+		return Directory{}, err
+	}
+	return Directory{
+		FileBase: r.BaseFromInode(i, name),
+		Entries:  entries,
+	}, nil
+}
+
+func (d Directory) Open(r Reader, path string) (FileBase, error) {
+	path = filepath.Clean(path)
+	if path == "." || path == "" {
+		return d.FileBase, nil
+	}
+	split := strings.Split(path, "/")
+	i, found := slices.BinarySearchFunc(d.Entries, split[0], func(e directory.Entry, name string) int {
+		return strings.Compare(e.Name, name)
+	})
+	if !found {
+		return FileBase{}, fs.ErrNotExist
+	}
+	b, err := r.BaseFromEntry(d.Entries[i])
+	if err != nil {
+		return FileBase{}, err
+	}
+	if len(split) == 1 {
+		return b, nil
+	} else if !b.IsDir() {
+		return FileBase{}, fs.ErrNotExist
+	}
+	dir, err := b.ToDir(r)
+	if err != nil {
+		return FileBase{}, err
+	}
+	return dir.Open(r, strings.Join(split[1:], "/"))
+}

low/directory/directory.go

@@ -0,0 +1,62 @@
+package directory
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+type header struct {
+	Count      uint32
+	BlockStart uint32
+	Num        uint32
+}
+
+type decEntry struct {
+	Offset    uint16
+	NumOffset int16
+	InodeType uint16
+	NameSize  uint16
+	// Name []byte (not decoded along with decEntry)
+}
+
+type Entry struct {
+	Name       string
+	BlockStart uint32
+	Offset     uint16
+	InodeType  uint16
+	Num        uint32
+}
+
+func ReadDirectory(r io.Reader, size uint32) (out []Entry, err error) {
+	size -= 3
+	var curRead uint32
+	var h header
+	var de decEntry
+	for curRead < size {
+		err = binary.Read(r, binary.LittleEndian, &h)
+		if err != nil {
+			return
+		}
+		curRead += 12
+		for i := uint32(0); i < h.Count+1 && curRead < size; i++ {
+			err = binary.Read(r, binary.LittleEndian, &de)
+			if err != nil {
+				return
+			}
+			nameTmp := make([]byte, de.NameSize+1)
+			err = binary.Read(r, binary.LittleEndian, &nameTmp)
+			if err != nil {
+				return
+			}
+			curRead += 8 + uint32(de.NameSize) + 1
+			out = append(out, Entry{
+				BlockStart: h.BlockStart,
+				Offset:     de.Offset,
+				Name:       string(nameTmp),
+				InodeType:  de.InodeType,
+				Num:        h.Num + uint32(de.NumOffset),
+			})
+		}
+	}
+	return
+}

low/file_base.go

@@ -0,0 +1,203 @@
+package squashfslow
+
+import (
+	"errors"
+	"io"
+
+	"github.com/CalebQ42/squashfs/internal/metadata"
+	"github.com/CalebQ42/squashfs/internal/toreader"
+	"github.com/CalebQ42/squashfs/low/data"
+	"github.com/CalebQ42/squashfs/low/directory"
+	"github.com/CalebQ42/squashfs/low/inode"
+)
+
+type FileBase struct {
+	Inode inode.Inode
+	Name  string
+}
+
+func (r Reader) BaseFromInode(i inode.Inode, name string) FileBase {
+	return FileBase{Inode: i, Name: name}
+}
+
+func (r Reader) BaseFromEntry(e directory.Entry) (FileBase, error) {
+	in, err := r.InodeFromEntry(e)
+	if err != nil {
+		return FileBase{}, err
+	}
+	return FileBase{Inode: in, Name: e.Name}, nil
+}
+
+func (r Reader) BaseFromRef(ref uint64, name string) (FileBase, error) {
+	in, err := r.InodeFromRef(ref)
+	if err != nil {
+		return FileBase{}, err
+	}
+	return FileBase{Inode: in, Name: name}, nil
+}
+
+func (b FileBase) Uid(r *Reader) (uint32, error) {
+	return r.Id(b.Inode.UidInd)
+}
+
+func (b FileBase) Gid(r *Reader) (uint32, error) {
+	return r.Id(b.Inode.GidInd)
+}
+
+func (b FileBase) IsDir() bool {
+	return b.Inode.Type == inode.Dir || b.Inode.Type == inode.EDir
+}
+
+func (b FileBase) ToDir(r Reader) (Directory, error) {
+	var blockStart uint32
+	var size uint32
+	var offset uint16
+	switch b.Inode.Type {
+	case inode.Dir:
+		blockStart = b.Inode.Data.(inode.Directory).BlockStart
+		size = uint32(b.Inode.Data.(inode.Directory).Size)
+		offset = b.Inode.Data.(inode.Directory).Offset
+	case inode.EDir:
+		blockStart = b.Inode.Data.(inode.EDirectory).BlockStart
+		size = b.Inode.Data.(inode.EDirectory).Size
+		offset = b.Inode.Data.(inode.EDirectory).Offset
+	default:
+		return Directory{}, errors.New("not a directory")
+	}
+	dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d)
+	defer dirRdr.Close()
+	_, err := dirRdr.Read(make([]byte, offset))
+	if err != nil {
+		return Directory{}, err
+	}
+	entries, err := directory.ReadDirectory(&dirRdr, size)
+	if err != nil {
+		return Directory{}, err
+	}
+	return Directory{
+		FileBase: b,
+		Entries:  entries,
+	}, nil
+}
+
+func (b FileBase) IsRegular() bool {
+	return b.Inode.Type == inode.Fil || b.Inode.Type == inode.EFil
+}
+
+func (b FileBase) GetRegFileReaders(r Reader) (data.Reader, data.FullReader, error) {
+	if !b.IsRegular() {
+		return data.Reader{}, data.FullReader{}, errors.New("not a regular file")
+	}
+	var blockStart uint64
+	var fragIndex uint32
+	var fragOffset uint32
+	var fragSize uint64
+	var sizes []uint32
+	if b.Inode.Type == inode.Fil {
+		blockStart = uint64(b.Inode.Data.(inode.File).BlockStart)
+		fragIndex = b.Inode.Data.(inode.File).FragInd
+		fragOffset = b.Inode.Data.(inode.File).FragOffset
+		sizes = b.Inode.Data.(inode.File).BlockSizes
+		fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize)
+	} else {
+		blockStart = b.Inode.Data.(inode.EFile).BlockStart
+		fragIndex = b.Inode.Data.(inode.EFile).FragInd
+		fragOffset = b.Inode.Data.(inode.EFile).FragOffset
+		sizes = b.Inode.Data.(inode.EFile).BlockSizes
+		fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize)
+	}
+	frag := func() (io.Reader, error) {
+		ent, err := r.fragEntry(fragIndex)
+		if err != nil {
+			return nil, err
+		}
+		frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
+		frag.Read(make([]byte, fragOffset))
+		return io.LimitReader(&frag, int64(fragSize)), nil
+	}
+	outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize)
+	if fragIndex != 0xffffffff {
+		f, err := frag()
+		if err != nil {
+			return data.Reader{}, data.FullReader{}, err
+		}
+		outRdr.AddFrag(f)
+	}
+	outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes, fragSize, r.Superblock.BlockSize)
+	if fragIndex != 0xffffffff {
+		outFull.AddFrag(frag)
+	}
+	return outRdr, outFull, nil
+}
+
+func (b FileBase) GetFullReader(r *Reader) (data.FullReader, error) {
+	if !b.IsRegular() {
+		return data.FullReader{}, errors.New("not a regular file")
+	}
+	var blockStart uint64
+	var fragIndex uint32
+	var fragOffset uint32
+	var fragSize uint64
+	var sizes []uint32
+	if b.Inode.Type == inode.Fil {
+		blockStart = uint64(b.Inode.Data.(inode.File).BlockStart)
+		fragIndex = b.Inode.Data.(inode.File).FragInd
+		fragOffset = b.Inode.Data.(inode.File).FragOffset
+		sizes = b.Inode.Data.(inode.File).BlockSizes
+		fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize)
+	} else {
+		blockStart = b.Inode.Data.(inode.EFile).BlockStart
+		fragIndex = b.Inode.Data.(inode.EFile).FragInd
+		fragOffset = b.Inode.Data.(inode.EFile).FragOffset
+		sizes = b.Inode.Data.(inode.EFile).BlockSizes
+		fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize)
+	}
+	outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes, fragSize, r.Superblock.BlockSize)
+	if fragIndex != 0xffffffff {
+		outFull.AddFrag(func() (io.Reader, error) {
+			ent, err := r.fragEntry(fragIndex)
+			if err != nil {
+				return nil, err
+			}
+			frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
+			frag.Read(make([]byte, fragOffset))
+			return io.LimitReader(&frag, int64(fragSize)), nil
+		})
+	}
+	return outFull, nil
+}
+
+func (b FileBase) GetReader(r *Reader) (data.Reader, error) {
+	if !b.IsRegular() {
+		return data.Reader{}, errors.New("not a regular file")
+	}
+	var blockStart uint64
+	var fragIndex uint32
+	var fragOffset uint32
+	var fragSize uint64
+	var sizes []uint32
+	if b.Inode.Type == inode.Fil {
+		blockStart = uint64(b.Inode.Data.(inode.File).BlockStart)
+		fragIndex = b.Inode.Data.(inode.File).FragInd
+		fragOffset = b.Inode.Data.(inode.File).FragOffset
+		sizes = b.Inode.Data.(inode.File).BlockSizes
+		fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize)
+	} else {
+		blockStart = b.Inode.Data.(inode.EFile).BlockStart
+		fragIndex = b.Inode.Data.(inode.EFile).FragInd
+		fragOffset = b.Inode.Data.(inode.EFile).FragOffset
+		sizes = b.Inode.Data.(inode.EFile).BlockSizes
+		fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize)
+	}
+	outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize)
+	if fragIndex != 0xffffffff {
+		ent, err := r.fragEntry(fragIndex)
+		if err != nil {
+			return data.Reader{}, err
+		}
+		frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
+		frag.Read(make([]byte, fragOffset))
+		outRdr.AddFrag(io.LimitReader(&frag, int64(fragSize)))
+	}
+	return outRdr, nil
+}

low/fragment.go

@@ -0,0 +1,7 @@
+package squashfslow
+
+type fragEntry struct {
+	Start uint64
+	Size  uint32
+	_     uint32
+}

low/inode.go

@@ -0,0 +1,26 @@
+package squashfslow
+
+import (
+	"github.com/CalebQ42/squashfs/internal/metadata"
+	"github.com/CalebQ42/squashfs/internal/toreader"
+	"github.com/CalebQ42/squashfs/low/directory"
+	"github.com/CalebQ42/squashfs/low/inode"
+)
+
+func (r Reader) InodeFromRef(ref uint64) (inode.Inode, error) {
+	offset, meta := (ref>>16)+r.Superblock.InodeTableStart, ref&0xFFFF
+	rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
+	defer rdr.Close()
+	_, err := rdr.Read(make([]byte, meta))
+	if err != nil {
+		return inode.Inode{}, err
+	}
+	return inode.Read(&rdr, r.Superblock.BlockSize)
+}
+
+func (r Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) {
+	rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d)
+	defer rdr.Close()
+	rdr.Read(make([]byte, e.Offset))
+	return inode.Read(&rdr, r.Superblock.BlockSize)
+}

low/inode/dir.go

@@ -0,0 +1,78 @@
+package inode
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+type Directory struct {
+	BlockStart uint32
+	LinkCount  uint32
+	Size       uint16
+	Offset     uint16
+	ParentNum  uint32
+}
+
+type EDirectory struct {
+	LinkCount  uint32
+	Size       uint32
+	BlockStart uint32
+	ParentNum  uint32
+	IndCount   uint16
+	Offset     uint16
+	XattrInd   uint32
+	Indexes    []DirectoryIndex
+}
+
+type DirectoryIndex struct {
+	Ind      uint32
+	Start    uint32
+	NameSize uint32
+	Name     []byte
+}
+
+func ReadDir(r io.Reader) (d Directory, err error) {
+	dat := make([]byte, 16)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	d.BlockStart = binary.LittleEndian.Uint32(dat)
+	d.LinkCount = binary.LittleEndian.Uint32(dat[4:])
+	d.Size = binary.LittleEndian.Uint16(dat[8:])
+	d.Offset = binary.LittleEndian.Uint16(dat[10:])
+	d.ParentNum = binary.LittleEndian.Uint32(dat[12:])
+	return
+}
+
+func ReadEDir(r io.Reader) (d EDirectory, err error) {
+	dat := make([]byte, 24)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	d.LinkCount = binary.LittleEndian.Uint32(dat)
+	d.Size = binary.LittleEndian.Uint32(dat[4:])
+	d.BlockStart = binary.LittleEndian.Uint32(dat[8:])
+	d.ParentNum = binary.LittleEndian.Uint32(dat[12:])
+	d.IndCount = binary.LittleEndian.Uint16(dat[16:])
+	d.Offset = binary.LittleEndian.Uint16(dat[18:])
+	d.XattrInd = binary.LittleEndian.Uint32(dat[20:])
+	d.Indexes = make([]DirectoryIndex, d.IndCount)
+	for i := range d.IndCount {
+		dat = make([]byte, 12)
+		_, err = r.Read(dat)
+		if err != nil {
+			return
+		}
+		d.Indexes[i].Ind = binary.LittleEndian.Uint32(dat)
+		d.Indexes[i].Start = binary.LittleEndian.Uint32(dat[4:])
+		d.Indexes[i].NameSize = binary.LittleEndian.Uint32(dat[8:])
+		d.Indexes[i].Name = make([]byte, d.Indexes[i].NameSize+1)
+		_, err = r.Read(d.Indexes[i].Name)
+		if err != nil {
+			return
+		}
+	}
+	return
+}

low/inode/file.go

@@ -0,0 +1,70 @@
+package inode
+
+import (
+	"encoding/binary"
+	"io"
+	"math"
+)
+
+type File struct {
+	BlockStart uint32
+	FragInd    uint32
+	FragOffset uint32
+	Size       uint32
+	BlockSizes []uint32
+}
+
+type eFileInit struct {
+	BlockStart uint64
+	Size       uint64
+	Sparse     uint64
+	LinkCount  uint32
+	FragInd    uint32
+	FragOffset uint32
+	XattrInd   uint32
+}
+
+type EFile struct {
+	eFileInit
+	BlockSizes []uint32
+}
+
+func ReadFile(r io.Reader, blockSize uint32) (f File, err error) {
+	dat := make([]byte, 16)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	f.BlockStart = binary.LittleEndian.Uint32(dat)
+	f.FragInd = binary.LittleEndian.Uint32(dat[4:])
+	f.FragOffset = binary.LittleEndian.Uint32(dat[8:])
+	f.Size = binary.LittleEndian.Uint32(dat[12:])
+	toRead := int(math.Floor(float64(f.Size) / float64(blockSize)))
+	if f.FragInd == 0xFFFFFFFF && f.Size%blockSize > 0 {
+		toRead++
+	}
+	dat = make([]byte, toRead*4)
+	_, err = r.Read(dat)
+	if err != nil {
+		return
+	}
+	f.BlockSizes = make([]uint32, toRead)
+	for i := range toRead {
+		f.BlockSizes[i] = binary.LittleEndian.Uint32(dat[i*4:])
+	}
+	return
+}
+
+func ReadEFile(r io.Reader, blockSize uint32) (f EFile, err error) {
+	err = binary.Read(r, binary.LittleEndian, &f.eFileInit)
+	if err != nil {
+		return
+	}
+	toRead := int(math.Floor(float64(f.Size) / float64(blockSize)))
+	if f.FragInd == 0xFFFFFFFF && f.Size%uint64(blockSize) > 0 {
+		toRead++
+	}
+	f.BlockSizes = make([]uint32, toRead)
+	err = binary.Read(r, binary.LittleEndian, &f.BlockSizes)
+	return
+}

low/inode/inode.go

@@ -0,0 +1,137 @@
+package inode
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+	"io/fs"
+	"strconv"
+)
+
+const (
+	Dir = uint16(iota + 1)
+	Fil
+	Sym
+	Block
+	Char
+	Fifo
+	Sock
+	EDir
+	EFil
+	ESym
+	EBlock
+	EChar
+	EFifo
+	ESock
+)
+
+type Header struct {
+	Type    uint16
+	Perm    uint16
+	UidInd  uint16
+	GidInd  uint16
+	ModTime uint32
+	Num     uint32
+}
+
+type Inode struct {
+	Header
+	Data any
+}
+
+func Read(r io.Reader, blockSize uint32) (i Inode, err error) {
+	err = binary.Read(r, binary.LittleEndian, &i.Header)
+	if err != nil {
+		return
+	}
+	switch i.Type {
+	case Dir:
+		i.Data, err = ReadDir(r)
+	case Fil:
+		i.Data, err = ReadFile(r, blockSize)
+	case Sym:
+		i.Data, err = ReadSym(r)
+	case Block:
+		fallthrough
+	case Char:
+		i.Data, err = ReadDevice(r)
+	case Fifo:
+		fallthrough
+	case Sock:
+		i.Data, err = ReadIPC(r)
+	case EDir:
+		i.Data, err = ReadEDir(r)
+	case EFil:
+		i.Data, err = ReadEFile(r, blockSize)
+	case ESym:
+		i.Data, err = ReadESym(r)
+	case EBlock:
+		fallthrough
+	case EChar:
+		i.Data, err = ReadEDevice(r)
+	case EFifo:
+		fallthrough
+	case ESock:
+		i.Data, err = ReadEIPC(r)
+	default:
+		return i, errors.New("invalid inode type " + strconv.Itoa(int(i.Type)))
+	}
+	return
+}
+
+func (i Inode) Mode() (out fs.FileMode) {
+	out = fs.FileMode(i.Perm)
+	switch i.Type {
+	case Dir, EDir:
+		out |= fs.ModeDir
+	case Sym, ESym:
+		out |= fs.ModeSymlink
+	case Char, EChar, Block, EBlock:
+		out |= fs.ModeDevice
+	case Fifo, EFifo:
+		out |= fs.ModeNamedPipe
+	case Sock, ESock:
+		out |= fs.ModeSocket
+	}
+	return
+}
+
+func (i Inode) LinkCount() uint32 {
+	switch i.Data.(type) {
+	case EFile:
+		return i.Data.(EFile).LinkCount
+	case Directory:
+		return i.Data.(Directory).LinkCount
+	case EDirectory:
+		return i.Data.(EDirectory).LinkCount
+	case Device:
+		return i.Data.(Device).LinkCount
+	case EDevice:
+		return i.Data.(EDevice).LinkCount
+	case IPC:
+		return i.Data.(IPC).LinkCount
+	case EIPC:
+		return i.Data.(EIPC).LinkCount
+	case Symlink:
+		return i.Data.(Symlink).LinkCount
+	case ESymlink:
+		return i.Data.(ESymlink).LinkCount
+	default:
+		return 0
+	}
+}
+
+func (i Inode) Size() uint64 {
+	switch i.Data.(type) {
+	case File:
+		return uint64(i.Data.(File).Size)
+	case EFile:
+		return i.Data.(EFile).Size
+	// case Directory:
+	// 	return uint64(i.Data.(Directory).Size)
+	// case EDirectory:
+	// 	return uint64(i.Data.(EDirectory).Size)
+	default:
+		return 0
+	}
+}

low/inode/misc.go

@@ -0,0 +1,45 @@
+package inode
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+type Device struct {
+	LinkCount uint32
+	Dev       uint32
+}
+
+type EDevice struct {
+	Device
+	XattrInd uint32
+}
+
+func ReadDevice(r io.Reader) (d Device, err error) {
+	err = binary.Read(r, binary.LittleEndian, &d)
+	return
+}
+
+func ReadEDevice(r io.Reader) (d EDevice, err error) {
+	err = binary.Read(r, binary.LittleEndian, &d)
+	return
+}
+
+type IPC struct {
+	LinkCount uint32
+}
+
+type EIPC struct {
+	IPC
+	XattrInd uint32
+}
+
+func ReadIPC(r io.Reader) (i IPC, err error) {
+	err = binary.Read(r, binary.LittleEndian, &i)
+	return
+}
+
+func ReadEIPC(r io.Reader) (i EIPC, err error) {
+	err = binary.Read(r, binary.LittleEndian, &i)
+	return
+}

low/inode/sym.go

@@ -0,0 +1,46 @@
+package inode
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+type symlinkInit struct {
+	LinkCount  uint32
+	TargetSize uint32
+}
+
+type Symlink struct {
+	symlinkInit
+	Target []byte
+}
+
+type ESymlink struct {
+	symlinkInit
+	Target   []byte
+	XattrInd uint32
+}
+
+func ReadSym(r io.Reader) (s Symlink, err error) {
+	err = binary.Read(r, binary.LittleEndian, &s.symlinkInit)
+	if err != nil {
+		return
+	}
+	s.Target = make([]byte, s.TargetSize)
+	err = binary.Read(r, binary.LittleEndian, &s.Target)
+	return
+}
+
+func ReadESym(r io.Reader) (s ESymlink, err error) {
+	err = binary.Read(r, binary.LittleEndian, &s.symlinkInit)
+	if err != nil {
+		return
+	}
+	s.Target = make([]byte, s.TargetSize)
+	err = binary.Read(r, binary.LittleEndian, &s.Target)
+	if err != nil {
+		return
+	}
+	err = binary.Read(r, binary.LittleEndian, &s.XattrInd)
+	return
+}

low/reader.go

@@ -0,0 +1,218 @@
+package squashfslow
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+	"math"
+
+	"github.com/CalebQ42/squashfs/internal/decompress"
+	"github.com/CalebQ42/squashfs/internal/metadata"
+	"github.com/CalebQ42/squashfs/internal/toreader"
+	"github.com/CalebQ42/squashfs/low/inode"
+)
+
+// The types of compression supported by squashfs
+const (
+	ZlibCompression = uint16(iota + 1)
+	LZMACompression
+	LZOCompression
+	XZCompression
+	LZ4Compression
+	ZSTDCompression
+)
+
+var (
+	ErrorMagic         = errors.New("magic incorrect. probably not reading squashfs archive or archive is corrupted")
+	ErrorLog           = errors.New("block log is incorrect. possible corrupted archive")
+	ErrorVersion       = errors.New("squashfs version of archive is not 4.0. may be corrupted")
+	ErrorNotExportable = errors.New("archive does not have an export table")
+)
+
+type Reader struct {
+	r           io.ReaderAt
+	d           decompress.Decompressor
+	Root        Directory
+	fragTable   []fragEntry
+	idTable     []uint32
+	exportTable []uint64
+	Superblock  superblock
+}
+
+func NewReader(r io.ReaderAt) (rdr Reader, err error) {
+	rdr.r = r
+	err = binary.Read(toreader.NewReader(r, 0), binary.LittleEndian, &rdr.Superblock)
+	if err != nil {
+		return rdr, errors.Join(errors.New("failed to read superblock"), err)
+	}
+	if !rdr.Superblock.ValidMagic() {
+		return rdr, ErrorMagic
+	}
+	if !rdr.Superblock.ValidBlockLog() {
+		return rdr, ErrorLog
+	}
+	if !rdr.Superblock.ValidVersion() {
+		return rdr, ErrorVersion
+	}
+	switch rdr.Superblock.CompType {
+	case ZlibCompression:
+		rdr.d = decompress.Zlib{}
+	case LZMACompression:
+		rdr.d, err = decompress.NewLzma()
+		if err != nil {
+			return rdr, err
+		}
+	case LZOCompression:
+		rdr.d, err = decompress.NewLzo()
+		if err != nil {
+			return rdr, err
+		}
+	case XZCompression:
+		rdr.d = decompress.NewXz()
+	case LZ4Compression:
+		rdr.d = decompress.NewLz4()
+	case ZSTDCompression:
+		rdr.d = decompress.Zstd{}
+	default:
+		return rdr, errors.New("invalid compression type. possible corrupted archive")
+	}
+	rdr.Root, err = rdr.directoryFromRef(rdr.Superblock.RootInodeRef, "")
+	if err != nil {
+		return rdr, errors.Join(errors.New("failed to read root directory"), err)
+	}
+	return
+}
+
+// Get a uid/gid at the given index. Lazily populates the reader's Id table as necessary.
+func (r *Reader) Id(i uint16) (uint32, error) {
+	if len(r.idTable) > int(i) {
+		return r.idTable[i], nil
+	} else if i >= r.Superblock.IdCount {
+		return 0, errors.New("id out of bounds")
+	}
+	// Populate the id table as needed
+	var blockNum uint32
+	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
+		blockNum = uint32(math.Ceil(float64(i+1)/2048)) - 1
+	} else {
+		blockNum = 0
+	}
+	blocksRead := len(r.idTable) / 2048
+	blocksToRead := int(blockNum) - blocksRead + 1
+
+	var offset uint64
+	var idsToRead uint16
+	var idsTmp []uint32
+	var err error
+	var rdr metadata.Reader
+	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
+	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
+		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset)
+		if err != nil {
+			return 0, err
+		}
+		idsToRead = min(r.Superblock.IdCount-uint16(len(r.idTable)), 2048)
+		idsTmp = make([]uint32, idsToRead)
+		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
+		err = binary.Read(&rdr, binary.LittleEndian, &idsTmp)
+		rdr.Close()
+		if err != nil {
+			return 0, err
+		}
+		r.idTable = append(r.idTable, idsTmp...)
+	}
+	return r.idTable[i], nil
+}
+
+// Get a fragment entry at the given index. Lazily populates the reader's fragment table as necessary.
+func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
+	if len(r.fragTable) > int(i) {
+		return r.fragTable[i], nil
+	} else if i >= r.Superblock.FragCount {
+		return fragEntry{}, errors.New("fragment out of bounds")
+	}
+	// Populate the fragment table as needed
+	var blockNum uint32
+	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
+		blockNum = uint32(math.Ceil(float64(i+1)/512)) - 1
+	} else {
+		blockNum = 0
+	}
+	blocksRead := len(r.fragTable) / 512
+	blocksToRead := int(blockNum) - blocksRead + 1
+
+	var offset uint64
+	var fragsToRead uint32
+	var fragsTmp []fragEntry
+	var err error
+	var rdr metadata.Reader
+	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
+	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
+		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset)
+		if err != nil {
+			return fragEntry{}, err
+		}
+		fragsToRead = min(r.Superblock.FragCount-uint32(len(r.fragTable)), 512)
+		fragsTmp = make([]fragEntry, fragsToRead)
+		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
+		err = binary.Read(&rdr, binary.LittleEndian, &fragsTmp)
+		rdr.Close()
+		if err != nil {
+			return fragEntry{}, err
+		}
+		r.fragTable = append(r.fragTable, fragsTmp...)
+	}
+	return r.fragTable[i], nil
+}
+
+// Get an inode reference at the given index. Lazily populates the reader's export table as necessary.
+func (r *Reader) inodeRef(i uint32) (uint64, error) {
+	if !r.Superblock.Exportable() {
+		return 0, ErrorNotExportable
+	}
+	if len(r.exportTable) > int(i) {
+		return r.exportTable[i], nil
+	} else if i >= r.Superblock.InodeCount {
+		return 0, errors.New("inode out of bounds")
+	}
+	// Populate the export table as needed
+	var blockNum uint32
+	if i != 0 { // If i == 0, we go negatives causing issues with uint32s
+		blockNum = uint32(math.Ceil(float64(i+1)/1024)) - 1
+	} else {
+		blockNum = 0
+	}
+	blocksRead := len(r.exportTable) / 1024
+	blocksToRead := int(blockNum) - blocksRead + 1
+
+	var offset uint64
+	var refsToRead uint32
+	var refsTmp []uint64
+	var err error
+	var rdr metadata.Reader
+	// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
+	for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
+		err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset)
+		if err != nil {
+			return 0, err
+		}
+		refsToRead = min(r.Superblock.InodeCount-uint32(len(r.exportTable)), 1024)
+		refsTmp = make([]uint64, refsToRead)
+		rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
+		err = binary.Read(&rdr, binary.LittleEndian, &refsTmp)
+		rdr.Close()
+		if err != nil {
+			return 0, err
+		}
+		r.exportTable = append(r.exportTable, refsTmp...)
+	}
+	return r.exportTable[i], nil
+}
+
+func (r Reader) Inode(i uint32) (inode.Inode, error) {
+	ref, err := r.inodeRef(i)
+	if err != nil {
+		return inode.Inode{}, err
+	}
+	return r.InodeFromRef(ref)
+}

low/reader_test.go

@@ -0,0 +1,146 @@
+package squashfslow
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+const (
+	squashfsURL  = "https://darkstorm.tech/files/LinuxPATest.sfs"
+	squashfsName = "airootfs.sfs"
+)
+
+func preTest(dir string) (fil *os.File, err error) {
+	fil, err = os.Open(filepath.Join(dir, squashfsName))
+	if err != nil {
+		_, err = os.Open(dir)
+		if os.IsNotExist(err) {
+			err = os.Mkdir(dir, 0755)
+		}
+		if err != nil {
+			return
+		}
+		os.Remove(filepath.Join(dir, squashfsName))
+		fil, err = os.Create(filepath.Join(dir, squashfsName))
+		if err != nil {
+			return
+		}
+		var resp *http.Response
+		resp, err = http.DefaultClient.Get(squashfsURL)
+		if err != nil {
+			return
+		}
+		_, err = io.Copy(fil, resp.Body)
+		if err != nil {
+			return
+		}
+	}
+	_, err = exec.LookPath("unsquashfs")
+	if err != nil {
+		return
+	}
+	_, err = exec.LookPath("mksquashfs")
+	return
+}
+
+func TestMisc(t *testing.T) {
+	tmpDir := "../testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fil.Close()
+	rdr, err := NewReader(fil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(rdr.Superblock.FragCount)
+	t.Fatal(rdr.fragEntry(1233))
+}
+
+func TestReader(t *testing.T) {
+	tmpDir := "../testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fil.Close()
+	rdr, err := NewReader(fil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(tmpDir, "extractTest")
+	os.RemoveAll(path)
+	os.MkdirAll(path, 0777)
+	err = extractToDir(rdr, rdr.Root.FileBase, path)
+	t.Fatal(err)
+}
+
+var singleFile = "PortableApps/CPU-X/CPU-X-v4.2.0-x86_64.AppImage"
+
+func TestSingleFile(t *testing.T) {
+	tmpDir := "../testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fil.Close()
+	rdr, err := NewReader(fil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(tmpDir, "extractTest")
+	os.RemoveAll(path)
+	os.MkdirAll(path, 0777)
+	b, err := rdr.Root.Open(rdr, singleFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = extractToDir(rdr, b, path)
+	t.Fatal(err)
+}
+
+func extractToDir(rdr Reader, b FileBase, folder string) error {
+	path := filepath.Join(folder, b.Name)
+	if b.IsDir() {
+		d, err := b.ToDir(rdr)
+		if err != nil {
+			return err
+		}
+		err = os.MkdirAll(path, 0777)
+		if err != nil {
+			return err
+		}
+		var nestBast FileBase
+		for _, e := range d.Entries {
+			nestBast, err = rdr.BaseFromEntry(e)
+			if err != nil {
+				return err
+			}
+			err = extractToDir(rdr, nestBast, path)
+			if err != nil {
+				return err
+			}
+		}
+	} else if b.IsRegular() {
+		_, full, err := b.GetRegFileReaders(rdr)
+		if err != nil {
+			return err
+		}
+		fil, err := os.Create(path)
+		if err != nil {
+			return err
+		}
+		_, err = full.WriteTo(fil)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Successfully extracted file:", b.Name)
+	}
+	return nil
+}

low/superblock.go

@@ -0,0 +1,80 @@
+package squashfslow
+
+import "math"
+
+type superblock struct {
+	Magic            uint32
+	InodeCount       uint32
+	ModTime          uint32
+	BlockSize        uint32
+	FragCount        uint32
+	CompType         uint16
+	BlockLog         uint16
+	Flags            uint16
+	IdCount          uint16
+	VerMaj           uint16
+	VerMin           uint16
+	RootInodeRef     uint64
+	Size             uint64
+	IdTableStart     uint64
+	XattrTableStart  uint64
+	InodeTableStart  uint64
+	DirTableStart    uint64
+	FragTableStart   uint64
+	ExportTableStart uint64
+}
+
+func (s superblock) ValidMagic() bool {
+	return s.Magic == 0x73717368
+}
+
+func (s superblock) ValidBlockLog() bool {
+	return s.BlockLog == uint16(math.Log2(float64(s.BlockSize)))
+}
+
+func (s superblock) ValidVersion() bool {
+	return s.VerMaj == 4 && s.VerMin == 0
+}
+
+func (s superblock) UncompressedInodes() bool {
+	return s.Flags&0x1 == 0x1
+}
+
+func (s superblock) UncompressedData() bool {
+	return s.Flags&0x2 == 0x2
+}
+func (s superblock) UncompressedFragments() bool {
+	return s.Flags&0x8 == 0x8
+}
+
+func (s superblock) NoFragments() bool {
+	return s.Flags&0x10 == 0x10
+}
+
+func (s superblock) AlwaysFragment() bool {
+	return s.Flags&0x20 == 0x20
+}
+
+func (s superblock) Duplicates() bool {
+	return s.Flags&0x40 == 0x40
+}
+
+func (s superblock) Exportable() bool {
+	return s.Flags&0x80 == 0x80
+}
+
+func (s superblock) UncompressedXattrs() bool {
+	return s.Flags&0x100 == 0x100
+}
+
+func (s superblock) NoXattrs() bool {
+	return s.Flags&0x200 == 0x200
+}
+
+func (s superblock) CompressionOptions() bool {
+	return s.Flags&0x400 == 0x400
+}
+
+func (s superblock) UncompressedIDs() bool {
+	return s.Flags&0x800 == 0x800
+}

metadata.go

@@ -1,188 +0,0 @@
-package squashfs
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"io"
-)
-
-type metadata struct {
-	raw        uint16
-	size       uint16
-	compressed bool
-}
-
-//MetadataReader is a block reader for metadata. It will automatically read the next block, when it reaches the end of a block.
-type metadataReader struct {
-	s          *Reader
-	headers    []*metadata
-	data       []byte
-	offset     int64
-	readOffset int
-}
-
-//NewMetadataReader creates a new MetadataReader, beginning to read at the given offset. It will automatically cache the first block at the location.
-func (s *Reader) newMetadataReader(offset int64) (*metadataReader, error) {
-	var br metadataReader
-	br.s = s
-	br.offset = offset
-	err := br.parseMetadata()
-	if err != nil {
-		return nil, err
-	}
-	err = br.readNextDataBlock()
-	if err != nil {
-		return nil, err
-	}
-	return &br, nil
-}
-
-//NewMetadataReaderFromInodeRef creates a new MetadataReader with the offsets set by the given inode reference.
-func (s *Reader) newMetadataReaderFromInodeRef(ref uint64) (*metadataReader, error) {
-	offset, metaOffset := processInodeRef(ref)
-	br, err := s.newMetadataReader(int64(s.super.InodeTableStart + offset))
-	if err != nil {
-		return nil, err
-	}
-	_, err = br.Seek(int64(metaOffset), io.SeekStart)
-	if err != nil {
-		return nil, err
-	}
-	return br, nil
-}
-
-//ProcessInodeRef processes an inode reference and returns two values
-//
-//The first value is the inode table offset. AKA, it's where the metadata block of the inode STARTS relative to the inode table.
-//
-//The second value is the offset of the inode, INSIDE of the metadata.
-func processInodeRef(inodeRef uint64) (tableOffset uint64, metaOffset uint64) {
-	tableOffset = inodeRef >> 16
-	metaOffset = inodeRef &^ 0xFFFFFFFF0000
-	return
-}
-
-func (br *metadataReader) parseMetadata() error {
-	var raw uint16
-	err := binary.Read(io.NewSectionReader(br.s.r, br.offset, 2), binary.LittleEndian, &raw)
-	if err != nil {
-		return err
-	}
-	br.offset += 2
-	compressed := raw&0x8000 != 0x8000
-	size := raw &^ 0x8000
-	br.headers = append(br.headers, &metadata{
-		raw:        raw,
-		size:       size,
-		compressed: compressed,
-	})
-	return nil
-}
-
-func (br *metadataReader) readNextDataBlock() error {
-	meta := br.headers[len(br.headers)-1]
-	r := io.NewSectionReader(br.s.r, br.offset, int64(meta.size))
-	if meta.compressed {
-		byts, err := br.s.decompressor.Decompress(r)
-		if err != nil {
-			return err
-		}
-		br.offset += int64(meta.size)
-		br.data = append(br.data, byts...)
-		return nil
-	}
-	var buf bytes.Buffer
-	_, err := io.Copy(&buf, r)
-	if err != nil {
-		return err
-	}
-	br.offset += int64(meta.size)
-	br.data = append(br.data, buf.Bytes()...)
-	return nil
-}
-
-//Read reads bytes into the given byte slice. Returns the amount of data read.
-func (br *metadataReader) Read(p []byte) (int, error) {
-	if br.readOffset+len(p) <= len(br.data) {
-		for i := 0; i < len(p); i++ {
-			p[i] = br.data[br.readOffset+i]
-		}
-		br.readOffset += len(p)
-		return len(p), nil
-	}
-	read := 0
-	for read < len(p) {
-		err := br.parseMetadata()
-		if err != nil {
-			br.readOffset += read
-			return read, err
-		}
-		err = br.readNextDataBlock()
-		if err != nil {
-			br.readOffset += read
-			return read, err
-		}
-		for ; read < len(p); read++ {
-			if br.readOffset+read < len(br.data) {
-				p[read] = br.data[br.readOffset+read]
-			} else {
-				break
-			}
-		}
-	}
-	br.readOffset += read
-	if read != len(p) {
-		return read, errors.New("didn't read enough data")
-	}
-	return read, nil
-}
-
-//Seek will seek to the specified location (if possible). Seeking is relative to the uncompressed data.
-//When io.SeekCurrent or io.SeekStart is set, if seeking would put the offset beyond the current cached data, it will try to read the next data blocks to accomodate. On a failure it will seek to the end of the data.
-//When io.SeekEnd is set, it wil seek reletive to the currently cached data.
-func (br *metadataReader) Seek(offset int64, whence int) (int64, error) {
-	switch whence {
-	case io.SeekCurrent:
-		br.readOffset += int(offset)
-		for {
-			if br.readOffset < len(br.data) {
-				break
-			}
-			err := br.parseMetadata()
-			if err != nil {
-				br.readOffset = len(br.data)
-				return int64(br.readOffset), err
-			}
-			err = br.readNextDataBlock()
-			if err != nil {
-				br.readOffset = len(br.data)
-				return int64(br.readOffset), err
-			}
-		}
-	case io.SeekStart:
-		br.readOffset = int(offset)
-		for {
-			if br.readOffset < len(br.data) {
-				break
-			}
-			err := br.parseMetadata()
-			if err != nil {
-				br.readOffset = len(br.data)
-				return int64(br.readOffset), err
-			}
-			err = br.readNextDataBlock()
-			if err != nil {
-				br.readOffset = len(br.data)
-				return int64(br.readOffset), err
-			}
-		}
-	case io.SeekEnd:
-		br.readOffset = len(br.data) - int(offset)
-		if br.readOffset < 0 {
-			br.readOffset = 0
-			return int64(br.readOffset), errors.New("trying to seek to a negative value")
-		}
-	}
-	return int64(br.readOffset), nil
-}

reader.go

@@ -1,203 +1,37 @@
 package squashfs
 
 import (
-	"encoding/binary"
-	"errors"
 	"io"
-	"math"
 	"time"
 
-	"github.com/CalebQ42/squashfs/internal/compression"
-	"github.com/CalebQ42/squashfs/internal/inode"
-	"github.com/CalebQ42/squashfs/internal/rawreader"
+	"github.com/CalebQ42/squashfs/internal/toreader"
+	squashfslow "github.com/CalebQ42/squashfs/low"
 )
 
-const (
-	magic uint32 = 0x73717368
-)
-
-var (
-	//ErrNoMagic is returned if the magic number in the superblock isn't correct.
-	errNoMagic = errors.New("magic number doesn't match. Either isn't a squashfs or corrupted")
-	//ErrIncompatibleCompression is returned if the compression type in the superblock doesn't work.
-	errIncompatibleCompression = errors.New("compression type unsupported")
-)
-
-//Reader processes and reads a squashfs archive.
 type Reader struct {
 	FS
-	r            rawreader.RawReader
-	decompressor compression.Decompressor
-	fragOffsets  []uint64
-	idTable      []uint32
-	super        superblock
-	flags        SuperblockFlags
+	Low squashfslow.Reader
 }
 
-//NewSquashfsReader returns a new squashfs.Reader from an io.ReaderAt
-func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
-	var rdr Reader
-	rdr.r = rawreader.ConvertReaderAt(r)
-	err := rdr.Init()
+func NewReader(r io.ReaderAt) (Reader, error) {
+	rdr, err := squashfslow.NewReader(r)
 	if err != nil {
-		return nil, err
+		return Reader{}, err
 	}
-	return &rdr, nil
+	out := Reader{
+		Low: rdr,
+	}
+	out.FS = FS{
+		d: rdr.Root,
+		r: &out,
+	}
+	return out, nil
 }
 
-//NewSquashfsReaderFromReader returns a new squashfs.Reader from an io.Reader.
-//If the io.Reader implements io.Seeker, the seek functions are used.
-//It is NOT recommended to use a pure io.Reader as due to how squashfs
-//archives are formatted, the ENTIRETY of the io.Reader's data is loaded into
-//memory first before it can be used.
-func NewSquashfsReaderFromReader(r io.Reader) (*Reader, error) {
-	var rdr Reader
-	var err error
-	rdr.r, err = rawreader.ConvertReader(r)
-	if err != nil {
-		return nil, err
-	}
-	err = rdr.Init()
-	if err != nil {
-		return nil, err
-	}
-	return &rdr, nil
+func NewReaderAtOffset(r io.ReaderAt, offset int64) (Reader, error) {
+	return NewReader(toreader.NewOffsetReader(r, offset))
 }
 
-func (r *Reader) Init() error {
-	err := binary.Read(r.r, binary.LittleEndian, &r.super)
-	if err != nil {
-		return err
-	}
-	if r.super.Magic != magic {
-		return errNoMagic
-	}
-	if r.super.BlockLog != uint16(math.Log2(float64(r.super.BlockSize))) {
-		return errors.New("BlockSize and BlockLog doesn't match. The archive is probably corrupt")
-	}
-	r.r.Seek(96, io.SeekStart)
-	r.flags = r.super.GetFlags()
-	if r.flags.compressorOptions {
-		switch r.super.CompressionType {
-		case GzipCompression:
-			var gzip *compression.Gzip
-			gzip, err = compression.NewGzipCompressorWithOptions(r.r)
-			if err != nil {
-				return err
-			}
-			r.decompressor = gzip
-		case XzCompression:
-			var xz *compression.Xz
-			xz, err = compression.NewXzCompressorWithOptions(r.r)
-			if err != nil {
-				return err
-			}
-			r.decompressor = xz
-		case LzoCompression:
-			var lz *compression.Lzo
-			lz, err = compression.NewLzoCompressorWithOptions(r.r)
-			if err != nil {
-				return err
-			}
-			r.decompressor = lz
-		case Lz4Compression:
-			var lz4 *compression.Lz4
-			lz4, err = compression.NewLz4CompressorWithOptions(r.r)
-			if err != nil {
-				return err
-			}
-			r.decompressor = lz4
-		case ZstdCompression:
-			var zstd *compression.Zstd
-			zstd, err = compression.NewZstdCompressorWithOptions(r.r)
-			if err != nil {
-				return err
-			}
-			r.decompressor = zstd
-		default:
-			return errIncompatibleCompression
-		}
-	} else {
-		switch r.super.CompressionType {
-		case GzipCompression:
-			r.decompressor = &compression.Gzip{}
-		case LzmaCompression:
-			r.decompressor = &compression.Lzma{}
-		case LzoCompression:
-			r.decompressor = &compression.Lzo{}
-		case XzCompression:
-			r.decompressor = &compression.Xz{}
-		case Lz4Compression:
-			r.decompressor = &compression.Lz4{}
-		case ZstdCompression:
-			r.decompressor = &compression.Zstd{}
-		default:
-			//TODO: all compression types.
-			return errIncompatibleCompression
-		}
-	}
-	fragBlocks := int(math.Ceil(float64(r.super.FragCount) / 512))
-	if fragBlocks > 0 {
-		offset := int64(r.super.FragTableStart)
-		for i := 0; i < fragBlocks; i++ {
-			tmp := make([]byte, 8)
-			_, err = r.r.ReadAt(tmp, offset)
-			if err != nil {
-				return err
-			}
-			r.fragOffsets = append(r.fragOffsets, binary.LittleEndian.Uint64(tmp))
-			offset += 8
-		}
-	}
-	unread := r.super.IDCount
-	blockOffsets := make([]uint64, int(math.Ceil(float64(r.super.IDCount)/2048)))
-	_, err = r.r.Seek(int64(r.super.IDTableStart), io.SeekStart)
-	if err != nil {
-		return err
-	}
-	for i := range blockOffsets {
-		err = binary.Read(r.r, binary.LittleEndian, &blockOffsets[i])
-		if err != nil {
-			return err
-		}
-		var idRdr *metadataReader
-		idRdr, err = r.newMetadataReader(int64(blockOffsets[i]))
-		if err != nil {
-			return err
-		}
-		read := uint16(math.Min(float64(unread), 2048))
-		for i := uint16(0); i < read; i++ {
-			var tmp uint32
-			err = binary.Read(idRdr, binary.LittleEndian, &tmp)
-			if err != nil {
-				return err
-			}
-			r.idTable = append(r.idTable, tmp)
-		}
-		unread -= read
-	}
-	metaRdr, err := r.newMetadataReaderFromInodeRef(r.super.RootInodeRef)
-	if err != nil {
-		return err
-	}
-	i, err := inode.ProcessInode(metaRdr, r.super.BlockSize)
-	if err != nil {
-		return err
-	}
-	entries, err := r.readDirFromInode(i)
-	if err != nil {
-		return err
-	}
-	r.FS = FS{
-		i:       i,
-		r:       r,
-		name:    "/",
-		entries: entries,
-	}
-	return nil
-}
-
-//ModTime is the last time the file was modified/created.
 func (r *Reader) ModTime() time.Time {
-	return time.Unix(int64(r.super.CreationTime), 0)
+	return time.Unix(int64(r.Low.Superblock.ModTime), 0)
 }

reader_datareader.go

@@ -1,242 +0,0 @@
-package squashfs
-
-import (
-	"bytes"
-	"errors"
-	"io"
-
-	"github.com/CalebQ42/squashfs/internal/inode"
-)
-
-var (
-	//ErrInodeNotFile is given when giving an inode, but the function requires a file inode.
-	errInodeNotFile = errors.New("given inode is NOT a file type")
-	//ErrInodeOnlyFragment is given when trying to make a DataReader from an inode, but the inode only had data in a fragment
-	errInodeOnlyFragment = errors.New("given inode ONLY has fragment data")
-)
-
-//DataReader reads data from data blocks.
-type dataReader struct {
-	r             *Reader
-	curData       []byte
-	sizes         []uint32
-	offset        int64 //offset relative to the beginning of the squash file
-	curBlock      int   //Which block in sizes is currently cached
-	curReadOffset int   //offset relative to the currently cached data
-}
-
-//NewDataReader creates a new data reader at the given offset, with the blocks defined by sizes
-func (r *Reader) newDataReader(offset int64, sizes []uint32) (*dataReader, error) {
-	var dr dataReader
-	dr.r = r
-	dr.offset = offset
-	dr.sizes = sizes
-	err := dr.readCurBlock()
-	if err != nil {
-		return nil, err
-	}
-	return &dr, nil
-}
-
-//NewDataReaderFromInode creates a new DataReader from a given inode. Inode must be of BasicFile or ExtendedFile types
-func (r *Reader) newDataReaderFromInode(i *inode.Inode) (*dataReader, error) {
-	var rdr dataReader
-	rdr.r = r
-	switch i.Type {
-	case inode.FileType:
-		fil := i.Info.(inode.File)
-		if fil.BlockStart == 0 {
-			return nil, errInodeOnlyFragment
-		}
-		rdr.offset = int64(fil.BlockStart)
-		rdr.sizes = append(rdr.sizes, fil.BlockSizes...)
-		if fil.Fragmented {
-			rdr.sizes = rdr.sizes[:len(rdr.sizes)-1]
-		}
-	case inode.ExtFileType:
-		fil := i.Info.(inode.ExtFile)
-		if fil.BlockStart == 0 {
-			return nil, errInodeOnlyFragment
-		}
-		rdr.offset = int64(fil.BlockStart)
-		rdr.sizes = append(rdr.sizes, fil.BlockSizes...)
-		if fil.Fragmented {
-			rdr.sizes = rdr.sizes[:len(rdr.sizes)-1]
-		}
-	default:
-		return nil, errInodeNotFile
-	}
-	err := rdr.readCurBlock()
-	if err != nil {
-		return nil, err
-	}
-	return &rdr, nil
-}
-
-//removed the compression bit from a data block size
-func actualDataSize(size uint32) uint32 {
-	return size &^ (1 << 24)
-}
-
-func (d *dataReader) readNextBlock() error {
-	d.curBlock++
-	if d.curBlock >= len(d.sizes) {
-		d.curBlock--
-		return io.EOF
-	}
-	err := d.readCurBlock()
-	if err != nil {
-		d.curBlock--
-		d.readCurBlock()
-		return err
-	}
-	return nil
-}
-
-func (d *dataReader) readBlockAt(offset int64, size uint32) ([]byte, error) {
-	compressed := size&(1<<24) != (1 << 24)
-	size = size &^ (1 << 24)
-	if d.sizes[d.curBlock] == 0 {
-		return make([]byte, d.r.super.BlockSize), nil
-	}
-	sec := io.NewSectionReader(d.r.r, offset, int64(size))
-	if compressed {
-		btys, err := d.r.decompressor.Decompress(sec)
-		if err != nil {
-			return nil, err
-		}
-		return btys, nil
-	}
-	var buf bytes.Buffer
-	_, err := io.Copy(&buf, sec)
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-func (d *dataReader) offsetForBlock(index int) int64 {
-	out := d.offset
-	for i := 0; i < index; i++ {
-		out += int64(actualDataSize(d.sizes[i]))
-	}
-	return out
-}
-
-func (d *dataReader) readCurBlock() error {
-	if d.curBlock >= len(d.sizes) {
-		return io.EOF
-	}
-	offset := d.offsetForBlock(d.curBlock)
-	data, err := d.readBlockAt(offset, d.sizes[d.curBlock])
-	if err != nil {
-		return err
-	}
-	d.curData = data
-	return nil
-}
-
-func (d *dataReader) Read(p []byte) (int, error) {
-	if d.curData == nil {
-		err := d.readCurBlock()
-		if err != nil {
-			return 0, err
-		}
-	}
-	if d.curReadOffset+len(p) <= len(d.curData) {
-		for i := 0; i < len(p); i++ {
-			p[i] = d.curData[d.curReadOffset+i]
-		}
-		d.curReadOffset += len(p)
-		return len(p), nil
-	}
-	read := 0
-	for read < len(p) {
-		if d.curReadOffset == len(d.curData) {
-			err := d.readNextBlock()
-			if err != nil {
-				return read, err
-			}
-			d.curReadOffset = 0
-		}
-		for ; read < len(p); read++ {
-			if d.curReadOffset < len(d.curData) {
-				p[read] = d.curData[d.curReadOffset]
-			} else {
-				break
-			}
-			d.curReadOffset++
-		}
-	}
-	if read != len(p) {
-		return read, errors.New("didn't read enough data")
-	}
-	return read, nil
-}
-
-// WriteTo writes all the data in the datablock to the writer. MUST BE USED ON A FRESH DATA READER.
-func (d *dataReader) WriteTo(w io.Writer) (int64, error) {
-	type dataCache struct {
-		err   error
-		data  []byte
-		index int
-	}
-	dataChan := make(chan *dataCache)
-	for i := range d.sizes {
-		go func(index int, c chan *dataCache) {
-			var cache dataCache
-			cache.index = index
-			defer func() {
-				c <- &cache
-			}()
-			data, err := d.readBlockAt(d.offsetForBlock(index), d.sizes[index])
-			if err != nil {
-				cache.err = err
-				return
-			}
-			cache.data = data
-		}(i, dataChan)
-	}
-	curIndex := 0
-	totalWrite := int64(0)
-	var backlog []*dataCache
-mainLoop:
-	for {
-		if curIndex == len(d.sizes) {
-			return totalWrite, nil
-		}
-		if len(backlog) > 0 {
-			for i, cache := range backlog {
-				if cache.index == curIndex {
-					writen, err := w.Write(cache.data)
-					totalWrite += int64(writen)
-					if err != nil {
-						return totalWrite, err
-					}
-					if len(backlog) > 0 {
-						backlog[i] = backlog[len(backlog)-1]
-						backlog = backlog[:len(backlog)-1]
-					} else {
-						backlog = nil
-					}
-					curIndex++
-					continue mainLoop
-				}
-			}
-		}
-		cache := <-dataChan
-		if cache.err != nil {
-			return totalWrite, cache.err
-		}
-		if cache.index == curIndex {
-			writen, err := w.Write(cache.data)
-			totalWrite += int64(writen)
-			if err != nil {
-				return totalWrite, err
-			}
-			curIndex++
-		} else {
-			backlog = append(backlog, cache)
-		}
-	}
-}

reader_file.go

@@ -1,382 +0,0 @@
-package squashfs
-
-import (
-	"errors"
-	"io"
-	"io/fs"
-	"log"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-
-	"github.com/CalebQ42/squashfs/internal/directory"
-	"github.com/CalebQ42/squashfs/internal/inode"
-)
-
-//File represents a file inside a squashfs archive.
-type File struct {
-	i        *inode.Inode
-	parent   *FS
-	r        *Reader
-	reader   *fileReader
-	name     string
-	dirsRead int
-}
-
-//File creates a File from the FileInfo.
-//*File satisfies fs.File and fs.ReadDirFile.
-func (f FileInfo) File() (file *File, err error) {
-	file = &File{
-		name:   f.name,
-		r:      f.r,
-		parent: f.parent,
-		i:      f.i,
-	}
-	if file.IsRegular() {
-		file.reader, err = f.r.newFileReader(f.i)
-	}
-	return
-}
-
-//File creates a File from the DirEntry.
-func (d DirEntry) File() (file *File, err error) {
-	return d.r.newFileFromDirEntry(d.en, d.parent)
-}
-
-func (r Reader) newFileFromDirEntry(en *directory.Entry, parent *FS) (file *File, err error) {
-	file = &File{
-		name:   en.Name,
-		r:      &r,
-		parent: parent,
-	}
-	file.i, err = r.getInodeFromEntry(en)
-	if err != nil {
-		return nil, err
-	}
-	if file.IsRegular() {
-		file.reader, err = r.newFileReader(file.i)
-	}
-	return
-}
-
-//Stat returns the File's fs.FileInfo
-func (f File) Stat() (fs.FileInfo, error) {
-	return &FileInfo{
-		i:      f.i,
-		name:   f.name,
-		parent: f.parent,
-		r:      f.r,
-	}, nil
-}
-
-//Read reads the data from the file. Only works if file is a normal file.
-func (f File) Read(p []byte) (int, error) {
-	if f.i.Type == inode.FileType || f.i.Type == inode.ExtFileType {
-		if f.reader == nil {
-			return 0, fs.ErrClosed
-		}
-		return f.reader.Read(p)
-	}
-	return 0, errors.New("can only read files")
-}
-
-//WriteTo writes all data from the file to the writer. This is multi-threaded.
-func (f File) WriteTo(w io.Writer) (int64, error) {
-	if f.i.Type == inode.FileType || f.i.Type == inode.ExtFileType {
-		if f.reader == nil {
-			return 0, fs.ErrClosed
-		}
-		return f.reader.WriteTo(w)
-	}
-	return 0, errors.New("can only read files")
-}
-
-//Close simply nils the underlying reader. Here mostly to satisfy fs.File
-func (f *File) Close() error {
-	f.reader = nil
-	return nil
-}
-
-//ReadDir returns n fs.DirEntry's that's contained in the File (if it's a directory).
-//If n <= 0 all fs.DirEntry's are returned.
-func (f File) ReadDir(n int) ([]fs.DirEntry, error) {
-	if !f.IsDir() {
-		return nil, errors.New("File is not a directory")
-	}
-	ffs, err := f.FS()
-	if err != nil {
-		return nil, err
-	}
-	var beg, end int
-	if n <= 0 {
-		beg, end = 0, len(ffs.entries)
-	} else {
-		beg, end = f.dirsRead, f.dirsRead+n
-		if end > len(ffs.entries) {
-			end = len(ffs.entries)
-			err = io.EOF
-		}
-	}
-	out := make([]fs.DirEntry, end-beg)
-	for i, ent := range ffs.entries[beg:end] {
-		out[i] = f.r.newDirEntry(ent, ffs)
-	}
-	return out, err
-}
-
-//FS returns the File as a FS.
-func (f File) FS() (*FS, error) {
-	if !f.IsDir() {
-		return nil, errors.New("File is not a directory")
-	}
-	ents, err := f.r.readDirFromInode(f.i)
-	if err != nil {
-		return nil, err
-	}
-	return &FS{
-		i:       f.i,
-		r:       f.r,
-		parent:  f.parent,
-		name:    f.name,
-		entries: ents,
-	}, nil
-}
-
-//IsDir Yep.
-func (f File) IsDir() bool {
-	return f.i.Type == inode.DirType || f.i.Type == inode.ExtDirType
-}
-
-func (f File) path() string {
-	if f.name == "/" {
-		return f.name
-	}
-	return f.parent.path() + "/" + f.name
-}
-
-//IsRegular yep.
-func (f File) IsRegular() bool {
-	return f.i.Type == inode.FileType || f.i.Type == inode.ExtFileType
-}
-
-//IsSymlink yep.
-func (f File) IsSymlink() bool {
-	return f.i.Type == inode.SymType || f.i.Type == inode.ExtSymType
-}
-
-//SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string.
-func (f File) SymlinkPath() string {
-	switch f.i.Type {
-	case inode.SymType:
-		return f.i.Info.(inode.Sym).Path
-	case inode.ExtSymType:
-		return f.i.Info.(inode.ExtSym).Path
-	}
-	return ""
-}
-
-//GetSymlinkFile returns the File the symlink is pointing to.
-//If not a symlink, or the target is unobtainable (such as it being outside the archive or it's absolute) returns nil
-func (f File) GetSymlinkFile() *File {
-	if !f.IsSymlink() {
-		return nil
-	}
-	if strings.HasPrefix(f.SymlinkPath(), "/") {
-		return nil
-	}
-	sym, err := f.parent.Open(f.SymlinkPath())
-	if err != nil {
-		return nil
-	}
-	return sym.(*File)
-}
-
-//ExtractionOptions are available options on how to extract.
-type ExtractionOptions struct {
-	notBase            bool
-	DereferenceSymlink bool        //Replace symlinks with the target file
-	UnbreakSymlink     bool        //Try to make sure symlinks remain unbroken when extracted, without changing the symlink
-	Verbose            bool        //Prints extra info to log on an error
-	FolderPerm         fs.FileMode //The permissions used when creating the extraction folder
-}
-
-//DefaultOptions is the default ExtractionOptions.
-func DefaultOptions() ExtractionOptions {
-	return ExtractionOptions{
-		DereferenceSymlink: false,
-		UnbreakSymlink:     false,
-		Verbose:            false,
-		FolderPerm:         fs.ModePerm,
-	}
-}
-
-//ExtractTo extracts the File to the given folder with the default options.
-//If the File is a directory, it instead extracts the directory's contents to the folder.
-func (f File) ExtractTo(folder string) error {
-	return f.ExtractWithOptions(folder, DefaultOptions())
-}
-
-//ExtractSymlink extracts the File to the folder with the DereferenceSymlink option.
-//If the File is a directory, it instead extracts the directory's contents to the folder.
-func (f File) ExtractSymlink(folder string) error {
-	return f.ExtractWithOptions(folder, ExtractionOptions{
-		DereferenceSymlink: true,
-		FolderPerm:         fs.ModePerm,
-	})
-}
-
-//ExtractWithOptions extracts the File to the given folder with the given ExtrationOptions.
-//If the File is a directory, it instead extracts the directory's contents to the folder.
-func (f File) ExtractWithOptions(folder string, op ExtractionOptions) error {
-	folder = path.Clean(folder)
-	if !op.notBase {
-		err := os.MkdirAll(folder, op.FolderPerm)
-		if err != nil {
-			return err
-		}
-	}
-	stat, err := f.Stat()
-	if err != nil {
-		return err
-	}
-	if f.IsDir() {
-		if op.notBase {
-			err = os.Mkdir(folder+"/"+f.name, stat.Mode())
-			if err != nil && !os.IsExist(err) {
-				return err
-			}
-		} else {
-			op.notBase = true
-		}
-		var ents []fs.DirEntry
-		ents, err = f.ReadDir(0)
-		if err != nil {
-			if op.Verbose {
-				log.Println("Error while reading children of", f.path())
-			}
-			return err
-		}
-		errChan := make(chan error)
-		for i := 0; i < len(ents); i++ {
-			go func(ent *DirEntry) {
-				fil, goErr := ent.File()
-				if goErr != nil {
-					errChan <- goErr
-					fil.Close()
-					return
-				}
-				errChan <- fil.ExtractWithOptions(folder+"/"+f.name, op)
-				fil.Close()
-			}(ents[i].(*DirEntry))
-		}
-		for i := 0; i < len(ents); i++ {
-			err = <-errChan
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	} else if f.IsRegular() {
-		var fil *os.File
-		fil, err = os.Create(folder + "/" + f.name)
-		if os.IsExist(err) {
-			os.Remove(folder + "/" + f.name)
-			fil, err = os.Create(folder + "/" + f.name)
-			if err != nil {
-				log.Println("Error while creating", folder+"/"+f.name)
-				return err
-			}
-		} else if err != nil {
-			return err
-		}
-		_, err = io.Copy(fil, f)
-		if err != nil {
-			log.Println("Error while copying data to", folder+"/"+f.name)
-			return err
-		}
-		return nil
-	} else if f.IsSymlink() {
-		symPath := f.SymlinkPath()
-		if op.DereferenceSymlink {
-			fil := f.GetSymlinkFile()
-			if fil == nil {
-				if op.Verbose {
-					log.Println("Symlink path(", symPath, ") is unobtainable:", folder+"/"+f.name)
-				}
-				return errors.New("cannot get symlink target")
-			}
-			fil.name = f.name
-			err = fil.ExtractWithOptions(folder, op)
-			if err != nil {
-				if op.Verbose {
-					log.Println("Error while extracting the symlink's file:", folder+"/"+f.name)
-				}
-				return err
-			}
-			return nil
-		} else if op.UnbreakSymlink {
-			fil := f.GetSymlinkFile()
-			if fil == nil {
-				if op.Verbose {
-					log.Println("Symlink path(", symPath, ") is unobtainable:", folder+"/"+f.name)
-				}
-				return errors.New("cannot get symlink target")
-			}
-			extractLoc := path.Clean(folder + "/" + path.Dir(symPath))
-			err = fil.ExtractWithOptions(extractLoc, op)
-			if err != nil {
-				if op.Verbose {
-					log.Println("Error while extracting ", folder+"/"+f.name)
-				}
-				return err
-			}
-		}
-		err = os.Symlink(f.SymlinkPath(), folder+"/"+f.name)
-		if os.IsExist(err) {
-			os.Remove(folder + "/" + f.name)
-			err = os.Symlink(f.SymlinkPath(), folder+"/"+f.name)
-		}
-		if err != nil {
-			if op.Verbose {
-				log.Println("Error while making symlink:", folder+"/"+f.name)
-			}
-			return err
-		}
-		return nil
-	}
-	return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.i.Type)))
-}
-
-//ReadDirFromInode returns a fully populated Directory from a given Inode.
-//If the given inode is not a directory it returns an error.
-func (r *Reader) readDirFromInode(i *inode.Inode) ([]*directory.Entry, error) {
-	var offset uint32
-	var metaOffset uint16
-	var size uint32
-	switch i.Type {
-	case inode.DirType:
-		offset = i.Info.(inode.Dir).DirectoryIndex
-		metaOffset = i.Info.(inode.Dir).DirectoryOffset
-		size = uint32(i.Info.(inode.Dir).DirectorySize)
-	case inode.ExtDirType:
-		offset = i.Info.(inode.ExtDir).DirectoryIndex
-		metaOffset = i.Info.(inode.ExtDir).DirectoryOffset
-		size = i.Info.(inode.ExtDir).DirectorySize
-	default:
-		return nil, errors.New("not a directory inode")
-	}
-	br, err := r.newMetadataReader(int64(r.super.DirTableStart + uint64(offset)))
-	if err != nil {
-		return nil, err
-	}
-	_, err = br.Seek(int64(metaOffset), io.SeekStart)
-	if err != nil {
-		return nil, err
-	}
-	ents, err := directory.NewDirectory(br, size)
-	if err != nil {
-		return nil, err
-	}
-	return ents, nil
-}

reader_filereader.go

@@ -1,107 +0,0 @@
-package squashfs
-
-import (
-	"bytes"
-	"errors"
-	"io"
-
-	"github.com/CalebQ42/squashfs/internal/inode"
-)
-
-//FileReader provides a io.Reader interface for files within a squashfs archive
-type fileReader struct {
-	r            *Reader
-	data         *dataReader
-	in           *inode.Inode
-	fragmentData []byte
-	fragged      bool
-	fragOnly     bool
-	read         int
-	FileSize     int //FileSize is the total size of the given file
-}
-
-var (
-	//ErrPathIsNotFile returns when trying to read from a file, but the given path is NOT a file.
-	errPathIsNotFile = errors.New("the given path is not a file")
-)
-
-//ReadFile provides a squashfs.FileReader for the file at the given location.
-func (r *Reader) newFileReader(in *inode.Inode) (*fileReader, error) {
-	var rdr fileReader
-	rdr.in = in
-	if in.Type != inode.FileType && in.Type != inode.ExtFileType {
-		return nil, errPathIsNotFile
-	}
-	switch in.Type {
-	case inode.FileType:
-		fil := in.Info.(inode.File)
-		rdr.fragged = fil.Fragmented
-		rdr.fragOnly = fil.BlockStart == 0
-		rdr.FileSize = int(fil.Size)
-	case inode.ExtFileType:
-		fil := in.Info.(inode.ExtFile)
-		rdr.fragged = fil.Fragmented
-		rdr.fragOnly = fil.BlockStart == 0
-		rdr.FileSize = int(fil.Size)
-	}
-	var err error
-	if rdr.fragged {
-		rdr.fragmentData, err = r.getFragmentDataFromInode(in)
-		if err != nil {
-			return nil, err
-		}
-	}
-	if !rdr.fragOnly {
-		rdr.data, err = r.newDataReaderFromInode(in)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return &rdr, nil
-}
-
-func (f *fileReader) Read(p []byte) (int, error) {
-	if f.fragOnly {
-		n, err := bytes.NewBuffer(f.fragmentData[f.read:]).Read(p)
-		f.read += n
-		if err != nil {
-			return n, err
-		}
-		return n, nil
-	}
-	var read int
-	n, err := f.data.Read(p)
-	read += n
-	if f.fragged && err == io.EOF {
-		if f.fragmentData == nil {
-			f.fragmentData, err = f.r.getFragmentDataFromInode(f.in)
-			if err != nil {
-				return read, err
-			}
-		}
-		n, err = bytes.NewBuffer(f.fragmentData).Read(p[read:])
-		read += n
-		if err != nil {
-			return read, err
-		}
-	} else if err != nil {
-		return read, err
-	}
-	return read, nil
-}
-
-func (f *fileReader) WriteTo(w io.Writer) (int64, error) {
-	if f.fragOnly {
-		n, err := w.Write(f.fragmentData)
-		return int64(n), err
-	}
-	if !f.fragged {
-		return f.data.WriteTo(w)
-	}
-	n, err := f.data.WriteTo(w)
-	if err != nil {
-		return int64(n), err
-	}
-	nn, err := w.Write(f.fragmentData)
-	return int64(nn) + n, err
-}

reader_fs.go

@@ -1,510 +0,0 @@
-package squashfs
-
-import (
-	"bytes"
-	"errors"
-	"io"
-	"io/fs"
-	"os"
-	"path"
-	"strings"
-
-	"github.com/CalebQ42/squashfs/internal/directory"
-	"github.com/CalebQ42/squashfs/internal/inode"
-)
-
-//FS is a fs.FS representation of a squashfs directory.
-//Implements fs.GlobFS, fs.ReadDirFS, fs.ReadFileFS, fs.StatFS, and fs.SubFS
-type FS struct {
-	i       *inode.Inode
-	r       *Reader
-	parent  *FS
-	name    string
-	entries []*directory.Entry
-}
-
-//Open opens the file at name. Returns a squashfs.File.
-func (f FS) Open(name string) (fs.File, error) {
-	if !fs.ValidPath(name) {
-		return nil, &fs.PathError{
-			Op:   "open",
-			Path: name,
-			Err:  fs.ErrInvalid,
-		}
-	}
-	name = path.Clean(strings.TrimPrefix(name, "/"))
-	split := strings.Split(name, "/")
-	if split[0] == ".." {
-		if f.parent == nil {
-			//This should only happen on the root FS
-			return nil, &fs.PathError{
-				Op:   "open",
-				Path: name,
-				//TODO: make error clearer
-				Err: errors.New("trying to get file outside of squashfs"),
-			}
-		}
-		return f.parent.Open(strings.Join(split[1:], "/"))
-	}
-	if split[0] == "." {
-		return &File{i: f.i, r: f.r, parent: f.parent, name: f.name}, nil
-	}
-	for i := 0; i < len(f.entries); i++ {
-		if split[0] == f.entries[i].Name {
-			if len(split) == 1 {
-				return f.r.newFileFromDirEntry(f.entries[i], &f)
-			}
-			sub, err := f.Sub(split[0])
-			if err != nil {
-				if pathErr, ok := err.(*fs.PathError); ok {
-					pathErr.Op = "open"
-					pathErr.Path = name
-					return nil, err
-				}
-				return nil, &fs.PathError{
-					Op:   "open",
-					Path: name,
-					Err:  err,
-				}
-			}
-			fil, err := sub.Open(strings.Join(split[1:], "/"))
-			if err != nil {
-				if pathErr, ok := err.(*fs.PathError); ok {
-					if pathErr.Err == fs.ErrNotExist {
-						continue
-					}
-					pathErr.Op = "open"
-					pathErr.Path = name
-					return nil, err
-				}
-				return nil, &fs.PathError{
-					Op:   "open",
-					Path: name,
-					Err:  err,
-				}
-			}
-			return fil, nil
-		}
-	}
-	return nil, &fs.PathError{
-		Op:   "open",
-		Path: name,
-		Err:  fs.ErrNotExist,
-	}
-}
-
-//Glob returns the name of the files at the given pattern.
-//All paths are relative to the FS.
-func (f FS) Glob(pattern string) (out []string, err error) {
-	if !fs.ValidPath(pattern) {
-		return nil, &fs.PathError{
-			Op:   "glob",
-			Path: pattern,
-			Err:  fs.ErrInvalid,
-		}
-	}
-	pattern = path.Clean(strings.TrimPrefix(pattern, "/"))
-	split := strings.Split(pattern, "/")
-	if split[0] == ".." {
-		if f.parent == nil {
-			//This should only happen on the root FS
-			return nil, &fs.PathError{
-				Op:   "readdir",
-				Path: pattern,
-				//TODO: make error clearer
-				Err: errors.New("trying to get file outside of squashfs"),
-			}
-		}
-		return f.parent.Glob(strings.Join(split[1:], "/"))
-	}
-	for i := 0; i < len(f.entries); i++ {
-		if match, _ := path.Match(split[0], f.entries[i].Name); match {
-			if len(split) == 1 {
-				out = append(out, f.entries[i].Name)
-				continue
-			}
-			sub, err := f.Sub(split[0])
-			if err != nil {
-				if pathErr, ok := err.(*fs.PathError); ok {
-					if pathErr.Err == fs.ErrNotExist {
-						continue
-					}
-					pathErr.Op = "glob"
-					pathErr.Path = pattern
-					return nil, pathErr
-				}
-				return nil, &fs.PathError{
-					Op:   "glob",
-					Path: pattern,
-					Err:  err,
-				}
-			}
-			subGlob, err := sub.(FS).Glob(strings.Join(split[1:], "/"))
-			if err != nil {
-				if pathErr, ok := err.(*fs.PathError); ok {
-					if pathErr.Err == fs.ErrNotExist {
-						continue
-					}
-					pathErr.Op = "glob"
-					pathErr.Path = pattern
-					return nil, pathErr
-				}
-				return nil, &fs.PathError{
-					Op:   "glob",
-					Path: pattern,
-					Err:  err,
-				}
-			}
-			for i := 0; i < len(subGlob); i++ {
-				subGlob[i] = f.name + "/" + subGlob[i]
-			}
-			out = append(out, subGlob...)
-		}
-	}
-	return
-}
-
-//ReadDir returns all the DirEntry returns all DirEntry's for the directory at name.
-//If name is not a directory, returns an error.
-func (f FS) ReadDir(name string) ([]fs.DirEntry, error) {
-	if !fs.ValidPath(name) {
-		return nil, &fs.PathError{
-			Op:   "readdir",
-			Path: name,
-			Err:  fs.ErrInvalid,
-		}
-	}
-	name = path.Clean(strings.TrimPrefix(name, "/"))
-	split := strings.Split(name, "/")
-	if split[0] == ".." {
-		if f.parent == nil {
-			//This should only happen on the root FS
-			return nil, &fs.PathError{
-				Op:   "readdir",
-				Path: name,
-				//TODO: make error clearer
-				Err: errors.New("trying to get file outside of squashfs"),
-			}
-		}
-		return f.parent.ReadDir(strings.Join(split[1:], "/"))
-	}
-	if split[0] == "." {
-		f := &File{i: f.i, r: f.r, parent: f.parent, name: f.name}
-		return f.ReadDir(-1)
-	}
-	for i := 0; i < len(f.entries); i++ {
-		if split[0] == f.entries[i].Name {
-			if len(split) == 1 {
-				in, err := f.r.getInodeFromEntry(f.entries[i])
-				if err != nil {
-					return nil, &fs.PathError{
-						Op:   "readdir",
-						Path: name,
-						Err:  err,
-					}
-				}
-				ents, err := f.r.readDirFromInode(in)
-				if err != nil {
-					return nil, &fs.PathError{
-						Op:   "readdir",
-						Path: name,
-						Err:  err,
-					}
-				}
-				out := make([]fs.DirEntry, len(ents))
-				for i, ent := range ents {
-					out[i] = &DirEntry{
-						en:     ent,
-						parent: &f,
-						r:      f.r,
-					}
-				}
-				return out, nil
-			}
-			sub, err := f.Sub(split[0])
-			if err != nil {
-				if pathErr, ok := err.(*fs.PathError); ok {
-					if pathErr.Err == fs.ErrNotExist {
-						continue
-					}
-					pathErr.Op = "readir"
-					pathErr.Path = name
-					return nil, pathErr
-				}
-				return nil, &fs.PathError{
-					Op:   "readdir",
-					Path: name,
-					Err:  err,
-				}
-			}
-			redDir, err := sub.(FS).ReadDir(strings.Join(split[1:], "/"))
-			if err != nil {
-				if pathErr, ok := err.(*fs.PathError); ok {
-					if pathErr.Err == fs.ErrNotExist {
-						continue
-					}
-					pathErr.Op = "readdir"
-					pathErr.Path = name
-					return nil, pathErr
-				}
-				return nil, &fs.PathError{
-					Op:   "readdir",
-					Path: name,
-					Err:  err,
-				}
-			}
-			return redDir, nil
-		}
-	}
-	return nil, &fs.PathError{
-		Op:   "readdir",
-		Path: name,
-		Err:  fs.ErrNotExist,
-	}
-}
-
-//ReadFile returns the data (in []byte) for the file at name.
-func (f FS) ReadFile(name string) ([]byte, error) {
-	fil, err := f.Open(name)
-	if err != nil {
-		if pathErr, ok := err.(*fs.PathError); ok {
-			pathErr.Op = "readfile"
-			pathErr.Path = name
-			return nil, pathErr
-		}
-	}
-	var buf bytes.Buffer
-	_, err = io.Copy(&buf, fil)
-	if err != nil {
-		return nil, &fs.PathError{
-			Op:   "readfile",
-			Path: name,
-			Err:  err,
-		}
-	}
-	return buf.Bytes(), nil
-}
-
-//Stat returns the fs.FileInfo for the file at name.
-func (f FS) Stat(name string) (fs.FileInfo, error) {
-	if !fs.ValidPath(name) {
-		return nil, &fs.PathError{
-			Op:   "stat",
-			Path: name,
-			Err:  fs.ErrInvalid,
-		}
-	}
-	name = path.Clean(strings.TrimPrefix(name, "/"))
-	split := strings.Split(name, "/")
-	if split[0] == ".." {
-		if f.parent == nil {
-			//This should only happen on the root FS
-			return nil, &fs.PathError{
-				Op:   "stat",
-				Path: name,
-				//TODO: make error clearer
-				Err: errors.New("trying to get file outside of squashfs"),
-			}
-		}
-		return f.parent.Stat(strings.Join(split[1:], "/"))
-	}
-	if split[0] == "." {
-		f := &File{i: f.i, r: f.r, parent: f.parent, name: f.name}
-		return f.Stat()
-	}
-	for i := 0; i < len(f.entries); i++ {
-		if split[0] == f.entries[i].Name {
-			if len(split) == 1 {
-				in, err := f.r.getInodeFromEntry(f.entries[i])
-				if err != nil {
-					return nil, &fs.PathError{
-						Op:   "stat",
-						Path: name,
-						Err:  err,
-					}
-				}
-				return FileInfo{
-					i:      in,
-					parent: &f,
-					r:      f.r,
-					name:   f.entries[i].Name,
-				}, nil
-			}
-			sub, err := f.Sub(split[0])
-			if err != nil {
-				if pathErr, ok := err.(*fs.PathError); ok {
-					if pathErr.Err == fs.ErrNotExist {
-						continue
-					}
-					pathErr.Op = "stat"
-					pathErr.Path = name
-					return nil, pathErr
-				}
-				return nil, &fs.PathError{
-					Op:   "stat",
-					Path: name,
-					Err:  err,
-				}
-			}
-			stat, err := sub.(FS).Stat(strings.Join(split[1:], "/"))
-			if err != nil {
-				if pathErr, ok := err.(*fs.PathError); ok {
-					if pathErr.Err == fs.ErrNotExist {
-						continue
-					}
-					pathErr.Op = "stat"
-					pathErr.Path = name
-					return nil, pathErr
-				}
-				return nil, &fs.PathError{
-					Op:   "stat",
-					Path: name,
-					Err:  err,
-				}
-			}
-			return stat, nil
-		}
-	}
-	return nil, &fs.PathError{
-		Op:   "stat",
-		Path: name,
-		Err:  fs.ErrNotExist,
-	}
-}
-
-//Sub returns the FS at dir
-func (f FS) Sub(dir string) (fs.FS, error) {
-	if !fs.ValidPath(dir) {
-		return nil, &fs.PathError{
-			Op:   "sub",
-			Path: dir,
-			Err:  fs.ErrInvalid,
-		}
-	}
-	dir = path.Clean(strings.TrimPrefix(dir, "/"))
-	split := strings.Split(dir, "/")
-	if split[0] == ".." {
-		if f.parent == nil {
-			//This should only happen on the root FS
-			return nil, &fs.PathError{
-				Op:   "sub",
-				Path: dir,
-				//TODO: make error clearer
-				Err: errors.New("trying to get file outside of squashfs"),
-			}
-		}
-		return f.parent.Sub(strings.Join(split[1:], "/"))
-	}
-	if split[0] == "." {
-		return f, nil
-	}
-	for i := 0; i < len(f.entries); i++ {
-		if split[0] == f.entries[i].Name {
-			if len(split) == 1 {
-				in, err := f.r.getInodeFromEntry(f.entries[i])
-				if err != nil {
-					return nil, &fs.PathError{
-						Op:   "sub",
-						Path: dir,
-						Err:  err,
-					}
-				}
-				ents, err := f.r.readDirFromInode(in)
-				if err != nil {
-					return nil, &fs.PathError{
-						Op:   "sub",
-						Path: dir,
-						Err:  err,
-					}
-				}
-				return FS{
-					i:       in,
-					r:       f.r,
-					parent:  &f,
-					name:    f.entries[i].Name,
-					entries: ents,
-				}, nil
-			}
-			sub, err := f.Sub(strings.Join(split[1:], "/"))
-			if err != nil {
-				if pathErr, ok := err.(*fs.PathError); ok {
-					if pathErr.Err == fs.ErrNotExist {
-						continue
-					}
-					pathErr.Op = "sub"
-					pathErr.Path = dir
-					return nil, pathErr
-				}
-				return nil, &fs.PathError{
-					Op:   "sub",
-					Path: dir,
-					Err:  err,
-				}
-			}
-			return sub, nil
-		}
-	}
-	return nil, &fs.PathError{
-		Op:   "sub",
-		Path: dir,
-		Err:  fs.ErrNotExist,
-	}
-}
-
-func (f FS) path() string {
-	if f.name == "/" {
-		return f.name
-	} else if f.parent.name == "/" {
-		return f.name
-	}
-	return f.parent.path() + "/" + f.name
-}
-
-//ExtractTo extracts the File to the given folder with the default options.
-//It extracts the directory's contents to the folder.
-func (f FS) ExtractTo(folder string) error {
-	return f.ExtractWithOptions(folder, DefaultOptions())
-}
-
-//ExtractSymlink extracts the File to the folder with the DereferenceSymlink option.
-//It extracts the directory's contents to the folder.
-func (f FS) ExtractSymlink(folder string) error {
-	return f.ExtractWithOptions(folder, ExtractionOptions{
-		DereferenceSymlink: true,
-		FolderPerm:         fs.ModePerm,
-	})
-}
-
-//ExtractWithOptions extracts the File to the given folder with the given ExtrationOptions.
-//It extracts the directory's contents to the folder.
-func (f FS) ExtractWithOptions(folder string, op ExtractionOptions) error {
-	op.notBase = true
-	folder = path.Clean(folder)
-	err := os.MkdirAll(folder, op.FolderPerm)
-	if err != nil {
-		return err
-	}
-	errChan := make(chan error)
-	for i := 0; i < len(f.entries); i++ {
-		go func(ent *DirEntry) {
-			fil, goErr := ent.File()
-			if goErr != nil {
-				errChan <- goErr
-				return
-			}
-			errChan <- fil.ExtractWithOptions(folder, op)
-			fil.Close()
-		}(&DirEntry{
-			en:     f.entries[i],
-			parent: &f,
-			r:      f.r,
-		})
-	}
-	for i := 0; i < len(f.entries); i++ {
-		err := <-errChan
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

reader_test.go

@@ -1,272 +0,0 @@
-package squashfs
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"os/exec"
-	"strconv"
-	"testing"
-	"time"
-
-	goappimage "github.com/CalebQ42/GoAppImage"
-)
-
-const (
-	appImageURL  = "https://github.com/srevinsaju/Firefox-Appimage/releases/download/firefox-v84.0.r20201221152838/firefox-84.0.r20201221152838-x86_64.AppImage"
-	appImageName = "firefox-84.0.r20201221152838-x86_64.AppImage"
-	squashfsURL  = "https://darkstorm.tech/LinuxPATest.sfs"
-	squashfsName = "LinuxPATest.sfs"
-)
-
-func TestSquashfs(t *testing.T) {
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatal(err)
-	}
-	squashFil, err := os.Open(wd + "/testing/" + squashfsName)
-	if os.IsNotExist(err) {
-		err = downloadTestSquash(wd + "/testing")
-		if err != nil {
-			t.Fatal(err)
-		}
-		squashFil, err = os.Open(wd + "/testing/" + squashfsName)
-	}
-	if err != nil {
-		t.Fatal(err)
-	}
-	rdr, err := NewSquashfsReader(squashFil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	os.RemoveAll(wd + "/testing/" + squashfsName + ".d")
-	op := DefaultOptions()
-	op.Verbose = true
-	err = rdr.ExtractWithOptions(wd+"/testing/"+squashfsName+".d", op)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Fatal("No Problems")
-}
-
-func TestSquashfsFromReader(t *testing.T) {
-	resp, err := http.DefaultClient.Get(squashfsURL)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer resp.Body.Close()
-	rdr, err := NewSquashfsReaderFromReader(resp.Body)
-	if err != nil {
-		t.Fatal(err)
-	}
-	os.RemoveAll("testing/" + squashfsName + ".d")
-	op := DefaultOptions()
-	op.Verbose = true
-	err = rdr.ExtractWithOptions("testing/"+squashfsName+".d", op)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Fatal("No Problems")
-}
-
-func TestAppImage(t *testing.T) {
-	t.Parallel()
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatal(err)
-	}
-	aiFil, err := os.Open(wd + "/testing/" + appImageName)
-	if os.IsNotExist(err) {
-		err = downloadTestAppImage(wd + "/testing")
-		if err != nil {
-			t.Fatal(err)
-		}
-		aiFil, err = os.Open(wd + "/testing/" + appImageName)
-		if err != nil {
-			t.Fatal(err)
-		}
-	} else if err != nil {
-		t.Fatal(err)
-	}
-	defer aiFil.Close()
-	stat, _ := aiFil.Stat()
-	ai := goappimage.NewAppImage(wd + "/testing/" + appImageName)
-	rdr, err := NewSquashfsReader(io.NewSectionReader(aiFil, ai.Offset, stat.Size()-ai.Offset))
-	if err != nil {
-		t.Fatal(err)
-	}
-	os.RemoveAll(wd + "/testing/firefox")
-	err = rdr.ExtractTo(wd + "/testing/firefox")
-	t.Fatal(err)
-}
-
-func TestUnsquashfs(t *testing.T) {
-	t.Parallel()
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatal(err)
-	}
-	aiFil, err := os.Open(wd + "/testing/" + appImageName)
-	if os.IsNotExist(err) {
-		err = downloadTestAppImage(wd + "/testing")
-		if err != nil {
-			t.Fatal(err)
-		}
-		aiFil, err = os.Open(wd + "/testing/" + appImageName)
-		if err != nil {
-			t.Fatal(err)
-		}
-	} else if err != nil {
-		t.Fatal(err)
-	}
-	os.RemoveAll(wd + "/testing/unsquashFirefox")
-	os.RemoveAll(wd + "/testing/firefox")
-	ai := goappimage.NewAppImage(wd + "/testing/" + appImageName)
-	fmt.Println("Command:", "unsquashfs", "-d", wd+"/testing/unsquashFirefox", "-o", strconv.Itoa(int(ai.Offset)), aiFil.Name())
-	cmd := exec.Command("unsquashfs", "-d", wd+"/testing/unsquashFirefox", "-o", strconv.Itoa(int(ai.Offset)), aiFil.Name())
-	start := time.Now()
-	err = cmd.Run()
-	if err != nil {
-		t.Fatal(err)
-	}
-	fmt.Println(time.Since(start))
-	t.Fatal("HI")
-}
-
-func BenchmarkDragRace(b *testing.B) {
-	wd, err := os.Getwd()
-	if err != nil {
-		b.Fatal(err)
-	}
-	aiFil, err := os.Open(wd + "/testing/" + appImageName)
-	if os.IsNotExist(err) {
-		err = downloadTestAppImage(wd + "/testing")
-		if err != nil {
-			b.Fatal(err)
-		}
-		aiFil, err = os.Open(wd + "/testing/" + appImageName)
-		if err != nil {
-			b.Fatal(err)
-		}
-	} else if err != nil {
-		b.Fatal(err)
-	}
-	stat, _ := aiFil.Stat()
-	ai := goappimage.NewAppImage(wd + "/testing/" + appImageName)
-	os.RemoveAll(wd + "/testing/unsquashFirefox")
-	os.RemoveAll(wd + "/testing/firefox")
-	cmd := exec.Command("unsquashfs", "-d", wd+"/testing/unsquashFirefox", "-o", strconv.Itoa(int(ai.Offset)), aiFil.Name())
-	start := time.Now()
-	err = cmd.Run()
-	if err != nil {
-		b.Fatal(err)
-	}
-	unsquashTime := time.Since(start)
-	start = time.Now()
-	rdr, err := NewSquashfsReader(io.NewSectionReader(aiFil, ai.Offset, stat.Size()-ai.Offset))
-	if err != nil {
-		b.Fatal(err)
-	}
-	err = rdr.ExtractTo(wd + "/testing/firefox")
-	if err != nil {
-		b.Fatal(err)
-	}
-	libTime := time.Since(start)
-	b.Log("Unsqushfs:", unsquashTime.Round(time.Millisecond))
-	b.Log("Library:", libTime.Round(time.Millisecond))
-	b.Log("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64)+"x faster")
-	b.Error("STOP ALREADY!")
-}
-
-func downloadTestAppImage(dir string) error {
-	//seems to time out on slow connections. Might fix that at some point... or not. It's just a test...
-	os.Mkdir(dir, os.ModePerm)
-	appImage, err := os.Create(dir + "/" + appImageName)
-	if err != nil {
-		return err
-	}
-	defer appImage.Close()
-	check := http.Client{
-		CheckRedirect: func(r *http.Request, _ []*http.Request) error {
-			r.URL.Opaque = r.URL.Path
-			return nil
-		},
-	}
-	resp, err := check.Get(appImageURL)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	_, err = io.Copy(appImage, resp.Body)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func downloadTestSquash(dir string) error {
-	//seems to time out on slow connections. Might fix that at some point... or not. It's just a test...
-	os.Mkdir(dir, os.ModePerm)
-	sfs, err := os.Create(dir + "/" + squashfsName)
-	if err != nil {
-		return err
-	}
-	defer sfs.Close()
-	check := http.Client{
-		CheckRedirect: func(r *http.Request, _ []*http.Request) error {
-			r.URL.Opaque = r.URL.Path
-			return nil
-		},
-	}
-	resp, err := check.Get(squashfsURL)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	_, err = io.Copy(sfs, resp.Body)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func TestCreateSquashFromAppImage(t *testing.T) {
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = os.Mkdir(wd+"/testing", 0777)
-	if err != nil && !os.IsExist(err) {
-		t.Fatal(err)
-	}
-	_, err = os.Open(wd + "/testing/" + appImageName)
-	if os.IsNotExist(err) {
-		err = downloadTestAppImage(wd + "/testing")
-		if err != nil {
-			t.Fatal(err)
-		}
-		_, err = os.Open(wd + "/testing/" + appImageName)
-		if err != nil {
-			t.Fatal(err)
-		}
-	} else if err != nil {
-		t.Fatal(err)
-	}
-	ai := goappimage.NewAppImage(wd + "/testing/" + appImageName)
-	aiFil, err := os.Open(wd + "/testing/" + appImageName)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer aiFil.Close()
-	aiFil.Seek(ai.Offset, 0)
-	os.Remove(wd + "/testing/" + appImageName + ".squashfs")
-	aiSquash, err := os.Create(wd + "/testing/" + appImageName + ".squashfs")
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, err = io.Copy(aiSquash, aiFil)
-	if err != nil {
-		t.Fatal(err)
-	}
-}

squashfs_test.go

@@ -0,0 +1,213 @@
+package squashfs
+
+//Actually proper tests go here.
+
+import (
+	"errors"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+)
+
+const (
+	squashfsURL  = "https://darkstorm.tech/files/LinuxPATest.sfs"
+	squashfsName = "tensorflow.sqfs"
+)
+
+func preTest(dir string) (fil *os.File, err error) {
+	fil, err = os.Open(filepath.Join(dir, squashfsName))
+	if err != nil {
+		_, err = os.Open(dir)
+		if os.IsNotExist(err) {
+			err = os.Mkdir(dir, 0755)
+		}
+		if err != nil {
+			return
+		}
+		os.Remove(filepath.Join(dir, squashfsName))
+		fil, err = os.Create(filepath.Join(dir, squashfsName))
+		if err != nil {
+			return
+		}
+		var resp *http.Response
+		resp, err = http.DefaultClient.Get(squashfsURL)
+		if err != nil {
+			return
+		}
+		_, err = io.Copy(fil, resp.Body)
+		if err != nil {
+			return
+		}
+	}
+	_, err = exec.LookPath("unsquashfs")
+	if err != nil {
+		return
+	}
+	_, err = exec.LookPath("mksquashfs")
+	return
+}
+
+func TestMisc(t *testing.T) {
+	tmpDir := "testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rdr, err := NewReader(fil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = rdr
+	// Put testing here
+	// t.Fatal("UM")
+}
+
+func BenchmarkExtract(b *testing.B) {
+	tmpDir := "testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		b.Fatal(err)
+	}
+	libPath := filepath.Join(tmpDir, "ExtractLib")
+	os.RemoveAll(libPath)
+	op := FastOptions()
+	op.IgnorePerm = true
+	rdr, err := NewReader(fil)
+	if err != nil {
+		b.Fatal(err)
+	}
+	err = rdr.ExtractWithOptions(libPath, op)
+	if err != nil {
+		b.Fatal(err)
+	}
+}
+
+func BenchmarkRace(b *testing.B) {
+	tmpDir := "testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		b.Fatal(err)
+	}
+	libPath := filepath.Join(tmpDir, "ExtractLib")
+	unsquashPath := filepath.Join(tmpDir, "ExtractSquashfs")
+	os.RemoveAll(libPath)
+	os.RemoveAll(unsquashPath)
+	var libTime, unsquashTime time.Duration
+	op := FastOptions()
+	op.IgnorePerm = true
+	start := time.Now()
+	rdr, err := NewReader(fil)
+	if err != nil {
+		b.Fatal(err)
+	}
+	err = rdr.ExtractWithOptions(libPath, op)
+	if err != nil {
+		b.Fatal(err)
+	}
+	libTime = time.Since(start)
+	cmd := exec.Command("unsquashfs", "-q", "-d", unsquashPath, fil.Name())
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	start = time.Now()
+	err = cmd.Run()
+	if err != nil {
+		b.Log("Unsquashfs error:", err)
+	}
+	unsquashTime = time.Since(start)
+	b.Log("Library took:", libTime.Round(time.Millisecond))
+	b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond))
+	b.Log("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster")
+}
+
+func TestExtractQuick(t *testing.T) {
+	//First, setup everything and extract the archive using the library and unsquashfs
+
+	// tmpDir := bTempDir()
+	tmpDir := "testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	libPath := filepath.Join(tmpDir, "ExtractLib")
+	unsquashPath := filepath.Join(tmpDir, "ExtractSquashfs")
+	os.RemoveAll(libPath)
+	os.RemoveAll(unsquashPath)
+	rdr, err := NewReader(fil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.RemoveAll(filepath.Join(tmpDir, "testLog.txt"))
+	logFil, _ := os.Create(filepath.Join(tmpDir, "testLog.txt"))
+	op := FastOptions()
+	op.Verbose = true
+	op.IgnorePerm = true
+	op.LogOutput = logFil
+	err = rdr.ExtractWithOptions(libPath, op)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name())
+	err = cmd.Run()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//Then compare the sizes and existance between the two (using unsquashfs as a reference).
+	//If the file doesn't exist, or the size is different, we exit.
+	//TODO: Add long test that checks contents.
+
+	squashFils := os.DirFS(unsquashPath)
+	err = fs.WalkDir(squashFils, ".", func(path string, _ fs.DirEntry, _ error) error {
+		libFil, e := os.Open(filepath.Join(libPath, path))
+		if e != nil {
+			return e
+		}
+		sfsFile, e := os.Open(filepath.Join(unsquashPath, path))
+		if e != nil {
+			return e
+		}
+		sfsStat, _ := sfsFile.Stat()
+		libStat, _ := libFil.Stat()
+		if sfsStat.Size() != libStat.Size() {
+			t.Log(libFil.Name(), "not the same size between library and unsquashfs")
+			t.Log("File is", libStat.Size())
+			t.Log("Should be", sfsStat.Size())
+			return errors.New("file not the correct size")
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+var filePath = "usr/sbin/add-shell"
+
+func TestSingleFile(t *testing.T) {
+	tmpDir := "testing"
+	fil, err := preTest(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.RemoveAll("testing/stuff")
+	rdr, err := NewReader(fil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	f, err := rdr.Open(filePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	op := DefaultOptions()
+	op.Verbose = true
+	err = f.(*File).ExtractWithOptions("testing/stuff", op)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

superblock.go

@@ -1,126 +0,0 @@
-package squashfs
-
-//The types of compression supported by squashfs.
-const (
-	GzipCompression = 1 + iota
-	LzmaCompression
-	LzoCompression
-	XzCompression
-	Lz4Compression
-	ZstdCompression
-)
-
-//Superblock contains important information about a squashfs file. Located at the very front of the archive.
-type superblock struct {
-	Magic            uint32
-	InodeCount       uint32
-	CreationTime     uint32
-	BlockSize        uint32
-	FragCount        uint32
-	CompressionType  uint16
-	BlockLog         uint16
-	Flags            uint16
-	IDCount          uint16
-	MajorVersion     uint16
-	MinorVersion     uint16
-	RootInodeRef     uint64
-	BytesUsed        uint64
-	IDTableStart     uint64
-	XattrTableStart  uint64
-	InodeTableStart  uint64
-	DirTableStart    uint64
-	FragTableStart   uint64
-	ExportTableStart uint64
-}
-
-//SuperblockFlags is the series of flags describing how a squashfs archive is packed.
-type SuperblockFlags struct {
-	//If true, inodes are stored uncompressed.
-	UncompressedInodes bool
-	//If true, data is stored uncompressed.
-	UncompressedData bool
-	check            bool
-	//If true, fragments are stored uncompressed.
-	UncompressedFragments bool
-	//If true, ALL data is stored in sequential data blocks instead of utilizing fragments.
-	NoFragments bool
-	//If true, the last block of data will always be stored as a fragment if it's less then the block size.
-	AlwaysFragment bool
-	//If true, duplicate files are only stored once. (Currently unsupported)
-	RemoveDuplicates bool
-	//If true, the export table is populated. (Currently unsupported)
-	Exportable bool
-	//If true, the xattr table is uncompressed. (Currently unsupported)
-	UncompressedXattr bool
-	//If true, the xattr table is not populated. (Currently unsupported)
-	NoXattr           bool
-	compressorOptions bool
-	//If true, the UID/GID table is stored uncompressed.
-	UncompressedIDs bool
-}
-
-//DefaultFlags are the default SuperblockFlags that are used.
-var DefaultFlags = SuperblockFlags{
-	RemoveDuplicates: true,
-	Exportable:       true,
-}
-
-//GetFlags returns a SuperblockFlags for a given superblock.
-func (s *superblock) GetFlags() SuperblockFlags {
-	return SuperblockFlags{
-		UncompressedInodes:    s.Flags&0x1 == 0x1,
-		UncompressedData:      s.Flags&0x2 == 0x2,
-		check:                 s.Flags&0x4 == 0x4,
-		UncompressedFragments: s.Flags&0x8 == 0x8,
-		NoFragments:           s.Flags&0x10 == 0x10,
-		AlwaysFragment:        s.Flags&0x20 == 0x20,
-		RemoveDuplicates:      s.Flags&0x40 == 0x40,
-		Exportable:            s.Flags&0x80 == 0x80,
-		UncompressedXattr:     s.Flags&0x100 == 0x100,
-		NoXattr:               s.Flags&0x200 == 0x200,
-		compressorOptions:     s.Flags&0x400 == 0x400,
-		UncompressedIDs:       s.Flags&0x800 == 0x800,
-	}
-}
-
-//ToUint returns the uint16 representation of the given SuperblockFlags
-func (s *SuperblockFlags) ToUint() uint16 {
-	var out uint16
-	if s.UncompressedInodes {
-		out = out | 0x1
-	}
-	if s.UncompressedData {
-		out = out | 0x2
-	}
-	if s.check {
-		out = out | 0x4
-	}
-	if s.UncompressedFragments {
-		out = out | 0x8
-	}
-	if s.NoFragments {
-		out = out | 0x10
-	}
-	if s.AlwaysFragment {
-		out = out | 0x20
-	}
-	if s.RemoveDuplicates {
-		out = out | 0x40
-	}
-	if s.Exportable {
-		out = out | 0x80
-	}
-	if s.UncompressedXattr {
-		out = out | 0x100
-	}
-	if s.NoXattr {
-		out = out | 0x200
-	}
-	if s.compressorOptions {
-		out = out | 0x400
-	}
-	if s.UncompressedIDs {
-		out = out | 0x800
-	}
-	return out
-}