Merge pull request #40 from CalebQ42/go-unsquashfs-hardlinks

go-unsquashfs hardlinks & squashfslow access

cmd/go-unsquashfs/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"io/fs"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -12,6 +11,7 @@ import (
 	"time"
 
 	"github.com/CalebQ42/squashfs"
+	squashfslow "github.com/CalebQ42/squashfs/low"
 )
 
 func userName(uid int, numeric bool) string {
@@ -36,32 +36,84 @@ func groupName(gid int, numeric bool) string {
 	return gs
 }
 
-func printEntry(root, path string, d fs.DirEntry, numeric bool) {
+var hardLinks = make(map[uint32]string)
-	fi, _ := d.Info()
+
+func printFile(rdr *squashfs.Reader, path string, f *squashfs.File) {
+	path = filepath.Join(path, f.Low.Name)
+	fi, _ := f.Stat()
 	sfi := fi.(squashfs.FileInfo)
 	owner := fmt.Sprintf("%s/%s",
-		userName(sfi.Uid(), numeric),
+		userName(sfi.Uid(), *numeric),
-		groupName(sfi.Gid(), numeric))
+		groupName(sfi.Gid(), *numeric))
-	link := ""
+	var link string
+	var isHardLink bool
+	if *showHardLinks {
+		link, isHardLink = hardLinks[f.Low.Inode.Num]
+		if !isHardLink {
+			hardLinks[f.Low.Inode.Num] = path
+		}
+	}
+	var size int64
+	if isHardLink {
+		size = 0
+	} else {
+		size = fi.Size()
+	}
 	if sfi.IsSymlink() {
 		link = " -> " + sfi.SymlinkPath()
+	} else if isHardLink {
+		link = " link to " + link
 	}
 	fmt.Printf("%s %s %*d %s %s%s\n",
 		strings.ToLower(fi.Mode().String()),
-		owner, 26-len(owner), fi.Size(),
+		owner, 26-len(owner), size,
 		fi.ModTime().Format("2006-01-02 15:04"),
-		filepath.Join(root, path), link)
+		path, link)
+	if f.IsDir() {
+		fs, _ := f.FS()
+		printDir(rdr, path, fs)
+	}
 }
 
+func printDir(rdr *squashfs.Reader, path string, f squashfs.FS) {
+	var base squashfslow.FileBase
+	var fil squashfs.File
+	var err error
+	for _, e := range f.LowDir.Entries {
+		base, err = rdr.Low.BaseFromEntry(e)
+		if err != nil {
+			panic(err)
+		}
+		fil = rdr.FileFromBase(base, f)
+		printFile(rdr, path, &fil)
+	}
+}
+
+var (
+	verbose       *bool
+	list          *bool
+	long          *bool
+	numeric       *bool
+	offset        *int64
+	ignore        *bool
+	file          *string
+	showHardLinks *bool
+)
+
 func main() {
-	verbose := flag.Bool("v", false, "Verbose")
+	verbose = flag.Bool("v", false, "Verbose")
-	list := flag.Bool("l", false, "List")
+	list = flag.Bool("l", false, "List")
-	long := flag.Bool("ll", false, "List with attributes")
+	long = flag.Bool("ll", false, "List with attributes")
-	numeric := flag.Bool("lln", false, "List with attributes and numeric ids")
+	numeric = flag.Bool("lln", false, "List with attributes and numeric ids")
-	offset := flag.Int64("o", 0, "Offset")
+	showHardLinks = flag.Bool("show-hard-links", false, "When used with ll or lln, shows hard links")
-	ignore := flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755")
+	offset = flag.Int64("o", 0, "Offset")
+	ignore = flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755")
+	file = flag.String("e", "", "File or folder to extract")
 	flag.Parse()
-	if len(flag.Args()) < 2 {
+	if (*list || *long || *numeric) && flag.NArg() < 1 {
+		fmt.Println("Please provide a file name")
+		os.Exit(0)
+	} else if (!*list && !*long && !*numeric) && flag.NArg() < 2 {
 		fmt.Println("Please provide a file name and extraction path")
 		os.Exit(0)
 	}
@@ -73,26 +125,22 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	if *list || *long || *numeric {
+	extractFil := r.File()
-		root := flag.Arg(1)
+	if *file != "" {
-		fs.WalkDir(r, ".", func(path string, d fs.DirEntry, err error) error {
+		extractFil, err = r.OpenFile(*file)
 		if err != nil {
 			panic(err)
 		}
-			if *long || *numeric {
-				printEntry(root, path, d, *numeric)
-			} else {
-				fmt.Println(filepath.Join(root, path))
 	}
-			return nil
+	if *list || *long || *numeric {
-		})
+		printFile(&r, "", extractFil)
 		return
 	}
 	op := squashfs.DefaultOptions()
 	op.Verbose = *verbose
 	op.IgnorePerm = *ignore
 	n := time.Now()
-	err = r.ExtractWithOptions(flag.Arg(1), op)
+	err = extractFil.ExtractWithOptions(flag.Arg(1), op)
 	if err != nil {
 		panic(err)
 	}

file.go

@@ -24,14 +24,14 @@ type File struct {
 	rdrInit  bool
 	parent   FS
 	r        *Reader
-	b        squashfslow.FileBase
+	Low      squashfslow.FileBase
 	dirsRead int
 }
 
 // Creates a new *File from the given *squashfs.Base
 func (r *Reader) FileFromBase(b squashfslow.FileBase, parent FS) File {
 	return File{
-		b:      b,
+		Low:    b,
 		parent: parent,
 		r:      r,
 	}
@@ -41,11 +41,11 @@ func (f File) FS() (FS, error) {
 	if !f.IsDir() {
 		return FS{}, errors.New("not a directory")
 	}
-	d, err := f.b.ToDir(f.r.Low)
+	d, err := f.Low.ToDir(f.r.Low)
 	if err != nil {
 		return FS{}, err
 	}
-	return FS{d: d, parent: &f.parent, r: f.r}, nil
+	return FS{LowDir: d, parent: &f.parent, r: f.r}, nil
 }
 
 // Closes the underlying readers.
@@ -75,21 +75,21 @@ func (f File) GetSymlinkFile() fs.File {
 
 // Returns whether the file is a directory.
 func (f File) IsDir() bool {
-	return f.b.IsDir()
+	return f.Low.IsDir()
 }
 
 // Returns whether the file is a regular file.
 func (f File) IsRegular() bool {
-	return f.b.IsRegular()
+	return f.Low.IsRegular()
 }
 
 // Returns whether the file is a symlink.
 func (f File) IsSymlink() bool {
-	return f.b.Inode.Type == inode.Sym || f.b.Inode.Type == inode.ESym
+	return f.Low.Inode.Type == inode.Sym || f.Low.Inode.Type == inode.ESym
 }
 
 func (f File) Mode() fs.FileMode {
-	return f.b.Inode.Mode()
+	return f.Low.Inode.Mode()
 }
 
 // Read reads the data from the file. Only works if file is a normal file.
@@ -112,7 +112,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
 	if !f.IsDir() {
 		return nil, errors.New("file is not a directory")
 	}
-	d, err := f.b.ToDir(f.r.Low)
+	d, err := f.Low.ToDir(f.r.Low)
 	if err != nil {
 		return nil, err
 	}
@@ -140,24 +140,24 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
 
 // Returns the file's fs.FileInfo
 func (f File) Stat() (fs.FileInfo, error) {
-	uid, err := f.b.Uid(&f.r.Low)
+	uid, err := f.Low.Uid(&f.r.Low)
 	if err != nil {
 		return nil, err
 	}
-	gid, err := f.b.Gid(&f.r.Low)
+	gid, err := f.Low.Gid(&f.r.Low)
 	if err != nil {
 		return nil, err
 	}
-	return newFileInfo(f.b.Name, uid, gid, &f.b.Inode), nil
+	return newFileInfo(f.Low.Name, uid, gid, &f.Low.Inode), nil
 }
 
 // SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string.
 func (f File) SymlinkPath() string {
-	switch f.b.Inode.Type {
+	switch f.Low.Inode.Type {
 	case inode.Sym:
-		return string(f.b.Inode.Data.(inode.Symlink).Target)
+		return string(f.Low.Inode.Data.(inode.Symlink).Target)
 	case inode.ESym:
-		return string(f.b.Inode.Data.(inode.ESymlink).Target)
+		return string(f.Low.Inode.Data.(inode.ESymlink).Target)
 	}
 	return ""
 }
@@ -179,7 +179,7 @@ func (f *File) WriteTo(w io.Writer) (int64, error) {
 
 func (f *File) initializeReaders() error {
 	var err error
-	f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.Low)
+	f.rdr, f.full, err = f.Low.GetRegFileReaders(f.r.Low)
 	if err == nil {
 		f.rdrInit = true
 	} else {
@@ -191,20 +191,20 @@ func (f *File) initializeReaders() error {
 
 func (f File) deviceDevices() (maj uint32, min uint32) {
 	var dev uint32
-	switch f.b.Inode.Type {
+	switch f.Low.Inode.Type {
 	case inode.Char, inode.Block:
-		dev = f.b.Inode.Data.(inode.Device).Dev
+		dev = f.Low.Inode.Data.(inode.Device).Dev
 	case inode.EChar, inode.EBlock:
-		dev = f.b.Inode.Data.(inode.EDevice).Dev
+		dev = f.Low.Inode.Data.(inode.EDevice).Dev
 	}
 	return dev >> 8, dev & 0x000FF
 }
 
 func (f File) path() string {
-	if f.parent.d.Name == "" {
+	if f.parent.LowDir.Name == "" {
-		return f.b.Name
+		return f.Low.Name
 	}
-	return filepath.Join(f.parent.path(), f.b.Name)
+	return filepath.Join(f.parent.path(), f.Low.Name)
 }
 
 // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
@@ -229,9 +229,9 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			return err
 		}
 	}
-	switch f.b.Inode.Type {
+	switch f.Low.Inode.Type {
 	case inode.Dir, inode.EDir:
-		d, err := f.b.ToDir(f.r.Low)
+		d, err := f.Low.ToDir(f.r.Low)
 		if err != nil {
 			if op.Verbose {
 				log.Println("Failed to create squashfs.Directory for", path)
@@ -289,7 +289,7 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			return errors.Join(errors.New("failed to extract folder: "+path), errors.Join(errCache...))
 		}
 	case inode.Fil, inode.EFil:
-		path = filepath.Join(path, f.b.Name)
+		path = filepath.Join(path, f.Low.Name)
 		outFil, err := os.Create(path)
 		if err != nil {
 			if op.Verbose {
@@ -298,7 +298,7 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			return errors.Join(errors.New("failed to create file: "+path), err)
 		}
 		defer outFil.Close()
-		full, err := f.b.GetFullReader(&f.r.Low)
+		full, err := f.Low.GetFullReader(&f.r.Low)
 		if err != nil {
 			if op.Verbose {
 				log.Println("Failed to create full reader for", path)
@@ -324,11 +324,11 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 				return errors.New("failed to get symlink's file")
 			}
 			fil := filTmp.(*File)
-			fil.b.Name = f.b.Name
+			fil.Low.Name = f.Low.Name
 			err := fil.ExtractWithOptions(path, op)
 			if err != nil {
 				if op.Verbose {
-					log.Println("Failed to extract symlink's file:", filepath.Join(path, f.b.Name))
+					log.Println("Failed to extract symlink's file:", filepath.Join(path, f.Low.Name))
 				}
 				return errors.Join(errors.New("failed to extract symlink's file: "+path), err)
 			}
@@ -351,7 +351,7 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 					return errors.Join(errors.New("failed to extract symlink's file: "+extractLoc), err)
 				}
 			}
-			path = filepath.Join(path, f.b.Name)
+			path = filepath.Join(path, f.Low.Name)
 			err := os.Symlink(f.SymlinkPath(), path)
 			if err != nil {
 				if op.Verbose {
@@ -374,9 +374,9 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 			}
 			return errors.Join(errors.New("mknot command not found"), err)
 		}
-		path = filepath.Join(path, f.b.Name)
+		path = filepath.Join(path, f.Low.Name)
 		var typ string
-		switch f.b.Inode.Type {
+		switch f.Low.Inode.Type {
 		case inode.Char, inode.EChar:
 			typ = "c"
 		case inode.Block, inode.EBlock:
@@ -412,7 +412,7 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 		}
 		return nil
 	default:
-		return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.b.Inode.Type)))
+		return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.Low.Inode.Type)))
 	}
 	if op.Verbose {
 		log.Println(f.path(), "extracted to", path)
@@ -420,7 +420,7 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 	if op.IgnorePerm {
 		return nil
 	}
-	uid, err := f.b.Uid(&f.r.Low)
+	uid, err := f.Low.Uid(&f.r.Low)
 	if err != nil {
 		if op.Verbose {
 			log.Println("Failed to get uid for", path)
@@ -428,7 +428,7 @@ func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
 		}
 		return nil
 	}
-	gid, err := f.b.Gid(&f.r.Low)
+	gid, err := f.Low.Gid(&f.r.Low)
 	if err != nil {
 		if op.Verbose {
 			log.Println("Failed to get gid for", path)

fs.go

@@ -17,13 +17,13 @@ import (
 type FS struct {
 	r      *Reader
 	parent *FS
-	d      squashfslow.Directory
+	LowDir squashfslow.Directory
 }
 
 // Creates a new *FS from the given squashfs.directory
 func (r *Reader) FSFromDirectory(d squashfslow.Directory, parent FS) FS {
 	return FS{
-		d:      d,
+		LowDir: d,
 		r:      r,
 		parent: &parent,
 	}
@@ -42,10 +42,10 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 		}
 	}
 	split := strings.Split(pattern, "/")
-	for i := range f.d.Entries {
+	for i := range f.LowDir.Entries {
-		if match, _ := path.Match(split[0], f.d.Entries[i].Name); match {
+		if match, _ := path.Match(split[0], f.LowDir.Entries[i].Name); match {
 			if len(split) == 1 {
-				out = append(out, f.d.Entries[i].Name)
+				out = append(out, f.LowDir.Entries[i].Name)
 				continue
 			}
 			sub, err := f.Sub(split[0])
@@ -81,7 +81,7 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 				}
 			}
 			for i := range subGlob {
-				subGlob[i] = f.d.Name + "/" + subGlob[i]
+				subGlob[i] = f.LowDir.Name + "/" + subGlob[i]
 			}
 			out = append(out, subGlob...)
 		}
@@ -91,6 +91,10 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
 
 // Opens the file at name. Returns a *File as an fs.File.
 func (f FS) Open(name string) (fs.File, error) {
+	return f.OpenFile(name)
+}
+
+func (f FS) OpenFile(name string) (*File, error) {
 	name = filepath.Clean(name)
 	if !fs.ValidPath(name) {
 		return nil, &fs.PathError{
@@ -111,10 +115,10 @@ func (f FS) Open(name string) (fs.File, error) {
 				Err:  fs.ErrNotExist,
 			}
 		} else {
-			return f.parent.Open(strings.Join(split[1:], "/"))
+			return f.parent.OpenFile(strings.Join(split[1:], "/"))
 		}
 	}
-	i, found := slices.BinarySearchFunc(f.d.Entries, split[0], func(e directory.Entry, name string) int {
+	i, found := slices.BinarySearchFunc(f.LowDir.Entries, split[0], func(e directory.Entry, name string) int {
 		return strings.Compare(e.Name, name)
 	})
 	if !found {
@@ -124,13 +128,13 @@ func (f FS) Open(name string) (fs.File, error) {
 			Err:  fs.ErrNotExist,
 		}
 	}
-	b, err := f.r.Low.BaseFromEntry(f.d.Entries[i])
+	b, err := f.r.Low.BaseFromEntry(f.LowDir.Entries[i])
 	if err != nil {
 		return nil, err
 	}
 	if len(split) == 1 {
 		return &File{
-			b:      b,
+			Low:    b,
 			r:      f.r,
 			parent: f,
 		}, nil
@@ -146,7 +150,7 @@ func (f FS) Open(name string) (fs.File, error) {
 	if err != nil {
 		return nil, err
 	}
-	return f.r.FSFromDirectory(d, f).Open(strings.Join(split[1:], "/"))
+	return f.r.FSFromDirectory(d, f).OpenFile(strings.Join(split[1:], "/"))
 }
 
 // Returns all DirEntry's for the directory at name.
@@ -256,20 +260,20 @@ func (f FS) ExtractWithOptions(folder string, op *ExtractionOptions) error {
 func (f FS) File() *File {
 	if f.parent != nil {
 		return &File{
-			b:      f.d.FileBase,
+			Low:    f.LowDir.FileBase,
 			parent: *f.parent,
 			r:      f.r,
 		}
 	}
 	return &File{
-		b: f.d.FileBase,
+		Low: f.LowDir.FileBase,
 		r:   f.r,
 	}
 }
 
 func (f FS) path() string {
 	if f.parent == nil {
-		return f.d.Name
+		return f.LowDir.Name
 	}
-	return filepath.Join(f.parent.path(), f.d.Name)
+	return filepath.Join(f.parent.path(), f.LowDir.Name)
 }

internal/decompress/zlib.go

@@ -2,17 +2,31 @@ package decompress
 
 import (
 	"bytes"
-	"compress/zlib"
 	"io"
+	"sync"
+
+	"github.com/klauspost/compress/zlib"
 )
 
-type Zlib struct{}
+type Zlib struct {
+	pool sync.Pool
+}
 
-func (z Zlib) Decompress(data []byte) ([]byte, error) {
+func NewZlib() *Zlib {
-	rdr, err := zlib.NewReader(bytes.NewReader(data))
+	return &Zlib{}
+}
+
+func (z *Zlib) Decompress(data []byte) ([]byte, error) {
+	rdr := z.pool.Get()
+	defer z.pool.Put(rdr)
+	var err error
+	if rdr == nil {
+		rdr, err = zlib.NewReader(bytes.NewReader(data))
+	} else {
+		err = rdr.(zlib.Resetter).Reset(bytes.NewReader(data), nil)
+	}
 	if err != nil {
 		return nil, err
 	}
-	defer rdr.Close()
+	return io.ReadAll(rdr.(io.ReadCloser))
-	return io.ReadAll(rdr)
 }

internal/decompress/zstd.go

@@ -4,13 +4,17 @@ import (
 	"github.com/klauspost/compress/zstd"
 )
 
-type Zstd struct{}
+type Zstd struct {
+	rdr *zstd.Decoder
+}
+
+func NewZstd() Zstd {
+	rdr, _ := zstd.NewReader(nil, zstd.WithDecoderLowmem(true))
+	return Zstd{
+		rdr: rdr,
+	}
+}
 
 func (z Zstd) Decompress(data []byte) ([]byte, error) {
-	rdr, err := zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderConcurrency(1))
+	return z.rdr.DecodeAll(data, nil)
-	if err != nil {
-		return nil, err
-	}
-	defer rdr.Close()
-	return rdr.DecodeAll(data, nil)
 }

low/reader.go

@@ -56,7 +56,7 @@ func NewReader(r io.ReaderAt) (rdr Reader, err error) {
 	}
 	switch rdr.Superblock.CompType {
 	case ZlibCompression:
-		rdr.d = decompress.Zlib{}
+		rdr.d = decompress.NewZlib()
 	case LZMACompression:
 		rdr.d, err = decompress.NewLzma()
 		if err != nil {
@@ -72,7 +72,7 @@ func NewReader(r io.ReaderAt) (rdr Reader, err error) {
 	case LZ4Compression:
 		rdr.d = decompress.NewLz4()
 	case ZSTDCompression:
-		rdr.d = decompress.Zstd{}
+		rdr.d = decompress.NewZstd()
 	default:
 		return rdr, errors.New("invalid compression type. possible corrupted archive")
 	}

reader.go

@@ -22,7 +22,7 @@ func NewReader(r io.ReaderAt) (Reader, error) {
 		Low: rdr,
 	}
 	out.FS = FS{
-		d: rdr.Root,
+		LowDir: rdr.Root,
 		r:   &out,
 	}
 	return out, nil

squashfs_test.go

@@ -76,13 +76,11 @@ func BenchmarkExtract(b *testing.B) {
 	}
 	libPath := filepath.Join(tmpDir, "ExtractLib")
 	os.RemoveAll(libPath)
-	op := FastOptions()
-	op.IgnorePerm = true
 	rdr, err := NewReader(fil)
 	if err != nil {
 		b.Fatal(err)
 	}
-	err = rdr.ExtractWithOptions(libPath, op)
+	err = rdr.ExtractWithOptions(libPath, FastOptions())
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -99,14 +97,12 @@ func BenchmarkRace(b *testing.B) {
 	os.RemoveAll(libPath)
 	os.RemoveAll(unsquashPath)
 	var libTime, unsquashTime time.Duration
-	op := FastOptions()
-	op.IgnorePerm = true
 	start := time.Now()
 	rdr, err := NewReader(fil)
 	if err != nil {
 		b.Fatal(err)
 	}
-	err = rdr.ExtractWithOptions(libPath, op)
+	err = rdr.ExtractWithOptions(libPath, FastOptions())
 	if err != nil {
 		b.Fatal(err)
 	}