Merge branch 'main' of https://github.com/CalebQ42/squashfs into main

Haven't implemented LZO due to limited libraries

README.md

@@ -1,8 +1,12 @@
-# squashfs [![PkgGoDev](https://pkg.go.dev/badge/github.com/CalebQ42/squashfs)](https://pkg.go.dev/github.com/CalebQ42/squashfs)
+# squashfs (WIP)
+
+[![PkgGoDev](https://pkg.go.dev/badge/github.com/CalebQ42/squashfs)](https://pkg.go.dev/github.com/CalebQ42/squashfs) [![Go Report Card](https://goreportcard.com/badge/github.com/CalebQ42/squashfs)](https://goreportcard.com/report/github.com/CalebQ42/squashfs)
 
 A PURE Go library to read and write squashfs.
 
-Currently, you can read a squashfs and extract files (folder extraction not supported. Yet).
+Currently has support for reading squashfs files and extracting files and folders. Supports all compression types except LZO, but additional compression options are hit or miss.
+
+The only major thing missing from squashfs reading is Xattr parsing.
 
 Special thanks to https://dr-emann.github.io/squashfs/ for some VERY important information in an easy to understand format.
 Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree/master/internal/squashfs) as I referenced it to figure some things out (and double check others).

file.go

@@ -211,23 +211,32 @@ func (f *File) Permission() os.FileMode {
 	return mode
 }
 
-//ExtractTo extracts the file to the given path. This is the same as ExtractWithOptions(path, false, os.ModePerm, false).
+//ExtractTo extracts the file to the given path. This is the same as ExtractWithOptions(path, false, false, os.ModePerm, false).
 //Will NOT try to keep symlinks valid, folders extracted will have the permissions set by the squashfs, but the folder to make path will have full permissions (777).
 //
 //Will try it's best to extract all files, and if any errors come up, they will be appended to the error slice that's returned.
 func (f *File) ExtractTo(path string) []error {
-	return f.ExtractWithOptions(path, false, os.ModePerm, false)
+	return f.ExtractWithOptions(path, false, false, os.ModePerm, false)
+}
+
+//ExtractSymlink is similar to ExtractTo, but when it extracts a symlink, it instead extracts the file associated with the symlink in it's place.
+//This is the same as ExtractWithOptions(path, true, false, os.ModePerm, false)
+func (f *File) ExtractSymlink(path string) []error {
+	return f.ExtractWithOptions(path, true, false, os.ModePerm, false)
 }
 
 //ExtractWithOptions will extract the file to the given path, while allowing customization on how it works. ExtractTo is the "default" options.
 //Will try it's best to extract all files, and if any errors come up, they will be appended to the error slice that's returned.
 //Should only return multiple errors if extracting a folder.
 //
+//If dereferenceSymlink is set, instead of extracting a symlink, it will extract the file the symlink is pointed to in it's place.
+//If both dereferenceSymlink and unbreakSymlink is set, dereferenceSymlink takes precendence.
+//
 //If unbreakSymlink is set, it will also try to extract the symlink's associated file. WARNING: the symlink's file may have to go up the directory to work.
 //If unbreakSymlink is set and the file cannot be extracted, a ErrBrokenSymlink will be appended to the returned error slice.
 //
 //folderPerm only applies to the folders created to get to path. Folders from the archive are given the correct permissions defined by the archive.
-func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm os.FileMode, verbose bool) (errs []error) {
+func (f *File) ExtractWithOptions(path string, dereferenceSymlink, unbreakSymlink bool, folderPerm os.FileMode, verbose bool) (errs []error) {
 	errs = make([]error, 0)
 	err := os.MkdirAll(path, folderPerm)
 	if err != nil {
@@ -287,9 +296,9 @@ func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm o
 		for _, child := range children {
 			go func(child *File) {
 				if f.Name == "" {
-					finishChan <- child.ExtractWithOptions(path, unbreakSymlink, folderPerm, verbose)
+					finishChan <- child.ExtractWithOptions(path, dereferenceSymlink, unbreakSymlink, folderPerm, verbose)
 				} else {
-					finishChan <- child.ExtractWithOptions(path+"/"+f.Name, unbreakSymlink, folderPerm, verbose)
+					finishChan <- child.ExtractWithOptions(path+"/"+f.Name, dereferenceSymlink, unbreakSymlink, folderPerm, verbose)
 				}
 			}(child)
 		}
@@ -357,12 +366,30 @@ func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm o
 		return
 	case f.IsSymlink():
 		symPath := f.SymlinkPath()
-		if unbreakSymlink {
+		if dereferenceSymlink {
+			fil := f.GetSymlinkFile()
+			if fil == nil {
+				if verbose {
+					fmt.Println("Symlink path(", symPath, ") is outside the archive:"+path+"/"+f.Name)
+				}
+				return
+			}
+			fil.Name = f.Name
+			extracSymErrs := fil.ExtractWithOptions(path, dereferenceSymlink, unbreakSymlink, folderPerm, verbose)
+			if len(extracSymErrs) > 0 {
+				if verbose {
+					fmt.Println("Error(s) while extracting the symlink's file:", path+"/"+f.Name)
+					fmt.Println(extracSymErrs)
+				}
+				errs = append(errs, extracSymErrs...)
+			}
+			return
+		} else if unbreakSymlink {
 			fil := f.GetSymlinkFile()
 			if fil != nil {
 				symPath = path + "/" + symPath
 				paths := strings.Split(symPath, "/")
-				extracSymErrs := fil.ExtractWithOptions(strings.Join(paths[:len(paths)-1], "/"), unbreakSymlink, folderPerm, verbose)
+				extracSymErrs := fil.ExtractWithOptions(strings.Join(paths[:len(paths)-1], "/"), dereferenceSymlink, unbreakSymlink, folderPerm, verbose)
 				if len(extracSymErrs) > 0 {
 					if verbose {
 						fmt.Println("Error(s) while extracting the symlink's file:", path+"/"+f.Name)
@@ -370,8 +397,11 @@ func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm o
 					}
 					errs = append(errs, extracSymErrs...)
 				}
-			} else if verbose {
+			} else {
-				fmt.Println("Symlink path(", symPath, ") is outside the archive:"+path+"/"+f.Name)
+				if verbose {
+					fmt.Println("Symlink path(", symPath, ") is outside the archive:"+path+"/"+f.Name)
+				}
+				return
 			}
 		}
 		err = os.Symlink(f.SymlinkPath(), path+"/"+f.Name)

go.mod

@@ -7,7 +7,9 @@ require (
 	github.com/adrg/xdg v0.2.3 // indirect
 	github.com/google/go-cmp v0.5.4 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
+	github.com/klauspost/compress v1.11.3
 	github.com/kr/text v0.2.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.1
 	github.com/smartystreets/assertions v1.2.0 // indirect
 	github.com/ulikunitz/xz v0.5.8
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect

go.sum

@@ -18,6 +18,8 @@ github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3
 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc=
+github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -27,6 +29,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/pierrec/lz4/v4 v4.1.1 h1:cS6aGkNLJr4u+UwaA21yp+gbWN3WJWtKo1axmPDObMA=
+github.com/pierrec/lz4/v4 v4.1.1/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=

internal/compression/gzip.go

@@ -3,11 +3,37 @@ package compression
 import (
 	"bytes"
 	"compress/zlib"
+	"encoding/binary"
 	"io"
 )
 
+type gzipInit struct {
+	CompressionLevel int32
+	WindowSize       int16
+	Strategies       int16
+}
+
 //Gzip is a decompressor for gzip type compression. Uses zlib for compression and decompression
-type Gzip struct{}
+type Gzip struct {
+	CompressionLevel int32
+	HasCustomWindow  bool
+	HasStrategies    bool
+}
+
+//NewGzipCompressorWithOptions creates a new gzip compressor/decompressor with options read from the given reader.
+func NewGzipCompressorWithOptions(r io.Reader) (*Gzip, error) {
+	var gzip Gzip
+	var init gzipInit
+	err := binary.Read(r, binary.LittleEndian, &init)
+	if err != nil {
+		return nil, err
+	}
+	gzip.CompressionLevel = init.CompressionLevel
+	//TODO: proper support for window size and strategies
+	gzip.HasCustomWindow = init.WindowSize != 15
+	gzip.HasStrategies = init.Strategies != 0 && init.Strategies != 1
+	return &gzip, nil
+}
 
 //Decompress reads the entirety of the given reader and returns it uncompressed as a byte slice.
 func (g *Gzip) Decompress(r io.Reader) ([]byte, error) {

internal/compression/lz4.go

@@ -0,0 +1,37 @@
+package compression
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+
+	"github.com/pierrec/lz4/v4"
+)
+
+//Lz4 is a Lz4 Compressor/Decompressor
+type Lz4 struct {
+	HC bool
+}
+
+//NewLz4CompressorWithOptions creates a new lz4 compressor/decompressor with options read from the given reader.
+func NewLz4CompressorWithOptions(r io.Reader) (*Lz4, error) {
+	var lz4 Lz4
+	var init struct {
+		Version int32
+		Flags   int32
+	}
+	err := binary.Read(r, binary.LittleEndian, &init)
+	if err != nil {
+		return nil, err
+	}
+	lz4.HC = init.Flags == 1
+	return &lz4, nil
+}
+
+//Decompress decompresses all data from r and returns the uncompressed bytes
+func (l *Lz4) Decompress(r io.Reader) ([]byte, error) {
+	rdr := lz4.NewReader(r)
+	var buf bytes.Buffer
+	_, err := io.Copy(&buf, rdr)
+	return buf.Bytes(), err
+}

internal/compression/zstd.go

@@ -0,0 +1,36 @@
+package compression
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+//Zstd is a zstd compressor/decompressor
+type Zstd struct {
+	CompressionLevel int32
+}
+
+//NewZstdCompressorWithOptions creates a new Zstd with options read from the given reader
+func NewZstdCompressorWithOptions(r io.Reader) (*Zstd, error) {
+	var zstd Zstd
+	err := binary.Read(r, binary.LittleEndian, &zstd)
+	if err != nil {
+		return nil, err
+	}
+	return &zstd, nil
+}
+
+//Decompress decompresses all data from the reader and returns the uncompressed data
+func (z *Zstd) Decompress(r io.Reader) ([]byte, error) {
+	rdr, err := zstd.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+	defer rdr.Close()
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, rdr)
+	return buf.Bytes(), err
+}

reader.go

@@ -21,6 +21,8 @@ var (
 	errIncompatibleCompression = errors.New("Compression type unsupported")
 	//ErrCompressorOptions is returned if compressor options is present. It's not currently supported.
 	errCompressorOptions = errors.New("Compressor options is not currently supported")
+	//ErrOptions is returned when compression options that I haven't tested is set. When this is returned, the Reader is also returned.
+	ErrOptions = errors.New("Possibly incompatible compressor options")
 )
 
 //Reader processes and reads a squashfs archive.
@@ -35,6 +37,7 @@ type Reader struct {
 
 //NewSquashfsReader returns a new squashfs.Reader from an io.ReaderAt
 func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
+	hasUnsupportedOptions := false
 	var rdr Reader
 	rdr.r = r
 	err := binary.Read(io.NewSectionReader(rdr.r, 0, int64(binary.Size(rdr.super))), binary.LittleEndian, &rdr.super)
@@ -50,8 +53,17 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 	rdr.flags = rdr.super.GetFlags()
 	if rdr.flags.CompressorOptions {
 		switch rdr.super.CompressionType {
+		case gzipCompression:
+			gzip, err := compression.NewGzipCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 8))
+			if err != nil {
+				return nil, err
+			}
+			if gzip.HasCustomWindow || gzip.HasStrategies {
+				hasUnsupportedOptions = true
+			}
+			rdr.decompressor = gzip
 		case xzCompression:
-			xz, err := compression.NewXzCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 1000)) //1000 is technically too much, but it's just an easy way to do it.
+			xz, err := compression.NewXzCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 8))
 			if err != nil {
 				return nil, err
 			}
@@ -59,6 +71,21 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 				return nil, errors.New("XZ compression options has filters. These are not yet supported")
 			}
 			rdr.decompressor = xz
+		case lz4Compression:
+			lz4, err := compression.NewLz4CompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 8))
+			if err != nil {
+				return nil, err
+			}
+			if lz4.HC {
+				hasUnsupportedOptions = true
+			}
+			rdr.decompressor = lz4
+		case zstdCompression:
+			zstd, err := compression.NewZstdCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 4))
+			if err != nil {
+				return nil, err
+			}
+			rdr.decompressor = zstd
 		default:
 			return nil, errCompressorOptions
 		}
@@ -70,6 +97,10 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 			rdr.decompressor = &compression.Lzma{}
 		case xzCompression:
 			rdr.decompressor = &compression.Xz{}
+		case lz4Compression:
+			rdr.decompressor = &compression.Lz4{}
+		case zstdCompression:
+			rdr.decompressor = &compression.Zstd{}
 		default:
 			//TODO: all compression types.
 			return nil, errIncompatibleCompression
@@ -111,9 +142,21 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 		}
 		unread -= read
 	}
+	if hasUnsupportedOptions {
+		return &rdr, ErrOptions
+	}
 	return &rdr, nil
 }
 
+//ExtractTo tries to extract ALL files to the given path. This is the same as getting the root folder and extracting that.
+func (r *Reader) ExtractTo(path string) []error {
+	root, err := r.GetRootFolder()
+	if err != nil {
+		return []error{err}
+	}
+	return root.ExtractTo(path)
+}
+
 //GetRootFolder returns a squashfs.File that references the root directory of the squashfs archive.
 func (r *Reader) GetRootFolder() (root *File, err error) {
 	root = new(File)

squash_test.go

@@ -1,7 +1,6 @@
 package squashfs
 
 import (
-	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -32,7 +31,7 @@ func TestSquashfs(t *testing.T) {
 	}
 	os.RemoveAll(wd + "/testing/" + squashfsName + ".d")
 	root, _ := rdr.GetRootFolder()
-	errs := root.ExtractWithOptions(wd+"/testing/"+squashfsName+".d", false, os.ModePerm, true)
+	errs := root.ExtractWithOptions(wd+"/testing/"+squashfsName+".d", false, false, os.ModePerm, true)
 	t.Fatal(errs)
 }
 
@@ -59,11 +58,10 @@ func TestAppImage(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	fil := rdr.GetFileAtPath("usr/q*/QtQ*k/Extras/Priv*/q*")
+	errs := rdr.ExtractTo(wd + "/testing/cool-retro")
-	if fil == nil {
+	if len(errs) > 0 {
-		t.Fatal("Can't find desktop file")
+		t.Fatal(errs)
 	}
-	fmt.Println("Fount:", fil.Path())
 	// os.RemoveAll(wd + "/testing/" + appImageName + ".d")
 	// root, _ := rdr.GetRootFolder()
 	// errs := root.ExtractWithOptions(wd+"/testing/"+appImageName+".d", true, os.ModePerm, true)