Some more work on ExtractTo

Added Xz compression support Started testing using a big squashfs fil (particularly the squashfs from an Arch Linux install img)
2020-11-30 03:53:57 -06:00
parent 508a33b323
commit c5c6291643
7 changed files with 156 additions and 31 deletions
@@ -2,6 +2,7 @@ package squashfs

 import (
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"strings"
@@ -26,7 +27,7 @@ type File struct {
 	Name    string       //The name of the file or folder. Root folder will not have a name ("")
 	Parent  *File        //The parent directory. Should ALWAYS be a folder. If it's the root directory, will be nil
 	Reader  io.Reader    //Underlying reader. When writing, will probably be an os.File. When reading this is kept nil UNTIL reading to save memory.
-	Path    string       //The path to the folder the File is located in.
+	path    string       //The path to the folder the File is located in.
 	r       *Reader      //The squashfs.Reader where this file is contained.
 	in      *inode.Inode //Underlyting inode when reading.
 	filType int          //The file's type, using inode types.
@@ -56,6 +57,7 @@ func (f *File) GetChildren() (children []*File, err error) {
 	}
 	dir, err := f.r.readDirFromInode(f.in)
 	if err != nil {
+		fmt.Println("err reading dir")
 		return
 	}
 	var fil *File
@@ -66,7 +68,7 @@ func (f *File) GetChildren() (children []*File, err error) {
 		}
 		fil.Parent = f
 		if f.Name != "" {
-			fil.Path = f.Path + "/" + f.Name
+			fil.path = f.Path()
 		}
 		children = append(children, fil)
 	}
@@ -84,6 +86,7 @@ func (f *File) GetChildrenRecursively() (children []*File, err error) {
 	}
 	chil, err := f.GetChildren()
 	if err != nil {
+		fmt.Println("err here", f.Path())
 		return
 	}
 	var childFolders []*File
@@ -97,6 +100,7 @@ func (f *File) GetChildrenRecursively() (children []*File, err error) {
 		var childs []*File
 		childs, err = folds.GetChildrenRecursively()
 		if err != nil {
+			fmt.Println("err here Recursive", folds.Path())
 			return
 		}
 		children = append(children, childs...)
@@ -104,6 +108,14 @@ func (f *File) GetChildrenRecursively() (children []*File, err error) {
 	return
 }

+//Path returns the path of the file within the archive.
+func (f *File) Path() string {
+	if f.Name == "" {
+		return f.path
+	}
+	return f.path + "/" + f.Name
+}
+
 //GetFileAtPath tries to return the File at the given path, relative to the file.
 //Returns nil if called on something other then a folder, OR if the path goes oustide the archive.
 func (f *File) GetFileAtPath(path string) *File {
@@ -150,6 +162,11 @@ func (f *File) IsSymlink() bool {
 	return f.filType == inode.BasicSymlinkType || f.filType == inode.ExtSymlinkType
 }

+//IsFile returns if the file is a file.
+func (f *File) IsFile() bool {
+	return f.filType == inode.BasicFileType || f.filType == inode.ExtFileType
+}
+
 //SymlinkPath returns the path the symlink is pointing to. If the file ISN'T a symlink, will return an empty string.
 //If a path begins with "/" then the symlink is pointing to an absolute path (starting from root, and not a file inside the archive)
 func (f *File) SymlinkPath() string {
@@ -174,18 +191,47 @@ func (f *File) GetSymlinkFile() *File {
 	return f.r.GetFileAtPath(f.SymlinkPath())
 }

-//Permission returns the os.FileMode of the File. Currently only has the permission bits (the last 9) populated.
+//Permission returns the os.FileMode of the File. Sets mode bits for directories and symlinks.
 func (f *File) Permission() os.FileMode {
-	//TODO: possibly populate more os.FileMode bits
-	return os.FileMode(f.in.Header.Permissions)
+	mode := os.FileMode(f.in.Header.Permissions)
+	switch {
+	case f.IsDir():
+		mode = mode | os.ModeDir
+	case f.IsSymlink():
+		mode = mode | os.ModeSymlink
+	}
+	return mode
 }

-func (f *File) ExtractTo(path string) error {
-	if f.IsDir() {
-		//TODO
-	} else if f.IsSymlink() {
+//ExtractTo extracts the file to the given path. This is the same as ExtractWithOptions(path, false, os.ModePerm).
+//Will NOT try to keep symlinks valid, folders extracted will have the permissions set by the squashfs, but the folder to make path will have full permissions (777).
+//
+//Will try it's best to extract all files, and if any errors come up, they will be appended to the error slice that's returned.
+func (f *File) ExtractTo(path string) []error {
+	return f.ExtractWithOptions(path, false, os.ModePerm)
+}

+//ExtractWithOptions will extract the file to the given path, while allowing customization on how it works. Usually just ExtractTo is good enough.
+//Will try it's best to extract all files, and if any errors come up, they will be appended to the error slice that's returned.
+//
+//If unbreakSymlink is set, it will also try to extract the symlink's associated file. WARNING: the symlink's file may have to go up the directory to work.
+//If unbreakSymlink is set and the file cannot be extracted, a ErrBrokenSymlink will be appended to the returned error slice.
+//
+//folderPerm only applies to the folders created to get to path. Folders from the archive are given the correct permissions defined by the archive.
+func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm os.FileMode) (errs []error) {
+	err := os.MkdirAll(path, folderPerm)
+	if err != nil {
+		return []error{err}
 	}
+	switch {
+	case f.IsDir():
+		return []error{errors.New("Dir not supported yet")}
+	case f.IsFile():
+
+	case f.IsSymlink():
+		return []error{errors.New("Symlink not supported yet")}
+	}
+	return errs
 }

 //Close frees up the memory held up by the underlying reader. Should NOT be called when writing.
@@ -4,4 +4,5 @@ go 1.15

 require (
 	github.com/CalebQ42/GoAppImage v0.4.0
+	github.com/ulikunitz/xz v0.5.8
 )
@@ -1,7 +1,5 @@
 github.com/CalebQ42/GoAppImage v0.4.0 h1:aF+Y/vyo/RGhoyZEW1CMY6WyRWrZZO4ydsRFAtIGnaY=
 github.com/CalebQ42/GoAppImage v0.4.0/go.mod h1:qHudJKAn/dlkNWNnH4h1YKXp29EZ7Bppsn7sNP2HuvU=
-github.com/CalebQ42/GoSquashfs v0.1.0 h1:1E6oeZLxGwjFgB0M5BcDD/IpKOQq1aO0gGsN0llCFoE=
-github.com/CalebQ42/GoSquashfs v0.1.0/go.mod h1:NzAR1YC1SVKOKhRao5IiWY3GhOMI+IxBy1xeZJeVKlQ=
 github.com/adrg/xdg v0.2.2 h1:A7ZHKRz5KGOLJX/bg7IPzStryhvCzAE1wX+KWawPiAo=
 github.com/adrg/xdg v0.2.2/go.mod h1:7I2hH/IT30IsupOpKZ5ue7/qNi3CoKzD6tL3HwpaRMQ=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -9,8 +7,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
@@ -29,6 +25,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ=
+github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 go.lsp.dev/uri v0.3.0 h1:KcZJmh6nFIBeJzTugn5JTU6OOyG0lDOo3R9KwTxTYbo=
 go.lsp.dev/uri v0.3.0/go.mod h1:P5sbO1IQR+qySTWOCnhnK7phBx+W3zbLqSMDJNTw88I=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
@@ -43,8 +41,6 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -0,0 +1,47 @@
+package compression
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/ulikunitz/xz"
+)
+
+//Xz is a decompressor for xz type compression
+type Xz struct{}
+
+//Decompress reads the entirety of the given reader and returns it uncompressed as a byte slice.
+func (z *Xz) Decompress(r io.Reader) ([]byte, error) {
+	rdr, err := xz.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+	err = rdr.Verify()
+	if err != nil {
+		return nil, err
+	}
+	var data bytes.Buffer
+	_, err = io.Copy(&data, rdr)
+	if err != nil {
+		return nil, err
+	}
+	return data.Bytes(), nil
+}
+
+//Compress compresses the given data (as a byte array) and returns the compressed data.
+func (z *Xz) Compress(data []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	wrt, err := xz.NewWriter(&buf)
+	if err != nil {
+		return nil, err
+	}
+	defer wrt.Close()
+	_, err = wrt.Write(data)
+	if err != nil {
+		return nil, err
+	}
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
@@ -2,7 +2,6 @@ package inode

 import (
 	"encoding/binary"
-	"fmt"
 	"io"
 )

@@ -66,15 +65,12 @@ func NewExtendedDirectory(rdr io.Reader) (ExtendedDirectory, error) {
 	if err != nil {
 		return inode, err
 	}
-	if inode.Init.IndexCount > 0 {
-		inode.Indexes = make([]DirectoryIndex, inode.Init.IndexCount)
-		for i := uint16(0); i < inode.Init.IndexCount; i++ {
-			inode.Indexes[i], err = NewDirectoryIndex(rdr)
-			if err != nil {
-				fmt.Println("Error while reading Directory Index ", i)
-				return inode, err
-			}
+	for i := uint16(0); i < inode.Init.IndexCount; i++ {
+		tmp, err := NewDirectoryIndex(rdr)
+		if err != nil {
+			return inode, err
 		}
+		inode.Indexes = append(inode.Indexes, tmp)
 	}
 	return inode, err
 }
@@ -89,7 +85,7 @@ type DirectoryIndexInit struct {
 //DirectoryIndex is a quick lookup provided by an ExtendedDirectory
 type DirectoryIndex struct {
 	Init DirectoryIndexInit
-	Name []byte
+	Name string
 }

 //NewDirectoryIndex return a new DirectoryIndex
@@ -99,9 +95,13 @@ func NewDirectoryIndex(rdr io.Reader) (DirectoryIndex, error) {
 	if err != nil {
 		return index, err
 	}
-	index.Name = make([]byte, index.Init.NameSize, index.Init.NameSize)
-	err = binary.Read(rdr, binary.LittleEndian, &index.Name)
-	return index, err
+	tmp := make([]byte, index.Init.NameSize+1, index.Init.NameSize+1)
+	err = binary.Read(rdr, binary.LittleEndian, &tmp)
+	if err != nil {
+		return index, err
+	}
+	index.Name = string(tmp)
+	return index, nil
 }

 //BasicFileInit is the information that can be directoy decoded
@@ -47,6 +47,8 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 	switch rdr.super.CompressionType {
 	case gzipCompression:
 		rdr.decompressor = &compression.Zlib{}
+	case xzCompression:
+		rdr.decompressor = &compression.Xz{}
 	default:
 		return nil, errIncompatibleCompression
 	}
@@ -81,7 +83,7 @@ func (r *Reader) GetRootFolder() (root *File, err error) {
 	if err != nil {
 		return nil, err
 	}
-	root.Path = "/"
+	root.path = "/"
 	root.filType = root.in.Type
 	root.r = r
 	return root, nil
@@ -8,14 +8,47 @@ import (
 	"testing"

 	goappimage "github.com/CalebQ42/GoAppImage"
+	"github.com/CalebQ42/squashfs/internal/inode"
 )

 const (
 	downloadURL  = "https://github.com/Swordfish90/cool-retro-term/releases/download/1.1.1/Cool-Retro-Term-1.1.1-x86_64.AppImage"
 	appImageName = "Cool-Retro-Term.AppImage"
+	squashfsName = "airootfs.sfs" //testing with a ArchLinux root fs from the live img
 )

-func TestMain(t *testing.T) {
+func TestSquashfs(t *testing.T) {
+	t.Parallel()
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	squashFil, err := os.Open(wd + "/testing/" + squashfsName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rdr, err := NewSquashfsReader(squashFil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fils, err := rdr.GetAllFiles()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, fil := range fils {
+		if fil.filType != inode.BasicFileType && fil.filType != inode.BasicDirectoryType && fil.filType != inode.BasicSymlinkType {
+			fmt.Println("Found non-standard")
+			fmt.Println(fil.Path())
+			fmt.Println("type:", fil.filType)
+		} else if fil.IsSymlink() {
+			fmt.Println("Symlink!")
+			fmt.Println(fil.Path())
+			fmt.Println("symlink path:", fil.SymlinkPath())
+		}
+	}
+}
+
+func TestAppImage(t *testing.T) {
 	t.Parallel()
 	wd, err := os.Getwd()
 	if err != nil {
@@ -40,7 +73,7 @@ func TestMain(t *testing.T) {
 	}
 	fil := rdr.GetFileAtPath("/usr/bin/cool-retro-term")
 	if fil != nil {
-		fmt.Println("Worked!", fil.Path+"/"+fil.Name)
+		fmt.Println("Worked!", fil.Path())
 	} else {
 		t.Fatal("NOOOOOO!")
 	}