From c5c62916434734338dbaf94a51d4ff1cee659cfe Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Mon, 30 Nov 2020 03:53:57 -0600 Subject: [PATCH] Some more work on ExtractTo Added Xz compression support Started testing using a big squashfs fil (particularly the squashfs from an Arch Linux install img) --- file.go | 64 +++++++++++++++++++++++++++++++----- go.mod | 1 + go.sum | 8 ++--- internal/compression/xz.go | 47 ++++++++++++++++++++++++++ internal/inode/inodetypes.go | 26 +++++++-------- reader.go | 4 ++- squash_test.go | 37 +++++++++++++++++++-- 7 files changed, 156 insertions(+), 31 deletions(-) create mode 100644 internal/compression/xz.go diff --git a/file.go b/file.go index a5d6d3d..0d4150e 100644 --- a/file.go +++ b/file.go @@ -2,6 +2,7 @@ package squashfs import ( "errors" + "fmt" "io" "os" "strings" @@ -26,7 +27,7 @@ type File struct { Name string //The name of the file or folder. Root folder will not have a name ("") Parent *File //The parent directory. Should ALWAYS be a folder. If it's the root directory, will be nil Reader io.Reader //Underlying reader. When writing, will probably be an os.File. When reading this is kept nil UNTIL reading to save memory. - Path string //The path to the folder the File is located in. + path string //The path to the folder the File is located in. r *Reader //The squashfs.Reader where this file is contained. in *inode.Inode //Underlyting inode when reading. filType int //The file's type, using inode types. @@ -56,6 +57,7 @@ func (f *File) GetChildren() (children []*File, err error) { } dir, err := f.r.readDirFromInode(f.in) if err != nil { + fmt.Println("err reading dir") return } var fil *File @@ -66,7 +68,7 @@ func (f *File) GetChildren() (children []*File, err error) { } fil.Parent = f if f.Name != "" { - fil.Path = f.Path + "/" + f.Name + fil.path = f.Path() } children = append(children, fil) } @@ -84,6 +86,7 @@ func (f *File) GetChildrenRecursively() (children []*File, err error) { } chil, err := f.GetChildren() if err != nil { + fmt.Println("err here", f.Path()) return } var childFolders []*File @@ -97,6 +100,7 @@ func (f *File) GetChildrenRecursively() (children []*File, err error) { var childs []*File childs, err = folds.GetChildrenRecursively() if err != nil { + fmt.Println("err here Recursive", folds.Path()) return } children = append(children, childs...) @@ -104,6 +108,14 @@ func (f *File) GetChildrenRecursively() (children []*File, err error) { return } +//Path returns the path of the file within the archive. +func (f *File) Path() string { + if f.Name == "" { + return f.path + } + return f.path + "/" + f.Name +} + //GetFileAtPath tries to return the File at the given path, relative to the file. //Returns nil if called on something other then a folder, OR if the path goes oustide the archive. func (f *File) GetFileAtPath(path string) *File { @@ -150,6 +162,11 @@ func (f *File) IsSymlink() bool { return f.filType == inode.BasicSymlinkType || f.filType == inode.ExtSymlinkType } +//IsFile returns if the file is a file. +func (f *File) IsFile() bool { + return f.filType == inode.BasicFileType || f.filType == inode.ExtFileType +} + //SymlinkPath returns the path the symlink is pointing to. If the file ISN'T a symlink, will return an empty string. //If a path begins with "/" then the symlink is pointing to an absolute path (starting from root, and not a file inside the archive) func (f *File) SymlinkPath() string { @@ -174,18 +191,47 @@ func (f *File) GetSymlinkFile() *File { return f.r.GetFileAtPath(f.SymlinkPath()) } -//Permission returns the os.FileMode of the File. Currently only has the permission bits (the last 9) populated. +//Permission returns the os.FileMode of the File. Sets mode bits for directories and symlinks. func (f *File) Permission() os.FileMode { - //TODO: possibly populate more os.FileMode bits - return os.FileMode(f.in.Header.Permissions) + mode := os.FileMode(f.in.Header.Permissions) + switch { + case f.IsDir(): + mode = mode | os.ModeDir + case f.IsSymlink(): + mode = mode | os.ModeSymlink + } + return mode } -func (f *File) ExtractTo(path string) error { - if f.IsDir() { - //TODO - } else if f.IsSymlink() { +//ExtractTo extracts the file to the given path. This is the same as ExtractWithOptions(path, false, os.ModePerm). +//Will NOT try to keep symlinks valid, folders extracted will have the permissions set by the squashfs, but the folder to make path will have full permissions (777). +// +//Will try it's best to extract all files, and if any errors come up, they will be appended to the error slice that's returned. +func (f *File) ExtractTo(path string) []error { + return f.ExtractWithOptions(path, false, os.ModePerm) +} +//ExtractWithOptions will extract the file to the given path, while allowing customization on how it works. Usually just ExtractTo is good enough. +//Will try it's best to extract all files, and if any errors come up, they will be appended to the error slice that's returned. +// +//If unbreakSymlink is set, it will also try to extract the symlink's associated file. WARNING: the symlink's file may have to go up the directory to work. +//If unbreakSymlink is set and the file cannot be extracted, a ErrBrokenSymlink will be appended to the returned error slice. +// +//folderPerm only applies to the folders created to get to path. Folders from the archive are given the correct permissions defined by the archive. +func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm os.FileMode) (errs []error) { + err := os.MkdirAll(path, folderPerm) + if err != nil { + return []error{err} } + switch { + case f.IsDir(): + return []error{errors.New("Dir not supported yet")} + case f.IsFile(): + + case f.IsSymlink(): + return []error{errors.New("Symlink not supported yet")} + } + return errs } //Close frees up the memory held up by the underlying reader. Should NOT be called when writing. diff --git a/go.mod b/go.mod index 332880b..0f61557 100644 --- a/go.mod +++ b/go.mod @@ -4,4 +4,5 @@ go 1.15 require ( github.com/CalebQ42/GoAppImage v0.4.0 + github.com/ulikunitz/xz v0.5.8 ) diff --git a/go.sum b/go.sum index 5ffe4c3..55af131 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/CalebQ42/GoAppImage v0.4.0 h1:aF+Y/vyo/RGhoyZEW1CMY6WyRWrZZO4ydsRFAtIGnaY= github.com/CalebQ42/GoAppImage v0.4.0/go.mod h1:qHudJKAn/dlkNWNnH4h1YKXp29EZ7Bppsn7sNP2HuvU= -github.com/CalebQ42/GoSquashfs v0.1.0 h1:1E6oeZLxGwjFgB0M5BcDD/IpKOQq1aO0gGsN0llCFoE= -github.com/CalebQ42/GoSquashfs v0.1.0/go.mod h1:NzAR1YC1SVKOKhRao5IiWY3GhOMI+IxBy1xeZJeVKlQ= github.com/adrg/xdg v0.2.2 h1:A7ZHKRz5KGOLJX/bg7IPzStryhvCzAE1wX+KWawPiAo= github.com/adrg/xdg v0.2.2/go.mod h1:7I2hH/IT30IsupOpKZ5ue7/qNi3CoKzD6tL3HwpaRMQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -9,8 +7,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -29,6 +25,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= go.lsp.dev/uri v0.3.0 h1:KcZJmh6nFIBeJzTugn5JTU6OOyG0lDOo3R9KwTxTYbo= go.lsp.dev/uri v0.3.0/go.mod h1:P5sbO1IQR+qySTWOCnhnK7phBx+W3zbLqSMDJNTw88I= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= @@ -43,8 +41,6 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/compression/xz.go b/internal/compression/xz.go new file mode 100644 index 0000000..c26dfdc --- /dev/null +++ b/internal/compression/xz.go @@ -0,0 +1,47 @@ +package compression + +import ( + "bytes" + "io" + + "github.com/ulikunitz/xz" +) + +//Xz is a decompressor for xz type compression +type Xz struct{} + +//Decompress reads the entirety of the given reader and returns it uncompressed as a byte slice. +func (z *Xz) Decompress(r io.Reader) ([]byte, error) { + rdr, err := xz.NewReader(r) + if err != nil { + return nil, err + } + err = rdr.Verify() + if err != nil { + return nil, err + } + var data bytes.Buffer + _, err = io.Copy(&data, rdr) + if err != nil { + return nil, err + } + return data.Bytes(), nil +} + +//Compress compresses the given data (as a byte array) and returns the compressed data. +func (z *Xz) Compress(data []byte) ([]byte, error) { + var buf bytes.Buffer + wrt, err := xz.NewWriter(&buf) + if err != nil { + return nil, err + } + defer wrt.Close() + _, err = wrt.Write(data) + if err != nil { + return nil, err + } + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/internal/inode/inodetypes.go b/internal/inode/inodetypes.go index f9b7680..fe679c3 100644 --- a/internal/inode/inodetypes.go +++ b/internal/inode/inodetypes.go @@ -2,7 +2,6 @@ package inode import ( "encoding/binary" - "fmt" "io" ) @@ -66,15 +65,12 @@ func NewExtendedDirectory(rdr io.Reader) (ExtendedDirectory, error) { if err != nil { return inode, err } - if inode.Init.IndexCount > 0 { - inode.Indexes = make([]DirectoryIndex, inode.Init.IndexCount) - for i := uint16(0); i < inode.Init.IndexCount; i++ { - inode.Indexes[i], err = NewDirectoryIndex(rdr) - if err != nil { - fmt.Println("Error while reading Directory Index ", i) - return inode, err - } + for i := uint16(0); i < inode.Init.IndexCount; i++ { + tmp, err := NewDirectoryIndex(rdr) + if err != nil { + return inode, err } + inode.Indexes = append(inode.Indexes, tmp) } return inode, err } @@ -89,7 +85,7 @@ type DirectoryIndexInit struct { //DirectoryIndex is a quick lookup provided by an ExtendedDirectory type DirectoryIndex struct { Init DirectoryIndexInit - Name []byte + Name string } //NewDirectoryIndex return a new DirectoryIndex @@ -99,9 +95,13 @@ func NewDirectoryIndex(rdr io.Reader) (DirectoryIndex, error) { if err != nil { return index, err } - index.Name = make([]byte, index.Init.NameSize, index.Init.NameSize) - err = binary.Read(rdr, binary.LittleEndian, &index.Name) - return index, err + tmp := make([]byte, index.Init.NameSize+1, index.Init.NameSize+1) + err = binary.Read(rdr, binary.LittleEndian, &tmp) + if err != nil { + return index, err + } + index.Name = string(tmp) + return index, nil } //BasicFileInit is the information that can be directoy decoded diff --git a/reader.go b/reader.go index e62264f..0ffbfe9 100644 --- a/reader.go +++ b/reader.go @@ -47,6 +47,8 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) { switch rdr.super.CompressionType { case gzipCompression: rdr.decompressor = &compression.Zlib{} + case xzCompression: + rdr.decompressor = &compression.Xz{} default: return nil, errIncompatibleCompression } @@ -81,7 +83,7 @@ func (r *Reader) GetRootFolder() (root *File, err error) { if err != nil { return nil, err } - root.Path = "/" + root.path = "/" root.filType = root.in.Type root.r = r return root, nil diff --git a/squash_test.go b/squash_test.go index 6afb907..64efb1e 100644 --- a/squash_test.go +++ b/squash_test.go @@ -8,14 +8,47 @@ import ( "testing" goappimage "github.com/CalebQ42/GoAppImage" + "github.com/CalebQ42/squashfs/internal/inode" ) const ( downloadURL = "https://github.com/Swordfish90/cool-retro-term/releases/download/1.1.1/Cool-Retro-Term-1.1.1-x86_64.AppImage" appImageName = "Cool-Retro-Term.AppImage" + squashfsName = "airootfs.sfs" //testing with a ArchLinux root fs from the live img ) -func TestMain(t *testing.T) { +func TestSquashfs(t *testing.T) { + t.Parallel() + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + squashFil, err := os.Open(wd + "/testing/" + squashfsName) + if err != nil { + t.Fatal(err) + } + rdr, err := NewSquashfsReader(squashFil) + if err != nil { + t.Fatal(err) + } + fils, err := rdr.GetAllFiles() + if err != nil { + t.Fatal(err) + } + for _, fil := range fils { + if fil.filType != inode.BasicFileType && fil.filType != inode.BasicDirectoryType && fil.filType != inode.BasicSymlinkType { + fmt.Println("Found non-standard") + fmt.Println(fil.Path()) + fmt.Println("type:", fil.filType) + } else if fil.IsSymlink() { + fmt.Println("Symlink!") + fmt.Println(fil.Path()) + fmt.Println("symlink path:", fil.SymlinkPath()) + } + } +} + +func TestAppImage(t *testing.T) { t.Parallel() wd, err := os.Getwd() if err != nil { @@ -40,7 +73,7 @@ func TestMain(t *testing.T) { } fil := rdr.GetFileAtPath("/usr/bin/cool-retro-term") if fil != nil { - fmt.Println("Worked!", fil.Path+"/"+fil.Name) + fmt.Println("Worked!", fil.Path()) } else { t.Fatal("NOOOOOO!") }