Compare commits

...

13 Commits

Author SHA1 Message Date
Caleb Gardner e5d4d0902f Finished wildcard support.
Realized that path.Match works perfectly for my wildcard needs.
2020-12-07 10:41:45 -06:00
Caleb Gardner c0f3695cca Working on wildcards before next release. 2020-12-05 15:05:19 -06:00
Caleb Gardner 89b0a41ab9 Figured out it was the filters for XZ that was causing problems
Added lzma and xz decompression.
Renamed Zlib to Gzip
2020-12-04 05:36:58 -06:00
Caleb Gardner a894e2efb9 Removed XZ compression.
When testing the archlinux, xz compression couldn't seem to extract ANY files.
Testing extraction with the appimage works though. So that's good.
2020-12-02 01:56:24 -06:00
Caleb Gardner fef7bec20d Extracting is mostly finished.
Fixed reading metadata and data blocks that are the exactly correct size
2020-12-02 01:05:31 -06:00
Caleb Gardner 28bb0f873a Things just aren't working right now...
Had to re-write a seemingly fine for loop because it wouldn't increment
Extracting just isn't working right now.
I'm waaaay too tired for this right now, lol.
2020-12-01 15:29:25 -06:00
Caleb Gardner e302d98665 Further work on extracting
Extracting files is threaded with goroutines.
Extracting also sets the proper UID and GUID
Made recursive children getting threaded with goroutines
Decided I will allow wildcards in paths. Hasn't been implemented though.
Fixed issues with ExtendedDirectory reading (I was using a uint16 instead of a uint32)
I couldn't test basically any of this. somehow a for loop isn't incrementing. For seemingly no reason.
2020-12-01 06:14:09 -06:00
Caleb Gardner 1254df2861 Working on extracting files. 2020-11-30 15:13:47 -06:00
Caleb Gardner c5c6291643 Some more work on ExtractTo
Added Xz compression support
Started testing using a big squashfs fil (particularly the squashfs from an Arch Linux install img)
2020-11-30 03:53:57 -06:00
Caleb Gardner 508a33b323 Starting work on ExtractTo from File
When using ExtractTo, will automatically set the correct permissions
Will also be able to extract folders
2020-11-29 15:55:06 -06:00
Caleb Gardner c5f1962e72 Some documentation changes 2020-11-29 03:07:34 -06:00
Caleb Gardner faf9153323 Added zlib compression for absolutely zero reason 2020-11-28 11:05:38 -06:00
Caleb Gardner e20213c3f7 Reorganization. 2020-11-28 05:03:56 -06:00
16 changed files with 616 additions and 229 deletions
-33
View File
@@ -1,33 +0,0 @@
package squashfs
import (
"bytes"
"compress/zlib"
"io"
)
//Decompressor is a squashfs decompressor interface. Allows for easy decompression no matter the type of compression.
type decompressor interface {
Decompress(io.Reader) ([]byte, error)
}
type compressor interface {
Compress(io.Reader) ([]byte, error)
}
//ZlibDecompressor is a decompressor for gzip type compression
type zlibDecompressor struct{}
//Decompress reads the entirety of the given reader and returns it uncompressed as a byte slice.
func (z *zlibDecompressor) Decompress(r io.Reader) ([]byte, error) {
rdr, err := zlib.NewReader(r)
if err != nil {
return nil, err
}
var data bytes.Buffer
_, err = io.Copy(&data, rdr)
if err != nil {
return nil, err
}
return data.Bytes(), nil
}
+5 -2
View File
@@ -153,9 +153,12 @@ func (d *dataReader) Close() error {
func (d *dataReader) Read(p []byte) (int, error) {
if d.curData == nil {
d.readCurBlock()
err := d.readCurBlock()
if err != nil {
return 0, err
}
}
if d.curReadOffset+len(p) < len(d.curData) {
if d.curReadOffset+len(p) <= len(d.curData) {
for i := 0; i < len(p); i++ {
p[i] = d.curData[d.curReadOffset+i]
}
+295 -29
View File
@@ -2,7 +2,10 @@ package squashfs
import (
"errors"
"fmt"
"io"
"os"
"path"
"strings"
"github.com/CalebQ42/squashfs/internal/directory"
@@ -11,11 +14,13 @@ import (
var (
//ErrNotDirectory is returned when you're trying to do directory things with a non-directory
ErrNotDirectory = errors.New("File is not a directory")
errNotDirectory = errors.New("File is not a directory")
//ErrNotFile is returned when you're trying to do file things with a directory
ErrNotFile = errors.New("File is not a file")
errNotFile = errors.New("File is not a file")
//ErrNotReading is returned when running functions that are only meant to be used when reading a squashfs
ErrNotReading = errors.New("Function only supported when reading a squashfs")
errNotReading = errors.New("Function only supported when reading a squashfs")
//ErrBrokenSymlink is returned when using ExtractWithOptions with the unbreakSymlink set to true, but the symlink's file cannot be extracted.
ErrBrokenSymlink = errors.New("Extracted symlink is probably broken")
)
//File is the main way to interact with files within squashfs, or when putting files into a squashfs.
@@ -25,7 +30,7 @@ type File struct {
Name string //The name of the file or folder. Root folder will not have a name ("")
Parent *File //The parent directory. Should ALWAYS be a folder. If it's the root directory, will be nil
Reader io.Reader //Underlying reader. When writing, will probably be an os.File. When reading this is kept nil UNTIL reading to save memory.
Path string //The path to the folder the File is located in.
path string //The path to the folder the File is located in.
r *Reader //The squashfs.Reader where this file is contained.
in *inode.Inode //Underlyting inode when reading.
filType int //The file's type, using inode types.
@@ -48,10 +53,10 @@ func (r *Reader) newFileFromDirEntry(entry *directory.Entry) (fil *File, err err
func (f *File) GetChildren() (children []*File, err error) {
children = make([]*File, 0)
if f.r == nil {
return nil, ErrNotReading
return nil, errNotReading
}
if !f.IsDir() {
return nil, ErrNotDirectory
return nil, errNotDirectory
}
dir, err := f.r.readDirFromInode(f.in)
if err != nil {
@@ -65,7 +70,7 @@ func (f *File) GetChildren() (children []*File, err error) {
}
fil.Parent = f
if f.Name != "" {
fil.Path = f.Path + "/" + f.Name
fil.path = f.Path()
}
children = append(children, fil)
}
@@ -76,48 +81,64 @@ func (f *File) GetChildren() (children []*File, err error) {
func (f *File) GetChildrenRecursively() (children []*File, err error) {
children = make([]*File, 0)
if f.r == nil {
return nil, ErrNotReading
return nil, errNotReading
}
if !f.IsDir() {
return nil, ErrNotDirectory
return nil, errNotDirectory
}
chil, err := f.GetChildren()
children, err = f.GetChildren()
if err != nil {
return
}
var childFolders []*File
for _, child := range chil {
children = append(children, child)
for _, child := range children {
if child.IsDir() {
childFolders = append(childFolders, child)
}
}
foldChil := make(chan []*File)
errChan := make(chan error)
for _, folds := range childFolders {
var childs []*File
childs, err = folds.GetChildrenRecursively()
go func(fil *File) {
childs, err := fil.GetChildrenRecursively()
errChan <- err
foldChil <- childs
}(folds)
}
for range childFolders {
err = <-errChan
if err != nil {
return
}
children = append(children, childs...)
children = append(children, <-foldChil...)
}
return
}
//Path returns the path of the file within the archive.
func (f *File) Path() string {
if f.Name == "" {
return f.path
}
return f.path + "/" + f.Name
}
//GetFileAtPath tries to return the File at the given path, relative to the file.
//Returns nil if called on something other then a folder, OR if the path goes oustide the archive.
func (f *File) GetFileAtPath(path string) *File {
if path == "" {
//Allows wildcards supported by path.Match (namely * and ?).
func (f *File) GetFileAtPath(dirPath string) *File {
if dirPath == "" {
return f
}
path = strings.TrimSuffix(strings.TrimPrefix(path, "/"), "/")
if path != "" && !f.IsDir() {
dirPath = strings.TrimSuffix(strings.TrimPrefix(dirPath, "/"), "/")
if dirPath != "" && !f.IsDir() {
return nil
}
for strings.HasSuffix(path, "./") {
for strings.HasSuffix(dirPath, "./") {
//since you can TECHNICALLY have an infinite amount of ./ and it would still be valid.
path = strings.TrimPrefix(path, "./")
dirPath = strings.TrimPrefix(dirPath, "./")
}
split := strings.Split(path, "/")
split := strings.Split(dirPath, "/")
if split[0] == ".." && f.Name == "" {
return nil
} else if split[0] == ".." {
@@ -130,9 +151,9 @@ func (f *File) GetFileAtPath(path string) *File {
if err != nil {
return nil
}
for _, child := range children {
if child.Name == split[0] {
eq, _ := path.Match(split[0], child.Name)
if eq {
return child.GetFileAtPath(strings.Join(split[1:], "/"))
}
}
@@ -149,7 +170,13 @@ func (f *File) IsSymlink() bool {
return f.filType == inode.BasicSymlinkType || f.filType == inode.ExtSymlinkType
}
//SymlinkPath returns the path the symlink is pointing to. If the file ISN'T a symlink, will return an empty string
//IsFile returns if the file is a file.
func (f *File) IsFile() bool {
return f.filType == inode.BasicFileType || f.filType == inode.ExtFileType
}
//SymlinkPath returns the path the symlink is pointing to. If the file ISN'T a symlink, will return an empty string.
//If a path begins with "/" then the symlink is pointing to an absolute path (starting from root, and not a file inside the archive)
func (f *File) SymlinkPath() string {
switch f.filType {
case inode.BasicSymlinkType:
@@ -172,16 +199,205 @@ func (f *File) GetSymlinkFile() *File {
return f.r.GetFileAtPath(f.SymlinkPath())
}
//Permission returns the os.FileMode of the File. Sets mode bits for directories and symlinks.
func (f *File) Permission() os.FileMode {
mode := os.FileMode(f.in.Header.Permissions)
switch {
case f.IsDir():
mode = mode | os.ModeDir
case f.IsSymlink():
mode = mode | os.ModeSymlink
}
return mode
}
//ExtractTo extracts the file to the given path. This is the same as ExtractWithOptions(path, false, os.ModePerm, false).
//Will NOT try to keep symlinks valid, folders extracted will have the permissions set by the squashfs, but the folder to make path will have full permissions (777).
//
//Will try it's best to extract all files, and if any errors come up, they will be appended to the error slice that's returned.
func (f *File) ExtractTo(path string) []error {
return f.ExtractWithOptions(path, false, os.ModePerm, false)
}
//ExtractWithOptions will extract the file to the given path, while allowing customization on how it works. ExtractTo is the "default" options.
//Will try it's best to extract all files, and if any errors come up, they will be appended to the error slice that's returned.
//Should only return multiple errors if extracting a folder.
//
//If unbreakSymlink is set, it will also try to extract the symlink's associated file. WARNING: the symlink's file may have to go up the directory to work.
//If unbreakSymlink is set and the file cannot be extracted, a ErrBrokenSymlink will be appended to the returned error slice.
//
//folderPerm only applies to the folders created to get to path. Folders from the archive are given the correct permissions defined by the archive.
func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm os.FileMode, verbose bool) (errs []error) {
errs = make([]error, 0)
err := os.MkdirAll(path, folderPerm)
if err != nil {
return []error{err}
}
switch {
case f.IsDir():
if f.Name != "" {
//TODO: check if folder is present, and if so, try to set it's permission
err = os.Mkdir(path+"/"+f.Name, os.ModePerm)
if err != nil {
if verbose {
fmt.Println("Error while making: ", path+"/"+f.Name)
fmt.Println(err)
}
errs = append(errs, err)
return
}
fil, err := os.Open(path + "/" + f.Name)
if err != nil {
if verbose {
fmt.Println("Error while opening:", path+"/"+f.Name)
fmt.Println(err)
}
errs = append(errs, err)
return
}
fil.Chown(int(f.r.idTable[f.in.Header.UID]), int(f.r.idTable[f.in.Header.GID]))
//don't mention anything when it fails. Because it fails often. Probably has something to do about uid & gid 0
// if err != nil {
// if verbose {
// fmt.Println("Error while changing owner:", path+"/"+f.Name)
// fmt.Println(err)
// }
// errs = append(errs, err)
// }
err = fil.Chmod(f.Permission())
if err != nil {
if verbose {
fmt.Println("Error while changing owner:", path+"/"+f.Name)
fmt.Println(err)
}
errs = append(errs, err)
}
}
children, err := f.GetChildren()
if err != nil {
if verbose {
fmt.Println("Error getting children for:", f.Path())
fmt.Println(err)
}
errs = append(errs, err)
return
}
finishChan := make(chan []error)
defer close(finishChan)
for _, child := range children {
go func(child *File) {
if f.Name == "" {
finishChan <- child.ExtractWithOptions(path, unbreakSymlink, folderPerm, verbose)
} else {
finishChan <- child.ExtractWithOptions(path+"/"+f.Name, unbreakSymlink, folderPerm, verbose)
}
}(child)
}
for range children {
errs = append(errs, (<-finishChan)...)
}
return
case f.IsFile():
fil, err := os.Create(path + "/" + f.Name)
if os.IsExist(err) {
err = os.Remove(path + "/" + f.Name)
if err != nil {
if verbose {
fmt.Println("Error while making:", path+"/"+f.Name)
fmt.Println(err)
}
errs = append(errs, err)
return
}
fil, err = os.Create(path + "/" + f.Name)
if err != nil {
if verbose {
fmt.Println("Error while making:", path+"/"+f.Name)
fmt.Println(err)
}
errs = append(errs, err)
return
}
} else if err != nil {
if verbose {
fmt.Println("Error while making:", path+"/"+f.Name)
fmt.Println(err)
}
errs = append(errs, err)
return
} //Since we will be reading from the file
_, err = io.Copy(fil, f)
if err != nil {
if verbose {
fmt.Println("Error while Copying data to:", path+"/"+f.Name)
fmt.Println(err)
}
errs = append(errs, err)
return
}
f.Close()
fil.Chown(int(f.r.idTable[f.in.Header.UID]), int(f.r.idTable[f.in.Header.GID]))
//don't mention anything when it fails. Because it fails often. Probably has something to do about uid & gid 0
// if err != nil {
// if verbose {
// fmt.Println("Error while changing owner:", path+"/"+f.Name)
// fmt.Println(err)
// }
// errs = append(errs, err)
// return
// }
err = fil.Chmod(f.Permission())
if err != nil {
if verbose {
fmt.Println("Error while setting permissions for:", path+"/"+f.Name)
fmt.Println(err)
}
errs = append(errs, err)
}
return
case f.IsSymlink():
symPath := f.SymlinkPath()
if unbreakSymlink {
fil := f.GetSymlinkFile()
if fil != nil {
symPath = path + "/" + symPath
paths := strings.Split(symPath, "/")
extracSymErrs := fil.ExtractWithOptions(strings.Join(paths[:len(paths)-1], "/"), unbreakSymlink, folderPerm, verbose)
if len(extracSymErrs) > 0 {
if verbose {
fmt.Println("Error(s) while extracting the symlink's file:", path+"/"+f.Name)
fmt.Println(extracSymErrs)
}
errs = append(errs, extracSymErrs...)
}
} else if verbose {
fmt.Println("Symlink path(", symPath, ") is outside the archive:"+path+"/"+f.Name)
}
}
err = os.Symlink(f.SymlinkPath(), path+"/"+f.Name)
if err != nil {
if verbose {
fmt.Println("Error while making symlink:", path+"/"+f.Name)
fmt.Println(err)
}
errs = append(errs, err)
}
}
return
}
//Close frees up the memory held up by the underlying reader. Should NOT be called when writing.
//When reading, Close is safe to use, but any subsequent Read calls resets to the beginning of the file.
func (f *File) Close() error {
if f.IsDir() {
return ErrNotFile
return errNotFile
}
if closer, is := f.Reader.(io.Closer); is {
closer.Close()
if f.Reader != nil {
if closer, is := f.Reader.(io.Closer); is {
closer.Close()
}
f.Reader = nil
}
f.Reader = nil
return nil
}
@@ -199,3 +415,53 @@ func (f *File) Read(p []byte) (int, error) {
}
return f.Reader.Read(p)
}
//ReadDirFromInode returns a fully populated Directory from a given Inode.
//If the given inode is not a directory it returns an error.
func (r *Reader) readDirFromInode(i *inode.Inode) (*directory.Directory, error) {
var offset uint32
var metaOffset uint16
var size uint32
switch i.Type {
case inode.BasicDirectoryType:
offset = i.Info.(inode.BasicDirectory).DirectoryIndex
metaOffset = i.Info.(inode.BasicDirectory).DirectoryOffset
size = uint32(i.Info.(inode.BasicDirectory).DirectorySize)
case inode.ExtDirType:
offset = i.Info.(inode.ExtendedDirectory).Init.DirectoryIndex
metaOffset = i.Info.(inode.ExtendedDirectory).Init.DirectoryOffset
size = i.Info.(inode.ExtendedDirectory).Init.DirectorySize
default:
return nil, errors.New("Not a directory inode")
}
br, err := r.newMetadataReader(int64(r.super.DirTableStart + uint64(offset)))
if err != nil {
return nil, err
}
_, err = br.Seek(int64(metaOffset), io.SeekStart)
if err != nil {
return nil, err
}
dir, err := directory.NewDirectory(br, size)
if err != nil {
return dir, err
}
return dir, nil
}
//GetInodeFromEntry returns the inode associated with a given directory.Entry
func (r *Reader) getInodeFromEntry(en *directory.Entry) (*inode.Inode, error) {
br, err := r.newMetadataReader(int64(r.super.InodeTableStart + uint64(en.Header.InodeOffset)))
if err != nil {
return nil, err
}
_, err = br.Seek(int64(en.Init.Offset), io.SeekStart)
if err != nil {
return nil, err
}
i, err := inode.ProcessInode(br, r.super.BlockSize)
if err != nil {
return nil, err
}
return i, nil
}
+2 -2
View File
@@ -22,7 +22,7 @@ type fileReader struct {
var (
//ErrPathIsNotFile returns when trying to read from a file, but the given path is NOT a file.
ErrPathIsNotFile = errors.New("The given path is not a file")
errPathIsNotFile = errors.New("The given path is not a file")
)
//ReadFile provides a squashfs.FileReader for the file at the given location.
@@ -30,7 +30,7 @@ func (r *Reader) newFileReader(in *inode.Inode) (*fileReader, error) {
var rdr fileReader
rdr.in = in
if in.Type != inode.BasicFileType && in.Type != inode.ExtFileType {
return nil, ErrPathIsNotFile
return nil, errPathIsNotFile
}
switch in.Type {
case inode.BasicFileType:
+9
View File
@@ -4,4 +4,13 @@ go 1.15
require (
github.com/CalebQ42/GoAppImage v0.4.0
github.com/adrg/xdg v0.2.3 // indirect
github.com/google/go-cmp v0.5.4 // indirect
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/smartystreets/assertions v1.2.0 // indirect
github.com/ulikunitz/xz v0.5.8
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
)
+17 -2
View File
@@ -1,9 +1,10 @@
github.com/CalebQ42/GoAppImage v0.4.0 h1:aF+Y/vyo/RGhoyZEW1CMY6WyRWrZZO4ydsRFAtIGnaY=
github.com/CalebQ42/GoAppImage v0.4.0/go.mod h1:qHudJKAn/dlkNWNnH4h1YKXp29EZ7Bppsn7sNP2HuvU=
github.com/CalebQ42/GoSquashfs v0.1.0 h1:1E6oeZLxGwjFgB0M5BcDD/IpKOQq1aO0gGsN0llCFoE=
github.com/CalebQ42/GoSquashfs v0.1.0/go.mod h1:NzAR1YC1SVKOKhRao5IiWY3GhOMI+IxBy1xeZJeVKlQ=
github.com/adrg/xdg v0.2.2 h1:A7ZHKRz5KGOLJX/bg7IPzStryhvCzAE1wX+KWawPiAo=
github.com/adrg/xdg v0.2.2/go.mod h1:7I2hH/IT30IsupOpKZ5ue7/qNi3CoKzD6tL3HwpaRMQ=
github.com/adrg/xdg v0.2.3 h1:GxXngdYxNDkoUvZXjNJGwqZxWXi43MKbOOlA/00qZi4=
github.com/adrg/xdg v0.2.3/go.mod h1:7I2hH/IT30IsupOpKZ5ue7/qNi3CoKzD6tL3HwpaRMQ=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -13,22 +14,32 @@ github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ=
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
go.lsp.dev/uri v0.3.0 h1:KcZJmh6nFIBeJzTugn5JTU6OOyG0lDOo3R9KwTxTYbo=
go.lsp.dev/uri v0.3.0/go.mod h1:P5sbO1IQR+qySTWOCnhnK7phBx+W3zbLqSMDJNTw88I=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
@@ -48,7 +59,11 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+40
View File
@@ -0,0 +1,40 @@
package compression
import (
"bytes"
"compress/zlib"
"io"
)
//Gzip is a decompressor for gzip type compression. Uses zlib for compression and decompression
type Gzip struct{}
//Decompress reads the entirety of the given reader and returns it uncompressed as a byte slice.
func (g *Gzip) Decompress(r io.Reader) ([]byte, error) {
rdr, err := zlib.NewReader(r)
if err != nil {
return nil, err
}
var data bytes.Buffer
_, err = io.Copy(&data, rdr)
if err != nil {
return nil, err
}
return data.Bytes(), nil
}
//Compress compresses the given data (as a byte array) and returns the compressed data.
func (g *Gzip) Compress(data []byte) ([]byte, error) {
var buf bytes.Buffer
wrt := zlib.NewWriter(&buf)
defer wrt.Close()
_, err := wrt.Write(data)
if err != nil {
return nil, err
}
err = wrt.Flush()
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
+25
View File
@@ -0,0 +1,25 @@
package compression
import (
"bytes"
"io"
"github.com/ulikunitz/xz/lzma"
)
//Lzma is a lzma decompressor
type Lzma struct{}
//Decompress decompresses all the data in the given reader and returns the uncompressed bytes.
func (l *Lzma) Decompress(rdr io.Reader) ([]byte, error) {
r, err := lzma.NewReader(rdr)
if err != nil {
return nil, err
}
var buf bytes.Buffer
_, err = io.Copy(&buf, r)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
+13
View File
@@ -0,0 +1,13 @@
package compression
import "io"
//Compressor is a squashfs decompressor interface. Allows for easy compression.
type Compressor interface {
Compress([]byte) ([]byte, error)
}
//Decompressor is a squashfs decompressor interface. Allows for easy decompression no matter the type of compression.
type Decompressor interface {
Decompress(io.Reader) ([]byte, error)
}
+55
View File
@@ -0,0 +1,55 @@
package compression
import (
"bytes"
"encoding/binary"
"io"
"github.com/ulikunitz/xz"
)
type xzInit struct {
DictionarySize int32
Filters int32
}
//Xz is a Xz decompressor.
type Xz struct {
DictionarySize int32
HasFilters bool
}
//NewXzCompressorWithOptions creates a new Xz compressor/decompressor that reads the compressor options from the given reader.
func NewXzCompressorWithOptions(rdr io.Reader) (*Xz, error) {
var x Xz
var init xzInit
err := binary.Read(rdr, binary.LittleEndian, &init)
if err != nil {
return nil, err
}
x.DictionarySize = init.DictionarySize
//TODO: When I can do filters, parse the filters
if init.Filters != 0 {
x.HasFilters = true
}
return &x, nil
}
//Decompress decompresses all the data from the rdr and returns the uncompressed bytes.
func (x *Xz) Decompress(rdr io.Reader) ([]byte, error) {
r, err := xz.NewReader(rdr)
if err != nil {
return nil, err
}
r.DictCap = int(x.DictionarySize)
err = r.Verify()
if err != nil {
return nil, err
}
var buf bytes.Buffer
_, err = io.Copy(&buf, r)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
+1 -1
View File
@@ -52,7 +52,7 @@ type Directory struct {
}
//NewDirectory reads the directory from rdr
func NewDirectory(base io.Reader, size uint16) (*Directory, error) {
func NewDirectory(base io.Reader, size uint32) (*Directory, error) {
var dir Directory
var err error
tmp := make([]byte, size)
+13 -13
View File
@@ -2,7 +2,6 @@ package inode
import (
"encoding/binary"
"fmt"
"io"
)
@@ -66,15 +65,12 @@ func NewExtendedDirectory(rdr io.Reader) (ExtendedDirectory, error) {
if err != nil {
return inode, err
}
if inode.Init.IndexCount > 0 {
inode.Indexes = make([]DirectoryIndex, inode.Init.IndexCount)
for i := uint16(0); i < inode.Init.IndexCount; i++ {
inode.Indexes[i], err = NewDirectoryIndex(rdr)
if err != nil {
fmt.Println("Error while reading Directory Index ", i)
return inode, err
}
for i := uint16(0); i < inode.Init.IndexCount; i++ {
tmp, err := NewDirectoryIndex(rdr)
if err != nil {
return inode, err
}
inode.Indexes = append(inode.Indexes, tmp)
}
return inode, err
}
@@ -89,7 +85,7 @@ type DirectoryIndexInit struct {
//DirectoryIndex is a quick lookup provided by an ExtendedDirectory
type DirectoryIndex struct {
Init DirectoryIndexInit
Name []byte
Name string
}
//NewDirectoryIndex return a new DirectoryIndex
@@ -99,9 +95,13 @@ func NewDirectoryIndex(rdr io.Reader) (DirectoryIndex, error) {
if err != nil {
return index, err
}
index.Name = make([]byte, index.Init.NameSize, index.Init.NameSize)
err = binary.Read(rdr, binary.LittleEndian, &index.Name)
return index, err
tmp := make([]byte, index.Init.NameSize+1, index.Init.NameSize+1)
err = binary.Read(rdr, binary.LittleEndian, &tmp)
if err != nil {
return index, err
}
index.Name = string(tmp)
return index, nil
}
//BasicFileInit is the information that can be directoy decoded
+13 -2
View File
@@ -52,6 +52,17 @@ func (s *Reader) newMetadataReaderFromInodeRef(ref uint64) (*metadataReader, err
return br, nil
}
//ProcessInodeRef processes an inode reference and returns two values
//
//The first value is the inode table offset. AKA, it's where the metadata block of the inode STARTS relative to the inode table.
//
//The second value is the offset of the inode, INSIDE of the metadata.
func processInodeRef(inodeRef uint64) (tableOffset uint64, metaOffset uint64) {
tableOffset = inodeRef >> 16
metaOffset = inodeRef &^ 0xFFFFFFFF0000
return
}
func (br *metadataReader) parseMetadata() error {
var raw uint16
err := binary.Read(io.NewSectionReader(br.s.r, br.offset, 2), binary.LittleEndian, &raw)
@@ -59,7 +70,7 @@ func (br *metadataReader) parseMetadata() error {
return err
}
br.offset += 2
compressed := !(raw&0x8000 == 0x8000)
compressed := raw&0x8000 != 0x8000
size := raw &^ 0x8000
br.headers = append(br.headers, &metadata{
raw: raw,
@@ -93,7 +104,7 @@ func (br *metadataReader) readNextDataBlock() error {
//Read reads bytes into the given byte slice. Returns the amount of data read.
func (br *metadataReader) Read(p []byte) (int, error) {
if br.readOffset+len(p) < len(br.data) {
if br.readOffset+len(p) <= len(br.data) {
for i := 0; i < len(p); i++ {
p[i] = br.data[br.readOffset+i]
}
+60 -19
View File
@@ -6,6 +6,7 @@ import (
"io"
"math"
"github.com/CalebQ42/squashfs/internal/compression"
"github.com/CalebQ42/squashfs/internal/inode"
)
@@ -15,24 +16,21 @@ const (
var (
//ErrNoMagic is returned if the magic number in the superblock isn't correct.
ErrNoMagic = errors.New("Magic number doesn't match. Either isn't a squashfs or corrupted")
errNoMagic = errors.New("Magic number doesn't match. Either isn't a squashfs or corrupted")
//ErrIncompatibleCompression is returned if the compression type in the superblock doesn't work.
ErrIncompatibleCompression = errors.New("Compression type unsupported")
errIncompatibleCompression = errors.New("Compression type unsupported")
//ErrCompressorOptions is returned if compressor options is present. It's not currently supported.
ErrCompressorOptions = errors.New("Compressor options is not currently supported")
//ErrFragmentTableIssues is returned if there's trouble reading the fragment table when creating a reader.
//When this is returned, the reader is still returned.
errFragmentTableIssues = errors.New("Trouble while reading the fragment table")
errCompressorOptions = errors.New("Compressor options is not currently supported")
)
//Reader processes and reads a squashfs archive.
//TODO: Give a way to actually read files :P
type Reader struct {
r io.ReaderAt
super superblock
flags superblockFlags
decompressor decompressor
decompressor compression.Decompressor
fragOffsets []uint64
idTable []uint32
}
//NewSquashfsReader returns a new squashfs.Reader from an io.ReaderAt
@@ -44,20 +42,40 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
return nil, err
}
if rdr.super.Magic != magic {
return nil, ErrNoMagic
return nil, errNoMagic
}
// if rdr.super.BlockLog == uint16(math.Log2(float64(rdr.super.BlockSize))) {
// return nil, errors.New("BlockSize and BlockLog doesn't match. The archive is probably corrupt")
// }
rdr.flags = rdr.super.GetFlags()
switch rdr.super.CompressionType {
case gzipCompression:
rdr.decompressor = &zlibDecompressor{}
default:
return nil, ErrIncompatibleCompression
}
if rdr.flags.CompressorOptions {
//TODO: parse compressor options
return nil, ErrCompressorOptions
switch rdr.super.CompressionType {
case xzCompression:
xz, err := compression.NewXzCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 1000)) //1000 is technically too much, but it's just an easy way to do it.
if err != nil {
return nil, err
}
if xz.HasFilters {
return nil, errors.New("XZ compression options has filters. These are not yet supported")
}
rdr.decompressor = xz
default:
return nil, errCompressorOptions
}
} else {
switch rdr.super.CompressionType {
case gzipCompression:
rdr.decompressor = &compression.Gzip{}
case lzmaCompression:
rdr.decompressor = &compression.Lzma{}
case xzCompression:
rdr.decompressor = &compression.Xz{}
default:
//TODO: all compression types.
return nil, errIncompatibleCompression
}
}
fragBlocks := int(math.Ceil(float64(rdr.super.FragCount) / 512.0))
fragBlocks := int(math.Ceil(float64(rdr.super.FragCount) / 512))
if fragBlocks > 0 {
offset := int64(rdr.super.FragTableStart)
for i := 0; i < fragBlocks; i++ {
@@ -70,6 +88,29 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
offset += 8
}
}
unread := rdr.super.IDCount
blockOffsets := make([]uint64, int(math.Ceil(float64(rdr.super.IDCount)/2048)))
for i := range blockOffsets {
secRdr := io.NewSectionReader(r, int64(rdr.super.IDTableStart)+(8*int64(i)), 8)
err = binary.Read(secRdr, binary.LittleEndian, &blockOffsets[i])
if err != nil {
return nil, err
}
idRdr, err := rdr.newMetadataReader(int64(blockOffsets[i]))
if err != nil {
return nil, err
}
read := uint16(math.Min(float64(unread), 2048))
for i := uint16(0); i < read; i++ {
var tmp uint32
err = binary.Read(idRdr, binary.LittleEndian, &tmp)
if err != nil {
return nil, err
}
rdr.idTable = append(rdr.idTable, tmp)
}
unread -= read
}
return &rdr, nil
}
@@ -84,7 +125,7 @@ func (r *Reader) GetRootFolder() (root *File, err error) {
if err != nil {
return nil, err
}
root.Path = "/"
root.path = "/"
root.filType = root.in.Type
root.r = r
return root, nil
+68 -7
View File
@@ -13,9 +13,30 @@ import (
const (
downloadURL = "https://github.com/Swordfish90/cool-retro-term/releases/download/1.1.1/Cool-Retro-Term-1.1.1-x86_64.AppImage"
appImageName = "Cool-Retro-Term.AppImage"
squashfsName = "airootfs.sfs" //testing with a ArchLinux root fs from the live img
)
func TestMain(t *testing.T) {
//Right now, don't use. Arch linux sfs uses XZ compression and when tested, most files just completely fail to extract.
func TestSquashfs(t *testing.T) {
wd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
squashFil, err := os.Open(wd + "/testing/" + squashfsName)
if err != nil {
t.Fatal(err)
}
rdr, err := NewSquashfsReader(squashFil)
if err != nil {
t.Fatal(err)
}
os.RemoveAll(wd + "/testing/" + squashfsName + ".d")
root, _ := rdr.GetRootFolder()
errs := root.ExtractWithOptions(wd+"/testing/"+squashfsName+".d", false, os.ModePerm, true)
t.Fatal(errs)
}
func TestAppImage(t *testing.T) {
t.Parallel()
wd, err := os.Getwd()
if err != nil {
@@ -38,13 +59,16 @@ func TestMain(t *testing.T) {
if err != nil {
t.Fatal(err)
}
fil := rdr.GetFileAtPath("/usr/bin/cool-retro-term")
if fil != nil {
fmt.Println("Worked!", fil.Path+"/"+fil.Name)
} else {
t.Fatal("NOOOOOO!")
fil := rdr.GetFileAtPath("usr/q*/QtQ*k/Extras/Priv*/q*")
if fil == nil {
t.Fatal("Can't find desktop file")
}
t.Fatal("No problems here!")
fmt.Println("Fount:", fil.Path())
// os.RemoveAll(wd + "/testing/" + appImageName + ".d")
// root, _ := rdr.GetRootFolder()
// errs := root.ExtractWithOptions(wd+"/testing/"+appImageName+".d", true, os.ModePerm, true)
// t.Fatal(errs)
t.Fatal("No problemo!")
}
func downloadTestAppImage(t *testing.T, dir string) {
@@ -71,3 +95,40 @@ func downloadTestAppImage(t *testing.T, dir string) {
t.Fatal(err)
}
}
func TestCreateSquashFromAppImage(t *testing.T) {
wd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
err = os.Mkdir(wd+"/testing", 0777)
if err != nil && !os.IsExist(err) {
t.Fatal(err)
}
_, err = os.Open(wd + "/testing/" + appImageName)
if os.IsNotExist(err) {
downloadTestAppImage(t, wd+"/testing")
_, err = os.Open(wd + "/testing/" + appImageName)
if err != nil {
t.Fatal(err)
}
} else if err != nil {
t.Fatal(err)
}
ai := goappimage.NewAppImage(wd + "/testing/" + appImageName)
aiFil, err := os.Open(wd + "/testing/" + appImageName)
if err != nil {
t.Fatal(err)
}
defer aiFil.Close()
aiFil.Seek(ai.Offset, 0)
os.Remove(wd + "/testing/" + appImageName + ".squashfs")
aiSquash, err := os.Create(wd + "/testing/" + appImageName + ".squashfs")
if err != nil {
t.Fatal(err)
}
_, err = io.Copy(aiSquash, aiFil)
if err != nil {
t.Fatal(err)
}
}
-119
View File
@@ -1,119 +0,0 @@
package squashfs
import (
"errors"
"io"
"strings"
"github.com/CalebQ42/squashfs/internal/directory"
"github.com/CalebQ42/squashfs/internal/inode"
)
var (
//ErrNotFound means that the given path is NOT present in the archive
ErrNotFound = errors.New("Path not found")
)
//ProcessInodeRef processes an inode reference and returns two values
//
//The first value is the inode table offset. AKA, it's where the metadata block of the inode STARTS relative to the inode table.
//
//The second value is the offset of the inode, INSIDE of the metadata.
func processInodeRef(inodeRef uint64) (tableOffset uint64, metaOffset uint64) {
tableOffset = inodeRef >> 16
metaOffset = inodeRef &^ 0xFFFFFFFF0000
return
}
//ReadDirFromInode returns a fully populated directory.Directory from a given inode.Inode.
//If the given inode is not a directory it returns an error.
func (r *Reader) readDirFromInode(i *inode.Inode) (*directory.Directory, error) {
var offset uint32
var metaOffset uint16
var size uint16
switch i.Type {
case inode.BasicDirectoryType:
offset = i.Info.(inode.BasicDirectory).DirectoryIndex
metaOffset = i.Info.(inode.BasicDirectory).DirectoryOffset
size = i.Info.(inode.BasicDirectory).DirectorySize
case inode.ExtDirType:
offset = i.Info.(inode.ExtendedDirectory).Init.DirectoryIndex
metaOffset = i.Info.(inode.ExtendedDirectory).Init.DirectoryOffset
size = uint16(i.Info.(inode.ExtendedDirectory).Init.DirectorySize)
default:
return nil, errors.New("Not a directory inode")
}
br, err := r.newMetadataReader(int64(r.super.DirTableStart + uint64(offset)))
if err != nil {
return nil, err
}
_, err = br.Seek(int64(metaOffset), io.SeekStart)
if err != nil {
return nil, err
}
dir, err := directory.NewDirectory(br, size)
if err != nil {
return dir, err
}
return dir, nil
}
//GetInodeFromEntry returns the inode associated with a given directory.Entry
func (r *Reader) getInodeFromEntry(en *directory.Entry) (*inode.Inode, error) {
br, err := r.newMetadataReader(int64(r.super.InodeTableStart + uint64(en.Header.InodeOffset)))
if err != nil {
return nil, err
}
_, err = br.Seek(int64(en.Init.Offset), io.SeekStart)
if err != nil {
return nil, err
}
i, err := inode.ProcessInode(br, r.super.BlockSize)
if err != nil {
return nil, err
}
return i, nil
}
//GetInodeFromPath returns the inode at the given path, relative to root.
//The given path can start with or without "/"
func (r *Reader) getInodeFromPath(path string) (*inode.Inode, error) {
path = strings.TrimSuffix(strings.TrimPrefix(path, "/"), "/")
pathDirs := strings.Split(path, "/")
rdr, err := r.newMetadataReaderFromInodeRef(r.super.RootInodeRef)
if err != nil {
return nil, err
}
curInodeDir, err := inode.ProcessInode(rdr, r.super.BlockSize)
if err != nil {
return nil, err
}
if path == "" {
return curInodeDir, nil
}
for depth := 0; depth < len(pathDirs); depth++ {
if curInodeDir.Type != inode.BasicDirectoryType && curInodeDir.Type != inode.ExtDirType {
return nil, ErrNotFound
}
dir, err := r.readDirFromInode(curInodeDir)
if err != nil {
return nil, err
}
for _, entry := range dir.Entries {
if entry.Name == pathDirs[depth] {
if depth == len(pathDirs)-1 {
in, err := r.getInodeFromEntry(&entry)
if err != nil {
return nil, err
}
return in, nil
}
curInodeDir, err = r.getInodeFromEntry(&entry)
if err != nil {
return nil, err
}
}
}
}
return nil, ErrNotFound
}