Belac/squashfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 55 Wiki Activity

Further work on extracting

Browse Source 
Extracting files is threaded with goroutines.
Extracting also sets the proper UID and GUID
Made recursive children getting threaded with goroutines
Decided I will allow wildcards in paths. Hasn't been implemented though.
Fixed issues with ExtendedDirectory reading (I was using a uint16 instead of a uint32)
I couldn't test basically any of this. somehow a for loop isn't incrementing. For seemingly no reason.
This commit is contained in:
Caleb Gardner
2020-12-01 06:14:09 -06:00
parent 1254df2861
commit e302d98665
4 changed files with 130 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
file.go
+86 -29
View File
@@ -59,7 +59,6 @@ func (f *File) GetChildren() (children []*File, err error) {
}
dir, err := f.r.readDirFromInode(f.in)
if err != nil {
fmt.Println("err reading dir")
return
}
var fil *File
@@ -86,26 +85,31 @@ func (f *File) GetChildrenRecursively() (children []*File, err error) {
if !f.IsDir() {
return nil, errNotDirectory
}
chil, err := f.GetChildren()
children, err = f.GetChildren()
if err != nil {
fmt.Println("err here", f.Path())
return
}
var childFolders []*File
for _, child := range chil {
children = append(children, child)
for _, child := range children {
if child.IsDir() {
childFolders = append(childFolders, child)
}
}
foldChil := make(chan []*File)
errChan := make(chan error)
for _, folds := range childFolders {
var childs []*File
childs, err = folds.GetChildrenRecursively()
go func(fil *File) {
childs, err := fil.GetChildrenRecursively()
errChan <- err
foldChil <- childs
}(folds)
}
for range childFolders {
err = <-errChan
if err != nil {
fmt.Println("err here Recursive", folds.Path())
return
}
children = append(children, childs...)
children = append(children, <-foldChil...)
}
return
}
@@ -120,6 +124,7 @@ func (f *File) Path() string {
//GetFileAtPath tries to return the File at the given path, relative to the file.
//Returns nil if called on something other then a folder, OR if the path goes oustide the archive.
//Allows * wildcards.
func (f *File) GetFileAtPath(path string) *File {
if path == "" {
return f
@@ -145,9 +150,10 @@ func (f *File) GetFileAtPath(path string) *File {
if err != nil {
return nil
}
for _, child := range children {
if child.Name == split[0] {
if strings.Contains(split[0], "*") {
//TODO: wildcards
} else if child.Name == split[0] {
return child.GetFileAtPath(strings.Join(split[1:], "/"))
}
}
@@ -229,18 +235,37 @@ func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm o
}
switch {
case f.IsDir():
err = os.Mkdir(path+"/"+f.Name, f.Permission())
if err != nil {
if verbose {
fmt.Println("Error while making: "+path+"/"+f.Name, f.Permission())
if f.Name != "" {
//TODO: check if folder is present, and if so, try to set it's permission
err = os.Mkdir(path+"/"+f.Name, f.Permission())
if err != nil {
if verbose {
fmt.Println("Error while making: ", path+"/"+f.Name)
}
errs = append(errs, err)
return
}
fil, err := os.Open(path + "/" + f.Name)
if err != nil {
if verbose {
fmt.Println("Error while opening:", path+"/"+f.Name)
}
errs = append(errs, err)
return
}
err = fil.Chown(int(f.r.idTable[f.in.Header.UID]), int(f.r.idTable[f.in.Header.GID]))
if err != nil {
if verbose {
fmt.Println("Error while changing owner:", path+"/"+f.Name)
}
errs = append(errs, err)
return
}
errs = append(errs, err)
return
}
children, err := f.GetChildren()
if err != nil {
if verbose {
fmt.Println("Error while making: "+path+"/"+f.Name, f.Permission())
fmt.Println("Error getting children for:", f.Path())
}
errs = append(errs, err)
return
@@ -249,7 +274,11 @@ func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm o
defer close(finishChan)
for _, child := range children {
go func(child *File) {
finishChan <- child.ExtractWithOptions(path, unbreakSymlink, folderPerm, verbose)
if f.Name == "" {
finishChan <- child.ExtractWithOptions(path, unbreakSymlink, folderPerm, verbose)
} else {
finishChan <- child.ExtractWithOptions(path+"/"+f.Name, unbreakSymlink, folderPerm, verbose)
}
}(child)
}
for range children {
@@ -258,17 +287,43 @@ func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm o
return
case f.IsFile():
fil, err := os.Create(path + "/" + f.Name)
if err != nil {
if os.IsExist(err) {
err = os.Remove(path + "/" + f.Name)
if err != nil {
if verbose {
fmt.Println("Error while making:", path+"/"+f.Name)
}
errs = append(errs, err)
return
}
fil, err = os.Create(path + "/" + f.Name)
if err != nil {
if verbose {
fmt.Println("Error while making:", path+"/"+f.Name)
}
errs = append(errs, err)
return
}
} else if err != nil {
if verbose {
fmt.Println("Error while making: "+path+"/"+f.Name, f.Permission())
fmt.Println("Error while making:", path+"/"+f.Name)
}
errs = append(errs, err)
return
}
defer f.Close() //Since we will be reading from the file
_, err = io.Copy(fil, f)
if err != nil {
if verbose {
fmt.Println("Error while Copying data to: "+path+"/"+f.Name, f.Permission())
fmt.Println("Error while Copying data to:", path+"/"+f.Name)
}
errs = append(errs, err)
return
}
err = fil.Chown(int(f.r.idTable[f.in.Header.UID]), int(f.r.idTable[f.in.Header.GID]))
if err != nil {
if verbose {
fmt.Println("Error while changing owner:", path+"/"+f.Name)
}
errs = append(errs, err)
return
@@ -276,7 +331,7 @@ func (f *File) ExtractWithOptions(path string, unbreakSymlink bool, folderPerm o
err = fil.Chmod(f.Permission())
if err != nil {
if verbose {
fmt.Println("Error while setting permissions for: "+path+"/"+f.Name, f.Permission())
fmt.Println("Error while setting permissions for:", path+"/"+f.Name)
}
errs = append(errs, err)
}
@@ -293,10 +348,12 @@ func (f *File) Close() error {
if f.IsDir() {
return errNotFile
}
if closer, is := f.Reader.(io.Closer); is {
closer.Close()
if f.Reader != nil {
if closer, is := f.Reader.(io.Closer); is {
closer.Close()
}
f.Reader = nil
}
f.Reader = nil
return nil
}
@@ -320,16 +377,16 @@ func (f *File) Read(p []byte) (int, error) {
func (r *Reader) readDirFromInode(i *inode.Inode) (*directory.Directory, error) {
var offset uint32
var metaOffset uint16
var size uint16
var size uint32
switch i.Type {
case inode.BasicDirectoryType:
offset = i.Info.(inode.BasicDirectory).DirectoryIndex
metaOffset = i.Info.(inode.BasicDirectory).DirectoryOffset
size = i.Info.(inode.BasicDirectory).DirectorySize
size = uint32(i.Info.(inode.BasicDirectory).DirectorySize)
case inode.ExtDirType:
offset = i.Info.(inode.ExtendedDirectory).Init.DirectoryIndex
metaOffset = i.Info.(inode.ExtendedDirectory).Init.DirectoryOffset
size = uint16(i.Info.(inode.ExtendedDirectory).Init.DirectorySize)
size = i.Info.(inode.ExtendedDirectory).Init.DirectorySize
default:
return nil, errors.New("Not a directory inode")
}
internal/directory/directory.go
+1 -1
View File
@@ -52,7 +52,7 @@ type Directory struct {
}
//NewDirectory reads the directory from rdr
func NewDirectory(base io.Reader, size uint16) (*Directory, error) {
func NewDirectory(base io.Reader, size uint32) (*Directory, error) {
var dir Directory
var err error
tmp := make([]byte, size)
reader.go
+35 -1
View File
@@ -3,6 +3,7 @@ package squashfs
import (
"encoding/binary"
"errors"
"fmt"
"io"
"math"
@@ -30,6 +31,7 @@ type Reader struct {
flags superblockFlags
decompressor compression.Decompressor
fragOffsets []uint64
idTable []uint32
}
//NewSquashfsReader returns a new squashfs.Reader from an io.ReaderAt
@@ -50,13 +52,14 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
case xzCompression:
rdr.decompressor = &compression.Xz{}
default:
//TODO: all compression types.
return nil, errIncompatibleCompression
}
if rdr.flags.CompressorOptions {
//TODO: parse compressor options
return nil, errCompressorOptions
}
fragBlocks := int(math.Ceil(float64(rdr.super.FragCount) / 512.0))
fragBlocks := int(math.Ceil(float64(rdr.super.FragCount) / 512))
if fragBlocks > 0 {
offset := int64(rdr.super.FragTableStart)
for i := 0; i < fragBlocks; i++ {
@@ -69,6 +72,37 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
offset += 8
}
}
idBlocks := int(math.Ceil(float64(rdr.super.IDCount) / 2048))
fmt.Println("ID Blocks", idBlocks)
for idBlocks > 0 {
unread := rdr.super.IDCount
offset := int64(rdr.super.IDTableStart)
for i := 0; i < idBlocks; i++ {
tmp := make([]byte, 8)
_, err = r.ReadAt(tmp, offset)
if err != nil {
return nil, err
}
offset += 8
idRdr, err := rdr.newMetadataReader(int64(binary.LittleEndian.Uint64(tmp)))
if err != nil {
return nil, err
}
for j := 0; j < int(math.Min(float64(unread), 2048)); j++ {
var id uint32
err = binary.Read(idRdr, binary.LittleEndian, &id)
if err != nil {
return nil, err
}
rdr.idTable = append(rdr.idTable, id)
}
if unread > 2048 {
unread -= 2048
} else {
unread = 0
}
}
}
return &rdr, nil
}
squash_test.go
+8 -17
View File
@@ -8,7 +8,6 @@ import (
"testing"
goappimage "github.com/CalebQ42/GoAppImage"
"github.com/CalebQ42/squashfs/internal/inode"
)
const (
@@ -18,7 +17,7 @@ const (
)
func TestSquashfs(t *testing.T) {
t.Parallel()
fmt.Println("YOOO")
wd, err := os.Getwd()
if err != nil {
t.Fatal(err)
@@ -31,21 +30,13 @@ func TestSquashfs(t *testing.T) {
if err != nil {
t.Fatal(err)
}
fils, err := rdr.GetAllFiles()
if err != nil {
t.Fatal(err)
}
for _, fil := range fils {
if fil.filType != inode.BasicFileType && fil.filType != inode.BasicDirectoryType && fil.filType != inode.BasicSymlinkType {
fmt.Println("Found non-standard")
fmt.Println(fil.Path())
fmt.Println("type:", fil.filType)
} else if fil.IsSymlink() {
fmt.Println("Symlink!")
fmt.Println(fil.Path())
fmt.Println("symlink path:", fil.SymlinkPath())
}
}
os.RemoveAll(wd + "/testing/" + squashfsName + ".d")
fmt.Println("Whaaaa")
root, _ := rdr.GetRootFolder()
fmt.Println("WHYYYY")
errs := root.ExtractWithOptions(wd+"/testing/"+squashfsName+".d", false, os.ModePerm, true)
fmt.Println(errs)
t.Fatal("No prolems here!")
}
func TestAppImage(t *testing.T) {