From d9132ab6a46ce16a71a6c9b38f152d341f8e5ee6 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sun, 24 Dec 2023 18:20:05 -0600 Subject: [PATCH] Finished. Now for bug fixes --- cmd/go-unsquashfs/main.go | 37 +++++ extraction_options.go | 13 +- file.go | 238 ++++++++++++++++++++++++++++- fs.go | 55 +++++-- internal/routinemanager/manager.go | 25 +++ squashfs/base.go | 37 +++++ 6 files changed, 385 insertions(+), 20 deletions(-) create mode 100644 cmd/go-unsquashfs/main.go create mode 100644 internal/routinemanager/manager.go diff --git a/cmd/go-unsquashfs/main.go b/cmd/go-unsquashfs/main.go new file mode 100644 index 0000000..75be53f --- /dev/null +++ b/cmd/go-unsquashfs/main.go @@ -0,0 +1,37 @@ +package main + +import ( + "flag" + "fmt" + "os" + "time" + + "github.com/CalebQ42/squashfs" +) + +func main() { + verbose := flag.Bool("v", false, "Verbose") + ignore := flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755") + flag.Parse() + if len(flag.Args()) < 2 { + fmt.Println("Please provide a file name and extraction path") + os.Exit(0) + } + f, err := os.Open(flag.Arg(0)) + if err != nil { + panic(err) + } + r, err := squashfs.NewReader(f) + if err != nil { + panic(err) + } + op := squashfs.DefaultOptions() + op.Verbose = *verbose + op.IgnorePerm = *ignore + n := time.Now() + err = r.ExtractWithOptions(flag.Arg(1), op) + if err != nil { + panic(err) + } + fmt.Println("Took:", time.Since(n)) +} diff --git a/extraction_options.go b/extraction_options.go index 56535b8..ad917de 100644 --- a/extraction_options.go +++ b/extraction_options.go @@ -3,19 +3,28 @@ package squashfs import ( "io" "io/fs" + "os" + + "github.com/CalebQ42/squashfs/internal/routinemanager" ) type ExtractionOptions struct { - LogOutput io.Writer //Where error log should write. + manager *routinemanager.Manager + LogOutput io.Writer //Where the verbose log should write. Defaults to os.Stdout. DereferenceSymlink bool //Replace symlinks with the target file. UnbreakSymlink bool //Try to make sure symlinks remain unbroken when extracted, without changing the symlink. Verbose bool //Prints extra info to log on an error. IgnorePerm bool //Ignore file's permissions and instead use Perm. Perm fs.FileMode //Permission to use when IgnorePerm. Defaults to 0777. + SimultaneousFiles uint16 //Number of files to process in parallel. Defaults to 10. + ExtractionRoutines uint16 //Number of goroutines to use for each file's extraction. Only applies to regular files. Defaults to 10. } func DefaultOptions() *ExtractionOptions { return &ExtractionOptions{ - Perm: 0777, + LogOutput: os.Stdout, + Perm: 0777, + SimultaneousFiles: 10, + ExtractionRoutines: 10, } } diff --git a/file.go b/file.go index 197e4bf..2c51214 100644 --- a/file.go +++ b/file.go @@ -4,8 +4,14 @@ import ( "errors" "io" "io/fs" + "log" + "os" + "os/exec" "path/filepath" + "runtime" + "strconv" + "github.com/CalebQ42/squashfs/internal/routinemanager" "github.com/CalebQ42/squashfs/squashfs" "github.com/CalebQ42/squashfs/squashfs/data" "github.com/CalebQ42/squashfs/squashfs/inode" @@ -162,6 +168,20 @@ func (f *File) initializeReaders() error { return err } +func (f *File) deviceDevices() (maj uint32, min uint32) { + var dev uint32 + if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.Block { + dev = f.b.Inode.Data.(inode.Device).Dev + } else if f.b.Inode.Type == inode.EChar || f.b.Inode.Type == inode.EBlock { + dev = f.b.Inode.Data.(inode.EDevice).Dev + } + return dev >> 8, dev & 0x000FF +} + +func (f *File) path() string { + return filepath.Join(f.parent.path(), f.b.Name) +} + // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. // Uses default extraction options. func (f *File) Extract(folder string) error { @@ -170,6 +190,220 @@ func (f *File) Extract(folder string) error { // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. // Allows setting various extraction options via ExtractionOptions. -func (f *File) ExtractWithOptions(folder string, op *ExtractionOptions) error { - //TODO +func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { + if op.manager == nil { + op.manager = routinemanager.NewManager(op.SimultaneousFiles) + log.SetOutput(op.LogOutput) + } + switch f.b.Inode.Type { + case inode.Dir, inode.EDir: + d, err := f.b.ToDir(f.r.r) + if err != nil { + if op.Verbose { + log.Println("Failed to create squashfs.Directory for", path) + } + return errors.Join(errors.New("failed to create squashfs.Directory: "+path), err) + } + errChan := make(chan error, len(d.Entries)) + files := len(d.Entries) + for i := range d.Entries { + b, err := f.r.r.BaseFromEntry(d.Entries[i]) + if err != nil { + if op.Verbose { + log.Println("Failed to get squashfs.Base from entry for", path) + } + return errors.Join(errors.New("failed to get base from entry: "+path), err) + } + if b.IsDir() { + files-- + extDir := filepath.Join(path, b.Name) + err = os.Mkdir(extDir, 0777) + if err != nil { + if op.Verbose { + log.Println("Failed to create directory", path) + } + return errors.Join(errors.New("failed to create directory: "+path), err) + } + err = f.ExtractWithOptions(extDir, op) + if err != nil { + if op.Verbose { + log.Println("Failed to extract directory", path) + } + return errors.Join(errors.New("failed to extract directory: "+path), err) + } + } else { + fil := &File{ + b: b, + r: f.r, + } + go func(fil *File, folder string) { + i := op.manager.Lock() + defer op.manager.Unlock(i) + errChan <- fil.ExtractWithOptions(folder, op) + }(fil, path) + } + } + var errCache []error + for i := 0; i < files; i++ { + err := <-errChan + if err != nil { + errCache = append(errCache, err) + } + } + if len(errCache) > 0 { + return errors.Join(errors.New("failed to extract folder: "+path), errors.Join(errCache...)) + } + case inode.Fil, inode.EFil: + path = filepath.Join(path, f.b.Name) + outFil, err := os.Create(path) + if err != nil { + if op.Verbose { + log.Println("Failed to create file", path) + } + return errors.Join(errors.New("failed to create file: "+path), err) + } + defer outFil.Close() + full, err := f.b.GetFullReader(f.r.r) + if err != nil { + if op.Verbose { + log.Println("Failed to create full reader for", path) + } + return errors.Join(errors.New("failed to create full reader: "+path), err) + } + full.SetGoroutineLimit(op.ExtractionRoutines) + _, err = full.WriteTo(outFil) + if err != nil { + if op.Verbose { + log.Println("Failed to write file", path) + } + return errors.Join(errors.New("failed to write file: "+path), err) + } + if op.Verbose { + log.Println(f.path(), "extracted to", path) + } + case inode.Sym, inode.ESym: + symPath := f.SymlinkPath() + if op.DereferenceSymlink { + filTmp := f.GetSymlinkFile() + if filTmp == nil { + if op.Verbose { + log.Println("Failed to get symlink's file:", f.path()) + } + return errors.New("failed to get symlink's file") + } + fil := filTmp.(*File) + fil.b.Name = f.b.Name + err := fil.ExtractWithOptions(path, op) + if err != nil { + if op.Verbose { + log.Println("Failed to extract symlink's file:", filepath.Join(path, f.b.Name)) + } + return errors.Join(errors.New("failed to extract symlink's file: "+path), err) + } + } else { + if op.UnbreakSymlink { + filTmp := f.GetSymlinkFile() + if filTmp == nil { + if op.Verbose { + log.Println("Failed to get symlink's file:", f.path()) + } + return errors.New("failed to get symlink's file") + } + extractLoc := filepath.Join(path, filepath.Dir(symPath)) + fil := filTmp.(*File) + err := fil.ExtractWithOptions(extractLoc, op) + if err != nil { + if op.Verbose { + log.Println("Error while extracting", fil.path(), "to make sure symlink at", f.path(), "is unbroken") + } + return errors.Join(errors.New("failed to extract symlink's file: "+extractLoc), err) + } + } + path = filepath.Join(path, f.b.Name) + err := os.Symlink(f.SymlinkPath(), path) + if err != nil { + if op.Verbose { + log.Println("Failed to create symlink:", path) + } + return errors.Join(errors.New("failed to create symlink: "+path), err) + } + } + case inode.Char, inode.EChar, inode.Block, inode.EBlock, inode.Fifo, inode.EFifo: + if runtime.GOOS == "windows" { + if op.Verbose { + log.Println(f.path(), "ignored. A device link and can't be created on Windows.") + } + return nil + } + _, err := exec.LookPath("mknod") + if err != nil { + if op.Verbose { + log.Println("mknot command not found, cannot create device link for", f.path()) + } + return errors.Join(errors.New("mknot command not found"), err) + } + path = filepath.Join(path, f.b.Name) + var typ string + if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.EChar { + typ = "c" + } else if f.b.Inode.Type == inode.Block || f.b.Inode.Type == inode.EBlock { + typ = "b" + } else { //Fifo IPC + if runtime.GOOS == "darwin" { + if op.Verbose { + log.Println(f.path(), "ignored. A Fifo file and can't be created on Darwin.") + } + return nil + } + typ = "p" + } + cmd := exec.Command("mknod", path, typ) + if typ != "p" { + maj, min := f.deviceDevices() + cmd.Args = append(cmd.Args, strconv.Itoa(int(maj)), strconv.Itoa(int(min))) + } + if op.Verbose { + cmd.Stdout = op.LogOutput + cmd.Stderr = op.LogOutput + } + err = cmd.Run() + if err != nil { + if op.Verbose { + log.Println("Error while running mknod for", path) + } + return errors.Join(errors.New("error while running mknod for "+path), err) + } + case inode.Sock, inode.ESock: + if op.Verbose { + log.Println(f.path(), "ignored since it's a socket file.") + } + return nil + default: + return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.b.Inode.Type))) + } + if op.Verbose { + log.Println(f.path(), "extracted to", path) + } + if op.IgnorePerm { + return nil + } + uid, err := f.b.Uid(f.r.r) + if err != nil { + if op.Verbose { + log.Println("Failed to get uid for", path) + log.Println(err) + } + return nil + } + gid, err := f.b.Gid(f.r.r) + if err != nil { + if op.Verbose { + log.Println("Failed to get gid for", path) + log.Println(err) + } + return nil + } + os.Chmod(path, f.Mode()) + os.Chown(path, int(uid), int(gid)) + return nil } diff --git a/fs.go b/fs.go index 7f62f44..39f08cd 100644 --- a/fs.go +++ b/fs.go @@ -91,13 +91,20 @@ func (f *FS) Open(name string) (fs.File, error) { } } if name == "." || name == "" { - return &File{ - b: &f.d.Base, - r: f.r, - parent: f.parent, - }, nil + return f.File(), nil } split := strings.Split(name, "/") + if split[0] == ".." { + if f.parent == nil { // root directory + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrNotExist, + } + } + } else { + return f.parent.Open(strings.Join(split[1:], "/")) + } i, found := slices.BinarySearchFunc(f.d.Entries, split[0], func(e directory.Entry, name string) int { return strings.Compare(e.Name, name) }) @@ -116,7 +123,7 @@ func (f *FS) Open(name string) (fs.File, error) { return &File{ b: b, r: f.r, - parent: f.parent, + parent: f, }, nil } if !b.IsDir() { @@ -149,11 +156,7 @@ func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) { } } if name == "." || name == "" { - return (&File{ - b: &f.d.Base, - parent: f.parent, - r: f.r, - }).ReadDir(-1) + return f.File().ReadDir(-1) } fil, err := f.Open(name) if err != nil { @@ -196,11 +199,7 @@ func (f *FS) Stat(name string) (fs.FileInfo, error) { } } if name == "." || name == "" { - return (&File{ - b: &f.d.Base, - parent: f.parent, - r: f.r, - }).Stat() + return f.File().Stat() } fil, err := f.Open(name) if err != nil { @@ -236,6 +235,30 @@ func (f *FS) Sub(dir string) (fs.FS, error) { return fil.(*File).FS() } +// Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. +// Uses default extraction options. +func (f *FS) Extract(folder string) error { + return f.File().Extract(folder) +} + +// Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. +// Allows setting various extraction options via ExtractionOptions. +func (f *FS) ExtractWithOptions(folder string, op *ExtractionOptions) error { + return f.File().ExtractWithOptions(folder, op) +} + +// Returns the FS as a *File +func (f *FS) File() *File { + return &File{ + b: &f.d.Base, + parent: f.parent, + r: f.r, + } +} + func (f *FS) path() string { + if f.parent == nil { + return f.d.Name + } return filepath.Join(f.parent.path(), f.d.Name) } diff --git a/internal/routinemanager/manager.go b/internal/routinemanager/manager.go new file mode 100644 index 0000000..4e47ed8 --- /dev/null +++ b/internal/routinemanager/manager.go @@ -0,0 +1,25 @@ +package routinemanager + +type Manager struct { + channel chan uint16 + maxRoutines uint16 +} + +func NewManager(maxRoutines uint16) *Manager { + m := &Manager{ + maxRoutines: maxRoutines, + channel: make(chan uint16, maxRoutines), + } + for i := uint16(0); i < maxRoutines; i++ { + m.channel <- i + } + return m +} + +func (m *Manager) Lock() uint16 { + return <-m.channel +} + +func (m *Manager) Unlock(i uint16) { + m.channel <- i +} diff --git a/squashfs/base.go b/squashfs/base.go index f799f1a..53c8207 100644 --- a/squashfs/base.go +++ b/squashfs/base.go @@ -129,3 +129,40 @@ func (b *Base) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, err } return outRdr, outFull, nil } + +func (b *Base) GetFullReader(r *Reader) (*data.FullReader, error) { + if !b.IsRegular() { + return nil, errors.New("not a regular file") + } + var blockStart uint64 + var fragIndex uint32 + var fragOffset uint32 + var fragSize uint64 + var sizes []uint32 + if b.Inode.Type == inode.Fil { + blockStart = uint64(b.Inode.Data.(inode.File).BlockStart) + fragIndex = b.Inode.Data.(inode.File).FragInd + fragOffset = b.Inode.Data.(inode.File).FragOffset + sizes = b.Inode.Data.(inode.File).BlockSizes + fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize) + } else { + blockStart = b.Inode.Data.(inode.EFile).BlockStart + fragIndex = b.Inode.Data.(inode.EFile).FragInd + fragOffset = b.Inode.Data.(inode.EFile).FragOffset + sizes = b.Inode.Data.(inode.EFile).BlockSizes + fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize) + } + outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes, fragSize, r.Superblock.BlockSize) + if fragIndex != 0xffffffff { + outFull.AddFrag(func() (io.Reader, error) { + ent, err := r.fragEntry(fragIndex) + if err != nil { + return nil, err + } + frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize) + frag.Read(make([]byte, fragOffset)) + return io.LimitReader(frag, int64(fragSize)), nil + }) + } + return outFull, nil +}