diff --git a/README.md b/README.md index eb6b0c0..da5e660 100644 --- a/README.md +++ b/README.md @@ -2,27 +2,36 @@ [![PkgGoDev](https://pkg.go.dev/badge/github.com/CalebQ42/squashfs)](https://pkg.go.dev/github.com/CalebQ42/squashfs) [![Go Report Card](https://goreportcard.com/badge/github.com/CalebQ42/squashfs)](https://goreportcard.com/report/github.com/CalebQ42/squashfs) -A PURE Go library to read squashfs. There is currently no plans to add archive creation support as it will almost always be better to just call `mksquashfs`. I could see some possible use cases, but probably won't spend time on it unless it's requested (open a discussion fi you want this feature). +A PURE Go library to read squashfs. There is currently no plans to add archive creation support as it will almost always be better to just call `mksquashfs`. I could see some possible use cases, but probably won't spend time on it unless it's requested (open a discussion if you want this feature). + +The library has two parts with this `github.com/CalebQ42/squashfs` being easy to use as it implements `io/fs` interfaces and doesn't expose unnecessary information. 95% this is the library you want. If you need lower level access to the information, use `github.com/CalebQ42/squashfs/low` where far more information is exposed. Currently has support for reading squashfs files and extracting files and folders. Special thanks to for some VERY important information in an easy to understand format. Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree/master/internal/squashfs) as I referenced it to figure some things out (and double check others). -## [TODO](https://github.com/CalebQ42/squashfs/projects/1?fullscreen=true) +## FUSE + +As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://github.com/CalebQ42/squashfuse). ## Limitations -* No Xattr parsing. This is simply because I haven't done any research on it and how to apply these in a pure go way. +* No Xattr parsing. * Socket files are not extracted. * From my research, it seems like a socket file would be useless if it could be created. * Fifo files are ignored on `darwin` ## Issues -* Significantly slower then `unsquashfs` when extracting folders (about 5 ~ 7 times slower on a ~100MB archive using zstd compression) +* Significantly slower then `unsquashfs` when extracting folders * This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries. - * The larger the file's tree, the slower the extraction will be. Arch Linux's Live USB's airootfs.sfs takes ~35x longer for a full extraction. + * Times seem to be largely dependent on file tree size and compression type. + * My main testing image (~100MB) using Zstd takes about 6x longer. + * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 32x longer. + * A Tensorflow docker image (~3.3GB) using Zstd takes about 12x longer. + +Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes about 2x longer. ## Recommendations on Usage diff --git a/extraction_options.go b/extraction_options.go new file mode 100644 index 0000000..c55a5ec --- /dev/null +++ b/extraction_options.go @@ -0,0 +1,49 @@ +package squashfs + +import ( + "io" + "io/fs" + "runtime" + + "github.com/CalebQ42/squashfs/internal/routinemanager" +) + +type ExtractionOptions struct { + manager *routinemanager.Manager + LogOutput io.Writer //Where the verbose log should write. + DereferenceSymlink bool //Replace symlinks with the target file. + UnbreakSymlink bool //Try to make sure symlinks remain unbroken when extracted, without changing the symlink. + Verbose bool //Prints extra info to log on an error. + IgnorePerm bool //Ignore file's permissions and instead use Perm. + Perm fs.FileMode //Permission to use when IgnorePerm. Defaults to 0777. + SimultaneousFiles uint16 //Number of files to process in parallel. Default set based on runtime.NumCPU(). + ExtractionRoutines uint16 //Number of goroutines to use for each file's extraction. Only applies to regular files. Default set based on runtime.NumCPU(). +} + +// The default extraction options. +func DefaultOptions() *ExtractionOptions { + cores := uint16(runtime.NumCPU() / 2) + var files, routines uint16 + if cores <= 4 { + files = 1 + routines = cores + } else { + files = cores - 4 + routines = 4 + } + return &ExtractionOptions{ + Perm: 0777, + SimultaneousFiles: files, + ExtractionRoutines: routines, + } +} + +// Less limited default options. Can run up 2x faster than DefaultOptions. +// Tends to use all available CPU resources. +func FastOptions() *ExtractionOptions { + return &ExtractionOptions{ + Perm: 0777, + SimultaneousFiles: uint16(runtime.NumCPU()), + ExtractionRoutines: uint16(runtime.NumCPU()), + } +} diff --git a/file.go b/file.go new file mode 100644 index 0000000..7b2fcdf --- /dev/null +++ b/file.go @@ -0,0 +1,428 @@ +package squashfs + +import ( + "errors" + "io" + "io/fs" + "log" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + + "github.com/CalebQ42/squashfs/internal/routinemanager" + squashfslow "github.com/CalebQ42/squashfs/low" + "github.com/CalebQ42/squashfs/low/data" + "github.com/CalebQ42/squashfs/low/inode" +) + +// File represents a file inside a squashfs archive. +type File struct { + b *squashfslow.FileBase + full *data.FullReader + rdr *data.Reader + parent *FS + r *Reader + dirsRead int +} + +// Creates a new *File from the given *squashfs.Base +func (r *Reader) FileFromBase(b *squashfslow.FileBase, parent *FS) *File { + return &File{ + b: b, + parent: parent, + r: r, + } +} + +func (f *File) FS() (*FS, error) { + if !f.IsDir() { + return nil, errors.New("not a directory") + } + d, err := f.b.ToDir(f.r.Low) + if err != nil { + return nil, err + } + return &FS{d: d, parent: f.parent, r: f.r}, nil +} + +// Closes the underlying readers. +// Further calls to Read and WriteTo will re-create the readers. +// Never returns an error. +func (f *File) Close() error { + if f.rdr != nil { + return f.rdr.Close() + } + f.rdr = nil + f.full = nil + return nil +} + +// Returns the file the symlink points to. +// If the file isn't a symlink, or points to a file outside the archive, returns nil. +func (f *File) GetSymlinkFile() fs.File { + if !f.IsSymlink() { + return nil + } + if filepath.IsAbs(f.SymlinkPath()) { + return nil + } + fil, err := f.parent.Open(f.SymlinkPath()) + if err != nil { + return nil + } + return fil +} + +// Returns whether the file is a directory. +func (f *File) IsDir() bool { + return f.b.IsDir() +} + +// Returns whether the file is a regular file. +func (f *File) IsRegular() bool { + return f.b.IsRegular() +} + +// Returns whether the file is a symlink. +func (f *File) IsSymlink() bool { + return f.b.Inode.Type == inode.Sym || f.b.Inode.Type == inode.ESym +} + +func (f *File) Mode() fs.FileMode { + return f.b.Inode.Mode() +} + +// Read reads the data from the file. Only works if file is a normal file. +func (f *File) Read(b []byte) (int, error) { + if !f.IsRegular() { + return 0, errors.New("file is not a regular file") + } + if f.rdr == nil { + err := f.initializeReaders() + if err != nil { + return 0, err + } + } + return f.rdr.Read(b) +} + +// ReadDir returns n fs.DirEntry's that's contained in the File (if it's a directory). +// If n <= 0 all fs.DirEntry's are returned. +func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { + if !f.IsDir() { + return nil, errors.New("file is not a directory") + } + d, err := f.b.ToDir(f.r.Low) + if err != nil { + return nil, err + } + start, end := 0, len(d.Entries) + if n > 0 { + start, end = f.dirsRead, f.dirsRead+n + if end > len(d.Entries) { + end = len(d.Entries) + err = io.EOF + } + } + var out []fs.DirEntry + var fi fileInfo + for _, e := range d.Entries[start:end] { + fi, err = f.r.newFileInfo(e) + if err != nil { + f.dirsRead += len(out) + return out, err + } + out = append(out, fs.FileInfoToDirEntry(fi)) + } + f.dirsRead += len(out) + return out, err +} + +// Returns the file's fs.FileInfo +func (f *File) Stat() (fs.FileInfo, error) { + return newFileInfo(f.b.Name, f.b.Inode), nil +} + +// SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string. +func (f *File) SymlinkPath() string { + switch f.b.Inode.Type { + case inode.Sym: + return string(f.b.Inode.Data.(inode.Symlink).Target) + case inode.ESym: + return string(f.b.Inode.Data.(inode.ESymlink).Target) + } + return "" +} + +// Writes all data from the file to the given writer in a multi-threaded manner. +// The underlying reader is separate +func (f *File) WriteTo(w io.Writer) (int64, error) { + if !f.IsRegular() { + return 0, errors.New("file is not a regular file") + } + if f.full == nil { + err := f.initializeReaders() + if err != nil { + return 0, err + } + } + return f.full.WriteTo(w) +} + +func (f *File) initializeReaders() error { + var err error + f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.Low) + return err +} + +func (f *File) deviceDevices() (maj uint32, min uint32) { + var dev uint32 + if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.Block { + dev = f.b.Inode.Data.(inode.Device).Dev + } else if f.b.Inode.Type == inode.EChar || f.b.Inode.Type == inode.EBlock { + dev = f.b.Inode.Data.(inode.EDevice).Dev + } + return dev >> 8, dev & 0x000FF +} + +func (f *File) path() string { + if f.parent == nil { + return f.b.Name + } + return filepath.Join(f.parent.path(), f.b.Name) +} + +// Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. +// Uses default extraction options. +func (f *File) Extract(folder string) error { + return f.ExtractWithOptions(folder, DefaultOptions()) +} + +// Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. +// Allows setting various extraction options via ExtractionOptions. +func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { + if op.manager == nil { + op.manager = routinemanager.NewManager(op.SimultaneousFiles) + if op.LogOutput != nil { + log.SetOutput(op.LogOutput) + } + err := os.MkdirAll(path, 0777) + if err != nil { + if op.Verbose { + log.Println("Failed to create initial directory", path) + } + return err + } + } + switch f.b.Inode.Type { + case inode.Dir, inode.EDir: + d, err := f.b.ToDir(f.r.Low) + if err != nil { + if op.Verbose { + log.Println("Failed to create squashfs.Directory for", path) + } + return errors.Join(errors.New("failed to create squashfs.Directory: "+path), err) + } + errChan := make(chan error, len(d.Entries)) + for i := range d.Entries { + b, err := f.r.Low.BaseFromEntry(d.Entries[i]) + if err != nil { + if op.Verbose { + log.Println("Failed to get squashfs.Base from entry for", path) + } + return errors.Join(errors.New("failed to get base from entry: "+path), err) + } + go func(b *squashfslow.FileBase, path string) { + i := op.manager.Lock() + if b.IsDir() { + extDir := filepath.Join(path, b.Name) + err = os.Mkdir(extDir, 0777) + op.manager.Unlock(i) + if err != nil { + if op.Verbose { + log.Println("Failed to create directory", path) + } + errChan <- errors.Join(errors.New("failed to create directory: "+path), err) + return + } + err = f.r.FileFromBase(b, f.r.FSFromDirectory(d, f.parent)).ExtractWithOptions(extDir, op) + if err != nil { + if op.Verbose { + log.Println("Failed to extract directory", path) + } + errChan <- errors.Join(errors.New("failed to extract directory: "+path), err) + return + } + errChan <- nil + } else { + fil := f.r.FileFromBase(b, f.r.FSFromDirectory(d, f.parent)) + err = fil.ExtractWithOptions(path, op) + op.manager.Unlock(i) + fil.Close() + errChan <- err + } + }(b, path) + } + var errCache []error + for i := 0; i < len(d.Entries); i++ { + err := <-errChan + if err != nil { + errCache = append(errCache, err) + } + } + if len(errCache) > 0 { + return errors.Join(errors.New("failed to extract folder: "+path), errors.Join(errCache...)) + } + case inode.Fil, inode.EFil: + path = filepath.Join(path, f.b.Name) + outFil, err := os.Create(path) + if err != nil { + if op.Verbose { + log.Println("Failed to create file", path) + } + return errors.Join(errors.New("failed to create file: "+path), err) + } + defer outFil.Close() + full, err := f.b.GetFullReader(f.r.Low) + if err != nil { + if op.Verbose { + log.Println("Failed to create full reader for", path) + } + return errors.Join(errors.New("failed to create full reader: "+path), err) + } + full.SetGoroutineLimit(op.ExtractionRoutines) + _, err = full.WriteTo(outFil) + if err != nil { + if op.Verbose { + log.Println("Failed to write file", path) + } + return errors.Join(errors.New("failed to write file: "+path), err) + } + case inode.Sym, inode.ESym: + symPath := f.SymlinkPath() + if op.DereferenceSymlink { + filTmp := f.GetSymlinkFile() + if filTmp == nil { + if op.Verbose { + log.Println("Failed to get symlink's file:", f.path()) + } + return errors.New("failed to get symlink's file") + } + fil := filTmp.(*File) + fil.b.Name = f.b.Name + err := fil.ExtractWithOptions(path, op) + if err != nil { + if op.Verbose { + log.Println("Failed to extract symlink's file:", filepath.Join(path, f.b.Name)) + } + return errors.Join(errors.New("failed to extract symlink's file: "+path), err) + } + } else { + if op.UnbreakSymlink { + filTmp := f.GetSymlinkFile() + if filTmp == nil { + if op.Verbose { + log.Println("Failed to get symlink's file:", f.path()) + } + return errors.New("failed to get symlink's file") + } + extractLoc := filepath.Join(path, filepath.Dir(symPath)) + fil := filTmp.(*File) + err := fil.ExtractWithOptions(extractLoc, op) + if err != nil { + if op.Verbose { + log.Println("Error while extracting", fil.path(), "to make sure symlink at", f.path(), "is unbroken") + } + return errors.Join(errors.New("failed to extract symlink's file: "+extractLoc), err) + } + } + path = filepath.Join(path, f.b.Name) + err := os.Symlink(f.SymlinkPath(), path) + if err != nil { + if op.Verbose { + log.Println("Failed to create symlink:", path) + } + return errors.Join(errors.New("failed to create symlink: "+path), err) + } + } + case inode.Char, inode.EChar, inode.Block, inode.EBlock, inode.Fifo, inode.EFifo: + if runtime.GOOS == "windows" { + if op.Verbose { + log.Println(f.path(), "ignored. A device link and can't be created on Windows.") + } + return nil + } + _, err := exec.LookPath("mknod") + if err != nil { + if op.Verbose { + log.Println("mknot command not found, cannot create device link for", f.path()) + } + return errors.Join(errors.New("mknot command not found"), err) + } + path = filepath.Join(path, f.b.Name) + var typ string + if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.EChar { + typ = "c" + } else if f.b.Inode.Type == inode.Block || f.b.Inode.Type == inode.EBlock { + typ = "b" + } else { //Fifo IPC + if runtime.GOOS == "darwin" { + if op.Verbose { + log.Println(f.path(), "ignored. A Fifo file and can't be created on Darwin.") + } + return nil + } + typ = "p" + } + cmd := exec.Command("mknod", path, typ) + if typ != "p" { + maj, min := f.deviceDevices() + cmd.Args = append(cmd.Args, strconv.Itoa(int(maj)), strconv.Itoa(int(min))) + } + if op.Verbose { + cmd.Stdout = op.LogOutput + cmd.Stderr = op.LogOutput + } + err = cmd.Run() + if err != nil { + if op.Verbose { + log.Println("Error while running mknod for", path) + } + return errors.Join(errors.New("error while running mknod for "+path), err) + } + case inode.Sock, inode.ESock: + if op.Verbose { + log.Println(f.path(), "ignored since it's a socket file.") + } + return nil + default: + return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.b.Inode.Type))) + } + if op.Verbose { + log.Println(f.path(), "extracted to", path) + } + if op.IgnorePerm { + return nil + } + uid, err := f.b.Uid(f.r.Low) + if err != nil { + if op.Verbose { + log.Println("Failed to get uid for", path) + log.Println(err) + } + return nil + } + gid, err := f.b.Gid(f.r.Low) + if err != nil { + if op.Verbose { + log.Println("Failed to get gid for", path) + log.Println(err) + } + return nil + } + os.Chmod(path, f.Mode()) + os.Chown(path, int(uid), int(gid)) + return nil +} diff --git a/reader_fileinfo.go b/file_info.go similarity index 61% rename from reader_fileinfo.go rename to file_info.go index 4fc875c..d7390a6 100644 --- a/reader_fileinfo.go +++ b/file_info.go @@ -4,26 +4,27 @@ import ( "io/fs" "time" - "github.com/CalebQ42/squashfs/internal/directory" - "github.com/CalebQ42/squashfs/internal/inode" + "github.com/CalebQ42/squashfs/low/directory" + "github.com/CalebQ42/squashfs/low/inode" ) type fileInfo struct { - e directory.Entry - size int64 - perm uint32 - modTime uint32 + name string + size int64 + perm uint32 + modTime uint32 + fileType uint16 } func (r Reader) newFileInfo(e directory.Entry) (fileInfo, error) { - i, err := r.inodeFromDir(e) + i, err := r.Low.InodeFromEntry(e) if err != nil { return fileInfo{}, err } - return newFileInfo(e, i), nil + return newFileInfo(e.Name, i), nil } -func newFileInfo(e directory.Entry, i inode.Inode) fileInfo { +func newFileInfo(name string, i *inode.Inode) fileInfo { var size int64 if i.Type == inode.Fil { size = int64(i.Data.(inode.File).Size) @@ -31,15 +32,16 @@ func newFileInfo(e directory.Entry, i inode.Inode) fileInfo { size = int64(i.Data.(inode.EFile).Size) } return fileInfo{ - e: e, - size: size, - perm: uint32(i.Perm), - modTime: i.ModTime, + name: name, + size: size, + perm: uint32(i.Perm), + modTime: i.ModTime, + fileType: i.Type, } } func (f fileInfo) Name() string { - return f.e.Name + return f.name } func (f fileInfo) Size() int64 { @@ -58,7 +60,7 @@ func (f fileInfo) ModTime() time.Time { } func (f fileInfo) IsDir() bool { - return f.e.Type == inode.Dir + return f.fileType == inode.Dir || f.fileType == inode.EDir } func (f fileInfo) Sys() any { diff --git a/fs.go b/fs.go new file mode 100644 index 0000000..43d9e1e --- /dev/null +++ b/fs.go @@ -0,0 +1,269 @@ +package squashfs + +import ( + "io" + "io/fs" + "path" + "path/filepath" + "slices" + "strings" + + squashfslow "github.com/CalebQ42/squashfs/low" + "github.com/CalebQ42/squashfs/low/directory" +) + +// FS is a fs.FS representation of a squashfs directory. +// Implements fs.GlobFS, fs.ReadDirFS, fs.ReadFileFS, fs.StatFS, and fs.SubFS +type FS struct { + d *squashfslow.Directory + r *Reader + parent *FS +} + +// Creates a new *FS from the given squashfs.directory +func (r *Reader) FSFromDirectory(d *squashfslow.Directory, parent *FS) *FS { + return &FS{ + d: d, + r: r, + parent: parent, + } +} + +// Glob returns the name of the files at the given pattern. +// All paths are relative to the FS. +// Uses filepath.Match to compare names. +func (f *FS) Glob(pattern string) (out []string, err error) { + pattern = filepath.Clean(pattern) + if !fs.ValidPath(pattern) { + return nil, &fs.PathError{ + Op: "glob", + Path: pattern, + Err: fs.ErrInvalid, + } + } + split := strings.Split(pattern, "/") + for i := 0; i < len(f.d.Entries); i++ { + if match, _ := path.Match(split[0], f.d.Entries[i].Name); match { + if len(split) == 1 { + out = append(out, f.d.Entries[i].Name) + continue + } + sub, err := f.Sub(split[0]) + if err != nil { + if pathErr, ok := err.(*fs.PathError); ok { + if pathErr.Err == fs.ErrNotExist { + continue + } + pathErr.Op = "glob" + pathErr.Path = pattern + return nil, pathErr + } + return nil, &fs.PathError{ + Op: "glob", + Path: pattern, + Err: err, + } + } + subGlob, err := sub.(fs.GlobFS).Glob(strings.Join(split[1:], "/")) + if err != nil { + if pathErr, ok := err.(*fs.PathError); ok { + if pathErr.Err == fs.ErrNotExist { + continue + } + pathErr.Op = "glob" + pathErr.Path = pattern + return nil, pathErr + } + return nil, &fs.PathError{ + Op: "glob", + Path: pattern, + Err: err, + } + } + for i := 0; i < len(subGlob); i++ { + subGlob[i] = f.d.Name + "/" + subGlob[i] + } + out = append(out, subGlob...) + } + } + return +} + +// Opens the file at name. Returns a *File as an fs.File. +func (f *FS) Open(name string) (fs.File, error) { + name = filepath.Clean(name) + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrInvalid, + } + } + if name == "." || name == "" { + return f.File(), nil + } + split := strings.Split(name, "/") + if split[0] == ".." { + if f.parent == nil { // root directory + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrNotExist, + } + } else { + return f.parent.Open(strings.Join(split[1:], "/")) + } + } + i, found := slices.BinarySearchFunc(f.d.Entries, split[0], func(e directory.Entry, name string) int { + return strings.Compare(e.Name, name) + }) + if !found { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrNotExist, + } + } + b, err := f.r.Low.BaseFromEntry(f.d.Entries[i]) + if err != nil { + return nil, err + } + if len(split) == 1 { + return &File{ + b: b, + r: f.r, + parent: f, + }, nil + } + if !b.IsDir() { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrNotExist, + } + } + d, err := b.ToDir(f.r.Low) + if err != nil { + return nil, err + } + return f.r.FSFromDirectory(d, f).Open(strings.Join(split[1:], "/")) +} + +// Returns all DirEntry's for the directory at name. +// If name is not a directory, returns an error. +func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) { + name = filepath.Clean(name) + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "readdir", + Path: name, + Err: fs.ErrInvalid, + } + } + if name == "." || name == "" { + return f.File().ReadDir(-1) + } + fil, err := f.Open(name) + if err != nil { + return nil, err + } + return fil.(*File).ReadDir(-1) +} + +// Returns the contents of the file at name. +func (f *FS) ReadFile(name string) (out []byte, err error) { + name = filepath.Clean(name) + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "readfile", + Path: name, + Err: fs.ErrInvalid, + } + } + if name == "." || name == "" { + return nil, fs.ErrInvalid + } + fil, err := f.Open(name) + if err != nil { + return nil, err + } + if !fil.(*File).IsRegular() { + return nil, fs.ErrInvalid + } + return io.ReadAll(fil) +} + +// Returns the fs.FileInfo for the file at name. +func (f *FS) Stat(name string) (fs.FileInfo, error) { + name = filepath.Clean(name) + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "stat", + Path: name, + Err: fs.ErrInvalid, + } + } + if name == "." || name == "" { + return f.File().Stat() + } + fil, err := f.Open(name) + if err != nil { + return nil, err + } + return fil.(*File).Stat() +} + +// Returns the FS at dir +func (f *FS) Sub(dir string) (fs.FS, error) { + dir = filepath.Clean(dir) + if !fs.ValidPath(dir) { + return nil, &fs.PathError{ + Op: "dir", + Path: dir, + Err: fs.ErrInvalid, + } + } + if dir == "." || dir == "" { + return f, nil + } + fil, err := f.Open(dir) + if err != nil { + return nil, err + } + if !fil.(*File).IsDir() { + return nil, &fs.PathError{ + Op: "dir", + Path: dir, + Err: fs.ErrInvalid, + } + } + return fil.(*File).FS() +} + +// Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. +// Uses default extraction options. +func (f *FS) Extract(folder string) error { + return f.File().Extract(folder) +} + +// Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. +// Allows setting various extraction options via ExtractionOptions. +func (f *FS) ExtractWithOptions(folder string, op *ExtractionOptions) error { + return f.File().ExtractWithOptions(folder, op) +} + +// Returns the FS as a *File +func (f *FS) File() *File { + return &File{ + b: &f.d.FileBase, + parent: f.parent, + r: f.r, + } +} + +func (f *FS) path() string { + if f.parent == nil { + return f.d.Name + } + return filepath.Join(f.parent.path(), f.d.Name) +} diff --git a/fuse2.go b/fuse2.go deleted file mode 100644 index 8338610..0000000 --- a/fuse2.go +++ /dev/null @@ -1,150 +0,0 @@ -package squashfs - -import ( - "bytes" - "context" - "errors" - "io" - - "github.com/CalebQ42/squashfs/internal/inode" - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" -) - -// Mounts the archive to the given mountpoint using fuse2. Non-blocking. -// If Unmount does not get called, the mount point must be unmounted using umount before the directory can be used again. -func (r *Reader) MountFuse2(mountpoint string) (err error) { - if r.con != nil { - return errors.New("squashfs archive already mounted") - } - r.con2, err = fuse.Mount(mountpoint, fuse.ReadOnly()) - if err != nil { - return - } - <-r.con2.Ready - r.mount2Done = make(chan struct{}) - go func() { - fs.Serve(r.con2, squashFuse2{r: r}) - close(r.mount2Done) - }() - return -} - -// Blocks until the mount ends. -// Fuse2 version. -func (r *Reader) MountWaitFuse2() { - if r.mount2Done != nil { - <-r.mount2Done - } -} - -// Unmounts the archive. -// Fuse2 version. -func (r *Reader) UnmountFuse2() error { - if r.con != nil { - defer func() { r.con = nil }() - return r.con.Close() - } - return errors.New("squashfs archive is not mounted") -} - -type squashFuse2 struct { - r *Reader -} - -func (s squashFuse2) Root() (fs.Node, error) { - return fileNode2{File: s.r.FS.File}, nil -} - -type fileNode2 struct { - *File -} - -func (f fileNode2) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Blocks = f.r.s.Size / 512 - if f.r.s.Size%512 > 0 { - attr.Blocks++ - } - attr.Gid = f.r.ids[f.i.GidInd] - attr.Inode = uint64(f.i.Num) - attr.Mode = f.i.Mode() - attr.Nlink = f.i.LinkCount() - attr.Size = f.i.Size() - attr.Uid = f.r.ids[f.i.UidInd] - return nil -} - -func (f fileNode2) Id() uint64 { - return uint64(f.i.Num) -} - -func (f fileNode2) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - return f.SymlinkPath(), nil -} - -func (f fileNode2) Lookup(ctx context.Context, name string) (fs.Node, error) { - asFS, err := f.FS() - if err != nil { - return nil, fuse.ENOTDIR - } - ret, err := asFS.OpenFile(name) - if err != nil { - return nil, fuse.ENOENT - } - return fileNode2{File: ret}, nil -} - -func (f fileNode2) ReadAll(ctx context.Context) ([]byte, error) { - if f.IsRegular() { - var buf bytes.Buffer - _, err := f.WriteTo(&buf) - return buf.Bytes(), err - } - return nil, ENODATA -} - -func (f fileNode2) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - if f.IsRegular() { - buf := make([]byte, req.Size) - n, err := f.File.ReadAt(buf, req.Offset) - if err == io.EOF { - resp.Data = buf[:n] - } - return nil - } - return ENODATA -} - -func (f fileNode2) ReadDirAll(ctx context.Context) (out []fuse.Dirent, err error) { - asFS, err := f.FS() - if err != nil { - return nil, fuse.ENOTDIR - } - var t fuse.DirentType - for i := range asFS.e { - switch asFS.e[i].Type { - case inode.Fil: - t = fuse.DT_File - case inode.Dir: - t = fuse.DT_Dir - case inode.Block: - t = fuse.DT_Block - case inode.Sym: - t = fuse.DT_Link - case inode.Char: - t = fuse.DT_Char - case inode.Fifo: - t = fuse.DT_FIFO - case inode.Sock: - t = fuse.DT_Socket - default: - t = fuse.DT_Unknown - } - out = append(out, fuse.Dirent{ - Inode: uint64(asFS.e[i].Num), - Type: t, - Name: asFS.e[i].Name, - }) - } - return -} diff --git a/fuse3.go b/fuse3.go deleted file mode 100644 index e0b8b28..0000000 --- a/fuse3.go +++ /dev/null @@ -1,148 +0,0 @@ -package squashfs - -import ( - "bytes" - "context" - "errors" - "io" - - "github.com/CalebQ42/fuse" - "github.com/CalebQ42/fuse/fs" - "github.com/CalebQ42/squashfs/internal/inode" -) - -// Mounts the archive to the given mountpoint using fuse3. Non-blocking. -// If Unmount does not get called, the mount point must be unmounted using umount before the directory can be used again. -func (r *Reader) Mount(mountpoint string) (err error) { - if r.con != nil { - return errors.New("squashfs archive already mounted") - } - r.con, err = fuse.Mount(mountpoint, fuse.ReadOnly()) - if err != nil { - return - } - <-r.con.Ready - r.mountDone = make(chan struct{}) - go func() { - fs.Serve(r.con, squashFuse{r: r}) - close(r.mountDone) - }() - return -} - -// Blocks until the mount ends. -func (r *Reader) MountWait() { - if r.mountDone != nil { - <-r.mountDone - } -} - -// Unmounts the archive. -func (r *Reader) Unmount() error { - if r.con != nil { - defer func() { r.con = nil }() - return r.con.Close() - } - return errors.New("squashfs archive is not mounted") -} - -type squashFuse struct { - r *Reader -} - -func (s squashFuse) Root() (fs.Node, error) { - return fileNode{File: s.r.FS.File}, nil -} - -type fileNode struct { - *File -} - -func (f fileNode) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Blocks = f.r.s.Size / 512 - if f.r.s.Size%512 > 0 { - attr.Blocks++ - } - attr.Gid = f.r.ids[f.i.GidInd] - attr.Inode = uint64(f.i.Num) - attr.Mode = f.i.Mode() - attr.Nlink = f.i.LinkCount() - attr.Size = f.i.Size() - attr.Uid = f.r.ids[f.i.UidInd] - return nil -} - -func (f fileNode) Id() uint64 { - return uint64(f.i.Num) -} - -func (f fileNode) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - return f.SymlinkPath(), nil -} - -func (f fileNode) Lookup(ctx context.Context, name string) (fs.Node, error) { - asFS, err := f.FS() - if err != nil { - return nil, fuse.ENOTDIR - } - ret, err := asFS.OpenFile(name) - if err != nil { - return nil, fuse.ENOENT - } - return fileNode{File: ret}, nil -} - -func (f fileNode) ReadAll(ctx context.Context) ([]byte, error) { - if f.IsRegular() { - var buf bytes.Buffer - _, err := f.WriteTo(&buf) - return buf.Bytes(), err - } - return nil, ENODATA -} - -func (f fileNode) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - if f.IsRegular() { - buf := make([]byte, req.Size) - n, err := f.File.ReadAt(buf, req.Offset) - if err == io.EOF { - resp.Data = buf[:n] - } - return nil - } - return ENODATA -} - -func (f fileNode) ReadDirAll(ctx context.Context) (out []fuse.Dirent, err error) { - asFS, err := f.FS() - if err != nil { - return nil, fuse.ENOTDIR - } - var t fuse.DirentType - for i := range asFS.e { - switch asFS.e[i].Type { - case inode.Fil: - t = fuse.DT_File - case inode.Dir: - t = fuse.DT_Dir - case inode.Block: - t = fuse.DT_Block - case inode.Sym: - t = fuse.DT_Link - case inode.Char: - t = fuse.DT_Char - case inode.Fifo: - t = fuse.DT_FIFO - case inode.Sock: - t = fuse.DT_Socket - default: - t = fuse.DT_Unknown - } - out = append(out, fuse.Dirent{ - Inode: uint64(asFS.e[i].Num), - Type: t, - Name: asFS.e[i].Name, - }) - } - return -} diff --git a/fuse_darwin.go b/fuse_darwin.go deleted file mode 100644 index 353aee1..0000000 --- a/fuse_darwin.go +++ /dev/null @@ -1,7 +0,0 @@ -package squashfs - -import ( - "golang.org/x/sys/unix" -) - -var ENODATA = unix.Errno(unix.ENODATA) diff --git a/fuse_linux.go b/fuse_linux.go deleted file mode 100644 index 816560a..0000000 --- a/fuse_linux.go +++ /dev/null @@ -1,5 +0,0 @@ -package squashfs - -import "github.com/CalebQ42/fuse" - -var ENODATA = fuse.ENODATA diff --git a/fuse_windows.go b/fuse_windows.go deleted file mode 100644 index c50a696..0000000 --- a/fuse_windows.go +++ /dev/null @@ -1,3 +0,0 @@ -package squashfs - -var ENODATA = windows.Errno(windows.ENODATA) diff --git a/go.mod b/go.mod index 8599653..39a24a7 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,11 @@ module github.com/CalebQ42/squashfs -go 1.21 +go 1.21.5 require ( - github.com/CalebQ42/fuse v0.1.0 - github.com/klauspost/compress v1.16.7 - github.com/pierrec/lz4/v4 v4.1.18 - github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e - github.com/seaweedfs/fuse v1.2.2 - github.com/therootcompany/xz v1.0.1 + github.com/pierrec/lz4/v4 v4.1.19 github.com/ulikunitz/xz v0.5.11 - golang.org/x/sys v0.11.0 + github.com/klauspost/compress v1.17.4 + github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e + github.com/therootcompany/xz v1.0.1 ) diff --git a/go.sum b/go.sum index 6e8a10b..894206e 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,10 @@ -github.com/CalebQ42/fuse v0.1.0 h1:KLCNjun7zcd2kBNVFfH+SWJyhuwJdE0nhw5/q8K8HGQ= -github.com/CalebQ42/fuse v0.1.0/go.mod h1:pJpoKG03HJKVhsp8o0YQYqmfbFsr3Eowt90yQGQVO+4= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4= +github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs= github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M= -github.com/seaweedfs/fuse v1.2.2 h1:01l8OjIdyATRNqVc/gDPgFobuC8ubQF3hRKOPColROw= -github.com/seaweedfs/fuse v1.2.2/go.mod h1:iwbDQv5BZACY54r6AO/6xsLNuMaYcBKSkLTZVfmK594= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/data/fullreader.go b/internal/data/fullreader.go deleted file mode 100644 index 2630357..0000000 --- a/internal/data/fullreader.go +++ /dev/null @@ -1,235 +0,0 @@ -package data - -import ( - "io" - - "github.com/CalebQ42/squashfs/internal/decompress" - "github.com/CalebQ42/squashfs/internal/toreader" -) - -type FullReader struct { - r io.ReaderAt - d decompress.Decompressor - fragRdr func() (io.Reader, error) - sizes []uint32 - blockSize uint32 - start uint64 - fileSize uint64 -} - -func NewFullReader(r io.ReaderAt, start uint64, d decompress.Decompressor, blockSizes []uint32, blockSize uint32, fileSize uint64) *FullReader { - return &FullReader{ - r: r, - start: start, - blockSize: blockSize, - sizes: blockSizes, - d: d, - fileSize: fileSize, - } -} - -func (r *FullReader) AddFragment(rdr func() (io.Reader, error)) { - r.fragRdr = rdr - r.sizes = append(r.sizes, 0) -} - -type outDat struct { - err error - data []byte - i int -} - -func (r FullReader) process(index int, offset int64, out chan outDat) { - var err error - var dat []byte - var rdr io.ReadCloser - size := realSize(r.sizes[index]) - if size == 0 { - outSize := r.blockSize - if r.fileSize < uint64(r.blockSize) { - outSize = uint32(r.fileSize) - } - out <- outDat{ - i: index, - err: nil, - data: make([]byte, outSize), - } - return - } - // rdr := io.LimitReader(toreader.NewReader(r.r, offset), int64(size)) - if size == r.sizes[index] { - if dec, ok := r.d.(decompress.Decoder); ok { - dat = make([]byte, size) - _, err = r.r.ReadAt(dat, offset) - if err == nil { - dat, err = dec.Decode(dat) - } - } else { - rdr, err = r.d.Reader(io.LimitReader(toreader.NewReader(r.r, offset), int64(size))) - if err == nil { - dat, err = io.ReadAll(rdr) - } - } - } else { - dat = make([]byte, size) - _, err = r.r.ReadAt(dat, offset) - } - out <- outDat{ - i: index, - err: err, - data: dat, - } - if clr, ok := rdr.(io.Closer); ok { - clr.Close() - } -} - -func (r FullReader) ReadAt(p []byte, off int64) (n int, err error) { - out := make(chan outDat, len(r.sizes)) - offset := r.start - num := len(r.sizes) - start := off / int64(r.blockSize) - end := len(p) / int(r.blockSize) - if end%int(r.blockSize) > 0 { - end++ - } - if end > len(r.sizes) { - if r.fragRdr != nil { - end = len(r.sizes) - } else { - end = len(r.sizes) + 1 - } - } - for i := 0; i < num; i++ { - if i < int(start) || i > end { - offset += uint64(realSize(r.sizes[i])) - continue - } - if i == num-1 && r.fragRdr != nil { - go func() { - rdr, e := r.fragRdr() - if e != nil { - out <- outDat{ - i: num - 1, - err: e, - } - return - } - dat, e := io.ReadAll(rdr) - out <- outDat{ - i: num - 1, - err: e, - data: dat, - } - if clr, ok := rdr.(io.Closer); ok { - clr.Close() - } - }() - continue - } - go r.process(i, int64(offset), out) - offset += uint64(realSize(r.sizes[i])) - } - cache := make(map[int]outDat) - for cur := start; cur < int64(end); { - dat := <-out - if dat.err != nil { - err = dat.err - return - } - if dat.i != int(cur) { - cache[dat.i] = dat - continue - } - if cur == start { - dat.data = dat.data[off%int64(r.blockSize):] - } - for i := range dat.data { - p[n+i] = dat.data[i] - } - n += len(dat.data) - cur++ - var ok bool - for { - dat, ok = cache[int(cur)] - if !ok { - break - } - for i := range dat.data { - p[n+i] = dat.data[i] - } - n += len(dat.data) - cur++ - delete(cache, int(cur)) - } - } - if n < len(p) { - err = io.EOF - } - return -} - -func (r FullReader) WriteTo(w io.Writer) (n int64, err error) { - out := make(chan outDat, len(r.sizes)) - offset := r.start - num := len(r.sizes) - for i := 0; i < num; i++ { - if i == num-1 && r.fragRdr != nil { - go func() { - rdr, e := r.fragRdr() - if err != nil { - out <- outDat{ - i: num - 1, - err: e, - } - return - } - dat, e := io.ReadAll(rdr) - out <- outDat{ - i: num - 1, - err: e, - data: dat, - } - if clr, ok := rdr.(io.Closer); ok { - clr.Close() - } - }() - continue - } - go r.process(i, int64(offset), out) - offset += uint64(realSize(r.sizes[i])) - } - cache := make(map[int]outDat) - var tmpN int - for cur := 0; cur < num; { - dat := <-out - if dat.err != nil { - err = dat.err - return - } - if dat.i != cur { - cache[dat.i] = dat - continue - } - tmpN, err = w.Write(dat.data) - n += int64(tmpN) - if err != nil { - return - } - cur++ - var ok bool - for { - dat, ok = cache[cur] - if !ok { - break - } - tmpN, err = w.Write(dat.data) - n += int64(tmpN) - if err != nil { - return - } - cur++ - } - } - return -} diff --git a/internal/data/reader.go b/internal/data/reader.go deleted file mode 100644 index a1ddd05..0000000 --- a/internal/data/reader.go +++ /dev/null @@ -1,104 +0,0 @@ -package data - -import ( - "bytes" - "io" - - "github.com/CalebQ42/squashfs/internal/decompress" -) - -type Reader struct { - master io.Reader - cur io.Reader - fragRdr io.Reader - d decompress.Decompressor - comRdr io.Reader - blockSizes []uint32 - blockSize uint32 - resetable bool - fileSize uint64 -} - -func NewReader(r io.Reader, d decompress.Decompressor, blockSizes []uint32, blockSize uint32, fileSize uint64) *Reader { - return &Reader{ - d: d, - master: r, - blockSizes: blockSizes, - blockSize: blockSize, - resetable: true, - fileSize: fileSize, - } -} - -func (r *Reader) AddFragment(rdr io.Reader) { - r.fragRdr = rdr - r.blockSizes = append(r.blockSizes, 0) -} - -func realSize(siz uint32) uint32 { - return siz &^ (1 << 24) -} - -func (r *Reader) advance() (err error) { - if clr, ok := r.cur.(io.Closer); ok { - clr.Close() - } - if len(r.blockSizes) == 0 { - return io.EOF - } - if len(r.blockSizes) == 1 && r.fragRdr != nil { - r.cur = r.fragRdr - } else { - size := realSize(r.blockSizes[0]) - if size == 0 { - outSize := r.blockSize - if r.fileSize < uint64(r.blockSize) { - outSize = uint32(r.fileSize) - } - r.cur = bytes.NewReader(make([]byte, outSize)) - } else { - r.cur = io.LimitReader(r.master, int64(size)) - if size == r.blockSizes[0] { - if rs, ok := r.d.(decompress.Resetable); ok { - if r.comRdr == nil { - r.cur, err = r.d.Reader(r.cur) - if err != nil { - return - } - } else { - err = rs.Reset(r.comRdr, r.cur) - r.cur = r.comRdr - } - } else { - r.cur, err = r.d.Reader(r.cur) - } - } - } - } - r.blockSizes = r.blockSizes[1:] - return -} - -func (r *Reader) Read(p []byte) (n int, err error) { - if r.cur == nil { - err = r.advance() - if err != nil { - return - } - } - n, err = r.cur.Read(p) - if err == io.EOF { - err = r.advance() - if err != nil { - return - } - var tmpN int - tmp := make([]byte, len(p)-n) - tmpN, err = r.Read(tmp) - for i := range tmp { - p[n+i] = tmp[i] - } - n += tmpN - } - return -} diff --git a/internal/decompress/decompress.go b/internal/decompress/decompress.go new file mode 100644 index 0000000..eb66221 --- /dev/null +++ b/internal/decompress/decompress.go @@ -0,0 +1,5 @@ +package decompress + +type Decompressor interface { + Decompress([]byte) ([]byte, error) +} diff --git a/internal/decompress/gzip.go b/internal/decompress/gzip.go deleted file mode 100644 index 26feccd..0000000 --- a/internal/decompress/gzip.go +++ /dev/null @@ -1,17 +0,0 @@ -package decompress - -import ( - "io" - - "github.com/klauspost/compress/zlib" -) - -type GZip struct{} - -func (g GZip) Reader(src io.Reader) (io.ReadCloser, error) { - return zlib.NewReader(src) -} - -func (g GZip) Reset(old, src io.Reader) error { - return old.(zlib.Resetter).Reset(src, nil) -} diff --git a/internal/decompress/interface.go b/internal/decompress/interface.go deleted file mode 100644 index f6ac915..0000000 --- a/internal/decompress/interface.go +++ /dev/null @@ -1,22 +0,0 @@ -package decompress - -import ( - "io" -) - -type Decompressor interface { - //Creates a new decompressor reading from src. - Reader(src io.Reader) (io.ReadCloser, error) -} - -type Resetable interface { - //Reset attempts to re-use an old decompressor with new data. - //Will return ErrNotResetable if not Resetable(). - //Must ALWAYS be provided with a reader created with Reader. - Reset(old, src io.Reader) error -} - -type Decoder interface { - //Decodes a chunk of data all at once. - Decode(in []byte) ([]byte, error) -} diff --git a/internal/decompress/lz4.go b/internal/decompress/lz4.go index 2a53ac7..e2f3bdb 100644 --- a/internal/decompress/lz4.go +++ b/internal/decompress/lz4.go @@ -1,6 +1,7 @@ package decompress import ( + "bytes" "io" "github.com/pierrec/lz4/v4" @@ -8,11 +9,7 @@ import ( type Lz4 struct{} -func (l Lz4) Reader(r io.Reader) (io.ReadCloser, error) { - return io.NopCloser(lz4.NewReader(r)), nil -} - -func (l Lz4) Reset(old, src io.Reader) error { - old.(*lz4.Reader).Reset(src) - return nil +func (l Lz4) Decompress(data []byte) ([]byte, error) { + rdr := lz4.NewReader(bytes.NewReader(data)) + return io.ReadAll(rdr) } diff --git a/internal/decompress/lzma.go b/internal/decompress/lzma.go index 9add4ae..e79d0db 100644 --- a/internal/decompress/lzma.go +++ b/internal/decompress/lzma.go @@ -1,6 +1,7 @@ package decompress import ( + "bytes" "io" "github.com/ulikunitz/xz/lzma" @@ -8,7 +9,10 @@ import ( type Lzma struct{} -func (l Lzma) Reader(r io.Reader) (io.ReadCloser, error) { - rdr, err := lzma.NewReader(r) - return io.NopCloser(rdr), err +func (l Lzma) Decompress(data []byte) ([]byte, error) { + rdr, err := lzma.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + return io.ReadAll(rdr) } diff --git a/internal/decompress/lzo.go b/internal/decompress/lzo.go index 76333e7..f5783b4 100644 --- a/internal/decompress/lzo.go +++ b/internal/decompress/lzo.go @@ -2,17 +2,12 @@ package decompress import ( "bytes" - "io" "github.com/rasky/go-lzo" ) type Lzo struct{} -func (l Lzo) Reader(r io.Reader) (io.ReadCloser, error) { - cache, err := lzo.Decompress1X(r, 0, 0) - if err != nil { - return nil, err - } - return io.NopCloser(bytes.NewReader(cache)), nil +func (l Lzo) Decompress(data []byte) ([]byte, error) { + return lzo.Decompress1X(bytes.NewReader(data), len(data), 0) } diff --git a/internal/decompress/xz.go b/internal/decompress/xz.go index 8bc2c80..02191db 100644 --- a/internal/decompress/xz.go +++ b/internal/decompress/xz.go @@ -1,6 +1,7 @@ package decompress import ( + "bytes" "io" "github.com/therootcompany/xz" @@ -8,11 +9,10 @@ import ( type Xz struct{} -func (x Xz) Reader(r io.Reader) (io.ReadCloser, error) { - rdr, err := xz.NewReader(r, 0) - return io.NopCloser(rdr), err -} - -func (x Xz) Reset(old, src io.Reader) error { - return old.(*xz.Reader).Reset(src) +func (x Xz) Decompress(data []byte) ([]byte, error) { + rdr, err := xz.NewReader(bytes.NewReader(data), 0) + if err != nil { + return nil, err + } + return io.ReadAll(rdr) } diff --git a/internal/decompress/zlib.go b/internal/decompress/zlib.go new file mode 100644 index 0000000..3da3415 --- /dev/null +++ b/internal/decompress/zlib.go @@ -0,0 +1,18 @@ +package decompress + +import ( + "bytes" + "compress/zlib" + "io" +) + +type Zlib struct{} + +func (z Zlib) Decompress(data []byte) ([]byte, error) { + rdr, err := zlib.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer rdr.Close() + return io.ReadAll(rdr) +} diff --git a/internal/decompress/zstd.go b/internal/decompress/zstd.go index dde62ff..ff24013 100644 --- a/internal/decompress/zstd.go +++ b/internal/decompress/zstd.go @@ -1,27 +1,19 @@ package decompress import ( + "bytes" "io" "github.com/klauspost/compress/zstd" ) -type Zstd struct { - writeToReader *zstd.Decoder -} +type Zstd struct{} -func (z Zstd) Reader(src io.Reader) (io.ReadCloser, error) { - r, err := zstd.NewReader(src) - return r.IOReadCloser(), err -} - -func (z Zstd) Reset(old, src io.Reader) error { - return old.(*zstd.Decoder).Reset(src) -} - -func (z Zstd) Decode(in []byte) ([]byte, error) { - if z.writeToReader == nil { - z.writeToReader, _ = zstd.NewReader(nil) +func (z Zstd) Decompress(data []byte) ([]byte, error) { + rdr, err := zstd.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err } - return z.writeToReader.DecodeAll(in, nil) + defer rdr.Close() + return io.ReadAll(rdr) } diff --git a/internal/directory/directory.go b/internal/directory/directory.go deleted file mode 100644 index 7f49bc3..0000000 --- a/internal/directory/directory.go +++ /dev/null @@ -1,80 +0,0 @@ -package directory - -import ( - "bytes" - "encoding/binary" - "io" -) - -type header struct { - Entries uint32 - InodeStart uint32 - Num uint32 -} - -type entryInit struct { - Offset uint16 - NumOffset int16 - Type uint16 - NameSize uint16 -} - -type entry struct { - entryInit - Name []byte -} - -type Entry struct { - Name string - BlockStart uint32 - Type uint16 - Offset uint16 - Num uint32 -} - -func readEntry(r io.Reader) (e entry, err error) { - err = binary.Read(r, binary.LittleEndian, &e.entryInit) - if err != nil { - return - } - e.Name = make([]byte, e.NameSize+1) - err = binary.Read(r, binary.LittleEndian, &e.Name) - return -} - -func ReadEntries(rdr io.Reader, size uint32) (e []Entry, err error) { - dat := make([]byte, size-3) - rdr.Read(dat) - r := bytes.NewReader(dat) - var h header - var en entry - for { - err = binary.Read(r, binary.LittleEndian, &h) - if err == io.EOF { - err = nil - return - } else if err != nil { - return - } - h.Entries++ - for i := 0; i < int(h.Entries); i++ { - if i != 0 && i%256 == 0 { - err = binary.Read(r, binary.LittleEndian, &h) - if err != nil { - return - } - } - en, err = readEntry(r) - if err != nil { - return - } - e = append(e, Entry{ - Name: string(en.Name), - BlockStart: h.InodeStart, - Type: en.Type, - Offset: en.Offset, - Num: h.Num + uint32(en.NumOffset), - }) - } - } -} diff --git a/internal/metadata/reader.go b/internal/metadata/reader.go index 3fd0499..b20d63c 100644 --- a/internal/metadata/reader.go +++ b/internal/metadata/reader.go @@ -8,74 +8,60 @@ import ( ) type Reader struct { - master io.Reader - cur io.Reader - d decompress.Decompressor - comRdr io.Reader + r io.Reader + d decompress.Decompressor + dat []byte + curOffset uint16 } -func NewReader(master io.Reader, d decompress.Decompressor) *Reader { +func NewReader(r io.Reader, d decompress.Decompressor) *Reader { return &Reader{ - master: master, - d: d, + r: r, + d: d, } } -func realSize(siz uint16) uint16 { - return siz &^ 0x8000 -} - -func (r *Reader) advance() (err error) { - if _, ok := r.d.(decompress.Resetable); !ok { - if clr, ok := r.cur.(io.Closer); ok { - clr.Close() - } - } - var raw uint16 - err = binary.Read(r.master, binary.LittleEndian, &raw) +func (r *Reader) advance() error { + r.curOffset = 0 + var size uint16 + err := binary.Read(r.r, binary.LittleEndian, &size) if err != nil { - return + return err } - size := realSize(raw) - r.cur = io.LimitReader(r.master, int64(size)) - if size == raw { - if rs, ok := r.d.(decompress.Resetable); ok { - if r.comRdr == nil { - r.cur, err = r.d.Reader(r.cur) - if err != nil { - return - } - } else { - err = rs.Reset(r.comRdr, r.cur) - r.cur = r.comRdr - } - } else { - r.cur, err = r.d.Reader(r.cur) - } + realSize := size &^ 0x8000 + r.dat = make([]byte, realSize) + err = binary.Read(r.r, binary.LittleEndian, &r.dat) + if err != nil { + return err } - return + if size != realSize { + return nil + } + r.dat, err = r.d.Decompress(r.dat) + return err } -func (r *Reader) Read(p []byte) (n int, err error) { - if r.cur == nil { - err = r.advance() - if err != nil { - return +func (r *Reader) Read(b []byte) (int, error) { + curRead := 0 + var toRead int + for curRead < len(b) { + if r.curOffset >= uint16(len(r.dat)) { + if err := r.advance(); err != nil { + return curRead, err + } } + toRead = len(b) - curRead + if toRead > len(r.dat)-int(r.curOffset) { + toRead = len(r.dat) - int(r.curOffset) + } + copy(b[curRead:], r.dat[r.curOffset:int(r.curOffset)+toRead]) + r.curOffset += uint16(toRead) + curRead += toRead } - n, err = r.cur.Read(p) - if err == io.EOF { - err = r.advance() - if err != nil { - return - } - var tmpN int - tmp := make([]byte, len(p)-n) - tmpN, err = r.Read(tmp) - for i := 0; i < tmpN; i++ { - p[n+i] = tmp[i] - } - n += tmpN - } - return + return curRead, nil +} + +func (r *Reader) Close() error { + r.dat = nil + return nil } diff --git a/internal/routinemanager/manager.go b/internal/routinemanager/manager.go new file mode 100644 index 0000000..4e47ed8 --- /dev/null +++ b/internal/routinemanager/manager.go @@ -0,0 +1,25 @@ +package routinemanager + +type Manager struct { + channel chan uint16 + maxRoutines uint16 +} + +func NewManager(maxRoutines uint16) *Manager { + m := &Manager{ + maxRoutines: maxRoutines, + channel: make(chan uint16, maxRoutines), + } + for i := uint16(0); i < maxRoutines; i++ { + m.channel <- i + } + return m +} + +func (m *Manager) Lock() uint16 { + return <-m.channel +} + +func (m *Manager) Unlock(i uint16) { + m.channel <- i +} diff --git a/internal/threadmanager/manager.go b/internal/threadmanager/manager.go deleted file mode 100644 index 6ee48be..0000000 --- a/internal/threadmanager/manager.go +++ /dev/null @@ -1,23 +0,0 @@ -package threadmanager - -type Manager struct { - c chan int -} - -func NewManager(maxRoutines int) *Manager { - m := &Manager{ - c: make(chan int, maxRoutines), - } - for i := 0; i < maxRoutines; i++ { - m.c <- i - } - return m -} - -func (m *Manager) Lock() int { - return <-m.c -} - -func (m *Manager) Unlock(n int) { - m.c <- n -} diff --git a/internal/toreader/offsetreader.go b/internal/toreader/offsetreader.go deleted file mode 100644 index 0192177..0000000 --- a/internal/toreader/offsetreader.go +++ /dev/null @@ -1,19 +0,0 @@ -package toreader - -import "io" - -type OffsetReader struct { - r io.ReaderAt - off int64 -} - -func NewOffsetReader(r io.ReaderAt, off int64) *OffsetReader { - return &OffsetReader{ - r: r, - off: off, - } -} - -func (r OffsetReader) ReadAt(p []byte, off int64) (n int, e error) { - return r.r.ReadAt(p, off+r.off) -} diff --git a/internal/toreader/reader.go b/internal/toreader/reader.go deleted file mode 100644 index 6f711fd..0000000 --- a/internal/toreader/reader.go +++ /dev/null @@ -1,25 +0,0 @@ -package toreader - -import "io" - -type Reader struct { - r io.ReaderAt - off int64 -} - -func NewReader(r io.ReaderAt, start int64) *Reader { - return &Reader{ - r: r, - off: start, - } -} - -func (r *Reader) Read(p []byte) (n int, err error) { - n, err = r.r.ReadAt(p, r.off) - r.off += int64(n) - return -} - -func (r Reader) Offset() int64 { - return r.off -} diff --git a/internal/toreader/readerat.go b/internal/toreader/readerat.go deleted file mode 100644 index b556ca7..0000000 --- a/internal/toreader/readerat.go +++ /dev/null @@ -1,24 +0,0 @@ -package toreader - -import "io" - -type ReaderAt struct { - d []byte -} - -func NewReaderAt(r io.Reader) (ra *ReaderAt, err error) { - ra = new(ReaderAt) - ra.d, err = io.ReadAll(r) - return -} - -func (r ReaderAt) ReadAt(p []byte, off int64) (n int, err error) { - if int(off) >= len(r.d) { - return 0, io.EOF - } - n = copy(p, r.d[off:]) - if n != len(p) { - err = io.EOF - } - return -} diff --git a/internal/toreader/toreader.go b/internal/toreader/toreader.go new file mode 100644 index 0000000..9600bd5 --- /dev/null +++ b/internal/toreader/toreader.go @@ -0,0 +1,21 @@ +package toreader + +import "io" + +type Reader struct { + r io.ReaderAt + offset int64 +} + +func NewReader(r io.ReaderAt, start int64) *Reader { + return &Reader{ + r: r, + offset: start, + } +} + +func (r *Reader) Read(b []byte) (int, error) { + n, err := r.r.ReadAt(b, r.offset) + r.offset += int64(n) + return n, err +} diff --git a/low/README.md b/low/README.md new file mode 100644 index 0000000..33eb8eb --- /dev/null +++ b/low/README.md @@ -0,0 +1,3 @@ +# Lower-Level Squashfs + +This library is a lower level version of the main [squashfs](https://github.com/CalebQ42/squashfs) library that doesn't try to be easy to use and exposes a lot of information that is not necesary for must use cases. diff --git a/low/data/fullreader.go b/low/data/fullreader.go new file mode 100644 index 0000000..d31853d --- /dev/null +++ b/low/data/fullreader.go @@ -0,0 +1,174 @@ +package data + +import ( + "encoding/binary" + "errors" + "io" + "math" + "runtime" + "sync" + + "github.com/CalebQ42/squashfs/internal/decompress" + "github.com/CalebQ42/squashfs/internal/toreader" +) + +type FragReaderConstructor func() (io.Reader, error) + +type FullReader struct { + r io.ReaderAt + d decompress.Decompressor + frag FragReaderConstructor + retPool *sync.Pool + sizes []uint32 + initialOffset int64 + finalBlockSize uint64 + blockSize uint32 + goroutineLimit uint16 +} + +func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) *FullReader { + return &FullReader{ + r: r, + d: d, + sizes: sizes, + initialOffset: initialOffset, + goroutineLimit: uint16(runtime.NumCPU()), + finalBlockSize: finalBlockSize, + blockSize: blockSize, + retPool: &sync.Pool{ + New: func() any { + return &retValue{} + }, + }, + } +} + +func (r *FullReader) AddFrag(frag FragReaderConstructor) { + r.frag = frag +} + +func (r *FullReader) SetGoroutineLimit(limit uint16) { + r.goroutineLimit = limit +} + +type retValue struct { + err error + data []byte + index uint64 +} + +func (r *FullReader) process(index uint64, fileOffset uint64, retChan chan *retValue) { + ret := r.retPool.Get().(*retValue) + ret.index = index + realSize := r.sizes[index] &^ (1 << 24) + if realSize == 0 { + if index == uint64(len(r.sizes))-1 && r.frag == nil { + ret.data = make([]byte, r.finalBlockSize) + } else { + ret.data = make([]byte, r.blockSize) + } + ret.err = nil + retChan <- ret + return + } + ret.data = make([]byte, realSize) + ret.err = binary.Read(toreader.NewReader(r.r, int64(r.initialOffset)+int64(fileOffset)), binary.LittleEndian, &ret.data) + if r.sizes[index] == realSize { + ret.data, ret.err = r.d.Decompress(ret.data) + } + retChan <- ret +} + +func (r *FullReader) WriteTo(w io.Writer) (int64, error) { + var curIndex uint64 + var curOffset uint64 + var toProcess uint16 + var wrote int64 + cache := make(map[uint64]*retValue) + var errCache []error + retChan := make(chan *retValue, r.goroutineLimit) + for i := uint64(0); i < uint64(math.Ceil(float64(len(r.sizes))/float64(r.goroutineLimit))); i++ { + toProcess = uint16(len(r.sizes)) - (uint16(i) * r.goroutineLimit) + if toProcess > r.goroutineLimit { + toProcess = r.goroutineLimit + } + // Start all the goroutines + for j := uint16(0); j < toProcess; j++ { + go r.process((i*uint64(r.goroutineLimit))+uint64(j), curOffset, retChan) + curOffset += uint64(r.sizes[(i*uint64(r.goroutineLimit))+uint64(j)]) &^ (1 << 24) + } + // Then consume the results on retChan + for j := uint16(0); j < toProcess; j++ { + res := <-retChan + // If there's an error, we don't care about the results. + if res.err != nil { + errCache = append(errCache, res.err) + if len(cache) > 0 { + clear(cache) + } + continue + } + // If there has been an error previously, we don't care about the results. + // We still want to wait for all the goroutines to prevent resources being wasted. + if len(errCache) > 0 { + continue + } + // If we don't need the data yet, we cache it and move on + if res.index != curIndex { + cache[res.index] = res + continue + } + // If we do need the data, we write it + wr, err := w.Write(res.data) + wrote += int64(wr) + if err != nil { + errCache = append(errCache, err) + if len(cache) > 0 { + clear(cache) + } + continue + } + r.retPool.Put(res) + curIndex++ + // Now we recursively try to clear the cache + for len(cache) > 0 { + res, ok := cache[curIndex] + if !ok { + break + } + wr, err := w.Write(res.data) + wrote += int64(wr) + if err != nil { + errCache = append(errCache, err) + if len(cache) > 0 { + clear(cache) + } + break + } + delete(cache, curIndex) + r.retPool.Put(res) + curIndex++ + } + } + if len(errCache) > 0 { + return wrote, errors.Join(errCache...) + } + } + if r.frag != nil { + rdr, err := r.frag() + if err != nil { + return wrote, err + } + wr, err := io.Copy(w, rdr) + wrote += wr + if l, ok := rdr.(*io.LimitedReader); ok { + if cl, ok := l.R.(io.Closer); ok { + cl.Close() + } + } + if err != nil { + return wrote, err + } + } + return wrote, nil +} diff --git a/low/data/reader.go b/low/data/reader.go new file mode 100644 index 0000000..922263d --- /dev/null +++ b/low/data/reader.go @@ -0,0 +1,97 @@ +package data + +import ( + "encoding/binary" + "io" + + "github.com/CalebQ42/squashfs/internal/decompress" +) + +type Reader struct { + r io.Reader + d decompress.Decompressor + frag io.Reader + sizes []uint32 + dat []byte + curOffset int + curIndex uint64 + finalBlockSize uint64 + blockSize uint32 +} + +func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) *Reader { + return &Reader{ + r: r, + d: d, + sizes: sizes, + finalBlockSize: finalBlockSize, + blockSize: blockSize, + } +} + +func (r *Reader) AddFrag(fragRdr io.Reader) { + r.frag = fragRdr +} + +func (r *Reader) advance() error { + r.curOffset = 0 + defer func() { r.curIndex++ }() + var err error + if r.curIndex == uint64(len(r.sizes)) && r.frag != nil { + r.dat, err = io.ReadAll(r.frag) + return err + } else if r.curIndex >= uint64(len(r.sizes)) { + return io.EOF + } + realSize := r.sizes[r.curIndex] &^ (1 << 24) + if realSize == 0 { + if r.curIndex == uint64(len(r.sizes))-1 && r.frag == nil { + r.dat = make([]byte, r.finalBlockSize) + } else { + r.dat = make([]byte, r.blockSize) + } + return nil + } + r.dat = make([]byte, realSize) + err = binary.Read(r.r, binary.LittleEndian, &r.dat) + if err != nil { + return err + } + if r.sizes[r.curIndex] != realSize { + return nil + } + r.dat, err = r.d.Decompress(r.dat) + return err +} + +func (r *Reader) Read(b []byte) (int, error) { + curRead := 0 + var toRead int + for curRead < len(b) { + if r.curOffset >= len(r.dat) { + if err := r.advance(); err != nil { + return curRead, err + } + } + toRead = len(b) - curRead + if toRead > len(r.dat)-r.curOffset { + toRead = len(r.dat) - r.curOffset + } + toRead = copy(b[curRead:], r.dat[r.curOffset:r.curOffset+toRead]) + r.curOffset += toRead + curRead += toRead + } + return curRead, nil +} + +func (r *Reader) Close() error { + if r.frag != nil { + if l, ok := r.frag.(*io.LimitedReader); ok { + if cl, ok := l.R.(io.Closer); ok { + cl.Close() + } + } + } + r.dat = nil + return nil +} diff --git a/low/directory.go b/low/directory.go new file mode 100644 index 0000000..b219d77 --- /dev/null +++ b/low/directory.go @@ -0,0 +1,83 @@ +package squashfslow + +import ( + "errors" + "io/fs" + "path/filepath" + "slices" + "strings" + + "github.com/CalebQ42/squashfs/internal/metadata" + "github.com/CalebQ42/squashfs/internal/toreader" + "github.com/CalebQ42/squashfs/low/directory" + "github.com/CalebQ42/squashfs/low/inode" +) + +type Directory struct { + FileBase + Entries []directory.Entry +} + +func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) { + i, err := r.InodeFromRef(ref) + if err != nil { + return nil, err + } + var blockStart uint32 + var size uint32 + var offset uint16 + switch i.Type { + case inode.Dir: + blockStart = i.Data.(inode.Directory).BlockStart + size = uint32(i.Data.(inode.Directory).Size) + offset = i.Data.(inode.Directory).Offset + case inode.EDir: + blockStart = i.Data.(inode.EDirectory).BlockStart + size = i.Data.(inode.EDirectory).Size + offset = i.Data.(inode.EDirectory).Offset + default: + return nil, errors.New("not a directory") + } + dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d) + defer dirRdr.Close() + _, err = dirRdr.Read(make([]byte, offset)) + if err != nil { + return nil, err + } + entries, err := directory.ReadDirectory(dirRdr, size) + if err != nil { + return nil, err + } + return &Directory{ + FileBase: *r.BaseFromInode(i, name), + Entries: entries, + }, nil +} + +func (d *Directory) Open(r *Reader, path string) (*FileBase, error) { + path = filepath.Clean(path) + if path == "." || path == "" { + return &d.FileBase, nil + } + split := strings.Split(path, "/") + i, found := slices.BinarySearchFunc(d.Entries, split[0], func(e directory.Entry, name string) int { + return strings.Compare(e.Name, name) + }) + if !found { + return nil, fs.ErrNotExist + } + b, err := r.BaseFromEntry(d.Entries[i]) + if err != nil { + return nil, err + } + if len(split) == 1 { + return b, nil + } else if !b.IsDir() { + return nil, fs.ErrNotExist + } + dir, err := b.ToDir(r) + if err != nil { + return nil, err + } + return dir.Open(r, strings.Join(split[1:], "/")) +} diff --git a/low/directory/directory.go b/low/directory/directory.go new file mode 100644 index 0000000..be58199 --- /dev/null +++ b/low/directory/directory.go @@ -0,0 +1,62 @@ +package directory + +import ( + "encoding/binary" + "io" +) + +type header struct { + Count uint32 + BlockStart uint32 + Num uint32 +} + +type decEntry struct { + Offset uint16 + NumOffset int16 + InodeType uint16 + NameSize uint16 + // Name []byte (not decoded along with decEntry) +} + +type Entry struct { + Name string + BlockStart uint32 + Offset uint16 + InodeType uint16 + Num uint32 +} + +func ReadDirectory(r io.Reader, size uint32) (out []Entry, err error) { + size -= 3 + var curRead uint32 + var h header + var de decEntry + for curRead < size { + err = binary.Read(r, binary.LittleEndian, &h) + if err != nil { + return + } + curRead += 12 + for i := uint32(0); i < h.Count+1 && curRead < size; i++ { + err = binary.Read(r, binary.LittleEndian, &de) + if err != nil { + return + } + nameTmp := make([]byte, de.NameSize+1) + err = binary.Read(r, binary.LittleEndian, &nameTmp) + if err != nil { + return + } + curRead += 8 + uint32(de.NameSize) + 1 + out = append(out, Entry{ + BlockStart: h.BlockStart, + Offset: de.Offset, + Name: string(nameTmp), + InodeType: de.InodeType, + Num: h.Num + uint32(de.NumOffset), + }) + } + } + return +} diff --git a/low/file_base.go b/low/file_base.go new file mode 100644 index 0000000..e87bc9e --- /dev/null +++ b/low/file_base.go @@ -0,0 +1,203 @@ +package squashfslow + +import ( + "errors" + "io" + + "github.com/CalebQ42/squashfs/internal/metadata" + "github.com/CalebQ42/squashfs/internal/toreader" + "github.com/CalebQ42/squashfs/low/data" + "github.com/CalebQ42/squashfs/low/directory" + "github.com/CalebQ42/squashfs/low/inode" +) + +type FileBase struct { + Inode *inode.Inode + Name string +} + +func (r *Reader) BaseFromInode(i *inode.Inode, name string) *FileBase { + return &FileBase{Inode: i, Name: name} +} + +func (r *Reader) BaseFromEntry(e directory.Entry) (*FileBase, error) { + in, err := r.InodeFromEntry(e) + if err != nil { + return nil, err + } + return &FileBase{Inode: in, Name: e.Name}, nil +} + +func (r *Reader) BaseFromRef(ref uint64, name string) (*FileBase, error) { + in, err := r.InodeFromRef(ref) + if err != nil { + return nil, err + } + return &FileBase{Inode: in, Name: name}, nil +} + +func (b *FileBase) Uid(r *Reader) (uint32, error) { + return r.Id(b.Inode.UidInd) +} + +func (b *FileBase) Gid(r *Reader) (uint32, error) { + return r.Id(b.Inode.GidInd) +} + +func (b *FileBase) IsDir() bool { + return b.Inode.Type == inode.Dir || b.Inode.Type == inode.EDir +} + +func (b *FileBase) ToDir(r *Reader) (*Directory, error) { + var blockStart uint32 + var size uint32 + var offset uint16 + switch b.Inode.Type { + case inode.Dir: + blockStart = b.Inode.Data.(inode.Directory).BlockStart + size = uint32(b.Inode.Data.(inode.Directory).Size) + offset = b.Inode.Data.(inode.Directory).Offset + case inode.EDir: + blockStart = b.Inode.Data.(inode.EDirectory).BlockStart + size = b.Inode.Data.(inode.EDirectory).Size + offset = b.Inode.Data.(inode.EDirectory).Offset + default: + return nil, errors.New("not a directory") + } + dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d) + defer dirRdr.Close() + _, err := dirRdr.Read(make([]byte, offset)) + if err != nil { + return nil, err + } + entries, err := directory.ReadDirectory(dirRdr, size) + if err != nil { + return nil, err + } + return &Directory{ + FileBase: *b, + Entries: entries, + }, nil +} + +func (b *FileBase) IsRegular() bool { + return b.Inode.Type == inode.Fil || b.Inode.Type == inode.EFil +} + +func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, error) { + if !b.IsRegular() { + return nil, nil, errors.New("not a regular file") + } + var blockStart uint64 + var fragIndex uint32 + var fragOffset uint32 + var fragSize uint64 + var sizes []uint32 + if b.Inode.Type == inode.Fil { + blockStart = uint64(b.Inode.Data.(inode.File).BlockStart) + fragIndex = b.Inode.Data.(inode.File).FragInd + fragOffset = b.Inode.Data.(inode.File).FragOffset + sizes = b.Inode.Data.(inode.File).BlockSizes + fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize) + } else { + blockStart = b.Inode.Data.(inode.EFile).BlockStart + fragIndex = b.Inode.Data.(inode.EFile).FragInd + fragOffset = b.Inode.Data.(inode.EFile).FragOffset + sizes = b.Inode.Data.(inode.EFile).BlockSizes + fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize) + } + frag := func() (io.Reader, error) { + ent, err := r.fragEntry(fragIndex) + if err != nil { + return nil, err + } + frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize) + frag.Read(make([]byte, fragOffset)) + return io.LimitReader(frag, int64(fragSize)), nil + } + outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize) + if fragIndex != 0xffffffff { + f, err := frag() + if err != nil { + return nil, nil, err + } + outRdr.AddFrag(f) + } + outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes, fragSize, r.Superblock.BlockSize) + if fragIndex != 0xffffffff { + outFull.AddFrag(frag) + } + return outRdr, outFull, nil +} + +func (b *FileBase) GetFullReader(r *Reader) (*data.FullReader, error) { + if !b.IsRegular() { + return nil, errors.New("not a regular file") + } + var blockStart uint64 + var fragIndex uint32 + var fragOffset uint32 + var fragSize uint64 + var sizes []uint32 + if b.Inode.Type == inode.Fil { + blockStart = uint64(b.Inode.Data.(inode.File).BlockStart) + fragIndex = b.Inode.Data.(inode.File).FragInd + fragOffset = b.Inode.Data.(inode.File).FragOffset + sizes = b.Inode.Data.(inode.File).BlockSizes + fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize) + } else { + blockStart = b.Inode.Data.(inode.EFile).BlockStart + fragIndex = b.Inode.Data.(inode.EFile).FragInd + fragOffset = b.Inode.Data.(inode.EFile).FragOffset + sizes = b.Inode.Data.(inode.EFile).BlockSizes + fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize) + } + outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes, fragSize, r.Superblock.BlockSize) + if fragIndex != 0xffffffff { + outFull.AddFrag(func() (io.Reader, error) { + ent, err := r.fragEntry(fragIndex) + if err != nil { + return nil, err + } + frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize) + frag.Read(make([]byte, fragOffset)) + return io.LimitReader(frag, int64(fragSize)), nil + }) + } + return outFull, nil +} + +func (b *FileBase) GetReader(r *Reader) (*data.Reader, error) { + if !b.IsRegular() { + return nil, errors.New("not a regular file") + } + var blockStart uint64 + var fragIndex uint32 + var fragOffset uint32 + var fragSize uint64 + var sizes []uint32 + if b.Inode.Type == inode.Fil { + blockStart = uint64(b.Inode.Data.(inode.File).BlockStart) + fragIndex = b.Inode.Data.(inode.File).FragInd + fragOffset = b.Inode.Data.(inode.File).FragOffset + sizes = b.Inode.Data.(inode.File).BlockSizes + fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize) + } else { + blockStart = b.Inode.Data.(inode.EFile).BlockStart + fragIndex = b.Inode.Data.(inode.EFile).FragInd + fragOffset = b.Inode.Data.(inode.EFile).FragOffset + sizes = b.Inode.Data.(inode.EFile).BlockSizes + fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize) + } + outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize) + if fragIndex != 0xffffffff { + ent, err := r.fragEntry(fragIndex) + if err != nil { + return nil, err + } + frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize) + frag.Read(make([]byte, fragOffset)) + outRdr.AddFrag(io.LimitReader(frag, int64(fragSize))) + } + return outRdr, nil +} diff --git a/low/fragment.go b/low/fragment.go new file mode 100644 index 0000000..7230ef1 --- /dev/null +++ b/low/fragment.go @@ -0,0 +1,7 @@ +package squashfslow + +type fragEntry struct { + Start uint64 + Size uint32 + _ uint32 +} diff --git a/low/inode.go b/low/inode.go new file mode 100644 index 0000000..a10a2d1 --- /dev/null +++ b/low/inode.go @@ -0,0 +1,26 @@ +package squashfslow + +import ( + "github.com/CalebQ42/squashfs/internal/metadata" + "github.com/CalebQ42/squashfs/internal/toreader" + "github.com/CalebQ42/squashfs/low/directory" + "github.com/CalebQ42/squashfs/low/inode" +) + +func (r *Reader) InodeFromRef(ref uint64) (*inode.Inode, error) { + offset, meta := (ref>>16)+r.Superblock.InodeTableStart, ref&0xFFFF + rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) + defer rdr.Close() + _, err := rdr.Read(make([]byte, meta)) + if err != nil { + return nil, err + } + return inode.Read(rdr, r.Superblock.BlockSize) +} + +func (r *Reader) InodeFromEntry(e directory.Entry) (*inode.Inode, error) { + rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d) + defer rdr.Close() + rdr.Read(make([]byte, e.Offset)) + return inode.Read(rdr, r.Superblock.BlockSize) +} diff --git a/internal/inode/dir.go b/low/inode/dir.go similarity index 100% rename from internal/inode/dir.go rename to low/inode/dir.go diff --git a/internal/inode/file.go b/low/inode/file.go similarity index 100% rename from internal/inode/file.go rename to low/inode/file.go diff --git a/internal/inode/inode.go b/low/inode/inode.go similarity index 96% rename from internal/inode/inode.go rename to low/inode/inode.go index 11976f5..21f0a2e 100644 --- a/internal/inode/inode.go +++ b/low/inode/inode.go @@ -39,7 +39,8 @@ type Inode struct { Data any } -func Read(r io.Reader, blockSize uint32) (i Inode, err error) { +func Read(r io.Reader, blockSize uint32) (i *Inode, err error) { + i = new(Inode) err = binary.Read(r, binary.LittleEndian, &i.Header) if err != nil { return diff --git a/internal/inode/misc.go b/low/inode/misc.go similarity index 100% rename from internal/inode/misc.go rename to low/inode/misc.go diff --git a/internal/inode/sym.go b/low/inode/sym.go similarity index 100% rename from internal/inode/sym.go rename to low/inode/sym.go diff --git a/low/reader.go b/low/reader.go new file mode 100644 index 0000000..d6d6bef --- /dev/null +++ b/low/reader.go @@ -0,0 +1,219 @@ +package squashfslow + +import ( + "encoding/binary" + "errors" + "io" + "math" + + "github.com/CalebQ42/squashfs/internal/decompress" + "github.com/CalebQ42/squashfs/internal/metadata" + "github.com/CalebQ42/squashfs/internal/toreader" + "github.com/CalebQ42/squashfs/low/inode" +) + +// The types of compression supported by squashfs +const ( + ZlibCompression = uint16(iota + 1) + LZMACompression + LZOCompression + XZCompression + LZ4Compression + ZSTDCompression +) + +var ( + ErrorMagic = errors.New("magic incorrect. probably not reading squashfs archive or archive is corrupted") + ErrorLog = errors.New("block log is incorrect. possible corrupted archive") + ErrorVersion = errors.New("squashfs version of archive is not 4.0. may be corrupted") + ErrorNotExportable = errors.New("archive does not have an export table") +) + +type Reader struct { + r io.ReaderAt + d decompress.Decompressor + Root *Directory + fragTable []fragEntry + idTable []uint32 + exportTable []uint64 + Superblock superblock +} + +func NewReader(r io.ReaderAt) (rdr *Reader, err error) { + rdr = new(Reader) + rdr.r = r + err = binary.Read(toreader.NewReader(r, 0), binary.LittleEndian, &rdr.Superblock) + if err != nil { + return nil, errors.Join(errors.New("failed to read superblock"), err) + } + if !rdr.Superblock.ValidMagic() { + return nil, ErrorMagic + } + if !rdr.Superblock.ValidBlockLog() { + return nil, ErrorLog + } + if !rdr.Superblock.ValidVersion() { + return nil, ErrorVersion + } + switch rdr.Superblock.CompType { + case ZlibCompression: + rdr.d = decompress.Zlib{} + case LZMACompression: + rdr.d = decompress.Lzma{} + case LZOCompression: + rdr.d = decompress.Lzo{} + case XZCompression: + rdr.d = decompress.Xz{} + case LZ4Compression: + rdr.d = decompress.Lz4{} + case ZSTDCompression: + rdr.d = &decompress.Zstd{} + default: + return nil, errors.New("invalid compression type. possible corrupted archive") + } + rdr.Root, err = rdr.directoryFromRef(rdr.Superblock.RootInodeRef, "") + if err != nil { + return nil, errors.Join(errors.New("failed to read root directory"), err) + } + return +} + +// Get a uid/gid at the given index. Lazily populates the reader's Id table as necessary. +func (r *Reader) Id(i uint16) (uint32, error) { + if len(r.idTable) > int(i) { + return r.idTable[i], nil + } else if i >= r.Superblock.IdCount { + return 0, errors.New("id out of bounds") + } + // Populate the id table as needed + var blockNum uint32 + if i != 0 { // If i == 0, we go negatives causing issues with uint32s + blockNum = uint32(math.Ceil(float64(i)/2048)) - 1 + } else { + blockNum = 0 + } + blocksRead := len(r.idTable) / 2048 + blocksToRead := int(blockNum) - blocksRead + 1 + + var offset uint64 + var idsToRead uint16 + var idsTmp []uint32 + var err error + var rdr *metadata.Reader + for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { + err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset) + if err != nil { + return 0, err + } + idsToRead = r.Superblock.IdCount - uint16(len(r.idTable)) + if idsToRead > 2048 { + idsToRead = 2048 + } + idsTmp = make([]uint32, idsToRead) + rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) + err = binary.Read(rdr, binary.LittleEndian, &idsTmp) + rdr.Close() + if err != nil { + return 0, err + } + r.idTable = append(r.idTable, idsTmp...) + } + return r.idTable[i], nil +} + +// Get a fragment entry at the given index. Lazily populates the reader's fragment table as necessary. +func (r *Reader) fragEntry(i uint32) (fragEntry, error) { + if len(r.fragTable) > int(i) { + return r.fragTable[i], nil + } else if i >= r.Superblock.FragCount { + return fragEntry{}, errors.New("fragment out of bounds") + } + // Populate the fragment table as needed + var blockNum uint32 + if i != 0 { // If i == 0, we go negatives causing issues with uint32s + blockNum = uint32(math.Ceil(float64(i)/512)) - 1 + } else { + blockNum = 0 + } + blocksRead := len(r.fragTable) / 512 + blocksToRead := int(blockNum) - blocksRead + 1 + + var offset uint64 + var fragsToRead uint32 + var fragsTmp []fragEntry + var err error + var rdr *metadata.Reader + for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { + err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset) + if err != nil { + return fragEntry{}, err + } + fragsToRead = r.Superblock.FragCount - uint32(len(r.fragTable)) + if fragsToRead > 512 { + fragsToRead = 512 + } + fragsTmp = make([]fragEntry, fragsToRead) + rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) + err = binary.Read(rdr, binary.LittleEndian, &fragsTmp) + rdr.Close() + if err != nil { + return fragEntry{}, err + } + r.fragTable = append(r.fragTable, fragsTmp...) + } + return r.fragTable[i], nil +} + +// Get an inode reference at the given index. Lazily populates the reader's export table as necessary. +func (r *Reader) inodeRef(i uint32) (uint64, error) { + if !r.Superblock.Exportable() { + return 0, ErrorNotExportable + } + if len(r.exportTable) > int(i) { + return r.exportTable[i], nil + } else if i >= r.Superblock.InodeCount { + return 0, errors.New("inode out of bounds") + } + // Populate the export table as needed + var blockNum uint32 + if i != 0 { // If i == 0, we go negatives causing issues with uint32s + blockNum = uint32(math.Ceil(float64(i)/1024)) - 1 + } else { + blockNum = 0 + } + blocksRead := len(r.exportTable) / 1024 + blocksToRead := int(blockNum) - blocksRead + 1 + + var offset uint64 + var refsToRead uint32 + var refsTmp []uint64 + var err error + var rdr *metadata.Reader + for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { + err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset) + if err != nil { + return 0, err + } + refsToRead = r.Superblock.InodeCount - uint32(len(r.exportTable)) + if refsToRead > 1024 { + refsToRead = 1024 + } + refsTmp = make([]uint64, refsToRead) + rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) + err = binary.Read(rdr, binary.LittleEndian, &refsTmp) + rdr.Close() + if err != nil { + return 0, err + } + r.exportTable = append(r.exportTable, refsTmp...) + } + return r.exportTable[i], nil +} + +func (r *Reader) Inode(i uint32) (*inode.Inode, error) { + ref, err := r.inodeRef(i) + if err != nil { + return nil, err + } + return r.InodeFromRef(ref) +} diff --git a/low/reader_test.go b/low/reader_test.go new file mode 100644 index 0000000..7fe237f --- /dev/null +++ b/low/reader_test.go @@ -0,0 +1,133 @@ +package squashfslow_test + +import ( + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "testing" + + squashfslow "github.com/CalebQ42/squashfs/low" +) + +const ( + squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs" + squashfsName = "LinuxPATest.sfs" +) + +func preTest(dir string) (fil *os.File, err error) { + fil, err = os.Open(filepath.Join(dir, squashfsName)) + if err != nil { + _, err = os.Open(dir) + if os.IsNotExist(err) { + err = os.Mkdir(dir, 0755) + } + if err != nil { + return + } + os.Remove(filepath.Join(dir, squashfsName)) + fil, err = os.Create(filepath.Join(dir, squashfsName)) + if err != nil { + return + } + var resp *http.Response + resp, err = http.DefaultClient.Get(squashfsURL) + if err != nil { + return + } + _, err = io.Copy(fil, resp.Body) + if err != nil { + return + } + } + _, err = exec.LookPath("unsquashfs") + if err != nil { + return + } + _, err = exec.LookPath("mksquashfs") + return +} + +func TestReader(t *testing.T) { + tmpDir := "../testing" + fil, err := preTest(tmpDir) + if err != nil { + t.Fatal(err) + } + defer fil.Close() + rdr, err := squashfslow.NewReader(fil) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(tmpDir, "extractTest") + os.RemoveAll(path) + os.MkdirAll(path, 0777) + err = extractToDir(rdr, &rdr.Root.FileBase, path) + t.Fatal(err) +} + +var singleFile = "PortableApps/CPU-X/CPU-X-v4.2.0-x86_64.AppImage" + +func TestSingleFile(t *testing.T) { + tmpDir := "../testing" + fil, err := preTest(tmpDir) + if err != nil { + t.Fatal(err) + } + defer fil.Close() + rdr, err := squashfslow.NewReader(fil) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(tmpDir, "extractTest") + os.RemoveAll(path) + os.MkdirAll(path, 0777) + b, err := rdr.Root.Open(rdr, singleFile) + if err != nil { + t.Fatal(err) + } + err = extractToDir(rdr, b, path) + t.Fatal(err) +} + +func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder string) error { + path := filepath.Join(folder, b.Name) + if b.IsDir() { + d, err := b.ToDir(rdr) + if err != nil { + return err + } + err = os.MkdirAll(path, 0777) + if err != nil { + return err + } + var nestBast *squashfslow.FileBase + for _, e := range d.Entries { + nestBast, err = rdr.BaseFromEntry(e) + if err != nil { + return err + } + err = extractToDir(rdr, nestBast, path) + if err != nil { + return err + } + } + } else if b.IsRegular() { + _, full, err := b.GetRegFileReaders(rdr) + if err != nil { + return err + } + fil, err := os.Create(path) + if err != nil { + return err + } + _, err = full.WriteTo(fil) + if err != nil { + return err + } + fmt.Println("Successfully extracted file:", b.Name) + } + return nil +} diff --git a/superblock.go b/low/superblock.go similarity index 61% rename from superblock.go rename to low/superblock.go index 7916759..dca087a 100644 --- a/superblock.go +++ b/low/superblock.go @@ -1,4 +1,4 @@ -package squashfs +package squashfslow import "math" @@ -24,57 +24,57 @@ type superblock struct { ExportTableStart uint64 } -func (s superblock) checkMagic() bool { +func (s superblock) ValidMagic() bool { return s.Magic == 0x73717368 } -func (s superblock) checkBlockLog() bool { +func (s superblock) ValidBlockLog() bool { return s.BlockLog == uint16(math.Log2(float64(s.BlockSize))) } -func (s superblock) checkVersion() bool { +func (s superblock) ValidVersion() bool { return s.VerMaj == 4 && s.VerMin == 0 } -func (s superblock) uncompressedInodes() bool { +func (s superblock) UncompressedInodes() bool { return s.Flags&0x1 == 0x1 } -func (s superblock) uncompressedData() bool { +func (s superblock) UncompressedData() bool { return s.Flags&0x2 == 0x2 } -func (s superblock) uncompressedFragments() bool { +func (s superblock) UncompressedFragments() bool { return s.Flags&0x8 == 0x8 } -func (s superblock) noFragments() bool { +func (s superblock) NoFragments() bool { return s.Flags&0x10 == 0x10 } -func (s superblock) alwaysFragment() bool { +func (s superblock) AlwaysFragment() bool { return s.Flags&0x20 == 0x20 } -func (s superblock) duplicates() bool { +func (s superblock) Duplicates() bool { return s.Flags&0x40 == 0x40 } -func (s superblock) exportable() bool { +func (s superblock) Exportable() bool { return s.Flags&0x80 == 0x80 } -func (s superblock) uncompressedXattrs() bool { +func (s superblock) UncompressedXattrs() bool { return s.Flags&0x100 == 0x100 } -func (s superblock) noXattrs() bool { +func (s superblock) NoXattrs() bool { return s.Flags&0x200 == 0x200 } -func (s superblock) compressionOptions() bool { +func (s superblock) CompressionOptions() bool { return s.Flags&0x400 == 0x400 } -func (s superblock) uncompressedIDs() bool { +func (s superblock) UncompressedIDs() bool { return s.Flags&0x800 == 0x800 } diff --git a/reader.go b/reader.go index 746cd02..ce499f8 100644 --- a/reader.go +++ b/reader.go @@ -1,234 +1,32 @@ package squashfs import ( - "encoding/binary" - "errors" "io" - "math" "time" - "github.com/CalebQ42/fuse" - "github.com/CalebQ42/squashfs/internal/decompress" - "github.com/CalebQ42/squashfs/internal/directory" - "github.com/CalebQ42/squashfs/internal/inode" - "github.com/CalebQ42/squashfs/internal/metadata" - "github.com/CalebQ42/squashfs/internal/toreader" - fuse2 "github.com/seaweedfs/fuse" + squashfslow "github.com/CalebQ42/squashfs/low" ) type Reader struct { *FS - con *fuse.Conn - con2 *fuse2.Conn - mountDone chan struct{} - mount2Done chan struct{} - d decompress.Decompressor - r io.ReaderAt - fragEntries []fragEntry - ids []uint32 - // exportTable []uint64 - s superblock + Low *squashfslow.Reader } -var ( - ErrorMagic = errors.New("magic incorrect. probably not reading squashfs archive") - ErrorLog = errors.New("block log is incorrect. possible corrupted archive") - ErrorVersion = errors.New("squashfs version of archive is not 4.0") -) - -// The types of compression supported by squashfs -const ( - GZipCompression = uint16(iota + 1) - LZMACompression - LZOCompression - XZCompression - LZ4Compression - ZSTDCompression -) - -func NewReaderAtOffset(r io.ReaderAt, off int64) (*Reader, error) { - return NewReader(toreader.NewOffsetReader(r, off)) -} - -// Creates a new squashfs.Reader from the given io.Reader. NOTE: All data from the io.Reader will be read and stored in memory. -func NewReaderFromReader(r io.Reader) (*Reader, error) { - rdr, err := toreader.NewReaderAt(r) - if err != nil { - return nil, err - } - return NewReader(rdr) -} - -// Creates a new squashfs.Reader from the given io.ReaderAt. func NewReader(r io.ReaderAt) (*Reader, error) { - var squash Reader - squash.r = r - err := binary.Read(toreader.NewReader(r, 0), binary.LittleEndian, &squash.s) + rdr, err := squashfslow.NewReader(r) if err != nil { return nil, err } - if !squash.s.checkMagic() { - return nil, ErrorMagic + out := &Reader{ + Low: rdr, } - if !squash.s.checkBlockLog() { - return nil, ErrorLog + out.FS = &FS{ + d: rdr.Root, + r: out, } - if !squash.s.checkVersion() { - return nil, ErrorVersion - } - switch squash.s.CompType { - case GZipCompression: - squash.d = decompress.GZip{} - case LZMACompression: - squash.d = decompress.Lzma{} - case LZOCompression: - squash.d = decompress.Lzo{} - case XZCompression: - squash.d = decompress.Xz{} - case LZ4Compression: - squash.d = decompress.Lz4{} - case ZSTDCompression: - squash.d = &decompress.Zstd{} - default: - return nil, errors.New("uh, I need to do this, OR something if very wrong") - } - if !squash.s.noFragments() && squash.s.FragCount > 0 { - fragOffsets := make([]uint64, int(math.Ceil(float64(squash.s.FragCount)/512))) - err = binary.Read(toreader.NewReader(r, int64(squash.s.FragTableStart)), binary.LittleEndian, &fragOffsets) - if err != nil { - return nil, err - } - squash.fragEntries = make([]fragEntry, squash.s.FragCount) - if len(fragOffsets) == 1 { - rdr := metadata.NewReader(toreader.NewReader(r, int64(fragOffsets[0])), squash.d) - err = binary.Read(rdr, binary.LittleEndian, &squash.fragEntries) - if err != nil { - return nil, err - } - } else { - toRead := squash.s.FragCount - var curRead uint32 - var tmp []fragEntry - var rdr *metadata.Reader - var offset int - for i := range fragOffsets { - curRead = uint32(math.Min(512, float64(toRead))) - tmp = make([]fragEntry, curRead) - rdr = metadata.NewReader(toreader.NewReader(r, int64(fragOffsets[i])), squash.d) - err = binary.Read(rdr, binary.LittleEndian, &tmp) - if err != nil { - return nil, err - } - offset = int(squash.s.FragCount - toRead) - for i := range tmp { - squash.fragEntries[offset+i] = tmp[i] - } - toRead -= curRead - } - } - } - if squash.s.IdCount > 0 { - idOffsets := make([]uint64, int(math.Ceil(float64(squash.s.IdCount)/2048))) - err = binary.Read(toreader.NewReader(r, int64(squash.s.IdTableStart)), binary.LittleEndian, &idOffsets) - if err != nil { - return nil, err - } - squash.ids = make([]uint32, squash.s.IdCount) - if len(idOffsets) == 1 { - rdr := metadata.NewReader(toreader.NewReader(r, int64(idOffsets[0])), squash.d) - err = binary.Read(rdr, binary.LittleEndian, &squash.ids) - if err != nil { - return nil, err - } - } else { - toRead := squash.s.IdCount - var curRead uint16 - var tmp []uint32 - var rdr *metadata.Reader - var offset int - for i := range idOffsets { - curRead = uint16(math.Min(2048, float64(toRead))) - tmp = make([]uint32, curRead) - rdr = metadata.NewReader(toreader.NewReader(r, int64(idOffsets[i])), squash.d) - err = binary.Read(rdr, binary.LittleEndian, &tmp) - if err != nil { - return nil, err - } - offset = int(squash.s.IdCount - toRead) - for i := range tmp { - squash.ids[offset+i] = tmp[i] - } - toRead -= curRead - } - } - } - root, err := squash.inodeFromRef(squash.s.RootInodeRef) - if err != nil { - return nil, err - } - rootEnts, err := squash.readDirectory(root) - if err != nil { - return nil, err - } - enType := root.Type - if enType == inode.EDir { - enType = inode.Dir - } - squash.FS = &FS{ - e: rootEnts, - File: &File{ - rdr: &squash, - i: root, - e: directory.Entry{ - Name: "", - Type: enType, - }, - r: &squash, - }, - } - return &squash, nil + return out, nil } -// func (r *Reader) initExport() (err error) { -// num := int(math.Ceil(float64(r.s.InodeCount) / 1024)) -// offsets := make([]uint64, num) -// err = binary.Read(toreader.NewReader(r.r, int64(r.s.ExportTableStart)), binary.LittleEndian, &offsets) -// if err != nil { -// return -// } -// left := r.s.InodeCount -// var toRead uint32 -// var new []uint64 -// var rdr *metadata.Reader -// for i := range offsets { -// rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offsets[i])), r.d) -// toRead = uint32(math.Min(1024, float64(left))) -// new = make([]uint64, toRead) -// err = binary.Read(rdr, binary.LittleEndian, &new) -// if err != nil { -// return -// } -// left -= toRead -// r.exportTable = append(r.exportTable, new...) -// } -// return nil -// } - -// func (r *Reader) inode(index uint32) (i inode.Inode, err error) { -// if r.s.exportable() { -// if r.exportTable == nil { -// err = r.initExport() -// if err != nil { -// return -// } -// } -// return r.inodeFromRef(r.exportTable[index-1]) -// } -// err = errors.New("archive is not exportable") -// return -// } - -// Returns the last time the archive was modified. -func (r Reader) ModTime() time.Time { - return time.Unix(int64(r.s.ModTime), 0) +func (r *Reader) ModTime() time.Time { + return time.Unix(int64(r.Low.Superblock.ModTime), 0) } diff --git a/reader_file.go b/reader_file.go deleted file mode 100644 index 73adac6..0000000 --- a/reader_file.go +++ /dev/null @@ -1,518 +0,0 @@ -package squashfs - -import ( - "errors" - "io" - "io/fs" - "log" - "os" - "os/exec" - "path/filepath" - "runtime" - "strconv" - "strings" - - "github.com/CalebQ42/squashfs/internal/data" - "github.com/CalebQ42/squashfs/internal/directory" - "github.com/CalebQ42/squashfs/internal/inode" - "github.com/CalebQ42/squashfs/internal/threadmanager" -) - -// File represents a file inside a squashfs archive. -type File struct { - i inode.Inode - rdr io.Reader - fullRdr *data.FullReader - r *Reader - parent *FS - e directory.Entry - dirsRead int -} - -var ( - ErrReadNotFile = errors.New("read called on non-file") -) - -func (r Reader) newFile(en directory.Entry, parent *FS) (*File, error) { - i, err := r.inodeFromDir(en) - if err != nil { - return nil, err - } - var rdr io.Reader - var full *data.FullReader - if i.Type == inode.Fil || i.Type == inode.EFil { - full, rdr, err = r.getReaders(i) - if err != nil { - return nil, err - } - } - return &File{ - e: en, - i: i, - rdr: rdr, - fullRdr: full, - r: &r, - parent: parent, - }, nil -} - -// Stat returns the File's fs.FileInfo -func (f File) Stat() (fs.FileInfo, error) { - return newFileInfo(f.e, f.i), nil -} - -// Mode returns the file's fs.FileMode -func (f File) Mode() fs.FileMode { - switch f.e.Type { - case inode.Dir: - return fs.FileMode(f.i.Perm) | fs.ModeDir - case inode.Char: - return fs.FileMode(f.i.Perm) | fs.ModeCharDevice - case inode.Block: - return fs.FileMode(f.i.Perm) | fs.ModeDevice - case inode.Sym: - return fs.FileMode(f.i.Perm) | fs.ModeSymlink - } - return fs.FileMode(f.i.Perm) -} - -// Read reads the data from the file. Only works if file is a normal file. -func (f File) Read(p []byte) (int, error) { - if f.i.Type != inode.Fil && f.i.Type != inode.EFil { - return 0, ErrReadNotFile - } - if f.rdr == nil { - return 0, fs.ErrClosed - } - return f.rdr.Read(p) -} - -func (f File) ReadAt(p []byte, off int64) (int, error) { - if f.i.Type != inode.Fil && f.i.Type != inode.EFil { - return 0, ErrReadNotFile - } - return f.fullRdr.ReadAt(p, off) -} - -// WriteTo writes all data from the file to the writer. This is multi-threaded. -// The underlying reader is seperate from the one used with Read and can be reused. -func (f File) WriteTo(w io.Writer) (int64, error) { - if f.i.Type != inode.Fil && f.i.Type != inode.EFil { - return 0, ErrReadNotFile - } - return f.fullRdr.WriteTo(w) -} - -// Close simply nils the underlying reader. -func (f *File) Close() error { - f.rdr = nil - return nil -} - -// ReadDir returns n fs.DirEntry's that's contained in the File (if it's a directory). -// If n <= 0 all fs.DirEntry's are returned. -func (f *File) ReadDir(n int) (out []fs.DirEntry, err error) { - if !f.IsDir() { - return nil, errors.New("file is not a directory") - } - ents, err := f.r.readDirectory(f.i) - if err != nil { - return nil, err - } - start, end := 0, len(ents) - if n > 0 { - start, end = f.dirsRead, f.dirsRead+n - if end > len(f.r.e) { - end = len(f.r.e) - err = io.EOF - } - } - var fi fileInfo - for _, e := range ents[start:end] { - fi, err = f.r.newFileInfo(e) - if err != nil { - f.dirsRead += len(out) - return - } - out = append(out, fs.FileInfoToDirEntry(fi)) - } - f.dirsRead += len(out) - return -} - -// FS returns the File as a FS. -func (f *File) FS() (*FS, error) { - if !f.IsDir() { - return nil, errors.New("File is not a directory") - } - ents, err := f.r.readDirectory(f.i) - if err != nil { - return nil, err - } - return &FS{ - File: f, - e: ents, - }, nil -} - -// IsDir Yep. -func (f File) IsDir() bool { - return f.i.Type == inode.Dir || f.i.Type == inode.EDir -} - -// IsRegular yep. -func (f File) IsRegular() bool { - return f.i.Type == inode.Fil || f.i.Type == inode.EFil -} - -// IsSymlink yep. -func (f File) IsSymlink() bool { - return f.i.Type == inode.Sym || f.i.Type == inode.ESym -} - -func (f File) isDeviceOrFifo() bool { - return f.i.Type == inode.Char || f.i.Type == inode.Block || f.i.Type == inode.EChar || f.i.Type == inode.EBlock || f.i.Type == inode.Fifo || f.i.Type == inode.EFifo -} - -func (f File) deviceDevices() (maj uint32, min uint32) { - var dev uint32 - if f.i.Type == inode.Char || f.i.Type == inode.Block { - dev = f.i.Data.(inode.Device).Dev - } else if f.i.Type == inode.EChar || f.i.Type == inode.EBlock { - dev = f.i.Data.(inode.EDevice).Dev - } - return dev >> 8, dev & 0x000FF -} - -// SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string. -func (f File) SymlinkPath() string { - switch f.i.Type { - case inode.Sym: - return string(f.i.Data.(inode.Symlink).Target) - case inode.ESym: - return string(f.i.Data.(inode.ESymlink).Target) - } - return "" -} - -func (f File) path() string { - if f.parent == nil { - return f.e.Name - } - return f.parent.path() + "/" + f.e.Name -} - -// GetSymlinkFile returns the File the symlink is pointing to. -// If not a symlink, or the target is unobtainable (such as it being outside the archive or it's absolute) returns nil -func (f File) GetSymlinkFile() *File { - if !f.IsSymlink() { - return nil - } - if strings.HasPrefix(f.SymlinkPath(), "/") { - return nil - } - sym, err := f.parent.Open(f.SymlinkPath()) - if err != nil { - return nil - } - return sym.(*File) -} - -// ExtractionOptions are available options on how to extract. -type ExtractionOptions struct { - manager *threadmanager.Manager - LogOutput io.Writer //Where error log should write. - DereferenceSymlink bool //Replace symlinks with the target file. - UnbreakSymlink bool //Try to make sure symlinks remain unbroken when extracted, without changing the symlink. - Verbose bool //Prints extra info to log on an error. - IgnorePerm bool //Ignore file's permissions and instead use Perm. - Perm fs.FileMode //Permission to use when IgnorePerm. Defaults to 0755. - notFirst bool -} - -func DefaultOptions() *ExtractionOptions { - return &ExtractionOptions{ - Perm: 0755, - } -} - -// ExtractTo extracts the File to the given folder with the default options. -// If the File is a directory, it instead extracts the directory's contents to the folder. -func (f File) ExtractTo(folder string) error { - return f.realExtract(folder, DefaultOptions()) -} - -// ExtractVerbose extracts the File to the folder with the Verbose option. -func (f File) ExtractVerbose(folder string) error { - op := DefaultOptions() - op.Verbose = true - return f.realExtract(folder, op) -} - -// ExtractIgnorePermissions extracts the File to the folder with the IgnorePerm option. -func (f File) ExtractIgnorePermissions(folder string) error { - op := DefaultOptions() - op.IgnorePerm = true - return f.realExtract(folder, op) -} - -// ExtractSymlink extracts the File to the folder with the DereferenceSymlink option. -// If the File is a directory, it instead extracts the directory's contents to the folder. -func (f File) ExtractSymlink(folder string) error { - op := DefaultOptions() - op.DereferenceSymlink = true - return f.realExtract(folder, op) -} - -// ExtractWithOptions extracts the File to the given folder with the given ExtrationOptions. -// If the File is a directory, it instead extracts the directory's contents to the folder. -func (f File) ExtractWithOptions(folder string, op *ExtractionOptions) error { - if op.Verbose && op.LogOutput != nil { - log.SetOutput(op.LogOutput) - } - return f.realExtract(folder, op) -} - -func (f File) realExtract(folder string, op *ExtractionOptions) (err error) { - if op.manager == nil { - op.manager = threadmanager.NewManager(runtime.NumCPU()) - } - extDir := filepath.Join(folder, f.e.Name) - if !op.notFirst { - op.notFirst = true - if f.IsDir() { - extDir = folder - _, err = os.Open(folder) - if err != nil && os.IsNotExist(err) { - err = os.Mkdir(extDir, op.Perm) - } - if err != nil { - if op.Verbose { - log.Println("Error while making", folder) - } - return - } - if !op.IgnorePerm { - defer os.Chmod(extDir, f.Mode()) - defer os.Chown(extDir, int(f.r.ids[f.i.UidInd]), int(f.r.ids[f.i.GidInd])) - } - } - } - switch { - case f.IsDir(): - if folder != extDir && f.e.Name != "" { - //First extract it with a permisive permission. - err = os.Mkdir(extDir, op.Perm) - if err != nil { - if op.Verbose { - log.Println("Error while making directory", extDir) - } - return - } - //Then set it to it's actual permissions once we're done with it - if !op.IgnorePerm { - defer os.Chmod(extDir, f.Mode()) - defer os.Chown(extDir, int(f.r.ids[f.i.UidInd]), int(f.r.ids[f.i.GidInd])) - } - } - var filFS *FS - filFS, err = f.FS() - if err != nil { - if op.Verbose { - log.Println("Error while converting", f.path(), "to FS") - } - return err - } - errChan := make(chan error, len(filFS.e)) - files := make([]directory.Entry, 0) - //Focus on making the folder tree first... - var i int - for i = 0; i < len(filFS.e); i++ { - if filFS.e[i].Type == inode.Fil { - files = append(files, filFS.e[i]) - } else { - go func(index int) { - subF, goErr := f.r.newFile(filFS.e[index], filFS) - if goErr != nil { - if op.Verbose { - log.Println("Error while resolving", extDir) - } - errChan <- goErr - return - } - errChan <- subF.ExtractWithOptions(extDir, op) - }(i) - } - } - for i = 0; i < len(filFS.e)-len(files); i++ { - err = <-errChan - if err != nil { - return err - } - } - //Then we extract the files. - for i = 0; i < len(files); i++ { - go func(index int) { - n := op.manager.Lock() - defer op.manager.Unlock(n) - subF, goErr := f.r.newFile(files[index], filFS) - if goErr != nil { - if op.Verbose { - log.Println("Error while resolving", extDir) - } - errChan <- goErr - return - } - errChan <- subF.ExtractWithOptions(extDir, op) - }(i) - } - for i = 0; i < len(files); i++ { - err = <-errChan - if err != nil { - return err - } - } - case f.IsRegular(): - var fil *os.File - fil, err = os.Create(extDir) - if os.IsExist(err) { - os.Remove(extDir) - fil, err = os.Create(extDir) - if err != nil { - if op.Verbose { - log.Println("Error while creating", extDir) - } - return err - } - } else if err != nil { - if op.Verbose { - log.Println("Error while creating", extDir) - } - return err - } - defer fil.Close() - _, err = io.Copy(fil, f) - if err != nil { - if op.Verbose { - log.Println("Error while copying data to", extDir) - } - return err - } - if op.IgnorePerm { - os.Chmod(extDir, op.Perm|(f.Mode()&fs.ModeType)) - } else { - os.Chmod(extDir, f.Mode()) - os.Chown(extDir, int(f.r.ids[f.i.UidInd]), int(f.r.ids[f.i.GidInd])) - } - case f.IsSymlink(): - symPath := f.SymlinkPath() - if op.DereferenceSymlink { - fil := f.GetSymlinkFile() - if fil == nil { - if op.Verbose { - log.Println("Symlink path(", symPath, ") is unobtainable:", extDir) - } - return errors.New("cannot get symlink target") - } - fil.e.Name = f.e.Name - err = fil.realExtract(folder, op) - if err != nil { - if op.Verbose { - log.Println("Error while extracting the symlink's file:", extDir) - } - return err - } - return nil - } else if op.UnbreakSymlink { - fil := f.GetSymlinkFile() - if fil == nil { - if op.Verbose { - log.Println("Symlink path(", symPath, ") is unobtainable:", extDir) - } - return errors.New("cannot get symlink target") - } - extractLoc := filepath.Join(folder, filepath.Dir(symPath)) - err = fil.realExtract(extractLoc, op) - if err != nil { - if op.Verbose { - log.Println("Error while extracting ", extDir) - } - return err - } - } - err = os.Symlink(f.SymlinkPath(), extDir) - if os.IsExist(err) { - os.Remove(extDir) - err = os.Symlink(f.SymlinkPath(), extDir) - } - if err != nil { - if op.Verbose { - log.Println("Error while making symlink:", extDir) - } - return err - } - if op.IgnorePerm { - os.Chmod(extDir, op.Perm|(f.Mode()&fs.ModeType)) - } else { - os.Chmod(extDir, f.Mode()) - os.Chown(extDir, int(f.r.ids[f.i.UidInd]), int(f.r.ids[f.i.GidInd])) - } - case f.isDeviceOrFifo(): - if runtime.GOOS == "windows" { - if op.Verbose { - log.Println(extDir, "ignored since it's a device link and can't be created on Windows.") - } - return nil - } - _, err = exec.LookPath("mknod") - if err != nil { - if op.Verbose { - log.Println("Extracting Fifo IPC or Device and mknod is not in PATH") - } - return err - } - var typ string - if f.i.Type == inode.Char || f.i.Type == inode.EChar { - typ = "c" - } else if f.i.Type == inode.Block || f.i.Type == inode.EBlock { - typ = "b" - } else { //Fifo IPC - if runtime.GOOS == "darwin" { - if op.Verbose { - log.Println(extDir, "ignored since it's a Fifo file and can't be created on Darwin.") - } - return nil - } - typ = "p" - } - cmd := exec.Command("mknod", extDir, typ) - if typ != "p" { - maj, min := f.deviceDevices() - cmd.Args = append(cmd.Args, strconv.Itoa(int(maj)), strconv.Itoa(int(min))) - } - if op.Verbose { - cmd.Stdout = op.LogOutput - cmd.Stderr = op.LogOutput - } - err = cmd.Run() - if err != nil { - if op.Verbose { - log.Println("Error while running mknod for", extDir) - } - return err - } - if op.IgnorePerm { - os.Chmod(extDir, op.Perm|(f.Mode()&fs.ModeType)) - } else { - os.Chmod(extDir, f.Mode()) - os.Chown(extDir, int(f.r.ids[f.i.UidInd]), int(f.r.ids[f.i.GidInd])) - } - case f.e.Type == inode.Sock: - if op.Verbose { - log.Println(extDir, "ignored since it's a socket file.") - } - default: - return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.i.Type))) - } - return nil -} diff --git a/reader_frag.go b/reader_frag.go deleted file mode 100644 index 0e94e2e..0000000 --- a/reader_frag.go +++ /dev/null @@ -1,26 +0,0 @@ -package squashfs - -import ( - "bytes" - "io" - - "github.com/CalebQ42/squashfs/internal/toreader" -) - -type fragEntry struct { - Start uint64 - Size uint32 - _ uint32 -} - -func (r Reader) fragReader(index uint32, fragSize uint32) (io.Reader, error) { - realSize := r.fragEntries[index].Size &^ (1 << 24) - if realSize == 0 { - return bytes.NewReader(make([]byte, fragSize)), nil - } - rdr := io.LimitReader(toreader.NewReader(r.r, int64(r.fragEntries[index].Start)), int64(realSize)) - if realSize != r.fragEntries[index].Size { - return rdr, nil - } - return r.d.Reader(rdr) -} diff --git a/reader_fs.go b/reader_fs.go deleted file mode 100644 index 60296b7..0000000 --- a/reader_fs.go +++ /dev/null @@ -1,380 +0,0 @@ -package squashfs - -import ( - "bytes" - "io" - "io/fs" - "path" - "path/filepath" - "strings" - - "github.com/CalebQ42/squashfs/internal/directory" - "github.com/CalebQ42/squashfs/internal/inode" -) - -// FS is a fs.FS representation of a squashfs directory. -// Implements fs.GlobFS, fs.ReadDirFS, fs.ReadFileFS, fs.StatFS, and fs.SubFS -type FS struct { - *File - e []directory.Entry -} - -func (r Reader) newFS(e directory.Entry, parent *FS) (*FS, error) { - i, err := r.inodeFromDir(e) - if err != nil { - return nil, err - } - ents, err := r.readDirectory(i) - if err != nil { - return nil, err - } - return &FS{ - File: &File{ - i: i, - r: &r, - parent: parent, - e: e, - }, - e: ents, - }, nil -} - -// Opens the file at name. Returns a squashfs.File. -func (f FS) OpenFile(name string) (*File, error) { - name = filepath.Clean(name) - if !fs.ValidPath(name) { - return nil, &fs.PathError{ - Op: "open", - Path: name, - Err: fs.ErrInvalid, - } - } - if name == "." || name == "" { - return f.File, nil - } - split := strings.Split(name, "/") - for i := range f.e { - if f.e[i].Name != split[0] { - continue - } - if len(split) > 1 && f.e[i].Type != inode.Dir { - return nil, &fs.PathError{ - Op: "open", - Path: name, - Err: fs.ErrNotExist, - } - } - if len(split) > 1 { - newFS, err := f.r.newFS(f.e[i], &f) - if err != nil { - return nil, &fs.PathError{ - Op: "open", - Path: name, - Err: err, - } - } - out, err := newFS.OpenFile(strings.Join(split[1:], "/")) - if err != nil { - err.(*fs.PathError).Path = name - } - return out, err - } - out, err := f.r.newFile(f.e[i], &f) - if err != nil { - err = &fs.PathError{ - Op: "open", - Path: name, - Err: err, - } - } - return out, err - } - return nil, &fs.PathError{ - Op: "open", - Path: name, - Err: fs.ErrNotExist, - } -} - -// Opens the file at name. Returns a io/fs.File. -func (f FS) Open(name string) (fs.File, error) { - return f.OpenFile(name) -} - -// Glob returns the name of the files at the given pattern. -// All paths are relative to the FS. -// Uses filepath.Match to compare names. -func (f FS) Glob(pattern string) (out []string, err error) { - pattern = filepath.Clean(pattern) - if !fs.ValidPath(pattern) { - return nil, &fs.PathError{ - Op: "glob", - Path: pattern, - Err: fs.ErrInvalid, - } - } - split := strings.Split(pattern, "/") - for i := 0; i < len(f.e); i++ { - if match, _ := path.Match(split[0], f.e[i].Name); match { - if len(split) == 1 { - out = append(out, f.e[i].Name) - continue - } - sub, err := f.Sub(split[0]) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "glob" - pathErr.Path = pattern - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "glob", - Path: pattern, - Err: err, - } - } - subGlob, err := sub.(fs.GlobFS).Glob(strings.Join(split[1:], "/")) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "glob" - pathErr.Path = pattern - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "glob", - Path: pattern, - Err: err, - } - } - for i := 0; i < len(subGlob); i++ { - subGlob[i] = f.File.e.Name + "/" + subGlob[i] - } - out = append(out, subGlob...) - } - } - return -} - -// ReadDir returns all the DirEntry returns all DirEntry's for the directory at name. -// If name is not a directory, returns an error. -func (f FS) ReadDir(name string) ([]fs.DirEntry, error) { - name = filepath.Clean(name) - if !fs.ValidPath(name) { - return nil, &fs.PathError{ - Op: "readdir", - Path: name, - Err: fs.ErrInvalid, - } - } - if name == "." || name == "" { - return f.File.ReadDir(-1) - } - split := strings.Split(name, "/") - for i := 0; i < len(f.e); i++ { - if split[0] == f.e[i].Name { - if len(split) == 1 { - fi, err := f.r.newFile(f.e[i], &f) - if err != nil { - return nil, &fs.PathError{ - Op: "readdir", - Path: name, - Err: err, - } - } - out, err := fi.ReadDir(-1) - if err != nil { - err = &fs.PathError{ - Op: "readdir", - Path: name, - Err: err, - } - } - return out, err - } - sub, err := f.Sub(split[0]) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "readir" - pathErr.Path = name - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "readdir", - Path: name, - Err: err, - } - } - redDir, err := sub.(fs.ReadDirFS).ReadDir(strings.Join(split[1:], "/")) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "readdir" - pathErr.Path = name - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "readdir", - Path: name, - Err: err, - } - } - return redDir, nil - } - } - return nil, &fs.PathError{ - Op: "readdir", - Path: name, - Err: fs.ErrNotExist, - } -} - -// ReadFile returns the data (in []byte) for the file at name. -func (f FS) ReadFile(name string) ([]byte, error) { - fil, err := f.Open(name) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - pathErr.Op = "readfile" - pathErr.Path = name - return nil, pathErr - } - } - var buf bytes.Buffer - _, err = io.Copy(&buf, fil) - if err != nil { - return nil, &fs.PathError{ - Op: "readfile", - Path: name, - Err: err, - } - } - return buf.Bytes(), nil -} - -// Stat returns the fs.FileInfo for the file at name. -func (f FS) Stat(name string) (fs.FileInfo, error) { - name = filepath.Clean(strings.TrimPrefix(name, "/")) - if !fs.ValidPath(name) { - return nil, &fs.PathError{ - Op: "stat", - Path: name, - Err: fs.ErrInvalid, - } - } - if name == "." || name == "" { - return f.File.Stat() - } - split := strings.Split(name, "/") - for i := 0; i < len(f.e); i++ { - if split[0] == f.e[i].Name { - if len(split) == 1 { - in, err := f.r.newFileInfo(f.e[i]) - if err != nil { - err = &fs.PathError{ - Op: "stat", - Path: name, - Err: err, - } - } - return in, err - } - sub, err := f.Sub(split[0]) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "stat" - pathErr.Path = name - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "stat", - Path: name, - Err: err, - } - } - stat, err := sub.(fs.StatFS).Stat(strings.Join(split[1:], "/")) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "stat" - pathErr.Path = name - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "stat", - Path: name, - Err: err, - } - } - return stat, nil - } - } - return nil, &fs.PathError{ - Op: "stat", - Path: name, - Err: fs.ErrNotExist, - } -} - -// Sub returns the FS at dir -func (f FS) Sub(dir string) (fs.FS, error) { - dir = filepath.Clean(dir) - if !fs.ValidPath(dir) { - return nil, &fs.PathError{ - Op: "sub", - Path: dir, - Err: fs.ErrInvalid, - } - } - if dir == "." || dir == "" { - return f, nil - } - split := strings.Split(dir, "/") - for i := range f.e { - if f.e[i].Name != split[0] { - continue - } - if f.e[i].Type != inode.Dir { - return nil, &fs.PathError{ - Op: "sub", - Path: dir, - Err: fs.ErrNotExist, - } - } - newFS, err := f.r.newFS(f.e[i], &f) - if err != nil { - return nil, &fs.PathError{ - Op: "sub", - Path: dir, - Err: err, - } - } - if len(split) > 1 { - ret, err := newFS.Sub(strings.Join(split[1:], "/")) - if err != nil { - err.(*fs.PathError).Path = dir - } - return ret, err - } - return newFS, nil - } - return nil, &fs.PathError{ - Op: "sub", - Path: dir, - Err: fs.ErrNotExist, - } -} diff --git a/reader_inode.go b/reader_inode.go deleted file mode 100644 index ec0516c..0000000 --- a/reader_inode.go +++ /dev/null @@ -1,117 +0,0 @@ -package squashfs - -import ( - "errors" - "io" - - "github.com/CalebQ42/squashfs/internal/data" - "github.com/CalebQ42/squashfs/internal/directory" - "github.com/CalebQ42/squashfs/internal/inode" - "github.com/CalebQ42/squashfs/internal/metadata" - "github.com/CalebQ42/squashfs/internal/toreader" -) - -func (r Reader) inodeFromRef(ref uint64) (i inode.Inode, err error) { - offset, meta := (ref>>16)+r.s.InodeTableStart, ref&0xFFFF - rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) - _, err = rdr.Read(make([]byte, meta)) - if err != nil { - return - } - return inode.Read(rdr, r.s.BlockSize) -} - -func (r Reader) inodeFromDir(e directory.Entry) (i inode.Inode, err error) { - rdr := metadata.NewReader(toreader.NewReader(r.r, int64(uint64(e.BlockStart)+r.s.InodeTableStart)), r.d) - _, err = rdr.Read(make([]byte, e.Offset)) - if err != nil { - return - } - return inode.Read(rdr, r.s.BlockSize) -} - -func (r Reader) getReaders(i inode.Inode) (full *data.FullReader, rdr *data.Reader, err error) { - var fragOffset uint64 - var blockOffset uint64 - var blockSizes []uint32 - var fragInd uint32 - var fragSize uint32 - var fileSize uint64 - if i.Type == inode.Fil { - fragOffset = uint64(i.Data.(inode.File).FragOffset) - blockOffset = uint64(i.Data.(inode.File).BlockStart) - blockSizes = i.Data.(inode.File).BlockSizes - fragInd = i.Data.(inode.File).FragInd - fragSize = i.Data.(inode.File).Size % r.s.BlockSize - fileSize = uint64(i.Data.(inode.File).Size) - } else if i.Type == inode.EFil { - fragOffset = uint64(i.Data.(inode.EFile).FragOffset) - blockOffset = i.Data.(inode.EFile).BlockStart - blockSizes = i.Data.(inode.EFile).BlockSizes - fragInd = i.Data.(inode.EFile).FragInd - fragSize = uint32(i.Data.(inode.EFile).Size % uint64(r.s.BlockSize)) - fileSize = i.Data.(inode.EFile).Size - } else { - return nil, nil, errors.New("getReaders called on non-file type") - } - rdr = data.NewReader(toreader.NewReader(r.r, int64(blockOffset)), r.d, blockSizes, r.s.BlockSize, fileSize) - full = data.NewFullReader(r.r, uint64(blockOffset), r.d, blockSizes, r.s.BlockSize, fileSize) - if fragInd != 0xFFFFFFFF { - full.AddFragment(func() (io.Reader, error) { - var fragRdr io.Reader - fragRdr, err = r.fragReader(fragInd, fragSize) - if err != nil { - return nil, err - } - var n, tmpN int - for n != int(fragOffset) { - tmpN, err = fragRdr.Read(make([]byte, int(fragOffset)-n)) - if err != nil { - return nil, err - } - n += tmpN - } - fragRdr = io.LimitReader(fragRdr, int64(fragSize)) - return fragRdr, nil - }) - var fragRdr io.Reader - fragRdr, err = r.fragReader(fragInd, fragSize) - if err != nil { - return nil, nil, err - } - var n, tmpN int - for n != int(fragOffset) { - tmpN, err = fragRdr.Read(make([]byte, int(fragOffset)-n)) - if err != nil { - return nil, nil, err - } - n += tmpN - } - fragRdr = io.LimitReader(fragRdr, int64(fragSize)) - rdr.AddFragment(fragRdr) - } - return -} - -func (r Reader) readDirectory(i inode.Inode) ([]directory.Entry, error) { - var offset uint64 - var blockOffset uint16 - var size uint32 - if i.Type == inode.Dir { - offset = uint64(i.Data.(inode.Directory).BlockStart) - blockOffset = i.Data.(inode.Directory).Offset - size = uint32(i.Data.(inode.Directory).Size) - } else if i.Type == inode.EDir { - offset = uint64(i.Data.(inode.EDirectory).BlockStart) - blockOffset = i.Data.(inode.EDirectory).Offset - size = i.Data.(inode.EDirectory).Size - } else { - return nil, errors.New("readDirectory called on non-directory type") - } - rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset+r.s.DirTableStart)), r.d) - _, err := rdr.Read(make([]byte, blockOffset)) - if err != nil { - return nil, err - } - return directory.ReadEntries(rdr, size) -} diff --git a/squashfs_test.go b/squashfs_test.go index 4d42960..0ae1658 100644 --- a/squashfs_test.go +++ b/squashfs_test.go @@ -19,9 +19,7 @@ import ( const ( squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs" - squashfsName = "bug.sqfs" - - filePath = "PortableApps/Notepad++Portable/App/DefaultData/Config/contextMenu.xml" + squashfsName = "airootfs.sfs" ) func preTest(dir string) (fil *os.File, err error) { @@ -83,21 +81,24 @@ func BenchmarkRace(b *testing.B) { os.RemoveAll(libPath) os.RemoveAll(unsquashPath) var libTime, unsquashTime time.Duration + op := squashfs.FastOptions() start := time.Now() rdr, err := squashfs.NewReader(fil) if err != nil { b.Fatal(err) } - err = rdr.ExtractTo(libPath) + err = rdr.ExtractWithOptions(libPath, op) if err != nil { b.Fatal(err) } libTime = time.Since(start) cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name()) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr start = time.Now() err = cmd.Run() if err != nil { - b.Fatal(err) + b.Log("Unsquashfs error:", err) } unsquashTime = time.Since(start) b.Log("Library took:", libTime.Round(time.Millisecond)) @@ -143,17 +144,21 @@ func TestExtractQuick(t *testing.T) { //TODO: Add long test that checks contents. squashFils := os.DirFS(unsquashPath) - err = fs.WalkDir(squashFils, ".", func(path string, d fs.DirEntry, _ error) error { + err = fs.WalkDir(squashFils, ".", func(path string, _ fs.DirEntry, _ error) error { libFil, e := os.Open(filepath.Join(libPath, path)) if e != nil { return e } - stat, _ := d.Info() + sfsFile, e := os.Open(filepath.Join(unsquashPath, path)) + if e != nil { + return e + } + sfsStat, _ := sfsFile.Stat() libStat, _ := libFil.Stat() - if stat.Size() != libStat.Size() { - t.Log(path, "not the same size between library and unsquashfs") + if sfsStat.Size() != libStat.Size() { + t.Log(libFil.Name(), "not the same size between library and unsquashfs") t.Log("File is", libStat.Size()) - t.Log("Should be", stat.Size()) + t.Log("Should be", sfsStat.Size()) return errors.New("file not the correct size") } return nil @@ -161,16 +166,17 @@ func TestExtractQuick(t *testing.T) { if err != nil { t.Fatal(err) } - t.Fatal("end") } +var filePath = "bin" + func TestSingleFile(t *testing.T) { tmpDir := "testing" fil, err := preTest(tmpDir) if err != nil { t.Fatal(err) } - os.Remove(filepath.Base(filePath)) + os.Remove(filepath.Join(tmpDir, filePath)) rdr, err := squashfs.NewReader(fil) if err != nil { t.Fatal(err) @@ -185,23 +191,3 @@ func TestSingleFile(t *testing.T) { } t.Fatal("HI") } - -func TestFuse(t *testing.T) { - tmpDir := "testing" - fil, err := preTest(tmpDir) - if err != nil { - t.Fatal(err) - } - os.Remove(filepath.Base(filePath)) - rdr, err := squashfs.NewReader(fil) - if err != nil { - t.Fatal(err) - } - err = rdr.Mount("testing/fuseTest") - if err != nil { - t.Fatal(err) - } - defer rdr.Unmount() - rdr.MountWait() - t.Fatal("testing") -}