Belac/squashfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 55 Wiki Activity

Merge branch 'main' into upstream-fix-out-of-range-panic

Browse Source
This commit is contained in:
Caleb Gardner
2025-06-06 14:26:10 -05:00
committed by GitHub
parent 0253a76dbe ebbbc9e87e
commit 02d98b610c
28 changed files with 743 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+9 -6
View File
@@ -11,6 +11,10 @@ Currently has support for reading squashfs files and extracting files and folder
Special thanks to <https://dr-emann.github.io/squashfs/> for some VERY important information in an easy to understand format.
Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree/master/internal/squashfs) as I referenced it to figure some things out (and double check others).
## Build tags
As of `v1.1.0` this library has two optional build tags: `no_gpl` and `no_obsolete`. `no_gpl` disables the ability to read archives with lzo compression due to the library's gpl license. `no_obsolete` removes "obsolete" compression types for a reduced compilation size; currently this only disable lzma compression since it's superseded by xz.
## FUSE
As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://github.com/CalebQ42/squashfuse).
@@ -24,15 +28,14 @@ As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://g
## Issues
* Significantly slower then `unsquashfs` when nested images
* Noticably slower then `unsquashfs` for extraction, especially on larger images.
* This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries.
* Not to mention it's written in C
* Times seem to be largely dependent on file tree size and compression type.
* My main testing image (~100MB) using Zstd takes about 5x longer.
* An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 30x longer.
* A Tensorflow docker image (~3.3GB) using Zstd takes about 12x longer.
* My main testing image (~100MB) using Zstd takes ~2x longer.
* An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes ~28x longer.
* A Tensorflow docker image (~3.3GB) using Zstd takes ~3x longer.
Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes about 2x longer.
Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes ~2x longer.
## Recommendations on Usage
cmd/go-unsquashfs/main.go
+116 -5
View File
@@ -4,16 +4,116 @@ import (
"flag"
"fmt"
"os"
"os/user"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/CalebQ42/squashfs"
squashfslow "github.com/CalebQ42/squashfs/low"
)
func userName(uid int, numeric bool) string {
us := strconv.Itoa(uid)
if numeric {
return us
}
if u, err := user.LookupId(us); err == nil {
return u.Username
}
return us
}
func groupName(gid int, numeric bool) string {
gs := strconv.Itoa(gid)
if numeric {
return gs
}
if g, err := user.LookupGroupId(gs); err == nil {
return g.Name
}
return gs
}
var hardLinks = make(map[uint32]string)
func printFile(rdr *squashfs.Reader, path string, f *squashfs.File) {
path = filepath.Join(path, f.Low.Name)
fi, _ := f.Stat()
sfi := fi.(squashfs.FileInfo)
owner := fmt.Sprintf("%s/%s",
userName(sfi.Uid(), *numeric),
groupName(sfi.Gid(), *numeric))
var link string
var isHardLink bool
if *showHardLinks {
link, isHardLink = hardLinks[f.Low.Inode.Num]
if !isHardLink {
hardLinks[f.Low.Inode.Num] = path
}
}
var size int64
if isHardLink {
size = 0
} else {
size = fi.Size()
}
if sfi.IsSymlink() {
link = " -> " + sfi.SymlinkPath()
} else if isHardLink {
link = " link to " + link
}
fmt.Printf("%s %s %*d %s %s%s\n",
strings.ToLower(fi.Mode().String()),
owner, 26-len(owner), size,
fi.ModTime().Format("2006-01-02 15:04"),
path, link)
if f.IsDir() {
fs, _ := f.FS()
printDir(rdr, path, fs)
}
}
func printDir(rdr *squashfs.Reader, path string, f squashfs.FS) {
var base squashfslow.FileBase
var fil squashfs.File
var err error
for _, e := range f.LowDir.Entries {
base, err = rdr.Low.BaseFromEntry(e)
if err != nil {
panic(err)
}
fil = rdr.FileFromBase(base, f)
printFile(rdr, path, &fil)
}
}
var (
verbose *bool
list *bool
long *bool
numeric *bool
offset *int64
ignore *bool
file *string
showHardLinks *bool
)
func main() {
verbose := flag.Bool("v", false, "Verbose")
ignore := flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755")
verbose = flag.Bool("v", false, "Verbose")
list = flag.Bool("l", false, "List")
long = flag.Bool("ll", false, "List with attributes")
numeric = flag.Bool("lln", false, "List with attributes and numeric ids")
showHardLinks = flag.Bool("show-hard-links", false, "When used with ll or lln, shows hard links")
offset = flag.Int64("o", 0, "Offset")
ignore = flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755")
file = flag.String("e", "", "File or folder to extract")
flag.Parse()
if len(flag.Args()) < 2 {
if (*list || *long || *numeric) && flag.NArg() < 1 {
fmt.Println("Please provide a file name")
os.Exit(0)
} else if (!*list && !*long && !*numeric) && flag.NArg() < 2 {
fmt.Println("Please provide a file name and extraction path")
os.Exit(0)
}
@@ -21,15 +121,26 @@ func main() {
if err != nil {
panic(err)
}
r, err := squashfs.NewReader(f)
r, err := squashfs.NewReaderAtOffset(f, *offset)
if err != nil {
panic(err)
}
extractFil := r.File()
if *file != "" {
extractFil, err = r.OpenFile(*file)
if err != nil {
panic(err)
}
}
if *list || *long || *numeric {
printFile(&r, "", extractFil)
return
}
op := squashfs.DefaultOptions()
op.Verbose = *verbose
op.IgnorePerm = *ignore
n := time.Now()
err = r.ExtractWithOptions(flag.Arg(1), op)
err = extractFil.ExtractWithOptions(flag.Arg(1), op)
if err != nil {
panic(err)
}
file.go
+77 -63
View File
@@ -19,49 +19,47 @@ import (
// File represents a file inside a squashfs archive.
type File struct {
full *data.FullReader
rdr *data.Reader
parent *FS
full data.FullReader
rdr data.Reader
rdrInit bool
parent FS
r *Reader
b squashfslow.FileBase
Low squashfslow.FileBase
dirsRead int
}
// Creates a new *File from the given *squashfs.Base
func (r *Reader) FileFromBase(b squashfslow.FileBase, parent *FS) *File {
return &File{
b: b,
func (r *Reader) FileFromBase(b squashfslow.FileBase, parent FS) File {
return File{
Low: b,
parent: parent,
r: r,
}
}
func (f *File) FS() (*FS, error) {
func (f File) FS() (FS, error) {
if !f.IsDir() {
return nil, errors.New("not a directory")
return FS{}, errors.New("not a directory")
}
d, err := f.b.ToDir(&f.r.Low)
d, err := f.Low.ToDir(f.r.Low)
if err != nil {
return nil, err
return FS{}, err
}
return &FS{d: d, parent: f.parent, r: f.r}, nil
return FS{LowDir: d, parent: &f.parent, r: f.r}, nil
}
// Closes the underlying readers.
// Further calls to Read and WriteTo will re-create the readers.
// Never returns an error.
func (f *File) Close() error {
if f.rdr != nil {
return f.rdr.Close()
}
f.rdr = nil
f.full = nil
f.rdr.Close()
f.full.Close()
return nil
}
// Returns the file the symlink points to.
// If the file isn't a symlink, or points to a file outside the archive, returns nil.
func (f *File) GetSymlinkFile() fs.File {
func (f File) GetSymlinkFile() fs.File {
if !f.IsSymlink() {
return nil
}
@@ -76,22 +74,22 @@ func (f *File) GetSymlinkFile() fs.File {
}
// Returns whether the file is a directory.
func (f *File) IsDir() bool {
return f.b.IsDir()
func (f File) IsDir() bool {
return f.Low.IsDir()
}
// Returns whether the file is a regular file.
func (f *File) IsRegular() bool {
return f.b.IsRegular()
func (f File) IsRegular() bool {
return f.Low.IsRegular()
}
// Returns whether the file is a symlink.
func (f *File) IsSymlink() bool {
return f.b.Inode.Type == inode.Sym || f.b.Inode.Type == inode.ESym
func (f File) IsSymlink() bool {
return f.Low.Inode.Type == inode.Sym || f.Low.Inode.Type == inode.ESym
}
func (f *File) Mode() fs.FileMode {
return f.b.Inode.Mode()
func (f File) Mode() fs.FileMode {
return f.Low.Inode.Mode()
}
// Read reads the data from the file. Only works if file is a normal file.
@@ -99,7 +97,7 @@ func (f *File) Read(b []byte) (int, error) {
if !f.IsRegular() {
return 0, errors.New("file is not a regular file")
}
if f.rdr == nil {
if !f.rdrInit {
err := f.initializeReaders()
if err != nil {
return 0, err
@@ -114,7 +112,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
if !f.IsDir() {
return nil, errors.New("file is not a directory")
}
d, err := f.b.ToDir(&f.r.Low)
d, err := f.Low.ToDir(f.r.Low)
if err != nil {
return nil, err
}
@@ -127,7 +125,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
}
}
var out []fs.DirEntry
var fi fileInfo
var fi FileInfo
for _, e := range d.Entries[start:end] {
fi, err = f.r.newFileInfo(e)
if err != nil {
@@ -141,17 +139,25 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
}
// Returns the file's fs.FileInfo
func (f *File) Stat() (fs.FileInfo, error) {
return newFileInfo(f.b.Name, &f.b.Inode), nil
func (f File) Stat() (fs.FileInfo, error) {
uid, err := f.Low.Uid(&f.r.Low)
if err != nil {
return nil, err
}
gid, err := f.Low.Gid(&f.r.Low)
if err != nil {
return nil, err
}
return newFileInfo(f.Low.Name, uid, gid, &f.Low.Inode), nil
}
// SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string.
func (f *File) SymlinkPath() string {
switch f.b.Inode.Type {
func (f File) SymlinkPath() string {
switch f.Low.Inode.Type {
case inode.Sym:
return string(f.b.Inode.Data.(inode.Symlink).Target)
return string(f.Low.Inode.Data.(inode.Symlink).Target)
case inode.ESym:
return string(f.b.Inode.Data.(inode.ESymlink).Target)
return string(f.Low.Inode.Data.(inode.ESymlink).Target)
}
return ""
}
@@ -162,7 +168,7 @@ func (f *File) WriteTo(w io.Writer) (int64, error) {
if !f.IsRegular() {
return 0, errors.New("file is not a regular file")
}
if f.full == nil {
if !f.rdrInit {
err := f.initializeReaders()
if err != nil {
return 0, err
@@ -173,36 +179,43 @@ func (f *File) WriteTo(w io.Writer) (int64, error) {
func (f *File) initializeReaders() error {
var err error
f.rdr, f.full, err = f.b.GetRegFileReaders(&f.r.Low)
f.rdr, f.full, err = f.Low.GetRegFileReaders(f.r.Low)
if err == nil {
f.rdrInit = true
} else {
f.rdr.Close()
f.full.Close()
}
return err
}
func (f *File) deviceDevices() (maj uint32, min uint32) {
func (f File) deviceDevices() (maj uint32, min uint32) {
var dev uint32
if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.Block {
dev = f.b.Inode.Data.(inode.Device).Dev
} else if f.b.Inode.Type == inode.EChar || f.b.Inode.Type == inode.EBlock {
dev = f.b.Inode.Data.(inode.EDevice).Dev
switch f.Low.Inode.Type {
case inode.Char, inode.Block:
dev = f.Low.Inode.Data.(inode.Device).Dev
case inode.EChar, inode.EBlock:
dev = f.Low.Inode.Data.(inode.EDevice).Dev
}
return dev >> 8, dev & 0x000FF
}
func (f *File) path() string {
if f.parent == nil {
return f.b.Name
func (f File) path() string {
if f.parent.LowDir.Name == "" {
return f.Low.Name
}
return filepath.Join(f.parent.path(), f.b.Name)
return filepath.Join(f.parent.path(), f.Low.Name)
}
// Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
// Uses default extraction options.
func (f *File) Extract(folder string) error {
func (f File) Extract(folder string) error {
return f.ExtractWithOptions(folder, DefaultOptions())
}
// Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
// Allows setting various extraction options via ExtractionOptions.
func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
func (f File) ExtractWithOptions(path string, op *ExtractionOptions) error {
if op.manager == nil {
op.manager = routinemanager.NewManager(op.SimultaneousFiles)
if op.LogOutput != nil {
@@ -216,9 +229,9 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
return err
}
}
switch f.b.Inode.Type {
switch f.Low.Inode.Type {
case inode.Dir, inode.EDir:
d, err := f.b.ToDir(&f.r.Low)
d, err := f.Low.ToDir(f.r.Low)
if err != nil {
if op.Verbose {
log.Println("Failed to create squashfs.Directory for", path)
@@ -266,7 +279,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
}(b, path)
}
var errCache []error
for i := 0; i < len(d.Entries); i++ {
for range d.Entries {
err := <-errChan
if err != nil {
errCache = append(errCache, err)
@@ -276,7 +289,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
return errors.Join(errors.New("failed to extract folder: "+path), errors.Join(errCache...))
}
case inode.Fil, inode.EFil:
path = filepath.Join(path, f.b.Name)
path = filepath.Join(path, f.Low.Name)
outFil, err := os.Create(path)
if err != nil {
if op.Verbose {
@@ -285,7 +298,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
return errors.Join(errors.New("failed to create file: "+path), err)
}
defer outFil.Close()
full, err := f.b.GetFullReader(&f.r.Low)
full, err := f.Low.GetFullReader(&f.r.Low)
if err != nil {
if op.Verbose {
log.Println("Failed to create full reader for", path)
@@ -311,11 +324,11 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
return errors.New("failed to get symlink's file")
}
fil := filTmp.(*File)
fil.b.Name = f.b.Name
fil.Low.Name = f.Low.Name
err := fil.ExtractWithOptions(path, op)
if err != nil {
if op.Verbose {
log.Println("Failed to extract symlink's file:", filepath.Join(path, f.b.Name))
log.Println("Failed to extract symlink's file:", filepath.Join(path, f.Low.Name))
}
return errors.Join(errors.New("failed to extract symlink's file: "+path), err)
}
@@ -338,7 +351,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
return errors.Join(errors.New("failed to extract symlink's file: "+extractLoc), err)
}
}
path = filepath.Join(path, f.b.Name)
path = filepath.Join(path, f.Low.Name)
err := os.Symlink(f.SymlinkPath(), path)
if err != nil {
if op.Verbose {
@@ -361,13 +374,14 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
}
return errors.Join(errors.New("mknot command not found"), err)
}
path = filepath.Join(path, f.b.Name)
path = filepath.Join(path, f.Low.Name)
var typ string
if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.EChar {
switch f.Low.Inode.Type {
case inode.Char, inode.EChar:
typ = "c"
} else if f.b.Inode.Type == inode.Block || f.b.Inode.Type == inode.EBlock {
case inode.Block, inode.EBlock:
typ = "b"
} else { //Fifo IPC
default: //Fifo IPC
if runtime.GOOS == "darwin" {
if op.Verbose {
log.Println(f.path(), "ignored. A Fifo file and can't be created on Darwin.")
@@ -398,7 +412,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
}
return nil
default:
return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.b.Inode.Type)))
return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.Low.Inode.Type)))
}
if op.Verbose {
log.Println(f.path(), "extracted to", path)
@@ -406,7 +420,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
if op.IgnorePerm {
return nil
}
uid, err := f.b.Uid(&f.r.Low)
uid, err := f.Low.Uid(&f.r.Low)
if err != nil {
if op.Verbose {
log.Println("Failed to get uid for", path)
@@ -414,7 +428,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
}
return nil
}
gid, err := f.b.Gid(&f.r.Low)
gid, err := f.Low.Gid(&f.r.Low)
if err != nil {
if op.Verbose {
log.Println("Failed to get gid for", path)
file_info.go
+74 -16
View File
@@ -8,61 +8,119 @@ import (
"github.com/CalebQ42/squashfs/low/inode"
)
type fileInfo struct {
type FileInfo struct {
name string
uid uint32
gid uint32
size int64
target string
perm uint32
modTime uint32
fileType uint16
}
func (r Reader) newFileInfo(e directory.Entry) (fileInfo, error) {
i, err := r.Low.InodeFromEntry(e)
func (r Reader) newFileInfo(e directory.Entry) (FileInfo, error) {
b, err := r.Low.BaseFromEntry(e)
if err != nil {
return fileInfo{}, err
return FileInfo{}, err
}
return newFileInfo(e.Name, &i), nil
uid, err := b.Uid(&r.Low)
if err != nil {
return FileInfo{}, err
}
gid, err := b.Gid(&r.Low)
if err != nil {
return FileInfo{}, err
}
return newFileInfo(e.Name, uid, gid, &b.Inode), nil
}
func newFileInfo(name string, i *inode.Inode) fileInfo {
func newFileInfo(name string, uid, gid uint32, i *inode.Inode) FileInfo {
var size int64
if i.Type == inode.Fil {
var target string
switch i.Type {
case inode.Fil:
size = int64(i.Data.(inode.File).Size)
} else if i.Type == inode.EFil {
case inode.EFil:
size = int64(i.Data.(inode.EFile).Size)
case inode.Sym:
target = string(i.Data.(inode.Symlink).Target)
case inode.ESym:
target = string(i.Data.(inode.ESymlink).Target)
}
return fileInfo{
return FileInfo{
name: name,
uid: uid,
gid: gid,
size: size,
target: target,
perm: uint32(i.Perm),
modTime: i.ModTime,
fileType: i.Type,
}
}
func (f fileInfo) Name() string {
func (f FileInfo) Name() string {
return f.name
}
func (f fileInfo) Size() int64 {
func (f FileInfo) Uid() int {
return int(f.uid)
}
func (f FileInfo) Gid() int {
return int(f.gid)
}
func (f FileInfo) Size() int64 {
return f.size
}
func (f fileInfo) Mode() fs.FileMode {
if f.IsDir() {
func (f FileInfo) SymlinkPath() string {
return f.target
}
func (f FileInfo) Mode() fs.FileMode {
switch f.fileType {
case inode.Dir, inode.EDir:
return fs.FileMode(f.perm | uint32(fs.ModeDir))
case inode.Sym, inode.ESym:
return fs.FileMode(f.perm | uint32(fs.ModeSymlink))
case inode.Char, inode.EChar, inode.Block, inode.EBlock:
return fs.FileMode(f.perm | uint32(fs.ModeDevice))
case inode.Fifo, inode.EFifo:
return fs.FileMode(f.perm | uint32(fs.ModeNamedPipe))
case inode.Sock, inode.ESock:
return fs.FileMode(f.perm | uint32(fs.ModeSocket))
}
return fs.FileMode(f.perm)
}
func (f fileInfo) ModTime() time.Time {
func (f FileInfo) ModTime() time.Time {
return time.Unix(int64(f.modTime), 0)
}
func (f fileInfo) IsDir() bool {
func (f FileInfo) IsDir() bool {
return f.fileType == inode.Dir || f.fileType == inode.EDir
}
func (f fileInfo) Sys() any {
func (f FileInfo) IsSymlink() bool {
return f.fileType == inode.Sym || f.fileType == inode.ESym
}
func (f FileInfo) IsDevice() bool {
return f.fileType == inode.Block || f.fileType == inode.EBlock ||
f.fileType == inode.Char || f.fileType == inode.EChar
}
func (f FileInfo) IsFifo() bool {
return f.fileType == inode.Fifo || f.fileType == inode.EFifo
}
func (f FileInfo) IsSocket() bool {
return f.fileType == inode.Sock || f.fileType == inode.ESock
}
func (f FileInfo) Sys() any {
return nil
}
fs.go
+40 -30
View File
@@ -17,15 +17,15 @@ import (
type FS struct {
r *Reader
parent *FS
d squashfslow.Directory
LowDir squashfslow.Directory
}
// Creates a new *FS from the given squashfs.directory
func (r *Reader) FSFromDirectory(d squashfslow.Directory, parent *FS) *FS {
return &FS{
d: d,
func (r *Reader) FSFromDirectory(d squashfslow.Directory, parent FS) FS {
return FS{
LowDir: d,
r: r,
parent: parent,
parent: &parent,
}
}
@@ -42,10 +42,10 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
}
}
split := strings.Split(pattern, "/")
for i := 0; i < len(f.d.Entries); i++ {
if match, _ := path.Match(split[0], f.d.Entries[i].Name); match {
for i := range f.LowDir.Entries {
if match, _ := path.Match(split[0], f.LowDir.Entries[i].Name); match {
if len(split) == 1 {
out = append(out, f.d.Entries[i].Name)
out = append(out, f.LowDir.Entries[i].Name)
continue
}
sub, err := f.Sub(split[0])
@@ -80,8 +80,8 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
Err: err,
}
}
for i := 0; i < len(subGlob); i++ {
subGlob[i] = f.d.Name + "/" + subGlob[i]
for i := range subGlob {
subGlob[i] = f.LowDir.Name + "/" + subGlob[i]
}
out = append(out, subGlob...)
}
@@ -90,7 +90,11 @@ func (f *FS) Glob(pattern string) (out []string, err error) {
}
// Opens the file at name. Returns a *File as an fs.File.
func (f *FS) Open(name string) (fs.File, error) {
func (f FS) Open(name string) (fs.File, error) {
return f.OpenFile(name)
}
func (f FS) OpenFile(name string) (*File, error) {
name = filepath.Clean(name)
if !fs.ValidPath(name) {
return nil, &fs.PathError{
@@ -111,10 +115,10 @@ func (f *FS) Open(name string) (fs.File, error) {
Err: fs.ErrNotExist,
}
} else {
return f.parent.Open(strings.Join(split[1:], "/"))
return f.parent.OpenFile(strings.Join(split[1:], "/"))
}
}
i, found := slices.BinarySearchFunc(f.d.Entries, split[0], func(e directory.Entry, name string) int {
i, found := slices.BinarySearchFunc(f.LowDir.Entries, split[0], func(e directory.Entry, name string) int {
return strings.Compare(e.Name, name)
})
if !found {
@@ -124,13 +128,13 @@ func (f *FS) Open(name string) (fs.File, error) {
Err: fs.ErrNotExist,
}
}
b, err := f.r.Low.BaseFromEntry(f.d.Entries[i])
b, err := f.r.Low.BaseFromEntry(f.LowDir.Entries[i])
if err != nil {
return nil, err
}
if len(split) == 1 {
return &File{
b: b,
Low: b,
r: f.r,
parent: f,
}, nil
@@ -142,16 +146,16 @@ func (f *FS) Open(name string) (fs.File, error) {
Err: fs.ErrNotExist,
}
}
d, err := b.ToDir(&f.r.Low)
d, err := b.ToDir(f.r.Low)
if err != nil {
return nil, err
}
return f.r.FSFromDirectory(d, f).Open(strings.Join(split[1:], "/"))
return f.r.FSFromDirectory(d, f).OpenFile(strings.Join(split[1:], "/"))
}
// Returns all DirEntry's for the directory at name.
// If name is not a directory, returns an error.
func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) {
func (f FS) ReadDir(name string) ([]fs.DirEntry, error) {
name = filepath.Clean(name)
if !fs.ValidPath(name) {
return nil, &fs.PathError{
@@ -171,7 +175,7 @@ func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) {
}
// Returns the contents of the file at name.
func (f *FS) ReadFile(name string) (out []byte, err error) {
func (f FS) ReadFile(name string) (out []byte, err error) {
name = filepath.Clean(name)
if !fs.ValidPath(name) {
return nil, &fs.PathError{
@@ -194,7 +198,7 @@ func (f *FS) ReadFile(name string) (out []byte, err error) {
}
// Returns the fs.FileInfo for the file at name.
func (f *FS) Stat(name string) (fs.FileInfo, error) {
func (f FS) Stat(name string) (fs.FileInfo, error) {
name = filepath.Clean(name)
if !fs.ValidPath(name) {
return nil, &fs.PathError{
@@ -214,7 +218,7 @@ func (f *FS) Stat(name string) (fs.FileInfo, error) {
}
// Returns the FS at dir
func (f *FS) Sub(dir string) (fs.FS, error) {
func (f FS) Sub(dir string) (fs.FS, error) {
dir = filepath.Clean(dir)
if !fs.ValidPath(dir) {
return nil, &fs.PathError{
@@ -242,28 +246,34 @@ func (f *FS) Sub(dir string) (fs.FS, error) {
// Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
// Uses default extraction options.
func (f *FS) Extract(folder string) error {
func (f FS) Extract(folder string) error {
return f.File().Extract(folder)
}
// Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder.
// Allows setting various extraction options via ExtractionOptions.
func (f *FS) ExtractWithOptions(folder string, op *ExtractionOptions) error {
func (f FS) ExtractWithOptions(folder string, op *ExtractionOptions) error {
return f.File().ExtractWithOptions(folder, op)
}
// Returns the FS as a *File
func (f *FS) File() *File {
func (f FS) File() *File {
if f.parent != nil {
return &File{
Low: f.LowDir.FileBase,
parent: *f.parent,
r: f.r,
}
}
return &File{
b: f.d.FileBase,
parent: f.parent,
r: f.r,
Low: f.LowDir.FileBase,
r: f.r,
}
}
func (f *FS) path() string {
func (f FS) path() string {
if f.parent == nil {
return f.d.Name
return f.LowDir.Name
}
return filepath.Join(f.parent.path(), f.d.Name)
return filepath.Join(f.parent.path(), f.LowDir.Name)
}
go.mod
+3 -3
View File
@@ -1,10 +1,10 @@
module github.com/CalebQ42/squashfs
go 1.22.5
go 1.24.0
require (
github.com/klauspost/compress v1.17.9
github.com/pierrec/lz4/v4 v4.1.21
github.com/klauspost/compress v1.18.0
github.com/pierrec/lz4/v4 v4.1.22
github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e
github.com/therootcompany/xz v1.0.1
github.com/ulikunitz/xz v0.5.12
go.sum
+4 -4
View File
@@ -1,7 +1,7 @@
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs=
github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
internal/decompress/lz4.go
+18 -3
View File
@@ -3,13 +3,28 @@ package decompress
import (
"bytes"
"io"
"sync"
"github.com/pierrec/lz4/v4"
)
type Lz4 struct{}
type Lz4 struct {
pool sync.Pool
}
func (l Lz4) Decompress(data []byte) ([]byte, error) {
rdr := lz4.NewReader(bytes.NewReader(data))
func NewLz4() *Lz4 {
return &Lz4{
pool: sync.Pool{
New: func() any {
return lz4.NewReader(nil)
},
},
}
}
func (l *Lz4) Decompress(data []byte) ([]byte, error) {
rdr := l.pool.Get().(*lz4.Reader)
defer l.pool.Put(rdr)
rdr.Reset(bytes.NewReader(data))
return io.ReadAll(rdr)
}
internal/decompress/lzma.go
+6
View File
@@ -1,3 +1,5 @@
//go:build !no_obsolete
package decompress
import (
@@ -9,6 +11,10 @@ import (
type Lzma struct{}
func NewLzma() (Lzma, error) {
return Lzma{}, nil
}
func (l Lzma) Decompress(data []byte) ([]byte, error) {
rdr, err := lzma.NewReader(bytes.NewReader(data))
if err != nil {
internal/decompress/lzma_disabled.go
+17
View File
@@ -0,0 +1,17 @@
//go:build no_obsolete
package decompress
import (
"errors"
)
type Lzma struct{}
func NewLzma() (Lzma, error) {
return Lzma{}, errors.New("lzma compression is disable in this build with no_obsolete")
}
func (l Lzma) Decompress(data []byte) ([]byte, error) {
return nil, errors.New("lzma compression is disable in this build with no_obsolete")
}
internal/decompress/lzo.go
+6
View File
@@ -1,3 +1,5 @@
//go:build !no_gpl
package decompress
import (
@@ -8,6 +10,10 @@ import (
type Lzo struct{}
func NewLzo() (Lzo, error) {
return Lzo{}, nil
}
func (l Lzo) Decompress(data []byte) ([]byte, error) {
return lzo.Decompress1X(bytes.NewReader(data), len(data), 0)
}
internal/decompress/lzo_disabled.go
+15
View File
@@ -0,0 +1,15 @@
//go:build no_gpl
package decompress
import "errors"
type Lzo struct{}
func NewLzo() (Lzo, error) {
return Lzo{}, errors.New("lzo compression is disable in this build with no_gpl")
}
func (l Lzo) Decompress(data []byte) ([]byte, error) {
return nil, errors.New("lzo compression is disable in this build with no_gpl")
}
internal/decompress/xz.go
+19 -3
View File
@@ -3,14 +3,30 @@ package decompress
import (
"bytes"
"io"
"sync"
"github.com/therootcompany/xz"
)
type Xz struct{}
type Xz struct {
pool sync.Pool
}
func (x Xz) Decompress(data []byte) ([]byte, error) {
rdr, err := xz.NewReader(bytes.NewReader(data), 0)
func NewXz() *Xz {
return &Xz{
pool: sync.Pool{
New: func() any {
rdr, _ := xz.NewReader(nil, 0)
return rdr
},
},
}
}
func (x *Xz) Decompress(data []byte) ([]byte, error) {
rdr := x.pool.Get().(*xz.Reader)
defer x.pool.Put(rdr)
err := rdr.Reset(bytes.NewReader(data))
if err != nil {
return nil, err
}
internal/decompress/zlib.go
+20 -6
View File
@@ -2,17 +2,31 @@ package decompress
import (
"bytes"
"compress/zlib"
"io"
"sync"
"github.com/klauspost/compress/zlib"
)
type Zlib struct{}
type Zlib struct {
pool sync.Pool
}
func (z Zlib) Decompress(data []byte) ([]byte, error) {
rdr, err := zlib.NewReader(bytes.NewReader(data))
func NewZlib() *Zlib {
return &Zlib{}
}
func (z *Zlib) Decompress(data []byte) ([]byte, error) {
rdr := z.pool.Get()
defer z.pool.Put(rdr)
var err error
if rdr == nil {
rdr, err = zlib.NewReader(bytes.NewReader(data))
} else {
err = rdr.(zlib.Resetter).Reset(bytes.NewReader(data), nil)
}
if err != nil {
return nil, err
}
defer rdr.Close()
return io.ReadAll(rdr)
return io.ReadAll(rdr.(io.ReadCloser))
}
internal/decompress/zstd.go
+11 -10
View File
@@ -1,19 +1,20 @@
package decompress
import (
"bytes"
"io"
"github.com/klauspost/compress/zstd"
)
type Zstd struct{}
type Zstd struct {
rdr *zstd.Decoder
}
func NewZstd() Zstd {
rdr, _ := zstd.NewReader(nil, zstd.WithDecoderLowmem(true))
return Zstd{
rdr: rdr,
}
}
func (z Zstd) Decompress(data []byte) ([]byte, error) {
rdr, err := zstd.NewReader(bytes.NewReader(data))
if err != nil {
return nil, err
}
defer rdr.Close()
return io.ReadAll(rdr)
return z.rdr.DecodeAll(data, nil)
}
internal/metadata/reader.go
+7 -9
View File
@@ -14,8 +14,8 @@ type Reader struct {
curOffset uint16
}
func NewReader(r io.Reader, d decompress.Decompressor) *Reader {
return &Reader{
func NewReader(r io.Reader, d decompress.Decompressor) Reader {
return Reader{
r: r,
d: d,
}
@@ -23,14 +23,15 @@ func NewReader(r io.Reader, d decompress.Decompressor) *Reader {
func (r *Reader) advance() error {
r.curOffset = 0
var size uint16
err := binary.Read(r.r, binary.LittleEndian, &size)
dat := make([]byte, 2)
_, err := r.r.Read(dat)
if err != nil {
return err
}
size := binary.LittleEndian.Uint16(dat)
realSize := size &^ 0x8000
r.dat = make([]byte, realSize)
err = binary.Read(r.r, binary.LittleEndian, &r.dat)
_, err = r.r.Read(r.dat)
if err != nil {
return err
}
@@ -50,10 +51,7 @@ func (r *Reader) Read(b []byte) (int, error) {
return curRead, err
}
}
toRead = len(b) - curRead
if toRead > len(r.dat)-int(r.curOffset) {
toRead = len(r.dat) - int(r.curOffset)
}
toRead = min(len(b)-curRead, len(r.dat)-int(r.curOffset))
copy(b[curRead:], r.dat[r.curOffset:int(r.curOffset)+toRead])
r.curOffset += uint16(toRead)
curRead += toRead
low/data/fullreader.go
+107 -21
View File
@@ -1,15 +1,14 @@
package data
import (
"encoding/binary"
"errors"
"io"
"io/fs"
"math"
"runtime"
"sync"
"github.com/CalebQ42/squashfs/internal/decompress"
"github.com/CalebQ42/squashfs/internal/toreader"
)
type FragReaderConstructor func() (io.Reader, error)
@@ -18,16 +17,16 @@ type FullReader struct {
r io.ReaderAt
d decompress.Decompressor
frag FragReaderConstructor
retPool *sync.Pool
sizes []uint32
initialOffset int64
finalBlockSize uint64
blockSize uint32
goroutineLimit uint16
closed bool
}
func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) *FullReader {
return &FullReader{
func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) FullReader {
return FullReader{
r: r,
d: d,
sizes: sizes,
@@ -35,19 +34,26 @@ func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor
goroutineLimit: uint16(runtime.NumCPU()),
finalBlockSize: finalBlockSize,
blockSize: blockSize,
retPool: &sync.Pool{
New: func() any {
return &retValue{}
},
},
}
}
func (r *FullReader) Close() error {
r.closed = true
r.r = nil
r.d = nil
r.frag = nil
r.sizes = nil
return nil
}
func (r *FullReader) AddFrag(frag FragReaderConstructor) {
r.frag = frag
}
func (r *FullReader) SetGoroutineLimit(limit uint16) {
if limit <= 0 {
r.goroutineLimit = 1
}
r.goroutineLimit = limit
}
@@ -57,8 +63,8 @@ type retValue struct {
index uint64
}
func (r *FullReader) process(index uint64, fileOffset uint64, retChan chan *retValue) {
ret := r.retPool.Get().(*retValue)
func (r FullReader) process(index uint64, fileOffset uint64, pool *sync.Pool, retChan chan *retValue) {
ret := pool.Get().(*retValue)
ret.index = index
realSize := r.sizes[index] &^ (1 << 24)
if realSize == 0 {
@@ -72,14 +78,20 @@ func (r *FullReader) process(index uint64, fileOffset uint64, retChan chan *retV
return
}
ret.data = make([]byte, realSize)
ret.err = binary.Read(toreader.NewReader(r.r, int64(r.initialOffset)+int64(fileOffset)), binary.LittleEndian, &ret.data)
_, ret.err = r.r.ReadAt(ret.data, r.initialOffset+int64(fileOffset))
if r.sizes[index] == realSize {
ret.data, ret.err = r.d.Decompress(ret.data)
}
retChan <- ret
}
func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
func (r FullReader) WriteTo(w io.Writer) (int64, error) {
if r.closed {
return 0, fs.ErrClosed
}
// if wa, is := w.(io.WriterAt); is {
// return r.writeToWriteAt(wa)
// }
var curIndex uint64
var curOffset uint64
var toProcess uint16
@@ -87,14 +99,16 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
cache := make(map[uint64]*retValue)
var errCache []error
retChan := make(chan *retValue, r.goroutineLimit)
pool := &sync.Pool{
New: func() any {
return &retValue{}
},
}
for i := uint64(0); i < uint64(math.Ceil(float64(len(r.sizes))/float64(r.goroutineLimit))); i++ {
toProcess = uint16(len(r.sizes)) - (uint16(i) * r.goroutineLimit)
if toProcess > r.goroutineLimit {
toProcess = r.goroutineLimit
}
toProcess = min(uint16(len(r.sizes))-(uint16(i)*r.goroutineLimit), r.goroutineLimit)
// Start all the goroutines
for j := uint16(0); j < toProcess; j++ {
go r.process((i*uint64(r.goroutineLimit))+uint64(j), curOffset, retChan)
go r.process((i*uint64(r.goroutineLimit))+uint64(j), curOffset, pool, retChan)
curOffset += uint64(r.sizes[(i*uint64(r.goroutineLimit))+uint64(j)]) &^ (1 << 24)
}
// Then consume the results on retChan
@@ -128,7 +142,7 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
}
continue
}
r.retPool.Put(res)
pool.Put(res)
curIndex++
// Now we recursively try to clear the cache
for len(cache) > 0 {
@@ -146,7 +160,7 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
break
}
delete(cache, curIndex)
r.retPool.Put(res)
pool.Put(res)
curIndex++
}
}
@@ -172,3 +186,75 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) {
}
return wrote, nil
}
// func (r FullReader) writeToWriteAt(w io.WriterAt) (out int64, outErr error) {
// wait := &sync.WaitGroup{}
// wait.Add(len(r.sizes))
// mgr := routinemanager.NewManager(r.goroutineLimit)
// curOffset := r.initialOffset
// for i := uint64(0); i < uint64(len(r.sizes)); i++ {
// go func(index uint64, fileOffset int64) {
// lckNum := mgr.Lock()
// defer mgr.Unlock(lckNum)
// defer wait.Done()
// realSize := r.sizes[index] &^ (1 << 24)
// if realSize == 0 {
// if index == uint64(len(r.sizes))-1 && r.frag == nil {
// _, err := w.WriteAt([]byte{0}, int64((uint64(r.blockSize)*index)+r.finalBlockSize)-1)
// if err != nil {
// outErr = errors.Join(outErr, err)
// return
// }
// out = max(out, int64((uint64(r.blockSize)*index)+r.finalBlockSize))
// }
// return
// }
// data := make([]byte, realSize)
// err := binary.Read(toreader.NewReader(r.r, int64(fileOffset)), binary.LittleEndian, &data)
// if err != nil {
// outErr = errors.Join(outErr, err)
// return
// }
// if r.sizes[index] == realSize {
// data, err = r.d.Decompress(data)
// }
// if err != nil {
// outErr = errors.Join(outErr, err)
// return
// }
// _, err = w.WriteAt(data, int64(uint64(r.blockSize)*index))
// if err != nil {
// outErr = errors.Join(outErr, err)
// return
// }
// out = max(out, int64(uint64(r.blockSize)*(index+1)))
// }(i, curOffset)
// curOffset += int64(r.sizes[i]) &^ (1 << 24)
// }
// if r.frag != nil {
// wait.Add(1)
// go func() {
// lckNum := mgr.Lock()
// defer mgr.Unlock(lckNum)
// defer wait.Done()
// rdr, err := r.frag()
// if err != nil {
// outErr = errors.Join(outErr, err)
// return
// }
// dat, err := io.ReadAll(rdr)
// if err != nil {
// outErr = errors.Join(outErr, err)
// return
// }
// _, err = w.WriteAt(dat, int64(int(r.blockSize)*len(r.sizes)))
// if err != nil {
// outErr = errors.Join(outErr, err)
// return
// }
// out = int64(int(r.blockSize)*len(r.sizes)) + int64(r.finalBlockSize)
// }()
// }
// wait.Wait()
// return
// }
low/data/reader.go
+15 -8
View File
@@ -1,8 +1,8 @@
package data
import (
"encoding/binary"
"io"
"io/fs"
"github.com/CalebQ42/squashfs/internal/decompress"
)
@@ -17,10 +17,11 @@ type Reader struct {
curIndex uint64
finalBlockSize uint64
blockSize uint32
closed bool
}
func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) *Reader {
return &Reader{
func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) Reader {
return Reader{
r: r,
d: d,
sizes: sizes,
@@ -41,6 +42,7 @@ func (r *Reader) advance() error {
r.dat, err = io.ReadAll(r.frag)
return err
} else if r.curIndex >= uint64(len(r.sizes)) {
r.dat = []byte{}
return io.EOF
}
realSize := r.sizes[r.curIndex] &^ (1 << 24)
@@ -53,7 +55,7 @@ func (r *Reader) advance() error {
return nil
}
r.dat = make([]byte, realSize)
err = binary.Read(r.r, binary.LittleEndian, &r.dat)
_, err = r.r.Read(r.dat)
if err != nil {
return err
}
@@ -65,6 +67,9 @@ func (r *Reader) advance() error {
}
func (r *Reader) Read(b []byte) (int, error) {
if r.closed {
return 0, fs.ErrClosed
}
curRead := 0
var toRead int
for curRead < len(b) {
@@ -73,10 +78,7 @@ func (r *Reader) Read(b []byte) (int, error) {
return curRead, err
}
}
toRead = len(b) - curRead
if toRead > len(r.dat)-r.curOffset {
toRead = len(r.dat) - r.curOffset
}
toRead = min(len(b)-curRead, len(r.dat)-r.curOffset)
toRead = copy(b[curRead:], r.dat[r.curOffset:r.curOffset+toRead])
r.curOffset += toRead
curRead += toRead
@@ -85,6 +87,9 @@ func (r *Reader) Read(b []byte) (int, error) {
}
func (r *Reader) Close() error {
r.closed = true
r.r = nil
r.d = nil
if r.frag != nil {
if l, ok := r.frag.(*io.LimitedReader); ok {
if cl, ok := l.R.(io.Closer); ok {
@@ -92,6 +97,8 @@ func (r *Reader) Close() error {
}
}
}
r.frag = nil
r.sizes = nil
r.dat = nil
return nil
}
low/directory.go
+3 -3
View File
@@ -18,7 +18,7 @@ type Directory struct {
Entries []directory.Entry
}
func (r *Reader) directoryFromRef(ref uint64, name string) (Directory, error) {
func (r Reader) directoryFromRef(ref uint64, name string) (Directory, error) {
i, err := r.InodeFromRef(ref)
if err != nil {
return Directory{}, err
@@ -44,7 +44,7 @@ func (r *Reader) directoryFromRef(ref uint64, name string) (Directory, error) {
if err != nil {
return Directory{}, err
}
entries, err := directory.ReadDirectory(dirRdr, size)
entries, err := directory.ReadDirectory(&dirRdr, size)
if err != nil {
return Directory{}, err
}
@@ -54,7 +54,7 @@ func (r *Reader) directoryFromRef(ref uint64, name string) (Directory, error) {
}, nil
}
func (d *Directory) Open(r *Reader, path string) (FileBase, error) {
func (d Directory) Open(r Reader, path string) (FileBase, error) {
path = filepath.Clean(path)
if path == "." || path == "" {
return d.FileBase, nil
low/file_base.go
+21 -21
View File
@@ -16,11 +16,11 @@ type FileBase struct {
Name string
}
func (r *Reader) BaseFromInode(i inode.Inode, name string) FileBase {
func (r Reader) BaseFromInode(i inode.Inode, name string) FileBase {
return FileBase{Inode: i, Name: name}
}
func (r *Reader) BaseFromEntry(e directory.Entry) (FileBase, error) {
func (r Reader) BaseFromEntry(e directory.Entry) (FileBase, error) {
in, err := r.InodeFromEntry(e)
if err != nil {
return FileBase{}, err
@@ -28,7 +28,7 @@ func (r *Reader) BaseFromEntry(e directory.Entry) (FileBase, error) {
return FileBase{Inode: in, Name: e.Name}, nil
}
func (r *Reader) BaseFromRef(ref uint64, name string) (FileBase, error) {
func (r Reader) BaseFromRef(ref uint64, name string) (FileBase, error) {
in, err := r.InodeFromRef(ref)
if err != nil {
return FileBase{}, err
@@ -36,19 +36,19 @@ func (r *Reader) BaseFromRef(ref uint64, name string) (FileBase, error) {
return FileBase{Inode: in, Name: name}, nil
}
func (b *FileBase) Uid(r *Reader) (uint32, error) {
func (b FileBase) Uid(r *Reader) (uint32, error) {
return r.Id(b.Inode.UidInd)
}
func (b *FileBase) Gid(r *Reader) (uint32, error) {
func (b FileBase) Gid(r *Reader) (uint32, error) {
return r.Id(b.Inode.GidInd)
}
func (b *FileBase) IsDir() bool {
func (b FileBase) IsDir() bool {
return b.Inode.Type == inode.Dir || b.Inode.Type == inode.EDir
}
func (b *FileBase) ToDir(r *Reader) (Directory, error) {
func (b FileBase) ToDir(r Reader) (Directory, error) {
var blockStart uint32
var size uint32
var offset uint16
@@ -70,23 +70,23 @@ func (b *FileBase) ToDir(r *Reader) (Directory, error) {
if err != nil {
return Directory{}, err
}
entries, err := directory.ReadDirectory(dirRdr, size)
entries, err := directory.ReadDirectory(&dirRdr, size)
if err != nil {
return Directory{}, err
}
return Directory{
FileBase: *b,
FileBase: b,
Entries: entries,
}, nil
}
func (b *FileBase) IsRegular() bool {
func (b FileBase) IsRegular() bool {
return b.Inode.Type == inode.Fil || b.Inode.Type == inode.EFil
}
func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, error) {
func (b FileBase) GetRegFileReaders(r Reader) (data.Reader, data.FullReader, error) {
if !b.IsRegular() {
return nil, nil, errors.New("not a regular file")
return data.Reader{}, data.FullReader{}, errors.New("not a regular file")
}
var blockStart uint64
var fragIndex uint32
@@ -113,13 +113,13 @@ func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader,
}
frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
frag.Read(make([]byte, fragOffset))
return io.LimitReader(frag, int64(fragSize)), nil
return io.LimitReader(&frag, int64(fragSize)), nil
}
outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize)
if fragIndex != 0xffffffff {
f, err := frag()
if err != nil {
return nil, nil, err
return data.Reader{}, data.FullReader{}, err
}
outRdr.AddFrag(f)
}
@@ -130,9 +130,9 @@ func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader,
return outRdr, outFull, nil
}
func (b *FileBase) GetFullReader(r *Reader) (*data.FullReader, error) {
func (b FileBase) GetFullReader(r *Reader) (data.FullReader, error) {
if !b.IsRegular() {
return nil, errors.New("not a regular file")
return data.FullReader{}, errors.New("not a regular file")
}
var blockStart uint64
var fragIndex uint32
@@ -161,15 +161,15 @@ func (b *FileBase) GetFullReader(r *Reader) (*data.FullReader, error) {
}
frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
frag.Read(make([]byte, fragOffset))
return io.LimitReader(frag, int64(fragSize)), nil
return io.LimitReader(&frag, int64(fragSize)), nil
})
}
return outFull, nil
}
func (b *FileBase) GetReader(r *Reader) (*data.Reader, error) {
func (b FileBase) GetReader(r *Reader) (data.Reader, error) {
if !b.IsRegular() {
return nil, errors.New("not a regular file")
return data.Reader{}, errors.New("not a regular file")
}
var blockStart uint64
var fragIndex uint32
@@ -193,11 +193,11 @@ func (b *FileBase) GetReader(r *Reader) (*data.Reader, error) {
if fragIndex != 0xffffffff {
ent, err := r.fragEntry(fragIndex)
if err != nil {
return nil, err
return data.Reader{}, err
}
frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize)
frag.Read(make([]byte, fragOffset))
outRdr.AddFrag(io.LimitReader(frag, int64(fragSize)))
outRdr.AddFrag(io.LimitReader(&frag, int64(fragSize)))
}
return outRdr, nil
}
low/inode.go
+4 -4
View File
@@ -7,7 +7,7 @@ import (
"github.com/CalebQ42/squashfs/low/inode"
)
func (r *Reader) InodeFromRef(ref uint64) (inode.Inode, error) {
func (r Reader) InodeFromRef(ref uint64) (inode.Inode, error) {
offset, meta := (ref>>16)+r.Superblock.InodeTableStart, ref&0xFFFF
rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
defer rdr.Close()
@@ -15,12 +15,12 @@ func (r *Reader) InodeFromRef(ref uint64) (inode.Inode, error) {
if err != nil {
return inode.Inode{}, err
}
return inode.Read(rdr, r.Superblock.BlockSize)
return inode.Read(&rdr, r.Superblock.BlockSize)
}
func (r *Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) {
func (r Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) {
rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d)
defer rdr.Close()
rdr.Read(make([]byte, e.Offset))
return inode.Read(rdr, r.Superblock.BlockSize)
return inode.Read(&rdr, r.Superblock.BlockSize)
}
low/inode/dir.go
+32 -19
View File
@@ -13,7 +13,7 @@ type Directory struct {
ParentNum uint32
}
type eDirectoryInit struct {
type EDirectory struct {
LinkCount uint32
Size uint32
BlockStart uint32
@@ -21,42 +21,55 @@ type eDirectoryInit struct {
IndCount uint16
Offset uint16
XattrInd uint32
}
type EDirectory struct {
eDirectoryInit
Indexes []DirectoryIndex
}
type directoryIndexInit struct {
Ind uint32
Start uint32
NameSize uint32
Indexes []DirectoryIndex
}
type DirectoryIndex struct {
directoryIndexInit
Name []byte
Ind uint32
Start uint32
NameSize uint32
Name []byte
}
func ReadDir(r io.Reader) (d Directory, err error) {
err = binary.Read(r, binary.LittleEndian, &d)
dat := make([]byte, 16)
_, err = r.Read(dat)
if err != nil {
return
}
d.BlockStart = binary.LittleEndian.Uint32(dat)
d.LinkCount = binary.LittleEndian.Uint32(dat[4:])
d.Size = binary.LittleEndian.Uint16(dat[8:])
d.Offset = binary.LittleEndian.Uint16(dat[10:])
d.ParentNum = binary.LittleEndian.Uint32(dat[12:])
return
}
func ReadEDir(r io.Reader) (d EDirectory, err error) {
err = binary.Read(r, binary.LittleEndian, &d.eDirectoryInit)
dat := make([]byte, 24)
_, err = r.Read(dat)
if err != nil {
return
}
d.LinkCount = binary.LittleEndian.Uint32(dat)
d.Size = binary.LittleEndian.Uint32(dat[4:])
d.BlockStart = binary.LittleEndian.Uint32(dat[8:])
d.ParentNum = binary.LittleEndian.Uint32(dat[12:])
d.IndCount = binary.LittleEndian.Uint16(dat[16:])
d.Offset = binary.LittleEndian.Uint16(dat[18:])
d.XattrInd = binary.LittleEndian.Uint32(dat[20:])
d.Indexes = make([]DirectoryIndex, d.IndCount)
for i := range d.Indexes {
err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].directoryIndexInit)
for i := range d.IndCount {
dat = make([]byte, 12)
_, err = r.Read(dat)
if err != nil {
return
}
d.Indexes[i].Ind = binary.LittleEndian.Uint32(dat)
d.Indexes[i].Start = binary.LittleEndian.Uint32(dat[4:])
d.Indexes[i].NameSize = binary.LittleEndian.Uint32(dat[8:])
d.Indexes[i].Name = make([]byte, d.Indexes[i].NameSize+1)
err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].Name)
_, err = r.Read(d.Indexes[i].Name)
if err != nil {
return
}
low/inode/file.go
+15 -7
View File
@@ -6,15 +6,11 @@ import (
"math"
)
type fileInit struct {
type File struct {
BlockStart uint32
FragInd uint32
FragOffset uint32
Size uint32
}
type File struct {
fileInit
BlockSizes []uint32
}
@@ -34,16 +30,28 @@ type EFile struct {
}
func ReadFile(r io.Reader, blockSize uint32) (f File, err error) {
err = binary.Read(r, binary.LittleEndian, &f.fileInit)
dat := make([]byte, 16)
_, err = r.Read(dat)
if err != nil {
return
}
f.BlockStart = binary.LittleEndian.Uint32(dat)
f.FragInd = binary.LittleEndian.Uint32(dat[4:])
f.FragOffset = binary.LittleEndian.Uint32(dat[8:])
f.Size = binary.LittleEndian.Uint32(dat[12:])
toRead := int(math.Floor(float64(f.Size) / float64(blockSize)))
if f.FragInd == 0xFFFFFFFF && f.Size%blockSize > 0 {
toRead++
}
dat = make([]byte, toRead*4)
_, err = r.Read(dat)
if err != nil {
return
}
f.BlockSizes = make([]uint32, toRead)
err = binary.Read(r, binary.LittleEndian, &f.BlockSizes)
for i := range toRead {
f.BlockSizes[i] = binary.LittleEndian.Uint32(dat[i*4:])
}
return
}
low/inode/inode.go
+7 -13
View File
@@ -81,23 +81,17 @@ func Read(r io.Reader, blockSize uint32) (i Inode, err error) {
func (i Inode) Mode() (out fs.FileMode) {
out = fs.FileMode(i.Perm)
switch i.Data.(type) {
case Directory:
switch i.Type {
case Dir, EDir:
out |= fs.ModeDir
case EDirectory:
out |= fs.ModeDir
case Symlink:
case Sym, ESym:
out |= fs.ModeSymlink
case ESymlink:
out |= fs.ModeSymlink
case Device:
case Char, EChar, Block, EBlock:
out |= fs.ModeDevice
case EDevice:
out |= fs.ModeDevice
case IPC:
out |= fs.ModeNamedPipe
case EIPC:
case Fifo, EFifo:
out |= fs.ModeNamedPipe
case Sock, ESock:
out |= fs.ModeSocket
}
return
}
low/reader.go
+30 -29
View File
@@ -39,41 +39,46 @@ type Reader struct {
Superblock superblock
}
func NewReader(r io.ReaderAt) (rdr *Reader, err error) {
rdr = new(Reader)
func NewReader(r io.ReaderAt) (rdr Reader, err error) {
rdr.r = r
err = binary.Read(toreader.NewReader(r, 0), binary.LittleEndian, &rdr.Superblock)
if err != nil {
return nil, errors.Join(errors.New("failed to read superblock"), err)
return rdr, errors.Join(errors.New("failed to read superblock"), err)
}
if !rdr.Superblock.ValidMagic() {
return nil, ErrorMagic
return rdr, ErrorMagic
}
if !rdr.Superblock.ValidBlockLog() {
return nil, ErrorLog
return rdr, ErrorLog
}
if !rdr.Superblock.ValidVersion() {
return nil, ErrorVersion
return rdr, ErrorVersion
}
switch rdr.Superblock.CompType {
case ZlibCompression:
rdr.d = decompress.Zlib{}
rdr.d = decompress.NewZlib()
case LZMACompression:
rdr.d = decompress.Lzma{}
rdr.d, err = decompress.NewLzma()
if err != nil {
return rdr, err
}
case LZOCompression:
rdr.d = decompress.Lzo{}
rdr.d, err = decompress.NewLzo()
if err != nil {
return rdr, err
}
case XZCompression:
rdr.d = decompress.Xz{}
rdr.d = decompress.NewXz()
case LZ4Compression:
rdr.d = decompress.Lz4{}
rdr.d = decompress.NewLz4()
case ZSTDCompression:
rdr.d = &decompress.Zstd{}
rdr.d = decompress.NewZstd()
default:
return nil, errors.New("invalid compression type. possible corrupted archive")
return rdr, errors.New("invalid compression type. possible corrupted archive")
}
rdr.Root, err = rdr.directoryFromRef(rdr.Superblock.RootInodeRef, "")
if err != nil {
return nil, errors.Join(errors.New("failed to read root directory"), err)
return rdr, errors.Join(errors.New("failed to read root directory"), err)
}
return
}
@@ -88,7 +93,7 @@ func (r *Reader) Id(i uint16) (uint32, error) {
// Populate the id table as needed
var blockNum uint32
if i != 0 { // If i == 0, we go negatives causing issues with uint32s
blockNum = uint32(math.Ceil(float64(i)/2048)) - 1
blockNum = uint32(math.Ceil(float64(i+1)/2048)) - 1
} else {
blockNum = 0
}
@@ -99,19 +104,17 @@ func (r *Reader) Id(i uint16) (uint32, error) {
var idsToRead uint16
var idsTmp []uint32
var err error
var rdr *metadata.Reader
var rdr metadata.Reader
// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset)
if err != nil {
return 0, err
}
idsToRead = r.Superblock.IdCount - uint16(len(r.idTable))
if idsToRead > 2048 {
idsToRead = 2048
}
idsToRead = min(r.Superblock.IdCount-uint16(len(r.idTable)), 2048)
idsTmp = make([]uint32, idsToRead)
rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
err = binary.Read(rdr, binary.LittleEndian, &idsTmp)
err = binary.Read(&rdr, binary.LittleEndian, &idsTmp)
rdr.Close()
if err != nil {
return 0, err
@@ -157,7 +160,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
// Populate the export table as needed
var blockNum uint32
if i != 0 { // If i == 0, we go negatives causing issues with uint32s
blockNum = uint32(math.Ceil(float64(i)/1024)) - 1
blockNum = uint32(math.Ceil(float64(i+1)/1024)) - 1
} else {
blockNum = 0
}
@@ -168,19 +171,17 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
var refsToRead uint32
var refsTmp []uint64
var err error
var rdr *metadata.Reader
var rdr metadata.Reader
// We can *maybe* have a slight speed increase by manually decoding instead of using reflection via binary.Read
for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ {
err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset)
if err != nil {
return 0, err
}
refsToRead = r.Superblock.InodeCount - uint32(len(r.exportTable))
if refsToRead > 1024 {
refsToRead = 1024
}
refsToRead = min(r.Superblock.InodeCount-uint32(len(r.exportTable)), 1024)
refsTmp = make([]uint64, refsToRead)
rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
err = binary.Read(rdr, binary.LittleEndian, &refsTmp)
err = binary.Read(&rdr, binary.LittleEndian, &refsTmp)
rdr.Close()
if err != nil {
return 0, err
@@ -190,7 +191,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
return r.exportTable[i], nil
}
func (r *Reader) Inode(i uint32) (inode.Inode, error) {
func (r Reader) Inode(i uint32) (inode.Inode, error) {
ref, err := r.inodeRef(i)
if err != nil {
return inode.Inode{}, err
low/reader_test.go
+22 -9
View File
@@ -1,4 +1,4 @@
package squashfslow_test
package squashfslow
import (
"fmt"
@@ -8,13 +8,11 @@ import (
"os/exec"
"path/filepath"
"testing"
squashfslow "github.com/CalebQ42/squashfs/low"
)
const (
squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs"
squashfsName = "LinuxPATest.sfs"
squashfsName = "airootfs.sfs"
)
func preTest(dir string) (fil *os.File, err error) {
@@ -50,6 +48,21 @@ func preTest(dir string) (fil *os.File, err error) {
return
}
func TestMisc(t *testing.T) {
tmpDir := "../testing"
fil, err := preTest(tmpDir)
if err != nil {
t.Fatal(err)
}
defer fil.Close()
rdr, err := NewReader(fil)
if err != nil {
t.Fatal(err)
}
t.Log(rdr.Superblock.FragCount)
t.Fatal(rdr.fragEntry(1233))
}
func TestReader(t *testing.T) {
tmpDir := "../testing"
fil, err := preTest(tmpDir)
@@ -57,7 +70,7 @@ func TestReader(t *testing.T) {
t.Fatal(err)
}
defer fil.Close()
rdr, err := squashfslow.NewReader(fil)
rdr, err := NewReader(fil)
if err != nil {
t.Fatal(err)
}
@@ -79,7 +92,7 @@ func TestSingleFile(t *testing.T) {
t.Fatal(err)
}
defer fil.Close()
rdr, err := squashfslow.NewReader(fil)
rdr, err := NewReader(fil)
if err != nil {
t.Fatal(err)
}
@@ -96,7 +109,7 @@ func TestSingleFile(t *testing.T) {
}
}
func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder string) error {
func extractToDir(rdr Reader, b FileBase, folder string) error {
path := filepath.Join(folder, b.Name)
if b.IsDir() {
d, err := b.ToDir(rdr)
@@ -107,13 +120,13 @@ func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder strin
if err != nil {
return err
}
var nestBast squashfslow.FileBase
var nestBast FileBase
for _, e := range d.Entries {
nestBast, err = rdr.BaseFromEntry(e)
if err != nil {
return err
}
err = extractToDir(rdr, &nestBast, path)
err = extractToDir(rdr, nestBast, path)
if err != nil {
return err
}
reader.go
+9 -9
View File
@@ -9,26 +9,26 @@ import (
)
type Reader struct {
*FS
FS
Low squashfslow.Reader
}
func NewReader(r io.ReaderAt) (*Reader, error) {
func NewReader(r io.ReaderAt) (Reader, error) {
rdr, err := squashfslow.NewReader(r)
if err != nil {
return nil, err
return Reader{}, err
}
out := &Reader{
Low: *rdr,
out := Reader{
Low: rdr,
}
out.FS = &FS{
d: rdr.Root,
r: out,
out.FS = FS{
LowDir: rdr.Root,
r: &out,
}
return out, nil
}
func NewReaderAtOffset(r io.ReaderAt, offset int64) (*Reader, error) {
func NewReaderAtOffset(r io.ReaderAt, offset int64) (Reader, error) {
return NewReader(toreader.NewOffsetReader(r, offset))
}
squashfs_test.go
+36 -21
View File
@@ -1,4 +1,4 @@
package squashfs_test
package squashfs
//Actually proper tests go here.
@@ -13,13 +13,11 @@ import (
"strconv"
"testing"
"time"
"github.com/CalebQ42/squashfs"
)
const (
squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs"
squashfsName = "airootfs.sfs"
squashfsName = "tensorflow.sqfs"
)
func preTest(dir string) (fil *os.File, err error) {
@@ -61,7 +59,7 @@ func TestMisc(t *testing.T) {
if err != nil {
t.Fatal(err)
}
rdr, err := squashfs.NewReader(fil)
rdr, err := NewReader(fil)
if err != nil {
t.Fatal(err)
}
@@ -70,6 +68,24 @@ func TestMisc(t *testing.T) {
// t.Fatal("UM")
}
func BenchmarkExtract(b *testing.B) {
tmpDir := "testing"
fil, err := preTest(tmpDir)
if err != nil {
b.Fatal(err)
}
libPath := filepath.Join(tmpDir, "ExtractLib")
os.RemoveAll(libPath)
rdr, err := NewReader(fil)
if err != nil {
b.Fatal(err)
}
err = rdr.ExtractWithOptions(libPath, FastOptions())
if err != nil {
b.Fatal(err)
}
}
func BenchmarkRace(b *testing.B) {
tmpDir := "testing"
fil, err := preTest(tmpDir)
@@ -81,19 +97,17 @@ func BenchmarkRace(b *testing.B) {
os.RemoveAll(libPath)
os.RemoveAll(unsquashPath)
var libTime, unsquashTime time.Duration
op := squashfs.FastOptions()
op.IgnorePerm = true
start := time.Now()
rdr, err := squashfs.NewReader(fil)
rdr, err := NewReader(fil)
if err != nil {
b.Fatal(err)
}
err = rdr.ExtractWithOptions(libPath, op)
err = rdr.ExtractWithOptions(libPath, FastOptions())
if err != nil {
b.Fatal(err)
}
libTime = time.Since(start)
cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name())
cmd := exec.Command("unsquashfs", "-q", "-d", unsquashPath, fil.Name())
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
start = time.Now()
@@ -102,15 +116,15 @@ func BenchmarkRace(b *testing.B) {
b.Log("Unsquashfs error:", err)
}
unsquashTime = time.Since(start)
// b.Log("Library took:", libTime.Round(time.Millisecond))
// b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond))
b.Fatal("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster")
b.Log("Library took:", libTime.Round(time.Millisecond))
b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond))
b.Log("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster")
}
func TestExtractQuick(t *testing.T) {
//First, setup everything and extract the archive using the library and unsquashfs
// tmpDir := b.TempDir()
// tmpDir := bTempDir()
tmpDir := "testing"
fil, err := preTest(tmpDir)
if err != nil {
@@ -120,13 +134,13 @@ func TestExtractQuick(t *testing.T) {
unsquashPath := filepath.Join(tmpDir, "ExtractSquashfs")
os.RemoveAll(libPath)
os.RemoveAll(unsquashPath)
rdr, err := squashfs.NewReader(fil)
rdr, err := NewReader(fil)
if err != nil {
t.Fatal(err)
}
os.RemoveAll(filepath.Join(tmpDir, "testLog.txt"))
logFil, _ := os.Create(filepath.Join(tmpDir, "testLog.txt"))
op := squashfs.DefaultOptions()
op := FastOptions()
op.Verbose = true
op.IgnorePerm = true
op.LogOutput = logFil
@@ -169,7 +183,7 @@ func TestExtractQuick(t *testing.T) {
}
}
var filePath = "bin"
var filePath = "usr/sbin/add-shell"
func TestSingleFile(t *testing.T) {
tmpDir := "testing"
@@ -177,8 +191,8 @@ func TestSingleFile(t *testing.T) {
if err != nil {
t.Fatal(err)
}
os.Remove(filepath.Join(tmpDir, filePath))
rdr, err := squashfs.NewReader(fil)
os.RemoveAll("testing/stuff")
rdr, err := NewReader(fil)
if err != nil {
t.Fatal(err)
}
@@ -186,9 +200,10 @@ func TestSingleFile(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = f.(*squashfs.File).ExtractWithOptions("testing", &squashfs.ExtractionOptions{Verbose: true})
op := DefaultOptions()
op.Verbose = true
err = f.(*File).ExtractWithOptions("testing/stuff", op)
if err != nil {
t.Fatal(err)
}
t.Fatal("HI")
}