Compare commits

...

5 Commits

Author SHA1 Message Date
Caleb Gardner e9fdd89c67 Merge pull request #31 from willmurphyscode/main
fix: remove stray println
2024-12-10 16:09:45 -06:00
Will Murphy c80d150fdc fix: remove stray println 2024-12-10 17:00:57 -05:00
Caleb Gardner 03266d0560 Fix frag, id, inode table values on block boundries
Fixes bug mention in #30
2024-11-26 17:09:39 -06:00
Caleb Gardner 0f8a4e0027 Re-added NewReaderAtOffset 2024-09-20 20:10:33 -05:00
Caleb Gardner 2a33cad709 PERFORMANCE
Changed some struct values from pointers to normal values for improved performance.
2024-07-17 09:30:16 -05:00
15 changed files with 121 additions and 83 deletions
+4 -3
View File
@@ -24,11 +24,12 @@ As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://g
## Issues
* Significantly slower then `unsquashfs` when extracting folders
* Significantly slower then `unsquashfs` when nested images
* This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries.
* Not to mention it's written in C
* Times seem to be largely dependent on file tree size and compression type.
* My main testing image (~100MB) using Zstd takes about 6x longer.
* An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 32x longer.
* My main testing image (~100MB) using Zstd takes about 5x longer.
* An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 30x longer.
* A Tensorflow docker image (~3.3GB) using Zstd takes about 12x longer.
Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes about 2x longer.
+11 -11
View File
@@ -19,16 +19,16 @@ import (
// File represents a file inside a squashfs archive.
type File struct {
b *squashfslow.FileBase
full *data.FullReader
rdr *data.Reader
parent *FS
r *Reader
b squashfslow.FileBase
dirsRead int
}
// Creates a new *File from the given *squashfs.Base
func (r *Reader) FileFromBase(b *squashfslow.FileBase, parent *FS) *File {
func (r *Reader) FileFromBase(b squashfslow.FileBase, parent *FS) *File {
return &File{
b: b,
parent: parent,
@@ -40,7 +40,7 @@ func (f *File) FS() (*FS, error) {
if !f.IsDir() {
return nil, errors.New("not a directory")
}
d, err := f.b.ToDir(f.r.Low)
d, err := f.b.ToDir(&f.r.Low)
if err != nil {
return nil, err
}
@@ -114,7 +114,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
if !f.IsDir() {
return nil, errors.New("file is not a directory")
}
d, err := f.b.ToDir(f.r.Low)
d, err := f.b.ToDir(&f.r.Low)
if err != nil {
return nil, err
}
@@ -142,7 +142,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
// Returns the file's fs.FileInfo
func (f *File) Stat() (fs.FileInfo, error) {
return newFileInfo(f.b.Name, f.b.Inode), nil
return newFileInfo(f.b.Name, &f.b.Inode), nil
}
// SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string.
@@ -173,7 +173,7 @@ func (f *File) WriteTo(w io.Writer) (int64, error) {
func (f *File) initializeReaders() error {
var err error
f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.Low)
f.rdr, f.full, err = f.b.GetRegFileReaders(&f.r.Low)
return err
}
@@ -218,7 +218,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
}
switch f.b.Inode.Type {
case inode.Dir, inode.EDir:
d, err := f.b.ToDir(f.r.Low)
d, err := f.b.ToDir(&f.r.Low)
if err != nil {
if op.Verbose {
log.Println("Failed to create squashfs.Directory for", path)
@@ -234,7 +234,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
}
return errors.Join(errors.New("failed to get base from entry: "+path), err)
}
go func(b *squashfslow.FileBase, path string) {
go func(b squashfslow.FileBase, path string) {
i := op.manager.Lock()
if b.IsDir() {
extDir := filepath.Join(path, b.Name)
@@ -285,7 +285,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
return errors.Join(errors.New("failed to create file: "+path), err)
}
defer outFil.Close()
full, err := f.b.GetFullReader(f.r.Low)
full, err := f.b.GetFullReader(&f.r.Low)
if err != nil {
if op.Verbose {
log.Println("Failed to create full reader for", path)
@@ -406,7 +406,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
if op.IgnorePerm {
return nil
}
uid, err := f.b.Uid(f.r.Low)
uid, err := f.b.Uid(&f.r.Low)
if err != nil {
if op.Verbose {
log.Println("Failed to get uid for", path)
@@ -414,7 +414,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error {
}
return nil
}
gid, err := f.b.Gid(f.r.Low)
gid, err := f.b.Gid(&f.r.Low)
if err != nil {
if op.Verbose {
log.Println("Failed to get gid for", path)
+1 -1
View File
@@ -21,7 +21,7 @@ func (r Reader) newFileInfo(e directory.Entry) (fileInfo, error) {
if err != nil {
return fileInfo{}, err
}
return newFileInfo(e.Name, i), nil
return newFileInfo(e.Name, &i), nil
}
func newFileInfo(name string, i *inode.Inode) fileInfo {
+4 -4
View File
@@ -15,13 +15,13 @@ import (
// FS is a fs.FS representation of a squashfs directory.
// Implements fs.GlobFS, fs.ReadDirFS, fs.ReadFileFS, fs.StatFS, and fs.SubFS
type FS struct {
d *squashfslow.Directory
r *Reader
parent *FS
d squashfslow.Directory
}
// Creates a new *FS from the given squashfs.directory
func (r *Reader) FSFromDirectory(d *squashfslow.Directory, parent *FS) *FS {
func (r *Reader) FSFromDirectory(d squashfslow.Directory, parent *FS) *FS {
return &FS{
d: d,
r: r,
@@ -142,7 +142,7 @@ func (f *FS) Open(name string) (fs.File, error) {
Err: fs.ErrNotExist,
}
}
d, err := b.ToDir(f.r.Low)
d, err := b.ToDir(&f.r.Low)
if err != nil {
return nil, err
}
@@ -255,7 +255,7 @@ func (f *FS) ExtractWithOptions(folder string, op *ExtractionOptions) error {
// Returns the FS as a *File
func (f *FS) File() *File {
return &File{
b: &f.d.FileBase,
b: f.d.FileBase,
parent: f.parent,
r: f.r,
}
+4 -4
View File
@@ -1,11 +1,11 @@
module github.com/CalebQ42/squashfs
go 1.21.5
go 1.22.5
require (
github.com/pierrec/lz4/v4 v4.1.19
github.com/ulikunitz/xz v0.5.11
github.com/klauspost/compress v1.17.4
github.com/klauspost/compress v1.17.9
github.com/pierrec/lz4/v4 v4.1.21
github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e
github.com/therootcompany/xz v1.0.1
github.com/ulikunitz/xz v0.5.12
)
+6 -6
View File
@@ -1,10 +1,10 @@
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4=
github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs=
github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+19
View File
@@ -0,0 +1,19 @@
package toreader
import "io"
type OffsetReader struct {
r io.ReaderAt
off int64
}
func NewOffsetReader(r io.ReaderAt, off int64) *OffsetReader {
return &OffsetReader{
r: r,
off: off,
}
}
func (r OffsetReader) ReadAt(p []byte, off int64) (n int, e error) {
return r.r.ReadAt(p, off+r.off)
}
+13 -13
View File
@@ -18,10 +18,10 @@ type Directory struct {
Entries []directory.Entry
}
func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) {
func (r *Reader) directoryFromRef(ref uint64, name string) (Directory, error) {
i, err := r.InodeFromRef(ref)
if err != nil {
return nil, err
return Directory{}, err
}
var blockStart uint32
var size uint32
@@ -36,48 +36,48 @@ func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) {
size = i.Data.(inode.EDirectory).Size
offset = i.Data.(inode.EDirectory).Offset
default:
return nil, errors.New("not a directory")
return Directory{}, errors.New("not a directory")
}
dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d)
defer dirRdr.Close()
_, err = dirRdr.Read(make([]byte, offset))
if err != nil {
return nil, err
return Directory{}, err
}
entries, err := directory.ReadDirectory(dirRdr, size)
if err != nil {
return nil, err
return Directory{}, err
}
return &Directory{
FileBase: *r.BaseFromInode(i, name),
return Directory{
FileBase: r.BaseFromInode(i, name),
Entries: entries,
}, nil
}
func (d *Directory) Open(r *Reader, path string) (*FileBase, error) {
func (d *Directory) Open(r *Reader, path string) (FileBase, error) {
path = filepath.Clean(path)
if path == "." || path == "" {
return &d.FileBase, nil
return d.FileBase, nil
}
split := strings.Split(path, "/")
i, found := slices.BinarySearchFunc(d.Entries, split[0], func(e directory.Entry, name string) int {
return strings.Compare(e.Name, name)
})
if !found {
return nil, fs.ErrNotExist
return FileBase{}, fs.ErrNotExist
}
b, err := r.BaseFromEntry(d.Entries[i])
if err != nil {
return nil, err
return FileBase{}, err
}
if len(split) == 1 {
return b, nil
} else if !b.IsDir() {
return nil, fs.ErrNotExist
return FileBase{}, fs.ErrNotExist
}
dir, err := b.ToDir(r)
if err != nil {
return nil, err
return FileBase{}, err
}
return dir.Open(r, strings.Join(split[1:], "/"))
}
+14 -14
View File
@@ -12,28 +12,28 @@ import (
)
type FileBase struct {
Inode *inode.Inode
Inode inode.Inode
Name string
}
func (r *Reader) BaseFromInode(i *inode.Inode, name string) *FileBase {
return &FileBase{Inode: i, Name: name}
func (r *Reader) BaseFromInode(i inode.Inode, name string) FileBase {
return FileBase{Inode: i, Name: name}
}
func (r *Reader) BaseFromEntry(e directory.Entry) (*FileBase, error) {
func (r *Reader) BaseFromEntry(e directory.Entry) (FileBase, error) {
in, err := r.InodeFromEntry(e)
if err != nil {
return nil, err
return FileBase{}, err
}
return &FileBase{Inode: in, Name: e.Name}, nil
return FileBase{Inode: in, Name: e.Name}, nil
}
func (r *Reader) BaseFromRef(ref uint64, name string) (*FileBase, error) {
func (r *Reader) BaseFromRef(ref uint64, name string) (FileBase, error) {
in, err := r.InodeFromRef(ref)
if err != nil {
return nil, err
return FileBase{}, err
}
return &FileBase{Inode: in, Name: name}, nil
return FileBase{Inode: in, Name: name}, nil
}
func (b *FileBase) Uid(r *Reader) (uint32, error) {
@@ -48,7 +48,7 @@ func (b *FileBase) IsDir() bool {
return b.Inode.Type == inode.Dir || b.Inode.Type == inode.EDir
}
func (b *FileBase) ToDir(r *Reader) (*Directory, error) {
func (b *FileBase) ToDir(r *Reader) (Directory, error) {
var blockStart uint32
var size uint32
var offset uint16
@@ -62,19 +62,19 @@ func (b *FileBase) ToDir(r *Reader) (*Directory, error) {
size = b.Inode.Data.(inode.EDirectory).Size
offset = b.Inode.Data.(inode.EDirectory).Offset
default:
return nil, errors.New("not a directory")
return Directory{}, errors.New("not a directory")
}
dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d)
defer dirRdr.Close()
_, err := dirRdr.Read(make([]byte, offset))
if err != nil {
return nil, err
return Directory{}, err
}
entries, err := directory.ReadDirectory(dirRdr, size)
if err != nil {
return nil, err
return Directory{}, err
}
return &Directory{
return Directory{
FileBase: *b,
Entries: entries,
}, nil
+3 -3
View File
@@ -7,18 +7,18 @@ import (
"github.com/CalebQ42/squashfs/low/inode"
)
func (r *Reader) InodeFromRef(ref uint64) (*inode.Inode, error) {
func (r *Reader) InodeFromRef(ref uint64) (inode.Inode, error) {
offset, meta := (ref>>16)+r.Superblock.InodeTableStart, ref&0xFFFF
rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d)
defer rdr.Close()
_, err := rdr.Read(make([]byte, meta))
if err != nil {
return nil, err
return inode.Inode{}, err
}
return inode.Read(rdr, r.Superblock.BlockSize)
}
func (r *Reader) InodeFromEntry(e directory.Entry) (*inode.Inode, error) {
func (r *Reader) InodeFromEntry(e directory.Entry) (inode.Inode, error) {
rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d)
defer rdr.Close()
rdr.Read(make([]byte, e.Offset))
+1 -2
View File
@@ -39,8 +39,7 @@ type Inode struct {
Data any
}
func Read(r io.Reader, blockSize uint32) (i *Inode, err error) {
i = new(Inode)
func Read(r io.Reader, blockSize uint32) (i Inode, err error) {
err = binary.Read(r, binary.LittleEndian, &i.Header)
if err != nil {
return
+6 -6
View File
@@ -32,7 +32,7 @@ var (
type Reader struct {
r io.ReaderAt
d decompress.Decompressor
Root *Directory
Root Directory
fragTable []fragEntry
idTable []uint32
exportTable []uint64
@@ -88,7 +88,7 @@ func (r *Reader) Id(i uint16) (uint32, error) {
// Populate the id table as needed
var blockNum uint32
if i != 0 { // If i == 0, we go negatives causing issues with uint32s
blockNum = uint32(math.Ceil(float64(i)/2048)) - 1
blockNum = uint32(math.Ceil(float64(i+1)/2048)) - 1
} else {
blockNum = 0
}
@@ -131,7 +131,7 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) {
// Populate the fragment table as needed
var blockNum uint32
if i != 0 { // If i == 0, we go negatives causing issues with uint32s
blockNum = uint32(math.Ceil(float64(i)/512)) - 1
blockNum = uint32(math.Ceil(float64(i+1)/512)) - 1
} else {
blockNum = 0
}
@@ -177,7 +177,7 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
// Populate the export table as needed
var blockNum uint32
if i != 0 { // If i == 0, we go negatives causing issues with uint32s
blockNum = uint32(math.Ceil(float64(i)/1024)) - 1
blockNum = uint32(math.Ceil(float64(i+1)/1024)) - 1
} else {
blockNum = 0
}
@@ -210,10 +210,10 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) {
return r.exportTable[i], nil
}
func (r *Reader) Inode(i uint32) (*inode.Inode, error) {
func (r *Reader) Inode(i uint32) (inode.Inode, error) {
ref, err := r.inodeRef(i)
if err != nil {
return nil, err
return inode.Inode{}, err
}
return r.InodeFromRef(ref)
}
+23 -10
View File
@@ -1,4 +1,4 @@
package squashfslow_test
package squashfslow
import (
"fmt"
@@ -8,13 +8,11 @@ import (
"os/exec"
"path/filepath"
"testing"
squashfslow "github.com/CalebQ42/squashfs/low"
)
const (
squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs"
squashfsName = "LinuxPATest.sfs"
squashfsName = "airootfs.sfs"
)
func preTest(dir string) (fil *os.File, err error) {
@@ -50,6 +48,21 @@ func preTest(dir string) (fil *os.File, err error) {
return
}
func TestMisc(t *testing.T) {
tmpDir := "../testing"
fil, err := preTest(tmpDir)
if err != nil {
t.Fatal(err)
}
defer fil.Close()
rdr, err := NewReader(fil)
if err != nil {
t.Fatal(err)
}
t.Log(rdr.Superblock.FragCount)
t.Fatal(rdr.fragEntry(1233))
}
func TestReader(t *testing.T) {
tmpDir := "../testing"
fil, err := preTest(tmpDir)
@@ -57,7 +70,7 @@ func TestReader(t *testing.T) {
t.Fatal(err)
}
defer fil.Close()
rdr, err := squashfslow.NewReader(fil)
rdr, err := NewReader(fil)
if err != nil {
t.Fatal(err)
}
@@ -77,7 +90,7 @@ func TestSingleFile(t *testing.T) {
t.Fatal(err)
}
defer fil.Close()
rdr, err := squashfslow.NewReader(fil)
rdr, err := NewReader(fil)
if err != nil {
t.Fatal(err)
}
@@ -88,11 +101,11 @@ func TestSingleFile(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = extractToDir(rdr, b, path)
err = extractToDir(rdr, &b, path)
t.Fatal(err)
}
func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder string) error {
func extractToDir(rdr *Reader, b *FileBase, folder string) error {
path := filepath.Join(folder, b.Name)
if b.IsDir() {
d, err := b.ToDir(rdr)
@@ -103,13 +116,13 @@ func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder strin
if err != nil {
return err
}
var nestBast *squashfslow.FileBase
var nestBast FileBase
for _, e := range d.Entries {
nestBast, err = rdr.BaseFromEntry(e)
if err != nil {
return err
}
err = extractToDir(rdr, nestBast, path)
err = extractToDir(rdr, &nestBast, path)
if err != nil {
return err
}
+7 -2
View File
@@ -4,12 +4,13 @@ import (
"io"
"time"
"github.com/CalebQ42/squashfs/internal/toreader"
squashfslow "github.com/CalebQ42/squashfs/low"
)
type Reader struct {
*FS
Low *squashfslow.Reader
Low squashfslow.Reader
}
func NewReader(r io.ReaderAt) (*Reader, error) {
@@ -18,7 +19,7 @@ func NewReader(r io.ReaderAt) (*Reader, error) {
return nil, err
}
out := &Reader{
Low: rdr,
Low: *rdr,
}
out.FS = &FS{
d: rdr.Root,
@@ -27,6 +28,10 @@ func NewReader(r io.ReaderAt) (*Reader, error) {
return out, nil
}
func NewReaderAtOffset(r io.ReaderAt, offset int64) (*Reader, error) {
return NewReader(toreader.NewOffsetReader(r, offset))
}
func (r *Reader) ModTime() time.Time {
return time.Unix(int64(r.Low.Superblock.ModTime), 0)
}
+5 -4
View File
@@ -67,7 +67,7 @@ func TestMisc(t *testing.T) {
}
_ = rdr
// Put testing here
t.Fatal("UM")
// t.Fatal("UM")
}
func BenchmarkRace(b *testing.B) {
@@ -82,6 +82,7 @@ func BenchmarkRace(b *testing.B) {
os.RemoveAll(unsquashPath)
var libTime, unsquashTime time.Duration
op := squashfs.FastOptions()
op.IgnorePerm = true
start := time.Now()
rdr, err := squashfs.NewReader(fil)
if err != nil {
@@ -101,9 +102,9 @@ func BenchmarkRace(b *testing.B) {
b.Log("Unsquashfs error:", err)
}
unsquashTime = time.Since(start)
b.Log("Library took:", libTime.Round(time.Millisecond))
b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond))
b.Log("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster")
// b.Log("Library took:", libTime.Round(time.Millisecond))
// b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond))
b.Fatal("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster")
}
func TestExtractQuick(t *testing.T) {