From d4d1b2c2b2a961da97b7b3dc1357fa82427c28d1 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Tue, 19 Dec 2023 03:23:24 -0600 Subject: [PATCH 01/11] Reset to zero --- cmd/go-unsquashfs/main.go | 37 --- fuse2.go | 150 --------- fuse3.go | 148 --------- fuse_darwin.go | 7 - fuse_linux.go | 5 - fuse_windows.go | 3 - go.mod | 14 - go.sum | 16 - internal/data/fullreader.go | 235 -------------- internal/data/reader.go | 104 ------ internal/decompress/gzip.go | 17 - internal/decompress/interface.go | 22 -- internal/decompress/lz4.go | 18 -- internal/decompress/lzma.go | 14 - internal/decompress/lzo.go | 18 -- internal/decompress/xz.go | 18 -- internal/decompress/zstd.go | 27 -- internal/directory/directory.go | 80 ----- internal/inode/dir.go | 65 ---- internal/inode/file.go | 62 ---- internal/inode/inode.go | 143 --------- internal/inode/misc.go | 45 --- internal/inode/sym.go | 46 --- internal/metadata/reader.go | 81 ----- internal/threadmanager/manager.go | 23 -- internal/toreader/offsetreader.go | 19 -- internal/toreader/reader.go | 25 -- internal/toreader/readerat.go | 24 -- reader.go | 234 -------------- reader_file.go | 518 ------------------------------ reader_fileinfo.go | 66 ---- reader_frag.go | 26 -- reader_fs.go | 380 ---------------------- reader_inode.go | 117 ------- squashfs_test.go | 207 ------------ 35 files changed, 3014 deletions(-) delete mode 100644 cmd/go-unsquashfs/main.go delete mode 100644 fuse2.go delete mode 100644 fuse3.go delete mode 100644 fuse_darwin.go delete mode 100644 fuse_linux.go delete mode 100644 fuse_windows.go delete mode 100644 go.mod delete mode 100644 go.sum delete mode 100644 internal/data/fullreader.go delete mode 100644 internal/data/reader.go delete mode 100644 internal/decompress/gzip.go delete mode 100644 internal/decompress/interface.go delete mode 100644 internal/decompress/lz4.go delete mode 100644 internal/decompress/lzma.go delete mode 100644 internal/decompress/lzo.go delete mode 100644 internal/decompress/xz.go delete mode 100644 internal/decompress/zstd.go delete mode 100644 internal/directory/directory.go delete mode 100644 internal/inode/dir.go delete mode 100644 internal/inode/file.go delete mode 100644 internal/inode/inode.go delete mode 100644 internal/inode/misc.go delete mode 100644 internal/inode/sym.go delete mode 100644 internal/metadata/reader.go delete mode 100644 internal/threadmanager/manager.go delete mode 100644 internal/toreader/offsetreader.go delete mode 100644 internal/toreader/reader.go delete mode 100644 internal/toreader/readerat.go delete mode 100644 reader.go delete mode 100644 reader_file.go delete mode 100644 reader_fileinfo.go delete mode 100644 reader_frag.go delete mode 100644 reader_fs.go delete mode 100644 reader_inode.go delete mode 100644 squashfs_test.go diff --git a/cmd/go-unsquashfs/main.go b/cmd/go-unsquashfs/main.go deleted file mode 100644 index 75be53f..0000000 --- a/cmd/go-unsquashfs/main.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - "time" - - "github.com/CalebQ42/squashfs" -) - -func main() { - verbose := flag.Bool("v", false, "Verbose") - ignore := flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755") - flag.Parse() - if len(flag.Args()) < 2 { - fmt.Println("Please provide a file name and extraction path") - os.Exit(0) - } - f, err := os.Open(flag.Arg(0)) - if err != nil { - panic(err) - } - r, err := squashfs.NewReader(f) - if err != nil { - panic(err) - } - op := squashfs.DefaultOptions() - op.Verbose = *verbose - op.IgnorePerm = *ignore - n := time.Now() - err = r.ExtractWithOptions(flag.Arg(1), op) - if err != nil { - panic(err) - } - fmt.Println("Took:", time.Since(n)) -} diff --git a/fuse2.go b/fuse2.go deleted file mode 100644 index 8338610..0000000 --- a/fuse2.go +++ /dev/null @@ -1,150 +0,0 @@ -package squashfs - -import ( - "bytes" - "context" - "errors" - "io" - - "github.com/CalebQ42/squashfs/internal/inode" - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" -) - -// Mounts the archive to the given mountpoint using fuse2. Non-blocking. -// If Unmount does not get called, the mount point must be unmounted using umount before the directory can be used again. -func (r *Reader) MountFuse2(mountpoint string) (err error) { - if r.con != nil { - return errors.New("squashfs archive already mounted") - } - r.con2, err = fuse.Mount(mountpoint, fuse.ReadOnly()) - if err != nil { - return - } - <-r.con2.Ready - r.mount2Done = make(chan struct{}) - go func() { - fs.Serve(r.con2, squashFuse2{r: r}) - close(r.mount2Done) - }() - return -} - -// Blocks until the mount ends. -// Fuse2 version. -func (r *Reader) MountWaitFuse2() { - if r.mount2Done != nil { - <-r.mount2Done - } -} - -// Unmounts the archive. -// Fuse2 version. -func (r *Reader) UnmountFuse2() error { - if r.con != nil { - defer func() { r.con = nil }() - return r.con.Close() - } - return errors.New("squashfs archive is not mounted") -} - -type squashFuse2 struct { - r *Reader -} - -func (s squashFuse2) Root() (fs.Node, error) { - return fileNode2{File: s.r.FS.File}, nil -} - -type fileNode2 struct { - *File -} - -func (f fileNode2) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Blocks = f.r.s.Size / 512 - if f.r.s.Size%512 > 0 { - attr.Blocks++ - } - attr.Gid = f.r.ids[f.i.GidInd] - attr.Inode = uint64(f.i.Num) - attr.Mode = f.i.Mode() - attr.Nlink = f.i.LinkCount() - attr.Size = f.i.Size() - attr.Uid = f.r.ids[f.i.UidInd] - return nil -} - -func (f fileNode2) Id() uint64 { - return uint64(f.i.Num) -} - -func (f fileNode2) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - return f.SymlinkPath(), nil -} - -func (f fileNode2) Lookup(ctx context.Context, name string) (fs.Node, error) { - asFS, err := f.FS() - if err != nil { - return nil, fuse.ENOTDIR - } - ret, err := asFS.OpenFile(name) - if err != nil { - return nil, fuse.ENOENT - } - return fileNode2{File: ret}, nil -} - -func (f fileNode2) ReadAll(ctx context.Context) ([]byte, error) { - if f.IsRegular() { - var buf bytes.Buffer - _, err := f.WriteTo(&buf) - return buf.Bytes(), err - } - return nil, ENODATA -} - -func (f fileNode2) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - if f.IsRegular() { - buf := make([]byte, req.Size) - n, err := f.File.ReadAt(buf, req.Offset) - if err == io.EOF { - resp.Data = buf[:n] - } - return nil - } - return ENODATA -} - -func (f fileNode2) ReadDirAll(ctx context.Context) (out []fuse.Dirent, err error) { - asFS, err := f.FS() - if err != nil { - return nil, fuse.ENOTDIR - } - var t fuse.DirentType - for i := range asFS.e { - switch asFS.e[i].Type { - case inode.Fil: - t = fuse.DT_File - case inode.Dir: - t = fuse.DT_Dir - case inode.Block: - t = fuse.DT_Block - case inode.Sym: - t = fuse.DT_Link - case inode.Char: - t = fuse.DT_Char - case inode.Fifo: - t = fuse.DT_FIFO - case inode.Sock: - t = fuse.DT_Socket - default: - t = fuse.DT_Unknown - } - out = append(out, fuse.Dirent{ - Inode: uint64(asFS.e[i].Num), - Type: t, - Name: asFS.e[i].Name, - }) - } - return -} diff --git a/fuse3.go b/fuse3.go deleted file mode 100644 index e0b8b28..0000000 --- a/fuse3.go +++ /dev/null @@ -1,148 +0,0 @@ -package squashfs - -import ( - "bytes" - "context" - "errors" - "io" - - "github.com/CalebQ42/fuse" - "github.com/CalebQ42/fuse/fs" - "github.com/CalebQ42/squashfs/internal/inode" -) - -// Mounts the archive to the given mountpoint using fuse3. Non-blocking. -// If Unmount does not get called, the mount point must be unmounted using umount before the directory can be used again. -func (r *Reader) Mount(mountpoint string) (err error) { - if r.con != nil { - return errors.New("squashfs archive already mounted") - } - r.con, err = fuse.Mount(mountpoint, fuse.ReadOnly()) - if err != nil { - return - } - <-r.con.Ready - r.mountDone = make(chan struct{}) - go func() { - fs.Serve(r.con, squashFuse{r: r}) - close(r.mountDone) - }() - return -} - -// Blocks until the mount ends. -func (r *Reader) MountWait() { - if r.mountDone != nil { - <-r.mountDone - } -} - -// Unmounts the archive. -func (r *Reader) Unmount() error { - if r.con != nil { - defer func() { r.con = nil }() - return r.con.Close() - } - return errors.New("squashfs archive is not mounted") -} - -type squashFuse struct { - r *Reader -} - -func (s squashFuse) Root() (fs.Node, error) { - return fileNode{File: s.r.FS.File}, nil -} - -type fileNode struct { - *File -} - -func (f fileNode) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Blocks = f.r.s.Size / 512 - if f.r.s.Size%512 > 0 { - attr.Blocks++ - } - attr.Gid = f.r.ids[f.i.GidInd] - attr.Inode = uint64(f.i.Num) - attr.Mode = f.i.Mode() - attr.Nlink = f.i.LinkCount() - attr.Size = f.i.Size() - attr.Uid = f.r.ids[f.i.UidInd] - return nil -} - -func (f fileNode) Id() uint64 { - return uint64(f.i.Num) -} - -func (f fileNode) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - return f.SymlinkPath(), nil -} - -func (f fileNode) Lookup(ctx context.Context, name string) (fs.Node, error) { - asFS, err := f.FS() - if err != nil { - return nil, fuse.ENOTDIR - } - ret, err := asFS.OpenFile(name) - if err != nil { - return nil, fuse.ENOENT - } - return fileNode{File: ret}, nil -} - -func (f fileNode) ReadAll(ctx context.Context) ([]byte, error) { - if f.IsRegular() { - var buf bytes.Buffer - _, err := f.WriteTo(&buf) - return buf.Bytes(), err - } - return nil, ENODATA -} - -func (f fileNode) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - if f.IsRegular() { - buf := make([]byte, req.Size) - n, err := f.File.ReadAt(buf, req.Offset) - if err == io.EOF { - resp.Data = buf[:n] - } - return nil - } - return ENODATA -} - -func (f fileNode) ReadDirAll(ctx context.Context) (out []fuse.Dirent, err error) { - asFS, err := f.FS() - if err != nil { - return nil, fuse.ENOTDIR - } - var t fuse.DirentType - for i := range asFS.e { - switch asFS.e[i].Type { - case inode.Fil: - t = fuse.DT_File - case inode.Dir: - t = fuse.DT_Dir - case inode.Block: - t = fuse.DT_Block - case inode.Sym: - t = fuse.DT_Link - case inode.Char: - t = fuse.DT_Char - case inode.Fifo: - t = fuse.DT_FIFO - case inode.Sock: - t = fuse.DT_Socket - default: - t = fuse.DT_Unknown - } - out = append(out, fuse.Dirent{ - Inode: uint64(asFS.e[i].Num), - Type: t, - Name: asFS.e[i].Name, - }) - } - return -} diff --git a/fuse_darwin.go b/fuse_darwin.go deleted file mode 100644 index 353aee1..0000000 --- a/fuse_darwin.go +++ /dev/null @@ -1,7 +0,0 @@ -package squashfs - -import ( - "golang.org/x/sys/unix" -) - -var ENODATA = unix.Errno(unix.ENODATA) diff --git a/fuse_linux.go b/fuse_linux.go deleted file mode 100644 index 816560a..0000000 --- a/fuse_linux.go +++ /dev/null @@ -1,5 +0,0 @@ -package squashfs - -import "github.com/CalebQ42/fuse" - -var ENODATA = fuse.ENODATA diff --git a/fuse_windows.go b/fuse_windows.go deleted file mode 100644 index c50a696..0000000 --- a/fuse_windows.go +++ /dev/null @@ -1,3 +0,0 @@ -package squashfs - -var ENODATA = windows.Errno(windows.ENODATA) diff --git a/go.mod b/go.mod deleted file mode 100644 index 8599653..0000000 --- a/go.mod +++ /dev/null @@ -1,14 +0,0 @@ -module github.com/CalebQ42/squashfs - -go 1.21 - -require ( - github.com/CalebQ42/fuse v0.1.0 - github.com/klauspost/compress v1.16.7 - github.com/pierrec/lz4/v4 v4.1.18 - github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e - github.com/seaweedfs/fuse v1.2.2 - github.com/therootcompany/xz v1.0.1 - github.com/ulikunitz/xz v0.5.11 - golang.org/x/sys v0.11.0 -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 6e8a10b..0000000 --- a/go.sum +++ /dev/null @@ -1,16 +0,0 @@ -github.com/CalebQ42/fuse v0.1.0 h1:KLCNjun7zcd2kBNVFfH+SWJyhuwJdE0nhw5/q8K8HGQ= -github.com/CalebQ42/fuse v0.1.0/go.mod h1:pJpoKG03HJKVhsp8o0YQYqmfbFsr3Eowt90yQGQVO+4= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs= -github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M= -github.com/seaweedfs/fuse v1.2.2 h1:01l8OjIdyATRNqVc/gDPgFobuC8ubQF3hRKOPColROw= -github.com/seaweedfs/fuse v1.2.2/go.mod h1:iwbDQv5BZACY54r6AO/6xsLNuMaYcBKSkLTZVfmK594= -github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= -github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= -github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/data/fullreader.go b/internal/data/fullreader.go deleted file mode 100644 index 2630357..0000000 --- a/internal/data/fullreader.go +++ /dev/null @@ -1,235 +0,0 @@ -package data - -import ( - "io" - - "github.com/CalebQ42/squashfs/internal/decompress" - "github.com/CalebQ42/squashfs/internal/toreader" -) - -type FullReader struct { - r io.ReaderAt - d decompress.Decompressor - fragRdr func() (io.Reader, error) - sizes []uint32 - blockSize uint32 - start uint64 - fileSize uint64 -} - -func NewFullReader(r io.ReaderAt, start uint64, d decompress.Decompressor, blockSizes []uint32, blockSize uint32, fileSize uint64) *FullReader { - return &FullReader{ - r: r, - start: start, - blockSize: blockSize, - sizes: blockSizes, - d: d, - fileSize: fileSize, - } -} - -func (r *FullReader) AddFragment(rdr func() (io.Reader, error)) { - r.fragRdr = rdr - r.sizes = append(r.sizes, 0) -} - -type outDat struct { - err error - data []byte - i int -} - -func (r FullReader) process(index int, offset int64, out chan outDat) { - var err error - var dat []byte - var rdr io.ReadCloser - size := realSize(r.sizes[index]) - if size == 0 { - outSize := r.blockSize - if r.fileSize < uint64(r.blockSize) { - outSize = uint32(r.fileSize) - } - out <- outDat{ - i: index, - err: nil, - data: make([]byte, outSize), - } - return - } - // rdr := io.LimitReader(toreader.NewReader(r.r, offset), int64(size)) - if size == r.sizes[index] { - if dec, ok := r.d.(decompress.Decoder); ok { - dat = make([]byte, size) - _, err = r.r.ReadAt(dat, offset) - if err == nil { - dat, err = dec.Decode(dat) - } - } else { - rdr, err = r.d.Reader(io.LimitReader(toreader.NewReader(r.r, offset), int64(size))) - if err == nil { - dat, err = io.ReadAll(rdr) - } - } - } else { - dat = make([]byte, size) - _, err = r.r.ReadAt(dat, offset) - } - out <- outDat{ - i: index, - err: err, - data: dat, - } - if clr, ok := rdr.(io.Closer); ok { - clr.Close() - } -} - -func (r FullReader) ReadAt(p []byte, off int64) (n int, err error) { - out := make(chan outDat, len(r.sizes)) - offset := r.start - num := len(r.sizes) - start := off / int64(r.blockSize) - end := len(p) / int(r.blockSize) - if end%int(r.blockSize) > 0 { - end++ - } - if end > len(r.sizes) { - if r.fragRdr != nil { - end = len(r.sizes) - } else { - end = len(r.sizes) + 1 - } - } - for i := 0; i < num; i++ { - if i < int(start) || i > end { - offset += uint64(realSize(r.sizes[i])) - continue - } - if i == num-1 && r.fragRdr != nil { - go func() { - rdr, e := r.fragRdr() - if e != nil { - out <- outDat{ - i: num - 1, - err: e, - } - return - } - dat, e := io.ReadAll(rdr) - out <- outDat{ - i: num - 1, - err: e, - data: dat, - } - if clr, ok := rdr.(io.Closer); ok { - clr.Close() - } - }() - continue - } - go r.process(i, int64(offset), out) - offset += uint64(realSize(r.sizes[i])) - } - cache := make(map[int]outDat) - for cur := start; cur < int64(end); { - dat := <-out - if dat.err != nil { - err = dat.err - return - } - if dat.i != int(cur) { - cache[dat.i] = dat - continue - } - if cur == start { - dat.data = dat.data[off%int64(r.blockSize):] - } - for i := range dat.data { - p[n+i] = dat.data[i] - } - n += len(dat.data) - cur++ - var ok bool - for { - dat, ok = cache[int(cur)] - if !ok { - break - } - for i := range dat.data { - p[n+i] = dat.data[i] - } - n += len(dat.data) - cur++ - delete(cache, int(cur)) - } - } - if n < len(p) { - err = io.EOF - } - return -} - -func (r FullReader) WriteTo(w io.Writer) (n int64, err error) { - out := make(chan outDat, len(r.sizes)) - offset := r.start - num := len(r.sizes) - for i := 0; i < num; i++ { - if i == num-1 && r.fragRdr != nil { - go func() { - rdr, e := r.fragRdr() - if err != nil { - out <- outDat{ - i: num - 1, - err: e, - } - return - } - dat, e := io.ReadAll(rdr) - out <- outDat{ - i: num - 1, - err: e, - data: dat, - } - if clr, ok := rdr.(io.Closer); ok { - clr.Close() - } - }() - continue - } - go r.process(i, int64(offset), out) - offset += uint64(realSize(r.sizes[i])) - } - cache := make(map[int]outDat) - var tmpN int - for cur := 0; cur < num; { - dat := <-out - if dat.err != nil { - err = dat.err - return - } - if dat.i != cur { - cache[dat.i] = dat - continue - } - tmpN, err = w.Write(dat.data) - n += int64(tmpN) - if err != nil { - return - } - cur++ - var ok bool - for { - dat, ok = cache[cur] - if !ok { - break - } - tmpN, err = w.Write(dat.data) - n += int64(tmpN) - if err != nil { - return - } - cur++ - } - } - return -} diff --git a/internal/data/reader.go b/internal/data/reader.go deleted file mode 100644 index a1ddd05..0000000 --- a/internal/data/reader.go +++ /dev/null @@ -1,104 +0,0 @@ -package data - -import ( - "bytes" - "io" - - "github.com/CalebQ42/squashfs/internal/decompress" -) - -type Reader struct { - master io.Reader - cur io.Reader - fragRdr io.Reader - d decompress.Decompressor - comRdr io.Reader - blockSizes []uint32 - blockSize uint32 - resetable bool - fileSize uint64 -} - -func NewReader(r io.Reader, d decompress.Decompressor, blockSizes []uint32, blockSize uint32, fileSize uint64) *Reader { - return &Reader{ - d: d, - master: r, - blockSizes: blockSizes, - blockSize: blockSize, - resetable: true, - fileSize: fileSize, - } -} - -func (r *Reader) AddFragment(rdr io.Reader) { - r.fragRdr = rdr - r.blockSizes = append(r.blockSizes, 0) -} - -func realSize(siz uint32) uint32 { - return siz &^ (1 << 24) -} - -func (r *Reader) advance() (err error) { - if clr, ok := r.cur.(io.Closer); ok { - clr.Close() - } - if len(r.blockSizes) == 0 { - return io.EOF - } - if len(r.blockSizes) == 1 && r.fragRdr != nil { - r.cur = r.fragRdr - } else { - size := realSize(r.blockSizes[0]) - if size == 0 { - outSize := r.blockSize - if r.fileSize < uint64(r.blockSize) { - outSize = uint32(r.fileSize) - } - r.cur = bytes.NewReader(make([]byte, outSize)) - } else { - r.cur = io.LimitReader(r.master, int64(size)) - if size == r.blockSizes[0] { - if rs, ok := r.d.(decompress.Resetable); ok { - if r.comRdr == nil { - r.cur, err = r.d.Reader(r.cur) - if err != nil { - return - } - } else { - err = rs.Reset(r.comRdr, r.cur) - r.cur = r.comRdr - } - } else { - r.cur, err = r.d.Reader(r.cur) - } - } - } - } - r.blockSizes = r.blockSizes[1:] - return -} - -func (r *Reader) Read(p []byte) (n int, err error) { - if r.cur == nil { - err = r.advance() - if err != nil { - return - } - } - n, err = r.cur.Read(p) - if err == io.EOF { - err = r.advance() - if err != nil { - return - } - var tmpN int - tmp := make([]byte, len(p)-n) - tmpN, err = r.Read(tmp) - for i := range tmp { - p[n+i] = tmp[i] - } - n += tmpN - } - return -} diff --git a/internal/decompress/gzip.go b/internal/decompress/gzip.go deleted file mode 100644 index 26feccd..0000000 --- a/internal/decompress/gzip.go +++ /dev/null @@ -1,17 +0,0 @@ -package decompress - -import ( - "io" - - "github.com/klauspost/compress/zlib" -) - -type GZip struct{} - -func (g GZip) Reader(src io.Reader) (io.ReadCloser, error) { - return zlib.NewReader(src) -} - -func (g GZip) Reset(old, src io.Reader) error { - return old.(zlib.Resetter).Reset(src, nil) -} diff --git a/internal/decompress/interface.go b/internal/decompress/interface.go deleted file mode 100644 index f6ac915..0000000 --- a/internal/decompress/interface.go +++ /dev/null @@ -1,22 +0,0 @@ -package decompress - -import ( - "io" -) - -type Decompressor interface { - //Creates a new decompressor reading from src. - Reader(src io.Reader) (io.ReadCloser, error) -} - -type Resetable interface { - //Reset attempts to re-use an old decompressor with new data. - //Will return ErrNotResetable if not Resetable(). - //Must ALWAYS be provided with a reader created with Reader. - Reset(old, src io.Reader) error -} - -type Decoder interface { - //Decodes a chunk of data all at once. - Decode(in []byte) ([]byte, error) -} diff --git a/internal/decompress/lz4.go b/internal/decompress/lz4.go deleted file mode 100644 index 2a53ac7..0000000 --- a/internal/decompress/lz4.go +++ /dev/null @@ -1,18 +0,0 @@ -package decompress - -import ( - "io" - - "github.com/pierrec/lz4/v4" -) - -type Lz4 struct{} - -func (l Lz4) Reader(r io.Reader) (io.ReadCloser, error) { - return io.NopCloser(lz4.NewReader(r)), nil -} - -func (l Lz4) Reset(old, src io.Reader) error { - old.(*lz4.Reader).Reset(src) - return nil -} diff --git a/internal/decompress/lzma.go b/internal/decompress/lzma.go deleted file mode 100644 index 9add4ae..0000000 --- a/internal/decompress/lzma.go +++ /dev/null @@ -1,14 +0,0 @@ -package decompress - -import ( - "io" - - "github.com/ulikunitz/xz/lzma" -) - -type Lzma struct{} - -func (l Lzma) Reader(r io.Reader) (io.ReadCloser, error) { - rdr, err := lzma.NewReader(r) - return io.NopCloser(rdr), err -} diff --git a/internal/decompress/lzo.go b/internal/decompress/lzo.go deleted file mode 100644 index 76333e7..0000000 --- a/internal/decompress/lzo.go +++ /dev/null @@ -1,18 +0,0 @@ -package decompress - -import ( - "bytes" - "io" - - "github.com/rasky/go-lzo" -) - -type Lzo struct{} - -func (l Lzo) Reader(r io.Reader) (io.ReadCloser, error) { - cache, err := lzo.Decompress1X(r, 0, 0) - if err != nil { - return nil, err - } - return io.NopCloser(bytes.NewReader(cache)), nil -} diff --git a/internal/decompress/xz.go b/internal/decompress/xz.go deleted file mode 100644 index 8bc2c80..0000000 --- a/internal/decompress/xz.go +++ /dev/null @@ -1,18 +0,0 @@ -package decompress - -import ( - "io" - - "github.com/therootcompany/xz" -) - -type Xz struct{} - -func (x Xz) Reader(r io.Reader) (io.ReadCloser, error) { - rdr, err := xz.NewReader(r, 0) - return io.NopCloser(rdr), err -} - -func (x Xz) Reset(old, src io.Reader) error { - return old.(*xz.Reader).Reset(src) -} diff --git a/internal/decompress/zstd.go b/internal/decompress/zstd.go deleted file mode 100644 index dde62ff..0000000 --- a/internal/decompress/zstd.go +++ /dev/null @@ -1,27 +0,0 @@ -package decompress - -import ( - "io" - - "github.com/klauspost/compress/zstd" -) - -type Zstd struct { - writeToReader *zstd.Decoder -} - -func (z Zstd) Reader(src io.Reader) (io.ReadCloser, error) { - r, err := zstd.NewReader(src) - return r.IOReadCloser(), err -} - -func (z Zstd) Reset(old, src io.Reader) error { - return old.(*zstd.Decoder).Reset(src) -} - -func (z Zstd) Decode(in []byte) ([]byte, error) { - if z.writeToReader == nil { - z.writeToReader, _ = zstd.NewReader(nil) - } - return z.writeToReader.DecodeAll(in, nil) -} diff --git a/internal/directory/directory.go b/internal/directory/directory.go deleted file mode 100644 index 7f49bc3..0000000 --- a/internal/directory/directory.go +++ /dev/null @@ -1,80 +0,0 @@ -package directory - -import ( - "bytes" - "encoding/binary" - "io" -) - -type header struct { - Entries uint32 - InodeStart uint32 - Num uint32 -} - -type entryInit struct { - Offset uint16 - NumOffset int16 - Type uint16 - NameSize uint16 -} - -type entry struct { - entryInit - Name []byte -} - -type Entry struct { - Name string - BlockStart uint32 - Type uint16 - Offset uint16 - Num uint32 -} - -func readEntry(r io.Reader) (e entry, err error) { - err = binary.Read(r, binary.LittleEndian, &e.entryInit) - if err != nil { - return - } - e.Name = make([]byte, e.NameSize+1) - err = binary.Read(r, binary.LittleEndian, &e.Name) - return -} - -func ReadEntries(rdr io.Reader, size uint32) (e []Entry, err error) { - dat := make([]byte, size-3) - rdr.Read(dat) - r := bytes.NewReader(dat) - var h header - var en entry - for { - err = binary.Read(r, binary.LittleEndian, &h) - if err == io.EOF { - err = nil - return - } else if err != nil { - return - } - h.Entries++ - for i := 0; i < int(h.Entries); i++ { - if i != 0 && i%256 == 0 { - err = binary.Read(r, binary.LittleEndian, &h) - if err != nil { - return - } - } - en, err = readEntry(r) - if err != nil { - return - } - e = append(e, Entry{ - Name: string(en.Name), - BlockStart: h.InodeStart, - Type: en.Type, - Offset: en.Offset, - Num: h.Num + uint32(en.NumOffset), - }) - } - } -} diff --git a/internal/inode/dir.go b/internal/inode/dir.go deleted file mode 100644 index bffe973..0000000 --- a/internal/inode/dir.go +++ /dev/null @@ -1,65 +0,0 @@ -package inode - -import ( - "encoding/binary" - "io" -) - -type Directory struct { - BlockStart uint32 - LinkCount uint32 - Size uint16 - Offset uint16 - ParentNum uint32 -} - -type eDirectoryInit struct { - LinkCount uint32 - Size uint32 - BlockStart uint32 - ParentNum uint32 - IndCount uint16 - Offset uint16 - XattrInd uint32 -} - -type EDirectory struct { - eDirectoryInit - Indexes []DirectoryIndex -} - -type directoryIndexInit struct { - Ind uint32 - Start uint32 - NameSize uint32 -} - -type DirectoryIndex struct { - directoryIndexInit - Name []byte -} - -func ReadDir(r io.Reader) (d Directory, err error) { - err = binary.Read(r, binary.LittleEndian, &d) - return -} - -func ReadEDir(r io.Reader) (d EDirectory, err error) { - err = binary.Read(r, binary.LittleEndian, &d.eDirectoryInit) - if err != nil { - return - } - d.Indexes = make([]DirectoryIndex, d.IndCount) - for i := range d.Indexes { - err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].directoryIndexInit) - if err != nil { - return - } - d.Indexes[i].Name = make([]byte, d.Indexes[i].NameSize+1) - err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].Name) - if err != nil { - return - } - } - return -} diff --git a/internal/inode/file.go b/internal/inode/file.go deleted file mode 100644 index 1b4d461..0000000 --- a/internal/inode/file.go +++ /dev/null @@ -1,62 +0,0 @@ -package inode - -import ( - "encoding/binary" - "io" - "math" -) - -type fileInit struct { - BlockStart uint32 - FragInd uint32 - FragOffset uint32 - Size uint32 -} - -type File struct { - fileInit - BlockSizes []uint32 -} - -type eFileInit struct { - BlockStart uint64 - Size uint64 - Sparse uint64 - LinkCount uint32 - FragInd uint32 - FragOffset uint32 - XattrInd uint32 -} - -type EFile struct { - eFileInit - BlockSizes []uint32 -} - -func ReadFile(r io.Reader, blockSize uint32) (f File, err error) { - err = binary.Read(r, binary.LittleEndian, &f.fileInit) - if err != nil { - return - } - toRead := int(math.Floor(float64(f.Size) / float64(blockSize))) - if f.FragInd == 0xFFFFFFFF && f.Size%blockSize > 0 { - toRead++ - } - f.BlockSizes = make([]uint32, toRead) - err = binary.Read(r, binary.LittleEndian, &f.BlockSizes) - return -} - -func ReadEFile(r io.Reader, blockSize uint32) (f EFile, err error) { - err = binary.Read(r, binary.LittleEndian, &f.eFileInit) - if err != nil { - return - } - toRead := int(math.Floor(float64(f.Size) / float64(blockSize))) - if f.FragInd == 0xFFFFFFFF && f.Size%uint64(blockSize) > 0 { - toRead++ - } - f.BlockSizes = make([]uint32, toRead) - err = binary.Read(r, binary.LittleEndian, &f.BlockSizes) - return -} diff --git a/internal/inode/inode.go b/internal/inode/inode.go deleted file mode 100644 index 11976f5..0000000 --- a/internal/inode/inode.go +++ /dev/null @@ -1,143 +0,0 @@ -package inode - -import ( - "encoding/binary" - "errors" - "io" - "io/fs" - "strconv" -) - -const ( - Dir = uint16(iota + 1) - Fil - Sym - Block - Char - Fifo - Sock - EDir - EFil - ESym - EBlock - EChar - EFifo - ESock -) - -type Header struct { - Type uint16 - Perm uint16 - UidInd uint16 - GidInd uint16 - ModTime uint32 - Num uint32 -} - -type Inode struct { - Header - Data any -} - -func Read(r io.Reader, blockSize uint32) (i Inode, err error) { - err = binary.Read(r, binary.LittleEndian, &i.Header) - if err != nil { - return - } - switch i.Type { - case Dir: - i.Data, err = ReadDir(r) - case Fil: - i.Data, err = ReadFile(r, blockSize) - case Sym: - i.Data, err = ReadSym(r) - case Block: - fallthrough - case Char: - i.Data, err = ReadDevice(r) - case Fifo: - fallthrough - case Sock: - i.Data, err = ReadIPC(r) - case EDir: - i.Data, err = ReadEDir(r) - case EFil: - i.Data, err = ReadEFile(r, blockSize) - case ESym: - i.Data, err = ReadESym(r) - case EBlock: - fallthrough - case EChar: - i.Data, err = ReadEDevice(r) - case EFifo: - fallthrough - case ESock: - i.Data, err = ReadEIPC(r) - default: - return i, errors.New("invalid inode type " + strconv.Itoa(int(i.Type))) - } - return -} - -func (i Inode) Mode() (out fs.FileMode) { - out = fs.FileMode(i.Perm) - switch i.Data.(type) { - case Directory: - out |= fs.ModeDir - case EDirectory: - out |= fs.ModeDir - case Symlink: - out |= fs.ModeSymlink - case ESymlink: - out |= fs.ModeSymlink - case Device: - out |= fs.ModeDevice - case EDevice: - out |= fs.ModeDevice - case IPC: - out |= fs.ModeNamedPipe - case EIPC: - out |= fs.ModeNamedPipe - } - return -} - -func (i Inode) LinkCount() uint32 { - switch i.Data.(type) { - case EFile: - return i.Data.(EFile).LinkCount - case Directory: - return i.Data.(Directory).LinkCount - case EDirectory: - return i.Data.(EDirectory).LinkCount - case Device: - return i.Data.(Device).LinkCount - case EDevice: - return i.Data.(EDevice).LinkCount - case IPC: - return i.Data.(IPC).LinkCount - case EIPC: - return i.Data.(EIPC).LinkCount - case Symlink: - return i.Data.(Symlink).LinkCount - case ESymlink: - return i.Data.(ESymlink).LinkCount - default: - return 0 - } -} - -func (i Inode) Size() uint64 { - switch i.Data.(type) { - case File: - return uint64(i.Data.(File).Size) - case EFile: - return i.Data.(EFile).Size - // case Directory: - // return uint64(i.Data.(Directory).Size) - // case EDirectory: - // return uint64(i.Data.(EDirectory).Size) - default: - return 0 - } -} diff --git a/internal/inode/misc.go b/internal/inode/misc.go deleted file mode 100644 index 8ba0b61..0000000 --- a/internal/inode/misc.go +++ /dev/null @@ -1,45 +0,0 @@ -package inode - -import ( - "encoding/binary" - "io" -) - -type Device struct { - LinkCount uint32 - Dev uint32 -} - -type EDevice struct { - Device - XattrInd uint32 -} - -func ReadDevice(r io.Reader) (d Device, err error) { - err = binary.Read(r, binary.LittleEndian, &d) - return -} - -func ReadEDevice(r io.Reader) (d EDevice, err error) { - err = binary.Read(r, binary.LittleEndian, &d) - return -} - -type IPC struct { - LinkCount uint32 -} - -type EIPC struct { - IPC - XattrInd uint32 -} - -func ReadIPC(r io.Reader) (i IPC, err error) { - err = binary.Read(r, binary.LittleEndian, &i) - return -} - -func ReadEIPC(r io.Reader) (i EIPC, err error) { - err = binary.Read(r, binary.LittleEndian, &i) - return -} diff --git a/internal/inode/sym.go b/internal/inode/sym.go deleted file mode 100644 index 43659bb..0000000 --- a/internal/inode/sym.go +++ /dev/null @@ -1,46 +0,0 @@ -package inode - -import ( - "encoding/binary" - "io" -) - -type symlinkInit struct { - LinkCount uint32 - TargetSize uint32 -} - -type Symlink struct { - symlinkInit - Target []byte -} - -type ESymlink struct { - symlinkInit - Target []byte - XattrInd uint32 -} - -func ReadSym(r io.Reader) (s Symlink, err error) { - err = binary.Read(r, binary.LittleEndian, &s.symlinkInit) - if err != nil { - return - } - s.Target = make([]byte, s.TargetSize) - err = binary.Read(r, binary.LittleEndian, &s.Target) - return -} - -func ReadESym(r io.Reader) (s ESymlink, err error) { - err = binary.Read(r, binary.LittleEndian, &s.symlinkInit) - if err != nil { - return - } - s.Target = make([]byte, s.TargetSize) - err = binary.Read(r, binary.LittleEndian, &s.Target) - if err != nil { - return - } - err = binary.Read(r, binary.LittleEndian, &s.XattrInd) - return -} diff --git a/internal/metadata/reader.go b/internal/metadata/reader.go deleted file mode 100644 index 3fd0499..0000000 --- a/internal/metadata/reader.go +++ /dev/null @@ -1,81 +0,0 @@ -package metadata - -import ( - "encoding/binary" - "io" - - "github.com/CalebQ42/squashfs/internal/decompress" -) - -type Reader struct { - master io.Reader - cur io.Reader - d decompress.Decompressor - comRdr io.Reader -} - -func NewReader(master io.Reader, d decompress.Decompressor) *Reader { - return &Reader{ - master: master, - d: d, - } -} - -func realSize(siz uint16) uint16 { - return siz &^ 0x8000 -} - -func (r *Reader) advance() (err error) { - if _, ok := r.d.(decompress.Resetable); !ok { - if clr, ok := r.cur.(io.Closer); ok { - clr.Close() - } - } - var raw uint16 - err = binary.Read(r.master, binary.LittleEndian, &raw) - if err != nil { - return - } - size := realSize(raw) - r.cur = io.LimitReader(r.master, int64(size)) - if size == raw { - if rs, ok := r.d.(decompress.Resetable); ok { - if r.comRdr == nil { - r.cur, err = r.d.Reader(r.cur) - if err != nil { - return - } - } else { - err = rs.Reset(r.comRdr, r.cur) - r.cur = r.comRdr - } - } else { - r.cur, err = r.d.Reader(r.cur) - } - } - return -} - -func (r *Reader) Read(p []byte) (n int, err error) { - if r.cur == nil { - err = r.advance() - if err != nil { - return - } - } - n, err = r.cur.Read(p) - if err == io.EOF { - err = r.advance() - if err != nil { - return - } - var tmpN int - tmp := make([]byte, len(p)-n) - tmpN, err = r.Read(tmp) - for i := 0; i < tmpN; i++ { - p[n+i] = tmp[i] - } - n += tmpN - } - return -} diff --git a/internal/threadmanager/manager.go b/internal/threadmanager/manager.go deleted file mode 100644 index 6ee48be..0000000 --- a/internal/threadmanager/manager.go +++ /dev/null @@ -1,23 +0,0 @@ -package threadmanager - -type Manager struct { - c chan int -} - -func NewManager(maxRoutines int) *Manager { - m := &Manager{ - c: make(chan int, maxRoutines), - } - for i := 0; i < maxRoutines; i++ { - m.c <- i - } - return m -} - -func (m *Manager) Lock() int { - return <-m.c -} - -func (m *Manager) Unlock(n int) { - m.c <- n -} diff --git a/internal/toreader/offsetreader.go b/internal/toreader/offsetreader.go deleted file mode 100644 index 0192177..0000000 --- a/internal/toreader/offsetreader.go +++ /dev/null @@ -1,19 +0,0 @@ -package toreader - -import "io" - -type OffsetReader struct { - r io.ReaderAt - off int64 -} - -func NewOffsetReader(r io.ReaderAt, off int64) *OffsetReader { - return &OffsetReader{ - r: r, - off: off, - } -} - -func (r OffsetReader) ReadAt(p []byte, off int64) (n int, e error) { - return r.r.ReadAt(p, off+r.off) -} diff --git a/internal/toreader/reader.go b/internal/toreader/reader.go deleted file mode 100644 index 6f711fd..0000000 --- a/internal/toreader/reader.go +++ /dev/null @@ -1,25 +0,0 @@ -package toreader - -import "io" - -type Reader struct { - r io.ReaderAt - off int64 -} - -func NewReader(r io.ReaderAt, start int64) *Reader { - return &Reader{ - r: r, - off: start, - } -} - -func (r *Reader) Read(p []byte) (n int, err error) { - n, err = r.r.ReadAt(p, r.off) - r.off += int64(n) - return -} - -func (r Reader) Offset() int64 { - return r.off -} diff --git a/internal/toreader/readerat.go b/internal/toreader/readerat.go deleted file mode 100644 index b556ca7..0000000 --- a/internal/toreader/readerat.go +++ /dev/null @@ -1,24 +0,0 @@ -package toreader - -import "io" - -type ReaderAt struct { - d []byte -} - -func NewReaderAt(r io.Reader) (ra *ReaderAt, err error) { - ra = new(ReaderAt) - ra.d, err = io.ReadAll(r) - return -} - -func (r ReaderAt) ReadAt(p []byte, off int64) (n int, err error) { - if int(off) >= len(r.d) { - return 0, io.EOF - } - n = copy(p, r.d[off:]) - if n != len(p) { - err = io.EOF - } - return -} diff --git a/reader.go b/reader.go deleted file mode 100644 index 746cd02..0000000 --- a/reader.go +++ /dev/null @@ -1,234 +0,0 @@ -package squashfs - -import ( - "encoding/binary" - "errors" - "io" - "math" - "time" - - "github.com/CalebQ42/fuse" - "github.com/CalebQ42/squashfs/internal/decompress" - "github.com/CalebQ42/squashfs/internal/directory" - "github.com/CalebQ42/squashfs/internal/inode" - "github.com/CalebQ42/squashfs/internal/metadata" - "github.com/CalebQ42/squashfs/internal/toreader" - fuse2 "github.com/seaweedfs/fuse" -) - -type Reader struct { - *FS - con *fuse.Conn - con2 *fuse2.Conn - mountDone chan struct{} - mount2Done chan struct{} - d decompress.Decompressor - r io.ReaderAt - fragEntries []fragEntry - ids []uint32 - // exportTable []uint64 - s superblock -} - -var ( - ErrorMagic = errors.New("magic incorrect. probably not reading squashfs archive") - ErrorLog = errors.New("block log is incorrect. possible corrupted archive") - ErrorVersion = errors.New("squashfs version of archive is not 4.0") -) - -// The types of compression supported by squashfs -const ( - GZipCompression = uint16(iota + 1) - LZMACompression - LZOCompression - XZCompression - LZ4Compression - ZSTDCompression -) - -func NewReaderAtOffset(r io.ReaderAt, off int64) (*Reader, error) { - return NewReader(toreader.NewOffsetReader(r, off)) -} - -// Creates a new squashfs.Reader from the given io.Reader. NOTE: All data from the io.Reader will be read and stored in memory. -func NewReaderFromReader(r io.Reader) (*Reader, error) { - rdr, err := toreader.NewReaderAt(r) - if err != nil { - return nil, err - } - return NewReader(rdr) -} - -// Creates a new squashfs.Reader from the given io.ReaderAt. -func NewReader(r io.ReaderAt) (*Reader, error) { - var squash Reader - squash.r = r - err := binary.Read(toreader.NewReader(r, 0), binary.LittleEndian, &squash.s) - if err != nil { - return nil, err - } - if !squash.s.checkMagic() { - return nil, ErrorMagic - } - if !squash.s.checkBlockLog() { - return nil, ErrorLog - } - if !squash.s.checkVersion() { - return nil, ErrorVersion - } - switch squash.s.CompType { - case GZipCompression: - squash.d = decompress.GZip{} - case LZMACompression: - squash.d = decompress.Lzma{} - case LZOCompression: - squash.d = decompress.Lzo{} - case XZCompression: - squash.d = decompress.Xz{} - case LZ4Compression: - squash.d = decompress.Lz4{} - case ZSTDCompression: - squash.d = &decompress.Zstd{} - default: - return nil, errors.New("uh, I need to do this, OR something if very wrong") - } - if !squash.s.noFragments() && squash.s.FragCount > 0 { - fragOffsets := make([]uint64, int(math.Ceil(float64(squash.s.FragCount)/512))) - err = binary.Read(toreader.NewReader(r, int64(squash.s.FragTableStart)), binary.LittleEndian, &fragOffsets) - if err != nil { - return nil, err - } - squash.fragEntries = make([]fragEntry, squash.s.FragCount) - if len(fragOffsets) == 1 { - rdr := metadata.NewReader(toreader.NewReader(r, int64(fragOffsets[0])), squash.d) - err = binary.Read(rdr, binary.LittleEndian, &squash.fragEntries) - if err != nil { - return nil, err - } - } else { - toRead := squash.s.FragCount - var curRead uint32 - var tmp []fragEntry - var rdr *metadata.Reader - var offset int - for i := range fragOffsets { - curRead = uint32(math.Min(512, float64(toRead))) - tmp = make([]fragEntry, curRead) - rdr = metadata.NewReader(toreader.NewReader(r, int64(fragOffsets[i])), squash.d) - err = binary.Read(rdr, binary.LittleEndian, &tmp) - if err != nil { - return nil, err - } - offset = int(squash.s.FragCount - toRead) - for i := range tmp { - squash.fragEntries[offset+i] = tmp[i] - } - toRead -= curRead - } - } - } - if squash.s.IdCount > 0 { - idOffsets := make([]uint64, int(math.Ceil(float64(squash.s.IdCount)/2048))) - err = binary.Read(toreader.NewReader(r, int64(squash.s.IdTableStart)), binary.LittleEndian, &idOffsets) - if err != nil { - return nil, err - } - squash.ids = make([]uint32, squash.s.IdCount) - if len(idOffsets) == 1 { - rdr := metadata.NewReader(toreader.NewReader(r, int64(idOffsets[0])), squash.d) - err = binary.Read(rdr, binary.LittleEndian, &squash.ids) - if err != nil { - return nil, err - } - } else { - toRead := squash.s.IdCount - var curRead uint16 - var tmp []uint32 - var rdr *metadata.Reader - var offset int - for i := range idOffsets { - curRead = uint16(math.Min(2048, float64(toRead))) - tmp = make([]uint32, curRead) - rdr = metadata.NewReader(toreader.NewReader(r, int64(idOffsets[i])), squash.d) - err = binary.Read(rdr, binary.LittleEndian, &tmp) - if err != nil { - return nil, err - } - offset = int(squash.s.IdCount - toRead) - for i := range tmp { - squash.ids[offset+i] = tmp[i] - } - toRead -= curRead - } - } - } - root, err := squash.inodeFromRef(squash.s.RootInodeRef) - if err != nil { - return nil, err - } - rootEnts, err := squash.readDirectory(root) - if err != nil { - return nil, err - } - enType := root.Type - if enType == inode.EDir { - enType = inode.Dir - } - squash.FS = &FS{ - e: rootEnts, - File: &File{ - rdr: &squash, - i: root, - e: directory.Entry{ - Name: "", - Type: enType, - }, - r: &squash, - }, - } - return &squash, nil -} - -// func (r *Reader) initExport() (err error) { -// num := int(math.Ceil(float64(r.s.InodeCount) / 1024)) -// offsets := make([]uint64, num) -// err = binary.Read(toreader.NewReader(r.r, int64(r.s.ExportTableStart)), binary.LittleEndian, &offsets) -// if err != nil { -// return -// } -// left := r.s.InodeCount -// var toRead uint32 -// var new []uint64 -// var rdr *metadata.Reader -// for i := range offsets { -// rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offsets[i])), r.d) -// toRead = uint32(math.Min(1024, float64(left))) -// new = make([]uint64, toRead) -// err = binary.Read(rdr, binary.LittleEndian, &new) -// if err != nil { -// return -// } -// left -= toRead -// r.exportTable = append(r.exportTable, new...) -// } -// return nil -// } - -// func (r *Reader) inode(index uint32) (i inode.Inode, err error) { -// if r.s.exportable() { -// if r.exportTable == nil { -// err = r.initExport() -// if err != nil { -// return -// } -// } -// return r.inodeFromRef(r.exportTable[index-1]) -// } -// err = errors.New("archive is not exportable") -// return -// } - -// Returns the last time the archive was modified. -func (r Reader) ModTime() time.Time { - return time.Unix(int64(r.s.ModTime), 0) -} diff --git a/reader_file.go b/reader_file.go deleted file mode 100644 index 73adac6..0000000 --- a/reader_file.go +++ /dev/null @@ -1,518 +0,0 @@ -package squashfs - -import ( - "errors" - "io" - "io/fs" - "log" - "os" - "os/exec" - "path/filepath" - "runtime" - "strconv" - "strings" - - "github.com/CalebQ42/squashfs/internal/data" - "github.com/CalebQ42/squashfs/internal/directory" - "github.com/CalebQ42/squashfs/internal/inode" - "github.com/CalebQ42/squashfs/internal/threadmanager" -) - -// File represents a file inside a squashfs archive. -type File struct { - i inode.Inode - rdr io.Reader - fullRdr *data.FullReader - r *Reader - parent *FS - e directory.Entry - dirsRead int -} - -var ( - ErrReadNotFile = errors.New("read called on non-file") -) - -func (r Reader) newFile(en directory.Entry, parent *FS) (*File, error) { - i, err := r.inodeFromDir(en) - if err != nil { - return nil, err - } - var rdr io.Reader - var full *data.FullReader - if i.Type == inode.Fil || i.Type == inode.EFil { - full, rdr, err = r.getReaders(i) - if err != nil { - return nil, err - } - } - return &File{ - e: en, - i: i, - rdr: rdr, - fullRdr: full, - r: &r, - parent: parent, - }, nil -} - -// Stat returns the File's fs.FileInfo -func (f File) Stat() (fs.FileInfo, error) { - return newFileInfo(f.e, f.i), nil -} - -// Mode returns the file's fs.FileMode -func (f File) Mode() fs.FileMode { - switch f.e.Type { - case inode.Dir: - return fs.FileMode(f.i.Perm) | fs.ModeDir - case inode.Char: - return fs.FileMode(f.i.Perm) | fs.ModeCharDevice - case inode.Block: - return fs.FileMode(f.i.Perm) | fs.ModeDevice - case inode.Sym: - return fs.FileMode(f.i.Perm) | fs.ModeSymlink - } - return fs.FileMode(f.i.Perm) -} - -// Read reads the data from the file. Only works if file is a normal file. -func (f File) Read(p []byte) (int, error) { - if f.i.Type != inode.Fil && f.i.Type != inode.EFil { - return 0, ErrReadNotFile - } - if f.rdr == nil { - return 0, fs.ErrClosed - } - return f.rdr.Read(p) -} - -func (f File) ReadAt(p []byte, off int64) (int, error) { - if f.i.Type != inode.Fil && f.i.Type != inode.EFil { - return 0, ErrReadNotFile - } - return f.fullRdr.ReadAt(p, off) -} - -// WriteTo writes all data from the file to the writer. This is multi-threaded. -// The underlying reader is seperate from the one used with Read and can be reused. -func (f File) WriteTo(w io.Writer) (int64, error) { - if f.i.Type != inode.Fil && f.i.Type != inode.EFil { - return 0, ErrReadNotFile - } - return f.fullRdr.WriteTo(w) -} - -// Close simply nils the underlying reader. -func (f *File) Close() error { - f.rdr = nil - return nil -} - -// ReadDir returns n fs.DirEntry's that's contained in the File (if it's a directory). -// If n <= 0 all fs.DirEntry's are returned. -func (f *File) ReadDir(n int) (out []fs.DirEntry, err error) { - if !f.IsDir() { - return nil, errors.New("file is not a directory") - } - ents, err := f.r.readDirectory(f.i) - if err != nil { - return nil, err - } - start, end := 0, len(ents) - if n > 0 { - start, end = f.dirsRead, f.dirsRead+n - if end > len(f.r.e) { - end = len(f.r.e) - err = io.EOF - } - } - var fi fileInfo - for _, e := range ents[start:end] { - fi, err = f.r.newFileInfo(e) - if err != nil { - f.dirsRead += len(out) - return - } - out = append(out, fs.FileInfoToDirEntry(fi)) - } - f.dirsRead += len(out) - return -} - -// FS returns the File as a FS. -func (f *File) FS() (*FS, error) { - if !f.IsDir() { - return nil, errors.New("File is not a directory") - } - ents, err := f.r.readDirectory(f.i) - if err != nil { - return nil, err - } - return &FS{ - File: f, - e: ents, - }, nil -} - -// IsDir Yep. -func (f File) IsDir() bool { - return f.i.Type == inode.Dir || f.i.Type == inode.EDir -} - -// IsRegular yep. -func (f File) IsRegular() bool { - return f.i.Type == inode.Fil || f.i.Type == inode.EFil -} - -// IsSymlink yep. -func (f File) IsSymlink() bool { - return f.i.Type == inode.Sym || f.i.Type == inode.ESym -} - -func (f File) isDeviceOrFifo() bool { - return f.i.Type == inode.Char || f.i.Type == inode.Block || f.i.Type == inode.EChar || f.i.Type == inode.EBlock || f.i.Type == inode.Fifo || f.i.Type == inode.EFifo -} - -func (f File) deviceDevices() (maj uint32, min uint32) { - var dev uint32 - if f.i.Type == inode.Char || f.i.Type == inode.Block { - dev = f.i.Data.(inode.Device).Dev - } else if f.i.Type == inode.EChar || f.i.Type == inode.EBlock { - dev = f.i.Data.(inode.EDevice).Dev - } - return dev >> 8, dev & 0x000FF -} - -// SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string. -func (f File) SymlinkPath() string { - switch f.i.Type { - case inode.Sym: - return string(f.i.Data.(inode.Symlink).Target) - case inode.ESym: - return string(f.i.Data.(inode.ESymlink).Target) - } - return "" -} - -func (f File) path() string { - if f.parent == nil { - return f.e.Name - } - return f.parent.path() + "/" + f.e.Name -} - -// GetSymlinkFile returns the File the symlink is pointing to. -// If not a symlink, or the target is unobtainable (such as it being outside the archive or it's absolute) returns nil -func (f File) GetSymlinkFile() *File { - if !f.IsSymlink() { - return nil - } - if strings.HasPrefix(f.SymlinkPath(), "/") { - return nil - } - sym, err := f.parent.Open(f.SymlinkPath()) - if err != nil { - return nil - } - return sym.(*File) -} - -// ExtractionOptions are available options on how to extract. -type ExtractionOptions struct { - manager *threadmanager.Manager - LogOutput io.Writer //Where error log should write. - DereferenceSymlink bool //Replace symlinks with the target file. - UnbreakSymlink bool //Try to make sure symlinks remain unbroken when extracted, without changing the symlink. - Verbose bool //Prints extra info to log on an error. - IgnorePerm bool //Ignore file's permissions and instead use Perm. - Perm fs.FileMode //Permission to use when IgnorePerm. Defaults to 0755. - notFirst bool -} - -func DefaultOptions() *ExtractionOptions { - return &ExtractionOptions{ - Perm: 0755, - } -} - -// ExtractTo extracts the File to the given folder with the default options. -// If the File is a directory, it instead extracts the directory's contents to the folder. -func (f File) ExtractTo(folder string) error { - return f.realExtract(folder, DefaultOptions()) -} - -// ExtractVerbose extracts the File to the folder with the Verbose option. -func (f File) ExtractVerbose(folder string) error { - op := DefaultOptions() - op.Verbose = true - return f.realExtract(folder, op) -} - -// ExtractIgnorePermissions extracts the File to the folder with the IgnorePerm option. -func (f File) ExtractIgnorePermissions(folder string) error { - op := DefaultOptions() - op.IgnorePerm = true - return f.realExtract(folder, op) -} - -// ExtractSymlink extracts the File to the folder with the DereferenceSymlink option. -// If the File is a directory, it instead extracts the directory's contents to the folder. -func (f File) ExtractSymlink(folder string) error { - op := DefaultOptions() - op.DereferenceSymlink = true - return f.realExtract(folder, op) -} - -// ExtractWithOptions extracts the File to the given folder with the given ExtrationOptions. -// If the File is a directory, it instead extracts the directory's contents to the folder. -func (f File) ExtractWithOptions(folder string, op *ExtractionOptions) error { - if op.Verbose && op.LogOutput != nil { - log.SetOutput(op.LogOutput) - } - return f.realExtract(folder, op) -} - -func (f File) realExtract(folder string, op *ExtractionOptions) (err error) { - if op.manager == nil { - op.manager = threadmanager.NewManager(runtime.NumCPU()) - } - extDir := filepath.Join(folder, f.e.Name) - if !op.notFirst { - op.notFirst = true - if f.IsDir() { - extDir = folder - _, err = os.Open(folder) - if err != nil && os.IsNotExist(err) { - err = os.Mkdir(extDir, op.Perm) - } - if err != nil { - if op.Verbose { - log.Println("Error while making", folder) - } - return - } - if !op.IgnorePerm { - defer os.Chmod(extDir, f.Mode()) - defer os.Chown(extDir, int(f.r.ids[f.i.UidInd]), int(f.r.ids[f.i.GidInd])) - } - } - } - switch { - case f.IsDir(): - if folder != extDir && f.e.Name != "" { - //First extract it with a permisive permission. - err = os.Mkdir(extDir, op.Perm) - if err != nil { - if op.Verbose { - log.Println("Error while making directory", extDir) - } - return - } - //Then set it to it's actual permissions once we're done with it - if !op.IgnorePerm { - defer os.Chmod(extDir, f.Mode()) - defer os.Chown(extDir, int(f.r.ids[f.i.UidInd]), int(f.r.ids[f.i.GidInd])) - } - } - var filFS *FS - filFS, err = f.FS() - if err != nil { - if op.Verbose { - log.Println("Error while converting", f.path(), "to FS") - } - return err - } - errChan := make(chan error, len(filFS.e)) - files := make([]directory.Entry, 0) - //Focus on making the folder tree first... - var i int - for i = 0; i < len(filFS.e); i++ { - if filFS.e[i].Type == inode.Fil { - files = append(files, filFS.e[i]) - } else { - go func(index int) { - subF, goErr := f.r.newFile(filFS.e[index], filFS) - if goErr != nil { - if op.Verbose { - log.Println("Error while resolving", extDir) - } - errChan <- goErr - return - } - errChan <- subF.ExtractWithOptions(extDir, op) - }(i) - } - } - for i = 0; i < len(filFS.e)-len(files); i++ { - err = <-errChan - if err != nil { - return err - } - } - //Then we extract the files. - for i = 0; i < len(files); i++ { - go func(index int) { - n := op.manager.Lock() - defer op.manager.Unlock(n) - subF, goErr := f.r.newFile(files[index], filFS) - if goErr != nil { - if op.Verbose { - log.Println("Error while resolving", extDir) - } - errChan <- goErr - return - } - errChan <- subF.ExtractWithOptions(extDir, op) - }(i) - } - for i = 0; i < len(files); i++ { - err = <-errChan - if err != nil { - return err - } - } - case f.IsRegular(): - var fil *os.File - fil, err = os.Create(extDir) - if os.IsExist(err) { - os.Remove(extDir) - fil, err = os.Create(extDir) - if err != nil { - if op.Verbose { - log.Println("Error while creating", extDir) - } - return err - } - } else if err != nil { - if op.Verbose { - log.Println("Error while creating", extDir) - } - return err - } - defer fil.Close() - _, err = io.Copy(fil, f) - if err != nil { - if op.Verbose { - log.Println("Error while copying data to", extDir) - } - return err - } - if op.IgnorePerm { - os.Chmod(extDir, op.Perm|(f.Mode()&fs.ModeType)) - } else { - os.Chmod(extDir, f.Mode()) - os.Chown(extDir, int(f.r.ids[f.i.UidInd]), int(f.r.ids[f.i.GidInd])) - } - case f.IsSymlink(): - symPath := f.SymlinkPath() - if op.DereferenceSymlink { - fil := f.GetSymlinkFile() - if fil == nil { - if op.Verbose { - log.Println("Symlink path(", symPath, ") is unobtainable:", extDir) - } - return errors.New("cannot get symlink target") - } - fil.e.Name = f.e.Name - err = fil.realExtract(folder, op) - if err != nil { - if op.Verbose { - log.Println("Error while extracting the symlink's file:", extDir) - } - return err - } - return nil - } else if op.UnbreakSymlink { - fil := f.GetSymlinkFile() - if fil == nil { - if op.Verbose { - log.Println("Symlink path(", symPath, ") is unobtainable:", extDir) - } - return errors.New("cannot get symlink target") - } - extractLoc := filepath.Join(folder, filepath.Dir(symPath)) - err = fil.realExtract(extractLoc, op) - if err != nil { - if op.Verbose { - log.Println("Error while extracting ", extDir) - } - return err - } - } - err = os.Symlink(f.SymlinkPath(), extDir) - if os.IsExist(err) { - os.Remove(extDir) - err = os.Symlink(f.SymlinkPath(), extDir) - } - if err != nil { - if op.Verbose { - log.Println("Error while making symlink:", extDir) - } - return err - } - if op.IgnorePerm { - os.Chmod(extDir, op.Perm|(f.Mode()&fs.ModeType)) - } else { - os.Chmod(extDir, f.Mode()) - os.Chown(extDir, int(f.r.ids[f.i.UidInd]), int(f.r.ids[f.i.GidInd])) - } - case f.isDeviceOrFifo(): - if runtime.GOOS == "windows" { - if op.Verbose { - log.Println(extDir, "ignored since it's a device link and can't be created on Windows.") - } - return nil - } - _, err = exec.LookPath("mknod") - if err != nil { - if op.Verbose { - log.Println("Extracting Fifo IPC or Device and mknod is not in PATH") - } - return err - } - var typ string - if f.i.Type == inode.Char || f.i.Type == inode.EChar { - typ = "c" - } else if f.i.Type == inode.Block || f.i.Type == inode.EBlock { - typ = "b" - } else { //Fifo IPC - if runtime.GOOS == "darwin" { - if op.Verbose { - log.Println(extDir, "ignored since it's a Fifo file and can't be created on Darwin.") - } - return nil - } - typ = "p" - } - cmd := exec.Command("mknod", extDir, typ) - if typ != "p" { - maj, min := f.deviceDevices() - cmd.Args = append(cmd.Args, strconv.Itoa(int(maj)), strconv.Itoa(int(min))) - } - if op.Verbose { - cmd.Stdout = op.LogOutput - cmd.Stderr = op.LogOutput - } - err = cmd.Run() - if err != nil { - if op.Verbose { - log.Println("Error while running mknod for", extDir) - } - return err - } - if op.IgnorePerm { - os.Chmod(extDir, op.Perm|(f.Mode()&fs.ModeType)) - } else { - os.Chmod(extDir, f.Mode()) - os.Chown(extDir, int(f.r.ids[f.i.UidInd]), int(f.r.ids[f.i.GidInd])) - } - case f.e.Type == inode.Sock: - if op.Verbose { - log.Println(extDir, "ignored since it's a socket file.") - } - default: - return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.i.Type))) - } - return nil -} diff --git a/reader_fileinfo.go b/reader_fileinfo.go deleted file mode 100644 index 4fc875c..0000000 --- a/reader_fileinfo.go +++ /dev/null @@ -1,66 +0,0 @@ -package squashfs - -import ( - "io/fs" - "time" - - "github.com/CalebQ42/squashfs/internal/directory" - "github.com/CalebQ42/squashfs/internal/inode" -) - -type fileInfo struct { - e directory.Entry - size int64 - perm uint32 - modTime uint32 -} - -func (r Reader) newFileInfo(e directory.Entry) (fileInfo, error) { - i, err := r.inodeFromDir(e) - if err != nil { - return fileInfo{}, err - } - return newFileInfo(e, i), nil -} - -func newFileInfo(e directory.Entry, i inode.Inode) fileInfo { - var size int64 - if i.Type == inode.Fil { - size = int64(i.Data.(inode.File).Size) - } else if i.Type == inode.EFil { - size = int64(i.Data.(inode.EFile).Size) - } - return fileInfo{ - e: e, - size: size, - perm: uint32(i.Perm), - modTime: i.ModTime, - } -} - -func (f fileInfo) Name() string { - return f.e.Name -} - -func (f fileInfo) Size() int64 { - return f.size -} - -func (f fileInfo) Mode() fs.FileMode { - if f.IsDir() { - return fs.FileMode(f.perm | uint32(fs.ModeDir)) - } - return fs.FileMode(f.perm) -} - -func (f fileInfo) ModTime() time.Time { - return time.Unix(int64(f.modTime), 0) -} - -func (f fileInfo) IsDir() bool { - return f.e.Type == inode.Dir -} - -func (f fileInfo) Sys() any { - return nil -} diff --git a/reader_frag.go b/reader_frag.go deleted file mode 100644 index 0e94e2e..0000000 --- a/reader_frag.go +++ /dev/null @@ -1,26 +0,0 @@ -package squashfs - -import ( - "bytes" - "io" - - "github.com/CalebQ42/squashfs/internal/toreader" -) - -type fragEntry struct { - Start uint64 - Size uint32 - _ uint32 -} - -func (r Reader) fragReader(index uint32, fragSize uint32) (io.Reader, error) { - realSize := r.fragEntries[index].Size &^ (1 << 24) - if realSize == 0 { - return bytes.NewReader(make([]byte, fragSize)), nil - } - rdr := io.LimitReader(toreader.NewReader(r.r, int64(r.fragEntries[index].Start)), int64(realSize)) - if realSize != r.fragEntries[index].Size { - return rdr, nil - } - return r.d.Reader(rdr) -} diff --git a/reader_fs.go b/reader_fs.go deleted file mode 100644 index 60296b7..0000000 --- a/reader_fs.go +++ /dev/null @@ -1,380 +0,0 @@ -package squashfs - -import ( - "bytes" - "io" - "io/fs" - "path" - "path/filepath" - "strings" - - "github.com/CalebQ42/squashfs/internal/directory" - "github.com/CalebQ42/squashfs/internal/inode" -) - -// FS is a fs.FS representation of a squashfs directory. -// Implements fs.GlobFS, fs.ReadDirFS, fs.ReadFileFS, fs.StatFS, and fs.SubFS -type FS struct { - *File - e []directory.Entry -} - -func (r Reader) newFS(e directory.Entry, parent *FS) (*FS, error) { - i, err := r.inodeFromDir(e) - if err != nil { - return nil, err - } - ents, err := r.readDirectory(i) - if err != nil { - return nil, err - } - return &FS{ - File: &File{ - i: i, - r: &r, - parent: parent, - e: e, - }, - e: ents, - }, nil -} - -// Opens the file at name. Returns a squashfs.File. -func (f FS) OpenFile(name string) (*File, error) { - name = filepath.Clean(name) - if !fs.ValidPath(name) { - return nil, &fs.PathError{ - Op: "open", - Path: name, - Err: fs.ErrInvalid, - } - } - if name == "." || name == "" { - return f.File, nil - } - split := strings.Split(name, "/") - for i := range f.e { - if f.e[i].Name != split[0] { - continue - } - if len(split) > 1 && f.e[i].Type != inode.Dir { - return nil, &fs.PathError{ - Op: "open", - Path: name, - Err: fs.ErrNotExist, - } - } - if len(split) > 1 { - newFS, err := f.r.newFS(f.e[i], &f) - if err != nil { - return nil, &fs.PathError{ - Op: "open", - Path: name, - Err: err, - } - } - out, err := newFS.OpenFile(strings.Join(split[1:], "/")) - if err != nil { - err.(*fs.PathError).Path = name - } - return out, err - } - out, err := f.r.newFile(f.e[i], &f) - if err != nil { - err = &fs.PathError{ - Op: "open", - Path: name, - Err: err, - } - } - return out, err - } - return nil, &fs.PathError{ - Op: "open", - Path: name, - Err: fs.ErrNotExist, - } -} - -// Opens the file at name. Returns a io/fs.File. -func (f FS) Open(name string) (fs.File, error) { - return f.OpenFile(name) -} - -// Glob returns the name of the files at the given pattern. -// All paths are relative to the FS. -// Uses filepath.Match to compare names. -func (f FS) Glob(pattern string) (out []string, err error) { - pattern = filepath.Clean(pattern) - if !fs.ValidPath(pattern) { - return nil, &fs.PathError{ - Op: "glob", - Path: pattern, - Err: fs.ErrInvalid, - } - } - split := strings.Split(pattern, "/") - for i := 0; i < len(f.e); i++ { - if match, _ := path.Match(split[0], f.e[i].Name); match { - if len(split) == 1 { - out = append(out, f.e[i].Name) - continue - } - sub, err := f.Sub(split[0]) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "glob" - pathErr.Path = pattern - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "glob", - Path: pattern, - Err: err, - } - } - subGlob, err := sub.(fs.GlobFS).Glob(strings.Join(split[1:], "/")) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "glob" - pathErr.Path = pattern - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "glob", - Path: pattern, - Err: err, - } - } - for i := 0; i < len(subGlob); i++ { - subGlob[i] = f.File.e.Name + "/" + subGlob[i] - } - out = append(out, subGlob...) - } - } - return -} - -// ReadDir returns all the DirEntry returns all DirEntry's for the directory at name. -// If name is not a directory, returns an error. -func (f FS) ReadDir(name string) ([]fs.DirEntry, error) { - name = filepath.Clean(name) - if !fs.ValidPath(name) { - return nil, &fs.PathError{ - Op: "readdir", - Path: name, - Err: fs.ErrInvalid, - } - } - if name == "." || name == "" { - return f.File.ReadDir(-1) - } - split := strings.Split(name, "/") - for i := 0; i < len(f.e); i++ { - if split[0] == f.e[i].Name { - if len(split) == 1 { - fi, err := f.r.newFile(f.e[i], &f) - if err != nil { - return nil, &fs.PathError{ - Op: "readdir", - Path: name, - Err: err, - } - } - out, err := fi.ReadDir(-1) - if err != nil { - err = &fs.PathError{ - Op: "readdir", - Path: name, - Err: err, - } - } - return out, err - } - sub, err := f.Sub(split[0]) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "readir" - pathErr.Path = name - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "readdir", - Path: name, - Err: err, - } - } - redDir, err := sub.(fs.ReadDirFS).ReadDir(strings.Join(split[1:], "/")) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "readdir" - pathErr.Path = name - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "readdir", - Path: name, - Err: err, - } - } - return redDir, nil - } - } - return nil, &fs.PathError{ - Op: "readdir", - Path: name, - Err: fs.ErrNotExist, - } -} - -// ReadFile returns the data (in []byte) for the file at name. -func (f FS) ReadFile(name string) ([]byte, error) { - fil, err := f.Open(name) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - pathErr.Op = "readfile" - pathErr.Path = name - return nil, pathErr - } - } - var buf bytes.Buffer - _, err = io.Copy(&buf, fil) - if err != nil { - return nil, &fs.PathError{ - Op: "readfile", - Path: name, - Err: err, - } - } - return buf.Bytes(), nil -} - -// Stat returns the fs.FileInfo for the file at name. -func (f FS) Stat(name string) (fs.FileInfo, error) { - name = filepath.Clean(strings.TrimPrefix(name, "/")) - if !fs.ValidPath(name) { - return nil, &fs.PathError{ - Op: "stat", - Path: name, - Err: fs.ErrInvalid, - } - } - if name == "." || name == "" { - return f.File.Stat() - } - split := strings.Split(name, "/") - for i := 0; i < len(f.e); i++ { - if split[0] == f.e[i].Name { - if len(split) == 1 { - in, err := f.r.newFileInfo(f.e[i]) - if err != nil { - err = &fs.PathError{ - Op: "stat", - Path: name, - Err: err, - } - } - return in, err - } - sub, err := f.Sub(split[0]) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "stat" - pathErr.Path = name - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "stat", - Path: name, - Err: err, - } - } - stat, err := sub.(fs.StatFS).Stat(strings.Join(split[1:], "/")) - if err != nil { - if pathErr, ok := err.(*fs.PathError); ok { - if pathErr.Err == fs.ErrNotExist { - continue - } - pathErr.Op = "stat" - pathErr.Path = name - return nil, pathErr - } - return nil, &fs.PathError{ - Op: "stat", - Path: name, - Err: err, - } - } - return stat, nil - } - } - return nil, &fs.PathError{ - Op: "stat", - Path: name, - Err: fs.ErrNotExist, - } -} - -// Sub returns the FS at dir -func (f FS) Sub(dir string) (fs.FS, error) { - dir = filepath.Clean(dir) - if !fs.ValidPath(dir) { - return nil, &fs.PathError{ - Op: "sub", - Path: dir, - Err: fs.ErrInvalid, - } - } - if dir == "." || dir == "" { - return f, nil - } - split := strings.Split(dir, "/") - for i := range f.e { - if f.e[i].Name != split[0] { - continue - } - if f.e[i].Type != inode.Dir { - return nil, &fs.PathError{ - Op: "sub", - Path: dir, - Err: fs.ErrNotExist, - } - } - newFS, err := f.r.newFS(f.e[i], &f) - if err != nil { - return nil, &fs.PathError{ - Op: "sub", - Path: dir, - Err: err, - } - } - if len(split) > 1 { - ret, err := newFS.Sub(strings.Join(split[1:], "/")) - if err != nil { - err.(*fs.PathError).Path = dir - } - return ret, err - } - return newFS, nil - } - return nil, &fs.PathError{ - Op: "sub", - Path: dir, - Err: fs.ErrNotExist, - } -} diff --git a/reader_inode.go b/reader_inode.go deleted file mode 100644 index ec0516c..0000000 --- a/reader_inode.go +++ /dev/null @@ -1,117 +0,0 @@ -package squashfs - -import ( - "errors" - "io" - - "github.com/CalebQ42/squashfs/internal/data" - "github.com/CalebQ42/squashfs/internal/directory" - "github.com/CalebQ42/squashfs/internal/inode" - "github.com/CalebQ42/squashfs/internal/metadata" - "github.com/CalebQ42/squashfs/internal/toreader" -) - -func (r Reader) inodeFromRef(ref uint64) (i inode.Inode, err error) { - offset, meta := (ref>>16)+r.s.InodeTableStart, ref&0xFFFF - rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) - _, err = rdr.Read(make([]byte, meta)) - if err != nil { - return - } - return inode.Read(rdr, r.s.BlockSize) -} - -func (r Reader) inodeFromDir(e directory.Entry) (i inode.Inode, err error) { - rdr := metadata.NewReader(toreader.NewReader(r.r, int64(uint64(e.BlockStart)+r.s.InodeTableStart)), r.d) - _, err = rdr.Read(make([]byte, e.Offset)) - if err != nil { - return - } - return inode.Read(rdr, r.s.BlockSize) -} - -func (r Reader) getReaders(i inode.Inode) (full *data.FullReader, rdr *data.Reader, err error) { - var fragOffset uint64 - var blockOffset uint64 - var blockSizes []uint32 - var fragInd uint32 - var fragSize uint32 - var fileSize uint64 - if i.Type == inode.Fil { - fragOffset = uint64(i.Data.(inode.File).FragOffset) - blockOffset = uint64(i.Data.(inode.File).BlockStart) - blockSizes = i.Data.(inode.File).BlockSizes - fragInd = i.Data.(inode.File).FragInd - fragSize = i.Data.(inode.File).Size % r.s.BlockSize - fileSize = uint64(i.Data.(inode.File).Size) - } else if i.Type == inode.EFil { - fragOffset = uint64(i.Data.(inode.EFile).FragOffset) - blockOffset = i.Data.(inode.EFile).BlockStart - blockSizes = i.Data.(inode.EFile).BlockSizes - fragInd = i.Data.(inode.EFile).FragInd - fragSize = uint32(i.Data.(inode.EFile).Size % uint64(r.s.BlockSize)) - fileSize = i.Data.(inode.EFile).Size - } else { - return nil, nil, errors.New("getReaders called on non-file type") - } - rdr = data.NewReader(toreader.NewReader(r.r, int64(blockOffset)), r.d, blockSizes, r.s.BlockSize, fileSize) - full = data.NewFullReader(r.r, uint64(blockOffset), r.d, blockSizes, r.s.BlockSize, fileSize) - if fragInd != 0xFFFFFFFF { - full.AddFragment(func() (io.Reader, error) { - var fragRdr io.Reader - fragRdr, err = r.fragReader(fragInd, fragSize) - if err != nil { - return nil, err - } - var n, tmpN int - for n != int(fragOffset) { - tmpN, err = fragRdr.Read(make([]byte, int(fragOffset)-n)) - if err != nil { - return nil, err - } - n += tmpN - } - fragRdr = io.LimitReader(fragRdr, int64(fragSize)) - return fragRdr, nil - }) - var fragRdr io.Reader - fragRdr, err = r.fragReader(fragInd, fragSize) - if err != nil { - return nil, nil, err - } - var n, tmpN int - for n != int(fragOffset) { - tmpN, err = fragRdr.Read(make([]byte, int(fragOffset)-n)) - if err != nil { - return nil, nil, err - } - n += tmpN - } - fragRdr = io.LimitReader(fragRdr, int64(fragSize)) - rdr.AddFragment(fragRdr) - } - return -} - -func (r Reader) readDirectory(i inode.Inode) ([]directory.Entry, error) { - var offset uint64 - var blockOffset uint16 - var size uint32 - if i.Type == inode.Dir { - offset = uint64(i.Data.(inode.Directory).BlockStart) - blockOffset = i.Data.(inode.Directory).Offset - size = uint32(i.Data.(inode.Directory).Size) - } else if i.Type == inode.EDir { - offset = uint64(i.Data.(inode.EDirectory).BlockStart) - blockOffset = i.Data.(inode.EDirectory).Offset - size = i.Data.(inode.EDirectory).Size - } else { - return nil, errors.New("readDirectory called on non-directory type") - } - rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset+r.s.DirTableStart)), r.d) - _, err := rdr.Read(make([]byte, blockOffset)) - if err != nil { - return nil, err - } - return directory.ReadEntries(rdr, size) -} diff --git a/squashfs_test.go b/squashfs_test.go deleted file mode 100644 index 4d42960..0000000 --- a/squashfs_test.go +++ /dev/null @@ -1,207 +0,0 @@ -package squashfs_test - -//Actually proper tests go here. - -import ( - "errors" - "io" - "io/fs" - "net/http" - "os" - "os/exec" - "path/filepath" - "strconv" - "testing" - "time" - - "github.com/CalebQ42/squashfs" -) - -const ( - squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs" - squashfsName = "bug.sqfs" - - filePath = "PortableApps/Notepad++Portable/App/DefaultData/Config/contextMenu.xml" -) - -func preTest(dir string) (fil *os.File, err error) { - fil, err = os.Open(filepath.Join(dir, squashfsName)) - if err != nil { - _, err = os.Open(dir) - if os.IsNotExist(err) { - err = os.Mkdir(dir, 0755) - } - if err != nil { - return - } - os.Remove(filepath.Join(dir, squashfsName)) - fil, err = os.Create(filepath.Join(dir, squashfsName)) - if err != nil { - return - } - var resp *http.Response - resp, err = http.DefaultClient.Get(squashfsURL) - if err != nil { - return - } - _, err = io.Copy(fil, resp.Body) - if err != nil { - return - } - } - _, err = exec.LookPath("unsquashfs") - if err != nil { - return - } - _, err = exec.LookPath("mksquashfs") - return -} - -func TestMisc(t *testing.T) { - tmpDir := "testing" - fil, err := preTest(tmpDir) - if err != nil { - t.Fatal(err) - } - rdr, err := squashfs.NewReader(fil) - if err != nil { - t.Fatal(err) - } - _ = rdr - // Put testing here - t.Fatal("UM") -} - -func BenchmarkRace(b *testing.B) { - tmpDir := "testing" - fil, err := preTest(tmpDir) - if err != nil { - b.Fatal(err) - } - libPath := filepath.Join(tmpDir, "ExtractLib") - unsquashPath := filepath.Join(tmpDir, "ExtractSquashfs") - os.RemoveAll(libPath) - os.RemoveAll(unsquashPath) - var libTime, unsquashTime time.Duration - start := time.Now() - rdr, err := squashfs.NewReader(fil) - if err != nil { - b.Fatal(err) - } - err = rdr.ExtractTo(libPath) - if err != nil { - b.Fatal(err) - } - libTime = time.Since(start) - cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name()) - start = time.Now() - err = cmd.Run() - if err != nil { - b.Fatal(err) - } - unsquashTime = time.Since(start) - b.Log("Library took:", libTime.Round(time.Millisecond)) - b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond)) - b.Log("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster") -} - -func TestExtractQuick(t *testing.T) { - //First, setup everything and extract the archive using the library and unsquashfs - - // tmpDir := b.TempDir() - tmpDir := "testing" - fil, err := preTest(tmpDir) - if err != nil { - t.Fatal(err) - } - libPath := filepath.Join(tmpDir, "ExtractLib") - unsquashPath := filepath.Join(tmpDir, "ExtractSquashfs") - os.RemoveAll(libPath) - os.RemoveAll(unsquashPath) - rdr, err := squashfs.NewReader(fil) - if err != nil { - t.Fatal(err) - } - os.RemoveAll(filepath.Join(tmpDir, "testLog.txt")) - logFil, _ := os.Create(filepath.Join(tmpDir, "testLog.txt")) - op := squashfs.DefaultOptions() - op.Verbose = true - op.IgnorePerm = true - op.LogOutput = logFil - err = rdr.ExtractWithOptions(libPath, op) - if err != nil { - t.Fatal(err) - } - cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name()) - err = cmd.Run() - if err != nil { - t.Fatal(err) - } - - //Then compare the sizes and existance between the two (using unsquashfs as a reference). - //If the file doesn't exist, or the size is different, we exit. - //TODO: Add long test that checks contents. - - squashFils := os.DirFS(unsquashPath) - err = fs.WalkDir(squashFils, ".", func(path string, d fs.DirEntry, _ error) error { - libFil, e := os.Open(filepath.Join(libPath, path)) - if e != nil { - return e - } - stat, _ := d.Info() - libStat, _ := libFil.Stat() - if stat.Size() != libStat.Size() { - t.Log(path, "not the same size between library and unsquashfs") - t.Log("File is", libStat.Size()) - t.Log("Should be", stat.Size()) - return errors.New("file not the correct size") - } - return nil - }) - if err != nil { - t.Fatal(err) - } - t.Fatal("end") -} - -func TestSingleFile(t *testing.T) { - tmpDir := "testing" - fil, err := preTest(tmpDir) - if err != nil { - t.Fatal(err) - } - os.Remove(filepath.Base(filePath)) - rdr, err := squashfs.NewReader(fil) - if err != nil { - t.Fatal(err) - } - f, err := rdr.Open(filePath) - if err != nil { - t.Fatal(err) - } - err = f.(*squashfs.File).ExtractWithOptions("testing", &squashfs.ExtractionOptions{Verbose: true}) - if err != nil { - t.Fatal(err) - } - t.Fatal("HI") -} - -func TestFuse(t *testing.T) { - tmpDir := "testing" - fil, err := preTest(tmpDir) - if err != nil { - t.Fatal(err) - } - os.Remove(filepath.Base(filePath)) - rdr, err := squashfs.NewReader(fil) - if err != nil { - t.Fatal(err) - } - err = rdr.Mount("testing/fuseTest") - if err != nil { - t.Fatal(err) - } - defer rdr.Unmount() - rdr.MountWait() - t.Fatal("testing") -} From 707391babae39aecfd14e8a560d6a0ec08c6977d Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sat, 23 Dec 2023 02:48:54 -0600 Subject: [PATCH 02/11] Initial work Create Reader Pulled back in Inode decoding and superblock New Data and Metadata readers Added getting of id, fragment, and export table data lazily Added README to squashfs/squashfs --- README.md | 4 +- go.mod | 11 ++ go.sum | 10 ++ internal/data/reader.go | 73 ++++++++++ internal/decompress/decompress.go | 5 + internal/decompress/lz4.go | 15 ++ internal/decompress/lzma.go | 18 +++ internal/decompress/lzo.go | 13 ++ internal/decompress/xz.go | 18 +++ internal/decompress/zlib.go | 18 +++ internal/decompress/zstd.go | 19 +++ internal/metadata/reader.go | 62 ++++++++ internal/toreader/toreader.go | 21 +++ squashfs/README.md | 3 + squashfs/fragment.go | 7 + squashfs/inode.go | 17 +++ squashfs/inode/dir.go | 65 +++++++++ squashfs/inode/file.go | 62 ++++++++ squashfs/inode/inode.go | 144 ++++++++++++++++++ squashfs/inode/misc.go | 45 ++++++ squashfs/inode/sym.go | 46 ++++++ squashfs/reader.go | 186 ++++++++++++++++++++++++ superblock.go => squashfs/superblock.go | 0 23 files changed, 860 insertions(+), 2 deletions(-) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/data/reader.go create mode 100644 internal/decompress/decompress.go create mode 100644 internal/decompress/lz4.go create mode 100644 internal/decompress/lzma.go create mode 100644 internal/decompress/lzo.go create mode 100644 internal/decompress/xz.go create mode 100644 internal/decompress/zlib.go create mode 100644 internal/decompress/zstd.go create mode 100644 internal/metadata/reader.go create mode 100644 internal/toreader/toreader.go create mode 100644 squashfs/README.md create mode 100644 squashfs/fragment.go create mode 100644 squashfs/inode.go create mode 100644 squashfs/inode/dir.go create mode 100644 squashfs/inode/file.go create mode 100644 squashfs/inode/inode.go create mode 100644 squashfs/inode/misc.go create mode 100644 squashfs/inode/sym.go create mode 100644 squashfs/reader.go rename superblock.go => squashfs/superblock.go (100%) diff --git a/README.md b/README.md index eb6b0c0..2206edf 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PkgGoDev](https://pkg.go.dev/badge/github.com/CalebQ42/squashfs)](https://pkg.go.dev/github.com/CalebQ42/squashfs) [![Go Report Card](https://goreportcard.com/badge/github.com/CalebQ42/squashfs)](https://goreportcard.com/report/github.com/CalebQ42/squashfs) -A PURE Go library to read squashfs. There is currently no plans to add archive creation support as it will almost always be better to just call `mksquashfs`. I could see some possible use cases, but probably won't spend time on it unless it's requested (open a discussion fi you want this feature). +A PURE Go library to read squashfs. There is currently no plans to add archive creation support as it will almost always be better to just call `mksquashfs`. I could see some possible use cases, but probably won't spend time on it unless it's requested (open a discussion if you want this feature). Currently has support for reading squashfs files and extracting files and folders. @@ -15,7 +15,7 @@ Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree * No Xattr parsing. This is simply because I haven't done any research on it and how to apply these in a pure go way. * Socket files are not extracted. - * From my research, it seems like a socket file would be useless if it could be created. + * From my research, it seems like a socket file would be useless if it could be created. They are still exposed when fuse mounted. * Fifo files are ignored on `darwin` ## Issues diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..39a24a7 --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module github.com/CalebQ42/squashfs + +go 1.21.5 + +require ( + github.com/pierrec/lz4/v4 v4.1.19 + github.com/ulikunitz/xz v0.5.11 + github.com/klauspost/compress v1.17.4 + github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e + github.com/therootcompany/xz v1.0.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..894206e --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4= +github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs= +github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M= +github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= +github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= +github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= +github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= diff --git a/internal/data/reader.go b/internal/data/reader.go new file mode 100644 index 0000000..e4d5c16 --- /dev/null +++ b/internal/data/reader.go @@ -0,0 +1,73 @@ +package data + +import ( + "encoding/binary" + "io" + + "github.com/CalebQ42/squashfs/internal/decompress" +) + +type Reader struct { + r io.Reader + d decompress.Decompressor + frag io.Reader + sizes []uint32 + dat []byte + curOffset uint16 + curIndex uint64 +} + +func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32) (*Reader, error) { + return &Reader{ + r: r, + d: d, + sizes: sizes, + }, nil +} + +func (r *Reader) AddFrag(fragRdr io.Reader) { + r.frag = fragRdr +} + +func (r *Reader) advance() error { + r.curOffset = 0 + defer func() { r.curIndex++ }() + var err error + if r.curIndex == uint64(len(r.sizes))-1 && r.frag != nil { + r.dat, err = io.ReadAll(r.frag) + return err + } else if r.curIndex >= uint64(len(r.sizes))-1 { + return io.EOF + } + realSize := r.sizes[r.curIndex] &^ 0x8000 + r.dat = make([]byte, realSize) + err = binary.Read(r.r, binary.LittleEndian, &r.dat) + if err != nil { + return err + } + if r.sizes[r.curIndex] != realSize { + return nil + } + r.dat, err = r.d.Decompress(r.dat) + return err +} + +func (r *Reader) Read(b []byte) (int, error) { + curRead := 0 + var toRead int + for curRead < len(b) { + if r.curOffset >= uint16(len(r.dat)) { + if err := r.advance(); err != nil { + return curRead, err + } + } + toRead = len(b) - curRead + if toRead > len(r.dat)-int(r.curOffset) { + toRead = len(r.dat) - int(r.curOffset) + } + copy(b[curRead:], r.dat[r.curOffset:int(r.curOffset)+toRead]) + r.curOffset += uint16(toRead) + curRead += toRead + } + return curRead, nil +} diff --git a/internal/decompress/decompress.go b/internal/decompress/decompress.go new file mode 100644 index 0000000..eb66221 --- /dev/null +++ b/internal/decompress/decompress.go @@ -0,0 +1,5 @@ +package decompress + +type Decompressor interface { + Decompress([]byte) ([]byte, error) +} diff --git a/internal/decompress/lz4.go b/internal/decompress/lz4.go new file mode 100644 index 0000000..e2f3bdb --- /dev/null +++ b/internal/decompress/lz4.go @@ -0,0 +1,15 @@ +package decompress + +import ( + "bytes" + "io" + + "github.com/pierrec/lz4/v4" +) + +type Lz4 struct{} + +func (l Lz4) Decompress(data []byte) ([]byte, error) { + rdr := lz4.NewReader(bytes.NewReader(data)) + return io.ReadAll(rdr) +} diff --git a/internal/decompress/lzma.go b/internal/decompress/lzma.go new file mode 100644 index 0000000..e79d0db --- /dev/null +++ b/internal/decompress/lzma.go @@ -0,0 +1,18 @@ +package decompress + +import ( + "bytes" + "io" + + "github.com/ulikunitz/xz/lzma" +) + +type Lzma struct{} + +func (l Lzma) Decompress(data []byte) ([]byte, error) { + rdr, err := lzma.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + return io.ReadAll(rdr) +} diff --git a/internal/decompress/lzo.go b/internal/decompress/lzo.go new file mode 100644 index 0000000..f5783b4 --- /dev/null +++ b/internal/decompress/lzo.go @@ -0,0 +1,13 @@ +package decompress + +import ( + "bytes" + + "github.com/rasky/go-lzo" +) + +type Lzo struct{} + +func (l Lzo) Decompress(data []byte) ([]byte, error) { + return lzo.Decompress1X(bytes.NewReader(data), len(data), 0) +} diff --git a/internal/decompress/xz.go b/internal/decompress/xz.go new file mode 100644 index 0000000..02191db --- /dev/null +++ b/internal/decompress/xz.go @@ -0,0 +1,18 @@ +package decompress + +import ( + "bytes" + "io" + + "github.com/therootcompany/xz" +) + +type Xz struct{} + +func (x Xz) Decompress(data []byte) ([]byte, error) { + rdr, err := xz.NewReader(bytes.NewReader(data), 0) + if err != nil { + return nil, err + } + return io.ReadAll(rdr) +} diff --git a/internal/decompress/zlib.go b/internal/decompress/zlib.go new file mode 100644 index 0000000..3da3415 --- /dev/null +++ b/internal/decompress/zlib.go @@ -0,0 +1,18 @@ +package decompress + +import ( + "bytes" + "compress/zlib" + "io" +) + +type Zlib struct{} + +func (z Zlib) Decompress(data []byte) ([]byte, error) { + rdr, err := zlib.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer rdr.Close() + return io.ReadAll(rdr) +} diff --git a/internal/decompress/zstd.go b/internal/decompress/zstd.go new file mode 100644 index 0000000..ff24013 --- /dev/null +++ b/internal/decompress/zstd.go @@ -0,0 +1,19 @@ +package decompress + +import ( + "bytes" + "io" + + "github.com/klauspost/compress/zstd" +) + +type Zstd struct{} + +func (z Zstd) Decompress(data []byte) ([]byte, error) { + rdr, err := zstd.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer rdr.Close() + return io.ReadAll(rdr) +} diff --git a/internal/metadata/reader.go b/internal/metadata/reader.go new file mode 100644 index 0000000..58baa51 --- /dev/null +++ b/internal/metadata/reader.go @@ -0,0 +1,62 @@ +package metadata + +import ( + "encoding/binary" + "io" + + "github.com/CalebQ42/squashfs/internal/decompress" +) + +type Reader struct { + r io.Reader + d decompress.Decompressor + dat []byte + curOffset uint16 +} + +func NewReader(r io.Reader, d decompress.Decompressor) *Reader { + return &Reader{ + r: r, + d: d, + } +} + +func (r *Reader) advance() error { + r.curOffset = 0 + var size uint16 + err := binary.Read(r.r, binary.LittleEndian, &size) + if err != nil { + return err + } + realSize := size &^ 0x8000 + r.dat = make([]byte, realSize) + err = binary.Read(r.r, binary.LittleEndian, &r.dat) + if err != nil { + return err + } + if size != realSize { + return nil + } + r.dat, err = r.d.Decompress(r.dat) + return err +} + +func (r *Reader) Read(b []byte) (int, error) { + curRead := 0 + var toRead int + for curRead < len(b) { + if r.curOffset >= uint16(len(r.dat)) { + if err := r.advance(); err != nil { + return curRead, err + } + } + toRead = len(b) - curRead + if toRead > len(r.dat)-int(r.curOffset) { + toRead = len(r.dat) - int(r.curOffset) + } + copy(b[curRead:], r.dat[r.curOffset:int(r.curOffset)+toRead]) + r.curOffset += uint16(toRead) + curRead += toRead + } + return curRead, nil +} diff --git a/internal/toreader/toreader.go b/internal/toreader/toreader.go new file mode 100644 index 0000000..9600bd5 --- /dev/null +++ b/internal/toreader/toreader.go @@ -0,0 +1,21 @@ +package toreader + +import "io" + +type Reader struct { + r io.ReaderAt + offset int64 +} + +func NewReader(r io.ReaderAt, start int64) *Reader { + return &Reader{ + r: r, + offset: start, + } +} + +func (r *Reader) Read(b []byte) (int, error) { + n, err := r.r.ReadAt(b, r.offset) + r.offset += int64(n) + return n, err +} diff --git a/squashfs/README.md b/squashfs/README.md new file mode 100644 index 0000000..33eb8eb --- /dev/null +++ b/squashfs/README.md @@ -0,0 +1,3 @@ +# Lower-Level Squashfs + +This library is a lower level version of the main [squashfs](https://github.com/CalebQ42/squashfs) library that doesn't try to be easy to use and exposes a lot of information that is not necesary for must use cases. diff --git a/squashfs/fragment.go b/squashfs/fragment.go new file mode 100644 index 0000000..91003e0 --- /dev/null +++ b/squashfs/fragment.go @@ -0,0 +1,7 @@ +package squashfs + +type fragEntry struct { + start uint64 + size uint32 + _ uint32 +} diff --git a/squashfs/inode.go b/squashfs/inode.go new file mode 100644 index 0000000..531c0e1 --- /dev/null +++ b/squashfs/inode.go @@ -0,0 +1,17 @@ +package squashfs + +import ( + "github.com/CalebQ42/squashfs/internal/metadata" + "github.com/CalebQ42/squashfs/internal/toreader" + "github.com/CalebQ42/squashfs/squashfs/inode" +) + +func (r *Reader) inodeFromRef(ref uint64) (*inode.Inode, error) { + offset, meta := (ref>>16)+r.sup.InodeTableStart, ref&0xFFFF + rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) + _, err := rdr.Read(make([]byte, meta)) + if err != nil { + return nil, err + } + return inode.Read(rdr, r.sup.BlockSize) +} diff --git a/squashfs/inode/dir.go b/squashfs/inode/dir.go new file mode 100644 index 0000000..bffe973 --- /dev/null +++ b/squashfs/inode/dir.go @@ -0,0 +1,65 @@ +package inode + +import ( + "encoding/binary" + "io" +) + +type Directory struct { + BlockStart uint32 + LinkCount uint32 + Size uint16 + Offset uint16 + ParentNum uint32 +} + +type eDirectoryInit struct { + LinkCount uint32 + Size uint32 + BlockStart uint32 + ParentNum uint32 + IndCount uint16 + Offset uint16 + XattrInd uint32 +} + +type EDirectory struct { + eDirectoryInit + Indexes []DirectoryIndex +} + +type directoryIndexInit struct { + Ind uint32 + Start uint32 + NameSize uint32 +} + +type DirectoryIndex struct { + directoryIndexInit + Name []byte +} + +func ReadDir(r io.Reader) (d Directory, err error) { + err = binary.Read(r, binary.LittleEndian, &d) + return +} + +func ReadEDir(r io.Reader) (d EDirectory, err error) { + err = binary.Read(r, binary.LittleEndian, &d.eDirectoryInit) + if err != nil { + return + } + d.Indexes = make([]DirectoryIndex, d.IndCount) + for i := range d.Indexes { + err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].directoryIndexInit) + if err != nil { + return + } + d.Indexes[i].Name = make([]byte, d.Indexes[i].NameSize+1) + err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].Name) + if err != nil { + return + } + } + return +} diff --git a/squashfs/inode/file.go b/squashfs/inode/file.go new file mode 100644 index 0000000..1b4d461 --- /dev/null +++ b/squashfs/inode/file.go @@ -0,0 +1,62 @@ +package inode + +import ( + "encoding/binary" + "io" + "math" +) + +type fileInit struct { + BlockStart uint32 + FragInd uint32 + FragOffset uint32 + Size uint32 +} + +type File struct { + fileInit + BlockSizes []uint32 +} + +type eFileInit struct { + BlockStart uint64 + Size uint64 + Sparse uint64 + LinkCount uint32 + FragInd uint32 + FragOffset uint32 + XattrInd uint32 +} + +type EFile struct { + eFileInit + BlockSizes []uint32 +} + +func ReadFile(r io.Reader, blockSize uint32) (f File, err error) { + err = binary.Read(r, binary.LittleEndian, &f.fileInit) + if err != nil { + return + } + toRead := int(math.Floor(float64(f.Size) / float64(blockSize))) + if f.FragInd == 0xFFFFFFFF && f.Size%blockSize > 0 { + toRead++ + } + f.BlockSizes = make([]uint32, toRead) + err = binary.Read(r, binary.LittleEndian, &f.BlockSizes) + return +} + +func ReadEFile(r io.Reader, blockSize uint32) (f EFile, err error) { + err = binary.Read(r, binary.LittleEndian, &f.eFileInit) + if err != nil { + return + } + toRead := int(math.Floor(float64(f.Size) / float64(blockSize))) + if f.FragInd == 0xFFFFFFFF && f.Size%uint64(blockSize) > 0 { + toRead++ + } + f.BlockSizes = make([]uint32, toRead) + err = binary.Read(r, binary.LittleEndian, &f.BlockSizes) + return +} diff --git a/squashfs/inode/inode.go b/squashfs/inode/inode.go new file mode 100644 index 0000000..21f0a2e --- /dev/null +++ b/squashfs/inode/inode.go @@ -0,0 +1,144 @@ +package inode + +import ( + "encoding/binary" + "errors" + "io" + "io/fs" + "strconv" +) + +const ( + Dir = uint16(iota + 1) + Fil + Sym + Block + Char + Fifo + Sock + EDir + EFil + ESym + EBlock + EChar + EFifo + ESock +) + +type Header struct { + Type uint16 + Perm uint16 + UidInd uint16 + GidInd uint16 + ModTime uint32 + Num uint32 +} + +type Inode struct { + Header + Data any +} + +func Read(r io.Reader, blockSize uint32) (i *Inode, err error) { + i = new(Inode) + err = binary.Read(r, binary.LittleEndian, &i.Header) + if err != nil { + return + } + switch i.Type { + case Dir: + i.Data, err = ReadDir(r) + case Fil: + i.Data, err = ReadFile(r, blockSize) + case Sym: + i.Data, err = ReadSym(r) + case Block: + fallthrough + case Char: + i.Data, err = ReadDevice(r) + case Fifo: + fallthrough + case Sock: + i.Data, err = ReadIPC(r) + case EDir: + i.Data, err = ReadEDir(r) + case EFil: + i.Data, err = ReadEFile(r, blockSize) + case ESym: + i.Data, err = ReadESym(r) + case EBlock: + fallthrough + case EChar: + i.Data, err = ReadEDevice(r) + case EFifo: + fallthrough + case ESock: + i.Data, err = ReadEIPC(r) + default: + return i, errors.New("invalid inode type " + strconv.Itoa(int(i.Type))) + } + return +} + +func (i Inode) Mode() (out fs.FileMode) { + out = fs.FileMode(i.Perm) + switch i.Data.(type) { + case Directory: + out |= fs.ModeDir + case EDirectory: + out |= fs.ModeDir + case Symlink: + out |= fs.ModeSymlink + case ESymlink: + out |= fs.ModeSymlink + case Device: + out |= fs.ModeDevice + case EDevice: + out |= fs.ModeDevice + case IPC: + out |= fs.ModeNamedPipe + case EIPC: + out |= fs.ModeNamedPipe + } + return +} + +func (i Inode) LinkCount() uint32 { + switch i.Data.(type) { + case EFile: + return i.Data.(EFile).LinkCount + case Directory: + return i.Data.(Directory).LinkCount + case EDirectory: + return i.Data.(EDirectory).LinkCount + case Device: + return i.Data.(Device).LinkCount + case EDevice: + return i.Data.(EDevice).LinkCount + case IPC: + return i.Data.(IPC).LinkCount + case EIPC: + return i.Data.(EIPC).LinkCount + case Symlink: + return i.Data.(Symlink).LinkCount + case ESymlink: + return i.Data.(ESymlink).LinkCount + default: + return 0 + } +} + +func (i Inode) Size() uint64 { + switch i.Data.(type) { + case File: + return uint64(i.Data.(File).Size) + case EFile: + return i.Data.(EFile).Size + // case Directory: + // return uint64(i.Data.(Directory).Size) + // case EDirectory: + // return uint64(i.Data.(EDirectory).Size) + default: + return 0 + } +} diff --git a/squashfs/inode/misc.go b/squashfs/inode/misc.go new file mode 100644 index 0000000..8ba0b61 --- /dev/null +++ b/squashfs/inode/misc.go @@ -0,0 +1,45 @@ +package inode + +import ( + "encoding/binary" + "io" +) + +type Device struct { + LinkCount uint32 + Dev uint32 +} + +type EDevice struct { + Device + XattrInd uint32 +} + +func ReadDevice(r io.Reader) (d Device, err error) { + err = binary.Read(r, binary.LittleEndian, &d) + return +} + +func ReadEDevice(r io.Reader) (d EDevice, err error) { + err = binary.Read(r, binary.LittleEndian, &d) + return +} + +type IPC struct { + LinkCount uint32 +} + +type EIPC struct { + IPC + XattrInd uint32 +} + +func ReadIPC(r io.Reader) (i IPC, err error) { + err = binary.Read(r, binary.LittleEndian, &i) + return +} + +func ReadEIPC(r io.Reader) (i EIPC, err error) { + err = binary.Read(r, binary.LittleEndian, &i) + return +} diff --git a/squashfs/inode/sym.go b/squashfs/inode/sym.go new file mode 100644 index 0000000..43659bb --- /dev/null +++ b/squashfs/inode/sym.go @@ -0,0 +1,46 @@ +package inode + +import ( + "encoding/binary" + "io" +) + +type symlinkInit struct { + LinkCount uint32 + TargetSize uint32 +} + +type Symlink struct { + symlinkInit + Target []byte +} + +type ESymlink struct { + symlinkInit + Target []byte + XattrInd uint32 +} + +func ReadSym(r io.Reader) (s Symlink, err error) { + err = binary.Read(r, binary.LittleEndian, &s.symlinkInit) + if err != nil { + return + } + s.Target = make([]byte, s.TargetSize) + err = binary.Read(r, binary.LittleEndian, &s.Target) + return +} + +func ReadESym(r io.Reader) (s ESymlink, err error) { + err = binary.Read(r, binary.LittleEndian, &s.symlinkInit) + if err != nil { + return + } + s.Target = make([]byte, s.TargetSize) + err = binary.Read(r, binary.LittleEndian, &s.Target) + if err != nil { + return + } + err = binary.Read(r, binary.LittleEndian, &s.XattrInd) + return +} diff --git a/squashfs/reader.go b/squashfs/reader.go new file mode 100644 index 0000000..c4d8fbe --- /dev/null +++ b/squashfs/reader.go @@ -0,0 +1,186 @@ +package squashfs + +import ( + "encoding/binary" + "errors" + "io" + "math" + "time" + + "github.com/CalebQ42/squashfs/internal/decompress" + "github.com/CalebQ42/squashfs/internal/toreader" +) + +// The types of compression supported by squashfs +const ( + ZlibCompression = uint16(iota + 1) + LZMACompression + LZOCompression + XZCompression + LZ4Compression + ZSTDCompression +) + +var ( + ErrorMagic = errors.New("magic incorrect. probably not reading squashfs archive or archive is corrupted") + ErrorLog = errors.New("block log is incorrect. possible corrupted archive") + ErrorVersion = errors.New("squashfs version of archive is not 4.0. may be corrupted") + ErrorNotExportable = errors.New("archive does not have an export table") +) + +type Reader struct { + r io.ReaderAt + d decompress.Decompressor + fragTable []fragEntry + idTable []uint32 + exportTable []uint64 + sup superblock +} + +func NewReader(r io.ReaderAt) (rdr *Reader, err error) { + rdr = new(Reader) + rdr.r = r + err = binary.Read(toreader.NewReader(r, 0), binary.LittleEndian, &rdr.sup) + if err != nil { + return nil, errors.Join(errors.New("failed to read superblock"), err) + } + if !rdr.sup.checkMagic() { + return nil, ErrorMagic + } + if !rdr.sup.checkBlockLog() { + return nil, ErrorLog + } + if !rdr.sup.checkVersion() { + return nil, ErrorVersion + } + switch rdr.sup.CompType { + case ZlibCompression: + rdr.d = decompress.Zlib{} + case LZMACompression: + rdr.d = decompress.Lzma{} + case LZOCompression: + rdr.d = decompress.Lzo{} + case XZCompression: + rdr.d = decompress.Xz{} + case LZ4Compression: + rdr.d = decompress.Lz4{} + case ZSTDCompression: + rdr.d = &decompress.Zstd{} + default: + return nil, errors.New("invalid compression type. possible corrupted archive") + } + return +} + +// Returns the last time the archive was modified. +func (r *Reader) ModTime() time.Time { + return time.Unix(int64(r.sup.ModTime), 0) +} + +// Get a uid/gid at the given index. Lazily populates the reader's id table as necessary. +func (r *Reader) id(i uint16) (uint32, error) { + if len(r.idTable) > int(i) { + return r.idTable[i], nil + } else if i >= r.sup.IdCount { + return 0, errors.New("id out of bounds") + } + // Populate the id table as needed + blockNum := uint16(math.Ceil(float64(i) / 2048)) + blocksRead := len(r.idTable) / 2048 + blocksToRead := int(blockNum) - blocksRead + + var offset uint64 + var idsToRead uint16 + var idsTmp []uint32 + var err error + for i := blocksRead; i < int(blockNum)+blocksToRead; i++ { + err = binary.Read(toreader.NewReader(r.r, int64(r.sup.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset) + if err != nil { + return 0, err + } + idsToRead = r.sup.IdCount - uint16(len(r.idTable)) + if idsToRead > 2048 { + idsToRead = 2048 + } + idsTmp = make([]uint32, idsToRead) + err = binary.Read(toreader.NewReader(r.r, int64(offset)), binary.LittleEndian, &idsTmp) + if err != nil { + return 0, err + } + r.idTable = append(r.idTable, idsTmp...) + } + return r.idTable[i], nil +} + +// Get a fragment entry at the given index. Lazily populates the reader's fragment table as necessary. +func (r *Reader) fragEntry(i uint32) (fragEntry, error) { + if len(r.fragTable) > int(i) { + return r.fragTable[i], nil + } else if i >= r.sup.FragCount { + return fragEntry{}, errors.New("fragment out of bounds") + } + // Populate the fragment table as needed + blockNum := uint32(math.Ceil(float64(i) / 512)) + blocksRead := len(r.fragTable) / 512 + blocksToRead := int(blockNum) - blocksRead + + var offset uint64 + var fragsToRead uint32 + var fragsTmp []fragEntry + var err error + for i := blocksRead; i < int(blockNum)+blocksToRead; i++ { + err = binary.Read(toreader.NewReader(r.r, int64(r.sup.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset) + if err != nil { + return fragEntry{}, err + } + fragsToRead = r.sup.FragCount - uint32(len(r.fragTable)) + if fragsToRead > 512 { + fragsToRead = 512 + } + fragsTmp = make([]fragEntry, fragsToRead) + err = binary.Read(toreader.NewReader(r.r, int64(offset)), binary.LittleEndian, &fragsTmp) + if err != nil { + return fragEntry{}, err + } + r.fragTable = append(r.fragTable, fragsTmp...) + } + return r.fragTable[i], nil +} + +// Get an inode reference at the given index. Lazily populates the reader's export table as necessary. +func (r *Reader) inodeRef(i uint32) (uint64, error) { + if !r.sup.exportable() { + return 0, ErrorNotExportable + } + if len(r.exportTable) > int(i) { + return r.exportTable[i], nil + } else if i >= r.sup.InodeCount { + return 0, errors.New("inode out of bounds") + } + // Populate the export table as neede + blockNum := uint32(math.Ceil(float64(i) / 1024)) + blocksRead := len(r.exportTable) / 1024 + blocksToRead := int(blockNum) - blocksRead + + var offset uint64 + var refsToRead uint32 + var refsTmp []uint64 + var err error + for i := blocksRead; i < int(blockNum)+blocksToRead; i++ { + err = binary.Read(toreader.NewReader(r.r, int64(r.sup.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset) + if err != nil { + return 0, err + } + refsToRead = r.sup.InodeCount - uint32(len(r.exportTable)) + if refsToRead > 1024 { + refsToRead = 1024 + } + refsTmp = make([]uint64, refsToRead) + err = binary.Read(toreader.NewReader(r.r, int64(offset)), binary.LittleEndian, &refsTmp) + if err != nil { + return 0, err + } + r.exportTable = append(r.exportTable, refsTmp...) + } + return r.exportTable[i], nil +} diff --git a/superblock.go b/squashfs/superblock.go similarity index 100% rename from superblock.go rename to squashfs/superblock.go From 0574bbed39933d1e57a169b94a8cf49781b9a34b Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sat, 23 Dec 2023 05:47:21 -0600 Subject: [PATCH 03/11] Inode parsing and directory decoding --- internal/data/fullreader.go | 5 ++ internal/data/reader.go | 16 ++++-- squashfs/base.go | 87 +++++++++++++++++++++++++++++++++ squashfs/directory.go | 86 ++++++++++++++++++++++++++++++++ squashfs/directory/directory.go | 60 +++++++++++++++++++++++ squashfs/reader.go | 5 ++ squashfs/reader_test.go | 87 +++++++++++++++++++++++++++++++++ 7 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 internal/data/fullreader.go create mode 100644 squashfs/base.go create mode 100644 squashfs/directory.go create mode 100644 squashfs/directory/directory.go create mode 100644 squashfs/reader_test.go diff --git a/internal/data/fullreader.go b/internal/data/fullreader.go new file mode 100644 index 0000000..8e3a79f --- /dev/null +++ b/internal/data/fullreader.go @@ -0,0 +1,5 @@ +package data + +type FullReader struct{ + +} \ No newline at end of file diff --git a/internal/data/reader.go b/internal/data/reader.go index e4d5c16..82c18cc 100644 --- a/internal/data/reader.go +++ b/internal/data/reader.go @@ -10,22 +10,22 @@ import ( type Reader struct { r io.Reader d decompress.Decompressor - frag io.Reader + frag *Reader sizes []uint32 dat []byte curOffset uint16 curIndex uint64 } -func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32) (*Reader, error) { +func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32) *Reader { return &Reader{ r: r, d: d, sizes: sizes, - }, nil + } } -func (r *Reader) AddFrag(fragRdr io.Reader) { +func (r *Reader) AddFrag(fragRdr *Reader) { r.frag = fragRdr } @@ -71,3 +71,11 @@ func (r *Reader) Read(b []byte) (int, error) { } return curRead, nil } + +func (r *Reader) Close() error { + if r.frag != nil { + return r.frag.Close() + } + r.dat = nil + return nil +} diff --git a/squashfs/base.go b/squashfs/base.go new file mode 100644 index 0000000..bd5f432 --- /dev/null +++ b/squashfs/base.go @@ -0,0 +1,87 @@ +package squashfs + +import ( + "errors" + "io" + + "github.com/CalebQ42/squashfs/internal/data" + "github.com/CalebQ42/squashfs/internal/metadata" + "github.com/CalebQ42/squashfs/internal/toreader" + "github.com/CalebQ42/squashfs/squashfs/directory" + "github.com/CalebQ42/squashfs/squashfs/inode" +) + +type Base struct { + Inode *inode.Inode + Name string +} + +func (r *Reader) baseFromInode(i *inode.Inode, name string) *Base { + return &Base{Inode: i, Name: name} +} + +func (r *Reader) baseFromEntry(e directory.Entry) (*Base, error) { + rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.sup.InodeTableStart)+int64(e.BlockStart)), r.d) + rdr.Read(make([]byte, e.Offset)) + in, err := inode.Read(rdr, r.sup.BlockSize) + if err != nil { + return nil, err + } + return &Base{Inode: in, Name: e.Name}, nil +} + +func (b *Base) Uid(r *Reader) (uint32, error) { + return r.id(b.Inode.UidInd) +} + +func (b *Base) Gid(r *Reader) (uint32, error) { + return r.id(b.Inode.GidInd) +} + +func (b *Base) IsDir() bool { + return b.Inode.Type == inode.Dir || b.Inode.Type == inode.EDir +} + +func (b *Base) ToDir(r *Reader) (*Directory, error) { + return r.directoryFromInode(b.Inode, b.Name) +} + +func (b *Base) IsRegular() bool { + return b.Inode.Type == inode.Fil || b.Inode.Type == inode.EFil +} + +func (b *Base) GetRegFileReaders(r *Reader) (io.ReadCloser, error) { + if !b.IsRegular() { + return nil, errors.New("not a regular file") + } + var blockStart uint64 + var fragIndex uint32 + var fragOffset uint32 + var sizes []uint32 + if b.Inode.Type == inode.Fil { + blockStart = uint64(b.Inode.Data.(inode.File).BlockStart) + fragIndex = b.Inode.Data.(inode.File).FragInd + fragOffset = b.Inode.Data.(inode.File).FragOffset + sizes = b.Inode.Data.(inode.File).BlockSizes + } else { + blockStart = b.Inode.Data.(inode.EFile).BlockStart + fragIndex = b.Inode.Data.(inode.EFile).FragInd + fragOffset = b.Inode.Data.(inode.EFile).FragOffset + sizes = b.Inode.Data.(inode.EFile).BlockSizes + } + var frag *data.Reader + if fragIndex != 0xFFFFFFFF { + ent, err := r.fragEntry(fragIndex) + if err != nil { + return nil, err + } + frag = data.NewReader(toreader.NewReader(r.r, int64(ent.start)), r.d, []uint32{ent.size}) + frag.Read(make([]byte, fragOffset)) + } + out := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes) + if frag != nil { + out.AddFrag(out) + } + //TODO: implement and add full reader + return out, nil +} diff --git a/squashfs/directory.go b/squashfs/directory.go new file mode 100644 index 0000000..3a41e18 --- /dev/null +++ b/squashfs/directory.go @@ -0,0 +1,86 @@ +package squashfs + +import ( + "errors" + "io/fs" + "path/filepath" + "slices" + "strings" + + "github.com/CalebQ42/squashfs/internal/metadata" + "github.com/CalebQ42/squashfs/internal/toreader" + "github.com/CalebQ42/squashfs/squashfs/directory" + "github.com/CalebQ42/squashfs/squashfs/inode" +) + +type Directory struct { + Base + Entries []directory.Entry +} + +func (r *Reader) directoryFromInode(i *inode.Inode, name string) (*Directory, error) { + var blockStart uint32 + var size uint32 + var offset uint16 + switch i.Type { + case inode.Dir: + blockStart = i.Data.(inode.Directory).BlockStart + size = uint32(i.Data.(inode.Directory).Size) + offset = i.Data.(inode.Directory).Offset + case inode.EDir: + blockStart = i.Data.(inode.EDirectory).BlockStart + size = i.Data.(inode.EDirectory).Size + offset = i.Data.(inode.EDirectory).Offset + default: + return nil, errors.New("not a directory") + } + dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.sup.DirTableStart)+int64(blockStart)), r.d) + _, err := dirRdr.Read(make([]byte, offset)) + if err != nil { + return nil, err + } + entries, err := directory.ReadDirectory(dirRdr, size) + if err != nil { + return nil, err + } + return &Directory{ + Base: *r.baseFromInode(i, name), + Entries: entries, + }, nil +} + +func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) { + in, err := r.inodeFromRef(ref) + if err != nil { + return nil, err + } + return r.directoryFromInode(in, name) +} + +func (d *Directory) Open(r *Reader, path string) (*Base, error) { + path = filepath.Clean(path) + if path == "." || path == "" { + return &d.Base, nil + } + split := strings.Split(path, "/") + i, found := slices.BinarySearchFunc(d.Entries, split[0], func(e directory.Entry, name string) int { + return strings.Compare(e.Name, name) + }) + if !found { + return nil, fs.ErrNotExist + } + b, err := r.baseFromEntry(d.Entries[i]) + if err != nil { + return nil, err + } + if len(split) == 1 { + return b, nil + } else if !b.IsDir() { + return nil, fs.ErrNotExist + } + dir, err := b.ToDir(r) + if err != nil { + return nil, err + } + return dir.Open(r, strings.Join(split[1:], "/")) +} diff --git a/squashfs/directory/directory.go b/squashfs/directory/directory.go new file mode 100644 index 0000000..1464335 --- /dev/null +++ b/squashfs/directory/directory.go @@ -0,0 +1,60 @@ +package directory + +import ( + "encoding/binary" + "io" +) + +type header struct { + Count uint32 + BlockStart uint32 + Num uint32 +} + +type decEntry struct { + Offset uint16 + NumOffset int16 + InodeType uint16 + NameSize uint16 + // Name []byte (not decoded along with decEntry) +} + +type Entry struct { + Name string + BlockStart uint32 + Offset uint16 + InodeType uint16 +} + +func ReadDirectory(r io.Reader, size uint32) (out []Entry, err error) { + size -= 3 + var curRead uint32 + var h header + var de decEntry + for curRead < size { + err = binary.Read(r, binary.LittleEndian, &h) + if err != nil { + return + } + curRead += 12 + for i := uint32(0); i < h.Count+1 && curRead < size; i++ { + err = binary.Read(r, binary.LittleEndian, &de) + if err != nil { + return + } + nameTmp := make([]byte, de.NameSize+1) + err = binary.Read(r, binary.LittleEndian, &nameTmp) + if err != nil { + return + } + curRead += 8 + uint32(de.NameSize) + 1 + out = append(out, Entry{ + BlockStart: h.BlockStart, + Offset: de.Offset, + Name: string(nameTmp), + InodeType: de.InodeType, + }) + } + } + return +} diff --git a/squashfs/reader.go b/squashfs/reader.go index c4d8fbe..b1c3a84 100644 --- a/squashfs/reader.go +++ b/squashfs/reader.go @@ -31,6 +31,7 @@ var ( type Reader struct { r io.ReaderAt d decompress.Decompressor + root *Directory fragTable []fragEntry idTable []uint32 exportTable []uint64 @@ -69,6 +70,10 @@ func NewReader(r io.ReaderAt) (rdr *Reader, err error) { default: return nil, errors.New("invalid compression type. possible corrupted archive") } + rdr.root, err = rdr.directoryFromRef(rdr.sup.RootInodeRef, "") + if err != nil { + return nil, errors.Join(errors.New("failed to read root directory"), err) + } return } diff --git a/squashfs/reader_test.go b/squashfs/reader_test.go new file mode 100644 index 0000000..bba97dd --- /dev/null +++ b/squashfs/reader_test.go @@ -0,0 +1,87 @@ +package squashfs + +import ( + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/CalebQ42/squashfs/squashfs/inode" +) + +const ( + squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs" + squashfsName = "LinuxPATest.sfs" + + // filePath = "PortableApps/Notepad++Portable/App/DefaultData/Config/contextMenu.xml" +) + +func preTest(dir string) (fil *os.File, err error) { + fil, err = os.Open(filepath.Join(dir, squashfsName)) + if err != nil { + _, err = os.Open(dir) + if os.IsNotExist(err) { + err = os.Mkdir(dir, 0755) + } + if err != nil { + return + } + os.Remove(filepath.Join(dir, squashfsName)) + fil, err = os.Create(filepath.Join(dir, squashfsName)) + if err != nil { + return + } + var resp *http.Response + resp, err = http.DefaultClient.Get(squashfsURL) + if err != nil { + return + } + _, err = io.Copy(fil, resp.Body) + if err != nil { + return + } + } + _, err = exec.LookPath("unsquashfs") + if err != nil { + return + } + _, err = exec.LookPath("mksquashfs") + return +} + +func TestReader(t *testing.T) { + tmpDir := "../testing" + fil, err := preTest(tmpDir) + if err != nil { + t.Fatal(err) + } + defer fil.Close() + rdr, err := NewReader(fil) + if err != nil { + t.Fatal(err) + } + err = checkDir(rdr, rdr.root) + t.Fatal(err) +} + +func checkDir(rdr *Reader, d *Directory) error { + for _, e := range d.Entries { + if e.InodeType == inode.Dir { + b, err := d.Open(rdr, e.Name) + if err != nil { + return err + } + d, err := b.ToDir(rdr) + if err != nil { + return err + } + err = checkDir(rdr, d) + if err != nil { + return err + } + } + } + return nil +} From 0449a034280590b592d875c15b48b78bf142e2bb Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sun, 24 Dec 2023 02:55:31 -0600 Subject: [PATCH 04/11] Added data.FullReader and moved to low level library Added ability to get readers from Base --- internal/data/fullreader.go | 5 - squashfs/base.go | 30 +++-- squashfs/data/fullreader.go | 153 ++++++++++++++++++++++++++ {internal => squashfs}/data/reader.go | 0 4 files changed, 172 insertions(+), 16 deletions(-) delete mode 100644 internal/data/fullreader.go create mode 100644 squashfs/data/fullreader.go rename {internal => squashfs}/data/reader.go (100%) diff --git a/internal/data/fullreader.go b/internal/data/fullreader.go deleted file mode 100644 index 8e3a79f..0000000 --- a/internal/data/fullreader.go +++ /dev/null @@ -1,5 +0,0 @@ -package data - -type FullReader struct{ - -} \ No newline at end of file diff --git a/squashfs/base.go b/squashfs/base.go index bd5f432..10351a1 100644 --- a/squashfs/base.go +++ b/squashfs/base.go @@ -4,9 +4,10 @@ import ( "errors" "io" - "github.com/CalebQ42/squashfs/internal/data" + "github.com/CalebQ42/squashfs/internal/decompress" "github.com/CalebQ42/squashfs/internal/metadata" "github.com/CalebQ42/squashfs/internal/toreader" + "github.com/CalebQ42/squashfs/squashfs/data" "github.com/CalebQ42/squashfs/squashfs/directory" "github.com/CalebQ42/squashfs/squashfs/inode" ) @@ -50,9 +51,9 @@ func (b *Base) IsRegular() bool { return b.Inode.Type == inode.Fil || b.Inode.Type == inode.EFil } -func (b *Base) GetRegFileReaders(r *Reader) (io.ReadCloser, error) { +func (b *Base) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, error) { if !b.IsRegular() { - return nil, errors.New("not a regular file") + return nil, nil, errors.New("not a regular file") } var blockStart uint64 var fragIndex uint32 @@ -69,19 +70,26 @@ func (b *Base) GetRegFileReaders(r *Reader) (io.ReadCloser, error) { fragOffset = b.Inode.Data.(inode.EFile).FragOffset sizes = b.Inode.Data.(inode.EFile).BlockSizes } - var frag *data.Reader - if fragIndex != 0xFFFFFFFF { + frag := func(rdr io.ReaderAt, d decompress.Decompressor) (*data.Reader, error) { ent, err := r.fragEntry(fragIndex) if err != nil { return nil, err } - frag = data.NewReader(toreader.NewReader(r.r, int64(ent.start)), r.d, []uint32{ent.size}) + frag := data.NewReader(toreader.NewReader(r.r, int64(ent.start)), r.d, []uint32{ent.size}) frag.Read(make([]byte, fragOffset)) + return frag, nil } - out := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes) - if frag != nil { - out.AddFrag(out) + outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes) + if fragIndex != 0xffffffff { + f, err := frag(r.r, r.d) + if err != nil { + return nil, nil, err + } + outRdr.AddFrag(f) } - //TODO: implement and add full reader - return out, nil + outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes) + if fragIndex != 0xffffffff { + outFull.AddFrag(frag) + } + return outRdr, outFull, nil } diff --git a/squashfs/data/fullreader.go b/squashfs/data/fullreader.go new file mode 100644 index 0000000..8c1d48b --- /dev/null +++ b/squashfs/data/fullreader.go @@ -0,0 +1,153 @@ +package data + +import ( + "encoding/binary" + "errors" + "io" + "sync" + + "github.com/CalebQ42/squashfs/internal/decompress" + "github.com/CalebQ42/squashfs/internal/toreader" +) + +type FragReaderConstructor func(io.ReaderAt, decompress.Decompressor) (*Reader, error) + +type FullReader struct { + r io.ReaderAt + d decompress.Decompressor + frag FragReaderConstructor + retPool *sync.Pool + sizes []uint32 + initialOffset int64 + goroutineLimit uint16 +} + +func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32) *FullReader { + return &FullReader{ + r: r, + d: d, + sizes: sizes, + initialOffset: initialOffset, + goroutineLimit: 10, + retPool: &sync.Pool{ + New: func() any { + return &retValue{} + }, + }, + } +} + +func (r *FullReader) AddFrag(frag FragReaderConstructor) { + r.frag = frag +} + +func (r *FullReader) SetGoroutineLimit(limit uint16) { + r.goroutineLimit = limit +} + +type retValue struct { + err error + data []byte + index uint64 +} + +func (r *FullReader) process(index uint64, fileOffset uint64, retChan chan *retValue) { + ret := r.retPool.Get().(*retValue) + ret.index = index + realSize := r.sizes[index] &^ (1 << 24) + ret.data = make([]byte, realSize) + ret.err = binary.Read(toreader.NewReader(r.r, int64(r.initialOffset)+int64(fileOffset)), binary.LittleEndian, &ret.data) + if r.sizes[index] == realSize { + ret.data, ret.err = r.d.Decompress(ret.data) + } + retChan <- ret +} + +func (r *FullReader) WriteTo(w io.Writer) (int64, error) { + var curIndex uint64 + var curOffset uint64 + var toProcess uint16 + var wrote int64 + cache := make(map[uint64]*retValue) + var errCache []error + retChan := make(chan *retValue, r.goroutineLimit) + for i := uint64(0); i < uint64(len(r.sizes))/uint64(r.goroutineLimit); i++ { + toProcess = uint16(len(r.sizes)) - (uint16(i) * r.goroutineLimit) + if toProcess > r.goroutineLimit { + toProcess = r.goroutineLimit + } + // Start all the goroutines + for j := uint16(0); j < toProcess; j++ { + go r.process((i*uint64(r.goroutineLimit))+uint64(j), curOffset, retChan) + curOffset += uint64(r.sizes[(i*uint64(r.goroutineLimit))+uint64(j)]) &^ (1 << 24) + } + // Then consume the results on retChan + for j := uint16(0); j < toProcess; j++ { + res := <-retChan + // If there's an error, we don't care about the results. + if res.err != nil { + errCache = append(errCache, res.err) + if len(cache) > 0 { + clear(cache) + } + continue + } + // If there has been an error previously, we don't care about the results. + // We still want to wait for all the goroutines to prevent resources being wasted. + if len(errCache) > 0 { + continue + } + // If we don't need the data yet, we cache it and move on + if res.index != curIndex { + cache[res.index] = res + continue + } + // If we do need the data, we write it + wr, err := w.Write(res.data) + wrote += int64(wr) + if err != nil { + errCache = append(errCache, err) + if len(cache) > 0 { + clear(cache) + } + continue + } + r.retPool.Put(res) + curIndex++ + // Now we recursively try to clear the cache + for len(cache) > 0 { + res, ok := cache[curIndex] + if !ok { + break + } + wr, err := w.Write(res.data) + wrote += int64(wr) + if err != nil { + errCache = append(errCache, err) + if len(cache) > 0 { + clear(cache) + } + break + } + delete(cache, curIndex) + r.retPool.Put(res) + curIndex++ + } + } + if len(errCache) > 0 { + return wrote, errors.Join(errCache...) + } + } + if r.frag != nil { + rdr, err := r.frag(r.r, r.d) + if err != nil { + return wrote, err + } + wr, err := io.Copy(w, rdr) + wrote += wr + if err != nil { + return wrote, err + } + } + return wrote, nil +} diff --git a/internal/data/reader.go b/squashfs/data/reader.go similarity index 100% rename from internal/data/reader.go rename to squashfs/data/reader.go From b2a3920c1f394395f40d845e06ce22e592e8e6ca Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sun, 24 Dec 2023 06:02:11 -0600 Subject: [PATCH 05/11] Finished? Everything seems to extract fine (though more testing is needed) --- internal/metadata/reader.go | 5 ++ squashfs/base.go | 67 ++++++++++++++++++----- squashfs/data/fullreader.go | 28 ++++++++-- squashfs/data/reader.go | 58 ++++++++++++-------- squashfs/directory.go | 25 +++++---- squashfs/fragment.go | 4 +- squashfs/inode.go | 5 +- squashfs/reader.go | 102 ++++++++++++++++++++++++------------ squashfs/reader_test.go | 73 +++++++++++++++++++++----- squashfs/superblock.go | 28 +++++----- 10 files changed, 277 insertions(+), 118 deletions(-) diff --git a/internal/metadata/reader.go b/internal/metadata/reader.go index 58baa51..b20d63c 100644 --- a/internal/metadata/reader.go +++ b/internal/metadata/reader.go @@ -60,3 +60,8 @@ func (r *Reader) Read(b []byte) (int, error) { } return curRead, nil } + +func (r *Reader) Close() error { + r.dat = nil + return nil +} diff --git a/squashfs/base.go b/squashfs/base.go index 10351a1..2bac355 100644 --- a/squashfs/base.go +++ b/squashfs/base.go @@ -4,7 +4,6 @@ import ( "errors" "io" - "github.com/CalebQ42/squashfs/internal/decompress" "github.com/CalebQ42/squashfs/internal/metadata" "github.com/CalebQ42/squashfs/internal/toreader" "github.com/CalebQ42/squashfs/squashfs/data" @@ -17,26 +16,35 @@ type Base struct { Name string } -func (r *Reader) baseFromInode(i *inode.Inode, name string) *Base { +func (r *Reader) BaseFromInode(i *inode.Inode, name string) *Base { return &Base{Inode: i, Name: name} } -func (r *Reader) baseFromEntry(e directory.Entry) (*Base, error) { - rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.sup.InodeTableStart)+int64(e.BlockStart)), r.d) +func (r *Reader) BaseFromEntry(e directory.Entry) (*Base, error) { + rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d) + defer rdr.Close() rdr.Read(make([]byte, e.Offset)) - in, err := inode.Read(rdr, r.sup.BlockSize) + in, err := inode.Read(rdr, r.Superblock.BlockSize) if err != nil { return nil, err } return &Base{Inode: in, Name: e.Name}, nil } +func (r *Reader) BaseFromRef(ref uint64, name string) (*Base, error) { + in, err := r.inodeFromRef(ref) + if err != nil { + return nil, err + } + return &Base{Inode: in, Name: name}, nil +} + func (b *Base) Uid(r *Reader) (uint32, error) { - return r.id(b.Inode.UidInd) + return r.Id(b.Inode.UidInd) } func (b *Base) Gid(r *Reader) (uint32, error) { - return r.id(b.Inode.GidInd) + return r.Id(b.Inode.GidInd) } func (b *Base) IsDir() bool { @@ -44,7 +52,35 @@ func (b *Base) IsDir() bool { } func (b *Base) ToDir(r *Reader) (*Directory, error) { - return r.directoryFromInode(b.Inode, b.Name) + var blockStart uint32 + var size uint32 + var offset uint16 + switch b.Inode.Type { + case inode.Dir: + blockStart = b.Inode.Data.(inode.Directory).BlockStart + size = uint32(b.Inode.Data.(inode.Directory).Size) + offset = b.Inode.Data.(inode.Directory).Offset + case inode.EDir: + blockStart = b.Inode.Data.(inode.EDirectory).BlockStart + size = b.Inode.Data.(inode.EDirectory).Size + offset = b.Inode.Data.(inode.EDirectory).Offset + default: + return nil, errors.New("not a directory") + } + dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d) + defer dirRdr.Close() + _, err := dirRdr.Read(make([]byte, offset)) + if err != nil { + return nil, err + } + entries, err := directory.ReadDirectory(dirRdr, size) + if err != nil { + return nil, err + } + return &Directory{ + Base: *b, + Entries: entries, + }, nil } func (b *Base) IsRegular() bool { @@ -58,36 +94,39 @@ func (b *Base) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, err var blockStart uint64 var fragIndex uint32 var fragOffset uint32 + var fragSize uint64 var sizes []uint32 if b.Inode.Type == inode.Fil { blockStart = uint64(b.Inode.Data.(inode.File).BlockStart) fragIndex = b.Inode.Data.(inode.File).FragInd fragOffset = b.Inode.Data.(inode.File).FragOffset sizes = b.Inode.Data.(inode.File).BlockSizes + fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize) } else { blockStart = b.Inode.Data.(inode.EFile).BlockStart fragIndex = b.Inode.Data.(inode.EFile).FragInd fragOffset = b.Inode.Data.(inode.EFile).FragOffset sizes = b.Inode.Data.(inode.EFile).BlockSizes + fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize) } - frag := func(rdr io.ReaderAt, d decompress.Decompressor) (*data.Reader, error) { + frag := func() (io.Reader, error) { ent, err := r.fragEntry(fragIndex) if err != nil { return nil, err } - frag := data.NewReader(toreader.NewReader(r.r, int64(ent.start)), r.d, []uint32{ent.size}) + frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize) frag.Read(make([]byte, fragOffset)) - return frag, nil + return io.LimitReader(frag, int64(fragSize)), nil } - outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes) + outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize) if fragIndex != 0xffffffff { - f, err := frag(r.r, r.d) + f, err := frag() if err != nil { return nil, nil, err } outRdr.AddFrag(f) } - outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes) + outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes, fragSize, r.Superblock.BlockSize) if fragIndex != 0xffffffff { outFull.AddFrag(frag) } diff --git a/squashfs/data/fullreader.go b/squashfs/data/fullreader.go index 8c1d48b..d085e90 100644 --- a/squashfs/data/fullreader.go +++ b/squashfs/data/fullreader.go @@ -4,13 +4,14 @@ import ( "encoding/binary" "errors" "io" + "math" "sync" "github.com/CalebQ42/squashfs/internal/decompress" "github.com/CalebQ42/squashfs/internal/toreader" ) -type FragReaderConstructor func(io.ReaderAt, decompress.Decompressor) (*Reader, error) +type FragReaderConstructor func() (io.Reader, error) type FullReader struct { r io.ReaderAt @@ -19,16 +20,20 @@ type FullReader struct { retPool *sync.Pool sizes []uint32 initialOffset int64 + finalBlockSize uint64 + blockSize uint32 goroutineLimit uint16 } -func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32) *FullReader { +func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) *FullReader { return &FullReader{ r: r, d: d, sizes: sizes, initialOffset: initialOffset, goroutineLimit: 10, + finalBlockSize: finalBlockSize, + blockSize: blockSize, retPool: &sync.Pool{ New: func() any { return &retValue{} @@ -55,6 +60,16 @@ func (r *FullReader) process(index uint64, fileOffset uint64, retChan chan *retV ret := r.retPool.Get().(*retValue) ret.index = index realSize := r.sizes[index] &^ (1 << 24) + if realSize == 0 { + if index == uint64(len(r.sizes))-1 && r.frag == nil { + ret.data = make([]byte, r.finalBlockSize) + } else { + ret.data = make([]byte, r.blockSize) + } + ret.err = nil + retChan <- ret + return + } ret.data = make([]byte, realSize) ret.err = binary.Read(toreader.NewReader(r.r, int64(r.initialOffset)+int64(fileOffset)), binary.LittleEndian, &ret.data) if r.sizes[index] == realSize { @@ -71,7 +86,7 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) { cache := make(map[uint64]*retValue) var errCache []error retChan := make(chan *retValue, r.goroutineLimit) - for i := uint64(0); i < uint64(len(r.sizes))/uint64(r.goroutineLimit); i++ { + for i := uint64(0); i < uint64(math.Ceil(float64(len(r.sizes))/float64(r.goroutineLimit))); i++ { toProcess = uint16(len(r.sizes)) - (uint16(i) * r.goroutineLimit) if toProcess > r.goroutineLimit { toProcess = r.goroutineLimit @@ -139,12 +154,17 @@ func (r *FullReader) WriteTo(w io.Writer) (int64, error) { } } if r.frag != nil { - rdr, err := r.frag(r.r, r.d) + rdr, err := r.frag() if err != nil { return wrote, err } wr, err := io.Copy(w, rdr) wrote += wr + if l, ok := rdr.(*io.LimitedReader); ok { + if cl, ok := l.R.(io.Closer); ok { + cl.Close() + } + } if err != nil { return wrote, err } diff --git a/squashfs/data/reader.go b/squashfs/data/reader.go index 82c18cc..922263d 100644 --- a/squashfs/data/reader.go +++ b/squashfs/data/reader.go @@ -8,24 +8,28 @@ import ( ) type Reader struct { - r io.Reader - d decompress.Decompressor - frag *Reader - sizes []uint32 - dat []byte - curOffset uint16 - curIndex uint64 + r io.Reader + d decompress.Decompressor + frag io.Reader + sizes []uint32 + dat []byte + curOffset int + curIndex uint64 + finalBlockSize uint64 + blockSize uint32 } -func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32) *Reader { +func NewReader(r io.Reader, d decompress.Decompressor, sizes []uint32, finalBlockSize uint64, blockSize uint32) *Reader { return &Reader{ - r: r, - d: d, - sizes: sizes, + r: r, + d: d, + sizes: sizes, + finalBlockSize: finalBlockSize, + blockSize: blockSize, } } -func (r *Reader) AddFrag(fragRdr *Reader) { +func (r *Reader) AddFrag(fragRdr io.Reader) { r.frag = fragRdr } @@ -33,13 +37,21 @@ func (r *Reader) advance() error { r.curOffset = 0 defer func() { r.curIndex++ }() var err error - if r.curIndex == uint64(len(r.sizes))-1 && r.frag != nil { + if r.curIndex == uint64(len(r.sizes)) && r.frag != nil { r.dat, err = io.ReadAll(r.frag) return err - } else if r.curIndex >= uint64(len(r.sizes))-1 { + } else if r.curIndex >= uint64(len(r.sizes)) { return io.EOF } - realSize := r.sizes[r.curIndex] &^ 0x8000 + realSize := r.sizes[r.curIndex] &^ (1 << 24) + if realSize == 0 { + if r.curIndex == uint64(len(r.sizes))-1 && r.frag == nil { + r.dat = make([]byte, r.finalBlockSize) + } else { + r.dat = make([]byte, r.blockSize) + } + return nil + } r.dat = make([]byte, realSize) err = binary.Read(r.r, binary.LittleEndian, &r.dat) if err != nil { @@ -56,17 +68,17 @@ func (r *Reader) Read(b []byte) (int, error) { curRead := 0 var toRead int for curRead < len(b) { - if r.curOffset >= uint16(len(r.dat)) { + if r.curOffset >= len(r.dat) { if err := r.advance(); err != nil { return curRead, err } } toRead = len(b) - curRead - if toRead > len(r.dat)-int(r.curOffset) { - toRead = len(r.dat) - int(r.curOffset) + if toRead > len(r.dat)-r.curOffset { + toRead = len(r.dat) - r.curOffset } - copy(b[curRead:], r.dat[r.curOffset:int(r.curOffset)+toRead]) - r.curOffset += uint16(toRead) + toRead = copy(b[curRead:], r.dat[r.curOffset:r.curOffset+toRead]) + r.curOffset += toRead curRead += toRead } return curRead, nil @@ -74,7 +86,11 @@ func (r *Reader) Read(b []byte) (int, error) { func (r *Reader) Close() error { if r.frag != nil { - return r.frag.Close() + if l, ok := r.frag.(*io.LimitedReader); ok { + if cl, ok := l.R.(io.Closer); ok { + cl.Close() + } + } } r.dat = nil return nil diff --git a/squashfs/directory.go b/squashfs/directory.go index 3a41e18..9e7db15 100644 --- a/squashfs/directory.go +++ b/squashfs/directory.go @@ -2,6 +2,7 @@ package squashfs import ( "errors" + "fmt" "io/fs" "path/filepath" "slices" @@ -18,7 +19,12 @@ type Directory struct { Entries []directory.Entry } -func (r *Reader) directoryFromInode(i *inode.Inode, name string) (*Directory, error) { +func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) { + i, err := r.inodeFromRef(ref) + if err != nil { + fmt.Println("yo") + return nil, err + } var blockStart uint32 var size uint32 var offset uint16 @@ -34,8 +40,9 @@ func (r *Reader) directoryFromInode(i *inode.Inode, name string) (*Directory, er default: return nil, errors.New("not a directory") } - dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.sup.DirTableStart)+int64(blockStart)), r.d) - _, err := dirRdr.Read(make([]byte, offset)) + dirRdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.DirTableStart)+int64(blockStart)), r.d) + defer dirRdr.Close() + _, err = dirRdr.Read(make([]byte, offset)) if err != nil { return nil, err } @@ -44,19 +51,11 @@ func (r *Reader) directoryFromInode(i *inode.Inode, name string) (*Directory, er return nil, err } return &Directory{ - Base: *r.baseFromInode(i, name), + Base: *r.BaseFromInode(i, name), Entries: entries, }, nil } -func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) { - in, err := r.inodeFromRef(ref) - if err != nil { - return nil, err - } - return r.directoryFromInode(in, name) -} - func (d *Directory) Open(r *Reader, path string) (*Base, error) { path = filepath.Clean(path) if path == "." || path == "" { @@ -69,7 +68,7 @@ func (d *Directory) Open(r *Reader, path string) (*Base, error) { if !found { return nil, fs.ErrNotExist } - b, err := r.baseFromEntry(d.Entries[i]) + b, err := r.BaseFromEntry(d.Entries[i]) if err != nil { return nil, err } diff --git a/squashfs/fragment.go b/squashfs/fragment.go index 91003e0..ad103d9 100644 --- a/squashfs/fragment.go +++ b/squashfs/fragment.go @@ -1,7 +1,7 @@ package squashfs type fragEntry struct { - start uint64 - size uint32 + Start uint64 + Size uint32 _ uint32 } diff --git a/squashfs/inode.go b/squashfs/inode.go index 531c0e1..205c5ce 100644 --- a/squashfs/inode.go +++ b/squashfs/inode.go @@ -7,11 +7,12 @@ import ( ) func (r *Reader) inodeFromRef(ref uint64) (*inode.Inode, error) { - offset, meta := (ref>>16)+r.sup.InodeTableStart, ref&0xFFFF + offset, meta := (ref>>16)+r.Superblock.InodeTableStart, ref&0xFFFF rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) + defer rdr.Close() _, err := rdr.Read(make([]byte, meta)) if err != nil { return nil, err } - return inode.Read(rdr, r.sup.BlockSize) + return inode.Read(rdr, r.Superblock.BlockSize) } diff --git a/squashfs/reader.go b/squashfs/reader.go index b1c3a84..3e8ec89 100644 --- a/squashfs/reader.go +++ b/squashfs/reader.go @@ -8,7 +8,9 @@ import ( "time" "github.com/CalebQ42/squashfs/internal/decompress" + "github.com/CalebQ42/squashfs/internal/metadata" "github.com/CalebQ42/squashfs/internal/toreader" + "github.com/CalebQ42/squashfs/squashfs/inode" ) // The types of compression supported by squashfs @@ -31,30 +33,30 @@ var ( type Reader struct { r io.ReaderAt d decompress.Decompressor - root *Directory + Root *Directory fragTable []fragEntry idTable []uint32 exportTable []uint64 - sup superblock + Superblock superblock } func NewReader(r io.ReaderAt) (rdr *Reader, err error) { rdr = new(Reader) rdr.r = r - err = binary.Read(toreader.NewReader(r, 0), binary.LittleEndian, &rdr.sup) + err = binary.Read(toreader.NewReader(r, 0), binary.LittleEndian, &rdr.Superblock) if err != nil { return nil, errors.Join(errors.New("failed to read superblock"), err) } - if !rdr.sup.checkMagic() { + if !rdr.Superblock.ValidMagic() { return nil, ErrorMagic } - if !rdr.sup.checkBlockLog() { + if !rdr.Superblock.ValidBlockLog() { return nil, ErrorLog } - if !rdr.sup.checkVersion() { + if !rdr.Superblock.ValidVersion() { return nil, ErrorVersion } - switch rdr.sup.CompType { + switch rdr.Superblock.CompType { case ZlibCompression: rdr.d = decompress.Zlib{} case LZMACompression: @@ -70,7 +72,7 @@ func NewReader(r io.ReaderAt) (rdr *Reader, err error) { default: return nil, errors.New("invalid compression type. possible corrupted archive") } - rdr.root, err = rdr.directoryFromRef(rdr.sup.RootInodeRef, "") + rdr.Root, err = rdr.directoryFromRef(rdr.Superblock.RootInodeRef, "") if err != nil { return nil, errors.Join(errors.New("failed to read root directory"), err) } @@ -79,36 +81,44 @@ func NewReader(r io.ReaderAt) (rdr *Reader, err error) { // Returns the last time the archive was modified. func (r *Reader) ModTime() time.Time { - return time.Unix(int64(r.sup.ModTime), 0) + return time.Unix(int64(r.Superblock.ModTime), 0) } -// Get a uid/gid at the given index. Lazily populates the reader's id table as necessary. -func (r *Reader) id(i uint16) (uint32, error) { +// Get a uid/gid at the given index. Lazily populates the reader's Id table as necessary. +func (r *Reader) Id(i uint16) (uint32, error) { if len(r.idTable) > int(i) { return r.idTable[i], nil - } else if i >= r.sup.IdCount { + } else if i >= r.Superblock.IdCount { return 0, errors.New("id out of bounds") } // Populate the id table as needed - blockNum := uint16(math.Ceil(float64(i) / 2048)) + var blockNum uint32 + if i != 0 { // If i == 0, we go negatives causing issues with uint32s + blockNum = uint32(math.Ceil(float64(i)/2048)) - 1 + } else { + blockNum = 0 + } blocksRead := len(r.idTable) / 2048 - blocksToRead := int(blockNum) - blocksRead + blocksToRead := int(blockNum) - blocksRead + 1 var offset uint64 var idsToRead uint16 var idsTmp []uint32 var err error - for i := blocksRead; i < int(blockNum)+blocksToRead; i++ { - err = binary.Read(toreader.NewReader(r.r, int64(r.sup.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset) + var rdr *metadata.Reader + for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { + err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.IdTableStart)+int64(8*i)), binary.LittleEndian, &offset) if err != nil { return 0, err } - idsToRead = r.sup.IdCount - uint16(len(r.idTable)) + idsToRead = r.Superblock.IdCount - uint16(len(r.idTable)) if idsToRead > 2048 { idsToRead = 2048 } idsTmp = make([]uint32, idsToRead) - err = binary.Read(toreader.NewReader(r.r, int64(offset)), binary.LittleEndian, &idsTmp) + rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) + err = binary.Read(rdr, binary.LittleEndian, &idsTmp) + rdr.Close() if err != nil { return 0, err } @@ -121,29 +131,37 @@ func (r *Reader) id(i uint16) (uint32, error) { func (r *Reader) fragEntry(i uint32) (fragEntry, error) { if len(r.fragTable) > int(i) { return r.fragTable[i], nil - } else if i >= r.sup.FragCount { + } else if i >= r.Superblock.FragCount { return fragEntry{}, errors.New("fragment out of bounds") } // Populate the fragment table as needed - blockNum := uint32(math.Ceil(float64(i) / 512)) + var blockNum uint32 + if i != 0 { // If i == 0, we go negatives causing issues with uint32s + blockNum = uint32(math.Ceil(float64(i)/512)) - 1 + } else { + blockNum = 0 + } blocksRead := len(r.fragTable) / 512 - blocksToRead := int(blockNum) - blocksRead + blocksToRead := int(blockNum) - blocksRead + 1 var offset uint64 var fragsToRead uint32 var fragsTmp []fragEntry var err error - for i := blocksRead; i < int(blockNum)+blocksToRead; i++ { - err = binary.Read(toreader.NewReader(r.r, int64(r.sup.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset) + var rdr *metadata.Reader + for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { + err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.FragTableStart)+int64(8*i)), binary.LittleEndian, &offset) if err != nil { return fragEntry{}, err } - fragsToRead = r.sup.FragCount - uint32(len(r.fragTable)) + fragsToRead = r.Superblock.FragCount - uint32(len(r.fragTable)) if fragsToRead > 512 { fragsToRead = 512 } fragsTmp = make([]fragEntry, fragsToRead) - err = binary.Read(toreader.NewReader(r.r, int64(offset)), binary.LittleEndian, &fragsTmp) + rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) + err = binary.Read(rdr, binary.LittleEndian, &fragsTmp) + rdr.Close() if err != nil { return fragEntry{}, err } @@ -154,34 +172,42 @@ func (r *Reader) fragEntry(i uint32) (fragEntry, error) { // Get an inode reference at the given index. Lazily populates the reader's export table as necessary. func (r *Reader) inodeRef(i uint32) (uint64, error) { - if !r.sup.exportable() { + if !r.Superblock.Exportable() { return 0, ErrorNotExportable } if len(r.exportTable) > int(i) { return r.exportTable[i], nil - } else if i >= r.sup.InodeCount { + } else if i >= r.Superblock.InodeCount { return 0, errors.New("inode out of bounds") } - // Populate the export table as neede - blockNum := uint32(math.Ceil(float64(i) / 1024)) + // Populate the export table as needed + var blockNum uint32 + if i != 0 { // If i == 0, we go negatives causing issues with uint32s + blockNum = uint32(math.Ceil(float64(i)/1024)) - 1 + } else { + blockNum = 0 + } blocksRead := len(r.exportTable) / 1024 - blocksToRead := int(blockNum) - blocksRead + blocksToRead := int(blockNum) - blocksRead + 1 var offset uint64 var refsToRead uint32 var refsTmp []uint64 var err error - for i := blocksRead; i < int(blockNum)+blocksToRead; i++ { - err = binary.Read(toreader.NewReader(r.r, int64(r.sup.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset) + var rdr *metadata.Reader + for i := blocksRead; i < int(blocksRead)+blocksToRead; i++ { + err = binary.Read(toreader.NewReader(r.r, int64(r.Superblock.ExportTableStart)+int64(8*i)), binary.LittleEndian, &offset) if err != nil { return 0, err } - refsToRead = r.sup.InodeCount - uint32(len(r.exportTable)) + refsToRead = r.Superblock.InodeCount - uint32(len(r.exportTable)) if refsToRead > 1024 { refsToRead = 1024 } refsTmp = make([]uint64, refsToRead) - err = binary.Read(toreader.NewReader(r.r, int64(offset)), binary.LittleEndian, &refsTmp) + rdr = metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) + err = binary.Read(rdr, binary.LittleEndian, &refsTmp) + rdr.Close() if err != nil { return 0, err } @@ -189,3 +215,11 @@ func (r *Reader) inodeRef(i uint32) (uint64, error) { } return r.exportTable[i], nil } + +func (r *Reader) Inode(i uint32) (*inode.Inode, error) { + ref, err := r.inodeRef(i) + if err != nil { + return nil, err + } + return r.inodeFromRef(ref) +} diff --git a/squashfs/reader_test.go b/squashfs/reader_test.go index bba97dd..38bd4ad 100644 --- a/squashfs/reader_test.go +++ b/squashfs/reader_test.go @@ -1,21 +1,18 @@ package squashfs import ( + "fmt" "io" "net/http" "os" "os/exec" "path/filepath" "testing" - - "github.com/CalebQ42/squashfs/squashfs/inode" ) const ( squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs" squashfsName = "LinuxPATest.sfs" - - // filePath = "PortableApps/Notepad++Portable/App/DefaultData/Config/contextMenu.xml" ) func preTest(dir string) (fil *os.File, err error) { @@ -62,26 +59,74 @@ func TestReader(t *testing.T) { if err != nil { t.Fatal(err) } - err = checkDir(rdr, rdr.root) + path := filepath.Join(tmpDir, "extractTest") + os.RemoveAll(path) + os.MkdirAll(path, 0777) + err = extractToDir(rdr, &rdr.Root.Base, path) t.Fatal(err) } -func checkDir(rdr *Reader, d *Directory) error { - for _, e := range d.Entries { - if e.InodeType == inode.Dir { - b, err := d.Open(rdr, e.Name) +var singleFile = "PortableApps/CPU-X/CPU-X-v4.2.0-x86_64.AppImage" + +func TestSingleFile(t *testing.T) { + tmpDir := "../testing" + fil, err := preTest(tmpDir) + if err != nil { + t.Fatal(err) + } + defer fil.Close() + rdr, err := NewReader(fil) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(tmpDir, "extractTest") + os.RemoveAll(path) + os.MkdirAll(path, 0777) + b, err := rdr.Root.Open(rdr, singleFile) + if err != nil { + t.Fatal(err) + } + err = extractToDir(rdr, b, path) + t.Fatal(err) +} + +func extractToDir(rdr *Reader, b *Base, folder string) error { + path := filepath.Join(folder, b.Name) + if b.IsDir() { + d, err := b.ToDir(rdr) + if err != nil { + return err + } + err = os.MkdirAll(path, 0777) + if err != nil { + return err + } + var nestBast *Base + for _, e := range d.Entries { + nestBast, err = rdr.BaseFromEntry(e) if err != nil { return err } - d, err := b.ToDir(rdr) - if err != nil { - return err - } - err = checkDir(rdr, d) + err = extractToDir(rdr, nestBast, path) if err != nil { return err } } + } else if b.IsRegular() { + _, full, err := b.GetRegFileReaders(rdr) + if err != nil { + fmt.Println("yo", path) + return err + } + fil, err := os.Create(path) + if err != nil { + return err + } + _, err = full.WriteTo(fil) + if err != nil { + return err + } + fmt.Println("Successfully extracted file:", b.Name) } return nil } diff --git a/squashfs/superblock.go b/squashfs/superblock.go index 7916759..0b19dd3 100644 --- a/squashfs/superblock.go +++ b/squashfs/superblock.go @@ -24,57 +24,57 @@ type superblock struct { ExportTableStart uint64 } -func (s superblock) checkMagic() bool { +func (s superblock) ValidMagic() bool { return s.Magic == 0x73717368 } -func (s superblock) checkBlockLog() bool { +func (s superblock) ValidBlockLog() bool { return s.BlockLog == uint16(math.Log2(float64(s.BlockSize))) } -func (s superblock) checkVersion() bool { +func (s superblock) ValidVersion() bool { return s.VerMaj == 4 && s.VerMin == 0 } -func (s superblock) uncompressedInodes() bool { +func (s superblock) UncompressedInodes() bool { return s.Flags&0x1 == 0x1 } -func (s superblock) uncompressedData() bool { +func (s superblock) UncompressedData() bool { return s.Flags&0x2 == 0x2 } -func (s superblock) uncompressedFragments() bool { +func (s superblock) UncompressedFragments() bool { return s.Flags&0x8 == 0x8 } -func (s superblock) noFragments() bool { +func (s superblock) NoFragments() bool { return s.Flags&0x10 == 0x10 } -func (s superblock) alwaysFragment() bool { +func (s superblock) AlwaysFragment() bool { return s.Flags&0x20 == 0x20 } -func (s superblock) duplicates() bool { +func (s superblock) Duplicates() bool { return s.Flags&0x40 == 0x40 } -func (s superblock) exportable() bool { +func (s superblock) Exportable() bool { return s.Flags&0x80 == 0x80 } -func (s superblock) uncompressedXattrs() bool { +func (s superblock) UncompressedXattrs() bool { return s.Flags&0x100 == 0x100 } -func (s superblock) noXattrs() bool { +func (s superblock) NoXattrs() bool { return s.Flags&0x200 == 0x200 } -func (s superblock) compressionOptions() bool { +func (s superblock) CompressionOptions() bool { return s.Flags&0x400 == 0x400 } -func (s superblock) uncompressedIDs() bool { +func (s superblock) UncompressedIDs() bool { return s.Flags&0x800 == 0x800 } From 5de59627df1c005122be980bb9626d2b23ee8e2e Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sun, 24 Dec 2023 08:05:56 -0600 Subject: [PATCH 06/11] Started working on the main library (nearly complete) --- README.md | 4 +- extraction_options.go | 21 ++++ file.go | 175 ++++++++++++++++++++++++++++++ file_info.go | 68 ++++++++++++ fs.go | 241 ++++++++++++++++++++++++++++++++++++++++++ reader.go | 30 ++++++ squashfs/base.go | 7 +- squashfs/directory.go | 2 +- squashfs/inode.go | 10 +- squashfs/reader.go | 8 +- 10 files changed, 550 insertions(+), 16 deletions(-) create mode 100644 extraction_options.go create mode 100644 file.go create mode 100644 file_info.go create mode 100644 fs.go create mode 100644 reader.go diff --git a/README.md b/README.md index 2206edf..fcd6f07 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ A PURE Go library to read squashfs. There is currently no plans to add archive creation support as it will almost always be better to just call `mksquashfs`. I could see some possible use cases, but probably won't spend time on it unless it's requested (open a discussion if you want this feature). +The library has two parts with this `github.com/CalebQ42/squashfs` being easy to use as it implements `io/fs` interfaces and doesn't expose unnecessary information. 95% this is the library you want. If you need lower level access to the information, use `github.com/CalebQ42/squashfs/squashfs` where far more information is exposed. + Currently has support for reading squashfs files and extracting files and folders. Special thanks to for some VERY important information in an easy to understand format. Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree/master/internal/squashfs) as I referenced it to figure some things out (and double check others). -## [TODO](https://github.com/CalebQ42/squashfs/projects/1?fullscreen=true) - ## Limitations * No Xattr parsing. This is simply because I haven't done any research on it and how to apply these in a pure go way. diff --git a/extraction_options.go b/extraction_options.go new file mode 100644 index 0000000..56535b8 --- /dev/null +++ b/extraction_options.go @@ -0,0 +1,21 @@ +package squashfs + +import ( + "io" + "io/fs" +) + +type ExtractionOptions struct { + LogOutput io.Writer //Where error log should write. + DereferenceSymlink bool //Replace symlinks with the target file. + UnbreakSymlink bool //Try to make sure symlinks remain unbroken when extracted, without changing the symlink. + Verbose bool //Prints extra info to log on an error. + IgnorePerm bool //Ignore file's permissions and instead use Perm. + Perm fs.FileMode //Permission to use when IgnorePerm. Defaults to 0777. +} + +func DefaultOptions() *ExtractionOptions { + return &ExtractionOptions{ + Perm: 0777, + } +} diff --git a/file.go b/file.go new file mode 100644 index 0000000..197e4bf --- /dev/null +++ b/file.go @@ -0,0 +1,175 @@ +package squashfs + +import ( + "errors" + "io" + "io/fs" + "path/filepath" + + "github.com/CalebQ42/squashfs/squashfs" + "github.com/CalebQ42/squashfs/squashfs/data" + "github.com/CalebQ42/squashfs/squashfs/inode" +) + +// File represents a file inside a squashfs archive. +type File struct { + b *squashfs.Base + full *data.FullReader + rdr *data.Reader + parent *FS + r *Reader + dirsRead int +} + +func (f *File) FS() (*FS, error) { + if !f.IsDir() { + return nil, errors.New("not a directory") + } + d, err := f.b.ToDir(f.r.r) + if err != nil { + return nil, err + } + return &FS{d: d, parent: f.parent, r: f.r}, nil +} + +// Closes the underlying readers. +// Further calls to Read and WriteTo will re-create the readers. +// Never returns an error. +func (f *File) Close() error { + if f.rdr != nil { + return f.rdr.Close() + } + f.rdr = nil + f.full = nil + return nil +} + +// Returns the file the symlink points to. +// If the file isn't a symlink, or points to a file outside the archive, returns nil. +func (f *File) GetSymlinkFile() fs.File { + if !f.IsSymlink() { + return nil + } + if filepath.IsAbs(f.SymlinkPath()) { + return nil + } + fil, err := f.parent.Open(f.SymlinkPath()) + if err != nil { + return nil + } + return fil +} + +// Returns whether the file is a directory. +func (f *File) IsDir() bool { + return f.b.IsDir() +} + +// Returns whether the file is a regular file. +func (f *File) IsRegular() bool { + return f.b.IsRegular() +} + +// Returns whether the file is a symlink. +func (f *File) IsSymlink() bool { + return f.b.Inode.Type == inode.Sym || f.b.Inode.Type == inode.ESym +} + +func (f *File) Mode() fs.FileMode { + return f.b.Inode.Mode() +} + +// Read reads the data from the file. Only works if file is a normal file. +func (f *File) Read(b []byte) (int, error) { + if !f.IsRegular() { + return 0, errors.New("file is not a regular file") + } + if f.rdr == nil { + err := f.initializeReaders() + if err != nil { + return 0, err + } + } + return f.rdr.Read(b) +} + +// ReadDir returns n fs.DirEntry's that's contained in the File (if it's a directory). +// If n <= 0 all fs.DirEntry's are returned. +func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { + if !f.IsDir() { + return nil, errors.New("file is not a directory") + } + d, err := f.b.ToDir(f.r.r) + if err != nil { + return nil, err + } + start, end := 0, len(d.Entries) + if n > 0 { + start, end = f.dirsRead, f.dirsRead+n + if end > len(d.Entries) { + end = len(d.Entries) + err = io.EOF + } + } + var out []fs.DirEntry + var fi fileInfo + for _, e := range d.Entries[start:end] { + fi, err = f.r.newFileInfo(e) + if err != nil { + f.dirsRead += len(out) + return out, err + } + out = append(out, fs.FileInfoToDirEntry(fi)) + } + f.dirsRead += len(out) + return out, err +} + +// Returns the file's fs.FileInfo +func (f *File) Stat() (fs.FileInfo, error) { + return newFileInfo(f.b.Name, f.b.Inode), nil +} + +// SymlinkPath returns the symlink's target path. Is the File isn't a symlink, returns an empty string. +func (f *File) SymlinkPath() string { + switch f.b.Inode.Type { + case inode.Sym: + return string(f.b.Inode.Data.(inode.Symlink).Target) + case inode.ESym: + return string(f.b.Inode.Data.(inode.ESymlink).Target) + } + return "" +} + +// Writes all data from the file to the given writer in a multi-threaded manner. +// The underlying reader is separate +func (f *File) WriteTo(w io.Writer) (int64, error) { + if !f.IsRegular() { + return 0, errors.New("file is not a regular file") + } + if f.full == nil { + err := f.initializeReaders() + if err != nil { + return 0, err + } + } + return f.full.WriteTo(w) +} + +func (f *File) initializeReaders() error { + var err error + f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.r) + return err +} + +// Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. +// Uses default extraction options. +func (f *File) Extract(folder string) error { + return f.ExtractWithOptions(folder, DefaultOptions()) +} + +// Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. +// Allows setting various extraction options via ExtractionOptions. +func (f *File) ExtractWithOptions(folder string, op *ExtractionOptions) error { + //TODO +} diff --git a/file_info.go b/file_info.go new file mode 100644 index 0000000..b4552a7 --- /dev/null +++ b/file_info.go @@ -0,0 +1,68 @@ +package squashfs + +import ( + "io/fs" + "time" + + "github.com/CalebQ42/squashfs/squashfs/directory" + "github.com/CalebQ42/squashfs/squashfs/inode" +) + +type fileInfo struct { + name string + size int64 + perm uint32 + modTime uint32 + fileType uint16 +} + +func (r Reader) newFileInfo(e directory.Entry) (fileInfo, error) { + i, err := r.r.InodeFromEntry(e) + if err != nil { + return fileInfo{}, err + } + return newFileInfo(e.Name, i), nil +} + +func newFileInfo(name string, i *inode.Inode) fileInfo { + var size int64 + if i.Type == inode.Fil { + size = int64(i.Data.(inode.File).Size) + } else if i.Type == inode.EFil { + size = int64(i.Data.(inode.EFile).Size) + } + return fileInfo{ + name: name, + size: size, + perm: uint32(i.Perm), + modTime: i.ModTime, + fileType: i.Type, + } +} + +func (f fileInfo) Name() string { + return f.name +} + +func (f fileInfo) Size() int64 { + return f.size +} + +func (f fileInfo) Mode() fs.FileMode { + if f.IsDir() { + return fs.FileMode(f.perm | uint32(fs.ModeDir)) + } + return fs.FileMode(f.perm) +} + +func (f fileInfo) ModTime() time.Time { + return time.Unix(int64(f.modTime), 0) +} + +func (f fileInfo) IsDir() bool { + return f.fileType == inode.Dir || f.fileType == inode.EDir +} + +func (f fileInfo) Sys() any { + return nil +} diff --git a/fs.go b/fs.go new file mode 100644 index 0000000..7f62f44 --- /dev/null +++ b/fs.go @@ -0,0 +1,241 @@ +package squashfs + +import ( + "io" + "io/fs" + "path" + "path/filepath" + "slices" + "strings" + + "github.com/CalebQ42/squashfs/squashfs" + "github.com/CalebQ42/squashfs/squashfs/directory" +) + +// FS is a fs.FS representation of a squashfs directory. +// Implements fs.GlobFS, fs.ReadDirFS, fs.ReadFileFS, fs.StatFS, and fs.SubFS +type FS struct { + d *squashfs.Directory + r *Reader + parent *FS +} + +// Glob returns the name of the files at the given pattern. +// All paths are relative to the FS. +// Uses filepath.Match to compare names. +func (f *FS) Glob(pattern string) (out []string, err error) { + pattern = filepath.Clean(pattern) + if !fs.ValidPath(pattern) { + return nil, &fs.PathError{ + Op: "glob", + Path: pattern, + Err: fs.ErrInvalid, + } + } + split := strings.Split(pattern, "/") + for i := 0; i < len(f.d.Entries); i++ { + if match, _ := path.Match(split[0], f.d.Entries[i].Name); match { + if len(split) == 1 { + out = append(out, f.d.Entries[i].Name) + continue + } + sub, err := f.Sub(split[0]) + if err != nil { + if pathErr, ok := err.(*fs.PathError); ok { + if pathErr.Err == fs.ErrNotExist { + continue + } + pathErr.Op = "glob" + pathErr.Path = pattern + return nil, pathErr + } + return nil, &fs.PathError{ + Op: "glob", + Path: pattern, + Err: err, + } + } + subGlob, err := sub.(fs.GlobFS).Glob(strings.Join(split[1:], "/")) + if err != nil { + if pathErr, ok := err.(*fs.PathError); ok { + if pathErr.Err == fs.ErrNotExist { + continue + } + pathErr.Op = "glob" + pathErr.Path = pattern + return nil, pathErr + } + return nil, &fs.PathError{ + Op: "glob", + Path: pattern, + Err: err, + } + } + for i := 0; i < len(subGlob); i++ { + subGlob[i] = f.d.Name + "/" + subGlob[i] + } + out = append(out, subGlob...) + } + } + return +} + +// Opens the file at name. Returns a *File as an fs.File. +func (f *FS) Open(name string) (fs.File, error) { + name = filepath.Clean(name) + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrInvalid, + } + } + if name == "." || name == "" { + return &File{ + b: &f.d.Base, + r: f.r, + parent: f.parent, + }, nil + } + split := strings.Split(name, "/") + i, found := slices.BinarySearchFunc(f.d.Entries, split[0], func(e directory.Entry, name string) int { + return strings.Compare(e.Name, name) + }) + if !found { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrNotExist, + } + } + b, err := f.r.r.BaseFromEntry(f.d.Entries[i]) + if err != nil { + return nil, err + } + if len(split) == 1 { + return &File{ + b: b, + r: f.r, + parent: f.parent, + }, nil + } + if !b.IsDir() { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrNotExist, + } + } + d, err := b.ToDir(f.r.r) + if err != nil { + return nil, err + } + return (&FS{ + d: d, + r: f.r, + parent: f, + }).Open(strings.Join(split[1:], "/")) +} + +// Returns all DirEntry's for the directory at name. +// If name is not a directory, returns an error. +func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) { + name = filepath.Clean(name) + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "readdir", + Path: name, + Err: fs.ErrInvalid, + } + } + if name == "." || name == "" { + return (&File{ + b: &f.d.Base, + parent: f.parent, + r: f.r, + }).ReadDir(-1) + } + fil, err := f.Open(name) + if err != nil { + return nil, err + } + return fil.(*File).ReadDir(-1) +} + +// Returns the contents of the file at name. +func (f *FS) ReadFile(name string) (out []byte, err error) { + name = filepath.Clean(name) + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "readfile", + Path: name, + Err: fs.ErrInvalid, + } + } + if name == "." || name == "" { + return nil, fs.ErrInvalid + } + fil, err := f.Open(name) + if err != nil { + return nil, err + } + if !fil.(*File).IsRegular() { + return nil, fs.ErrInvalid + } + return io.ReadAll(fil) +} + +// Returns the fs.FileInfo for the file at name. +func (f *FS) Stat(name string) (fs.FileInfo, error) { + name = filepath.Clean(name) + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "stat", + Path: name, + Err: fs.ErrInvalid, + } + } + if name == "." || name == "" { + return (&File{ + b: &f.d.Base, + parent: f.parent, + r: f.r, + }).Stat() + } + fil, err := f.Open(name) + if err != nil { + return nil, err + } + return fil.(*File).Stat() +} + +// Returns the FS at dir +func (f *FS) Sub(dir string) (fs.FS, error) { + dir = filepath.Clean(dir) + if !fs.ValidPath(dir) { + return nil, &fs.PathError{ + Op: "dir", + Path: dir, + Err: fs.ErrInvalid, + } + } + if dir == "." || dir == "" { + return f, nil + } + fil, err := f.Open(dir) + if err != nil { + return nil, err + } + if !fil.(*File).IsDir() { + return nil, &fs.PathError{ + Op: "dir", + Path: dir, + Err: fs.ErrInvalid, + } + } + return fil.(*File).FS() +} + +func (f *FS) path() string { + return filepath.Join(f.parent.path(), f.d.Name) +} diff --git a/reader.go b/reader.go new file mode 100644 index 0000000..61946f4 --- /dev/null +++ b/reader.go @@ -0,0 +1,30 @@ +package squashfs + +import ( + "io" + "time" + + "github.com/CalebQ42/squashfs/squashfs" +) + +type Reader struct { + *FS + r *squashfs.Reader +} + +func NewReader(r io.ReaderAt) (*Reader, error) { + rdr, err := squashfs.NewReader(r) + if err != nil { + return nil, err + } + return &Reader{ + r: rdr, + FS: &FS{ + d: rdr.Root, + }, + }, nil +} + +func (r *Reader) ModTime() time.Time { + return time.Unix(int64(r.r.Superblock.ModTime), 0) +} diff --git a/squashfs/base.go b/squashfs/base.go index 2bac355..f799f1a 100644 --- a/squashfs/base.go +++ b/squashfs/base.go @@ -21,10 +21,7 @@ func (r *Reader) BaseFromInode(i *inode.Inode, name string) *Base { } func (r *Reader) BaseFromEntry(e directory.Entry) (*Base, error) { - rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d) - defer rdr.Close() - rdr.Read(make([]byte, e.Offset)) - in, err := inode.Read(rdr, r.Superblock.BlockSize) + in, err := r.InodeFromEntry(e) if err != nil { return nil, err } @@ -32,7 +29,7 @@ func (r *Reader) BaseFromEntry(e directory.Entry) (*Base, error) { } func (r *Reader) BaseFromRef(ref uint64, name string) (*Base, error) { - in, err := r.inodeFromRef(ref) + in, err := r.InodeFromRef(ref) if err != nil { return nil, err } diff --git a/squashfs/directory.go b/squashfs/directory.go index 9e7db15..07fcb82 100644 --- a/squashfs/directory.go +++ b/squashfs/directory.go @@ -20,7 +20,7 @@ type Directory struct { } func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) { - i, err := r.inodeFromRef(ref) + i, err := r.InodeFromRef(ref) if err != nil { fmt.Println("yo") return nil, err diff --git a/squashfs/inode.go b/squashfs/inode.go index 205c5ce..d1cb5a2 100644 --- a/squashfs/inode.go +++ b/squashfs/inode.go @@ -3,10 +3,11 @@ package squashfs import ( "github.com/CalebQ42/squashfs/internal/metadata" "github.com/CalebQ42/squashfs/internal/toreader" + "github.com/CalebQ42/squashfs/squashfs/directory" "github.com/CalebQ42/squashfs/squashfs/inode" ) -func (r *Reader) inodeFromRef(ref uint64) (*inode.Inode, error) { +func (r *Reader) InodeFromRef(ref uint64) (*inode.Inode, error) { offset, meta := (ref>>16)+r.Superblock.InodeTableStart, ref&0xFFFF rdr := metadata.NewReader(toreader.NewReader(r.r, int64(offset)), r.d) defer rdr.Close() @@ -16,3 +17,10 @@ func (r *Reader) inodeFromRef(ref uint64) (*inode.Inode, error) { } return inode.Read(rdr, r.Superblock.BlockSize) } + +func (r *Reader) InodeFromEntry(e directory.Entry) (*inode.Inode, error) { + rdr := metadata.NewReader(toreader.NewReader(r.r, int64(r.Superblock.InodeTableStart)+int64(e.BlockStart)), r.d) + defer rdr.Close() + rdr.Read(make([]byte, e.Offset)) + return inode.Read(rdr, r.Superblock.BlockSize) +} diff --git a/squashfs/reader.go b/squashfs/reader.go index 3e8ec89..302cbc9 100644 --- a/squashfs/reader.go +++ b/squashfs/reader.go @@ -5,7 +5,6 @@ import ( "errors" "io" "math" - "time" "github.com/CalebQ42/squashfs/internal/decompress" "github.com/CalebQ42/squashfs/internal/metadata" @@ -79,11 +78,6 @@ func NewReader(r io.ReaderAt) (rdr *Reader, err error) { return } -// Returns the last time the archive was modified. -func (r *Reader) ModTime() time.Time { - return time.Unix(int64(r.Superblock.ModTime), 0) -} - // Get a uid/gid at the given index. Lazily populates the reader's Id table as necessary. func (r *Reader) Id(i uint16) (uint32, error) { if len(r.idTable) > int(i) { @@ -221,5 +215,5 @@ func (r *Reader) Inode(i uint32) (*inode.Inode, error) { if err != nil { return nil, err } - return r.inodeFromRef(ref) + return r.InodeFromRef(ref) } From d9132ab6a46ce16a71a6c9b38f152d341f8e5ee6 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sun, 24 Dec 2023 18:20:05 -0600 Subject: [PATCH 07/11] Finished. Now for bug fixes --- cmd/go-unsquashfs/main.go | 37 +++++ extraction_options.go | 13 +- file.go | 238 ++++++++++++++++++++++++++++- fs.go | 55 +++++-- internal/routinemanager/manager.go | 25 +++ squashfs/base.go | 37 +++++ 6 files changed, 385 insertions(+), 20 deletions(-) create mode 100644 cmd/go-unsquashfs/main.go create mode 100644 internal/routinemanager/manager.go diff --git a/cmd/go-unsquashfs/main.go b/cmd/go-unsquashfs/main.go new file mode 100644 index 0000000..75be53f --- /dev/null +++ b/cmd/go-unsquashfs/main.go @@ -0,0 +1,37 @@ +package main + +import ( + "flag" + "fmt" + "os" + "time" + + "github.com/CalebQ42/squashfs" +) + +func main() { + verbose := flag.Bool("v", false, "Verbose") + ignore := flag.Bool("ip", false, "Ignore Permissions and extract all files/folders with 0755") + flag.Parse() + if len(flag.Args()) < 2 { + fmt.Println("Please provide a file name and extraction path") + os.Exit(0) + } + f, err := os.Open(flag.Arg(0)) + if err != nil { + panic(err) + } + r, err := squashfs.NewReader(f) + if err != nil { + panic(err) + } + op := squashfs.DefaultOptions() + op.Verbose = *verbose + op.IgnorePerm = *ignore + n := time.Now() + err = r.ExtractWithOptions(flag.Arg(1), op) + if err != nil { + panic(err) + } + fmt.Println("Took:", time.Since(n)) +} diff --git a/extraction_options.go b/extraction_options.go index 56535b8..ad917de 100644 --- a/extraction_options.go +++ b/extraction_options.go @@ -3,19 +3,28 @@ package squashfs import ( "io" "io/fs" + "os" + + "github.com/CalebQ42/squashfs/internal/routinemanager" ) type ExtractionOptions struct { - LogOutput io.Writer //Where error log should write. + manager *routinemanager.Manager + LogOutput io.Writer //Where the verbose log should write. Defaults to os.Stdout. DereferenceSymlink bool //Replace symlinks with the target file. UnbreakSymlink bool //Try to make sure symlinks remain unbroken when extracted, without changing the symlink. Verbose bool //Prints extra info to log on an error. IgnorePerm bool //Ignore file's permissions and instead use Perm. Perm fs.FileMode //Permission to use when IgnorePerm. Defaults to 0777. + SimultaneousFiles uint16 //Number of files to process in parallel. Defaults to 10. + ExtractionRoutines uint16 //Number of goroutines to use for each file's extraction. Only applies to regular files. Defaults to 10. } func DefaultOptions() *ExtractionOptions { return &ExtractionOptions{ - Perm: 0777, + LogOutput: os.Stdout, + Perm: 0777, + SimultaneousFiles: 10, + ExtractionRoutines: 10, } } diff --git a/file.go b/file.go index 197e4bf..2c51214 100644 --- a/file.go +++ b/file.go @@ -4,8 +4,14 @@ import ( "errors" "io" "io/fs" + "log" + "os" + "os/exec" "path/filepath" + "runtime" + "strconv" + "github.com/CalebQ42/squashfs/internal/routinemanager" "github.com/CalebQ42/squashfs/squashfs" "github.com/CalebQ42/squashfs/squashfs/data" "github.com/CalebQ42/squashfs/squashfs/inode" @@ -162,6 +168,20 @@ func (f *File) initializeReaders() error { return err } +func (f *File) deviceDevices() (maj uint32, min uint32) { + var dev uint32 + if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.Block { + dev = f.b.Inode.Data.(inode.Device).Dev + } else if f.b.Inode.Type == inode.EChar || f.b.Inode.Type == inode.EBlock { + dev = f.b.Inode.Data.(inode.EDevice).Dev + } + return dev >> 8, dev & 0x000FF +} + +func (f *File) path() string { + return filepath.Join(f.parent.path(), f.b.Name) +} + // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. // Uses default extraction options. func (f *File) Extract(folder string) error { @@ -170,6 +190,220 @@ func (f *File) Extract(folder string) error { // Extract the file to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. // Allows setting various extraction options via ExtractionOptions. -func (f *File) ExtractWithOptions(folder string, op *ExtractionOptions) error { - //TODO +func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { + if op.manager == nil { + op.manager = routinemanager.NewManager(op.SimultaneousFiles) + log.SetOutput(op.LogOutput) + } + switch f.b.Inode.Type { + case inode.Dir, inode.EDir: + d, err := f.b.ToDir(f.r.r) + if err != nil { + if op.Verbose { + log.Println("Failed to create squashfs.Directory for", path) + } + return errors.Join(errors.New("failed to create squashfs.Directory: "+path), err) + } + errChan := make(chan error, len(d.Entries)) + files := len(d.Entries) + for i := range d.Entries { + b, err := f.r.r.BaseFromEntry(d.Entries[i]) + if err != nil { + if op.Verbose { + log.Println("Failed to get squashfs.Base from entry for", path) + } + return errors.Join(errors.New("failed to get base from entry: "+path), err) + } + if b.IsDir() { + files-- + extDir := filepath.Join(path, b.Name) + err = os.Mkdir(extDir, 0777) + if err != nil { + if op.Verbose { + log.Println("Failed to create directory", path) + } + return errors.Join(errors.New("failed to create directory: "+path), err) + } + err = f.ExtractWithOptions(extDir, op) + if err != nil { + if op.Verbose { + log.Println("Failed to extract directory", path) + } + return errors.Join(errors.New("failed to extract directory: "+path), err) + } + } else { + fil := &File{ + b: b, + r: f.r, + } + go func(fil *File, folder string) { + i := op.manager.Lock() + defer op.manager.Unlock(i) + errChan <- fil.ExtractWithOptions(folder, op) + }(fil, path) + } + } + var errCache []error + for i := 0; i < files; i++ { + err := <-errChan + if err != nil { + errCache = append(errCache, err) + } + } + if len(errCache) > 0 { + return errors.Join(errors.New("failed to extract folder: "+path), errors.Join(errCache...)) + } + case inode.Fil, inode.EFil: + path = filepath.Join(path, f.b.Name) + outFil, err := os.Create(path) + if err != nil { + if op.Verbose { + log.Println("Failed to create file", path) + } + return errors.Join(errors.New("failed to create file: "+path), err) + } + defer outFil.Close() + full, err := f.b.GetFullReader(f.r.r) + if err != nil { + if op.Verbose { + log.Println("Failed to create full reader for", path) + } + return errors.Join(errors.New("failed to create full reader: "+path), err) + } + full.SetGoroutineLimit(op.ExtractionRoutines) + _, err = full.WriteTo(outFil) + if err != nil { + if op.Verbose { + log.Println("Failed to write file", path) + } + return errors.Join(errors.New("failed to write file: "+path), err) + } + if op.Verbose { + log.Println(f.path(), "extracted to", path) + } + case inode.Sym, inode.ESym: + symPath := f.SymlinkPath() + if op.DereferenceSymlink { + filTmp := f.GetSymlinkFile() + if filTmp == nil { + if op.Verbose { + log.Println("Failed to get symlink's file:", f.path()) + } + return errors.New("failed to get symlink's file") + } + fil := filTmp.(*File) + fil.b.Name = f.b.Name + err := fil.ExtractWithOptions(path, op) + if err != nil { + if op.Verbose { + log.Println("Failed to extract symlink's file:", filepath.Join(path, f.b.Name)) + } + return errors.Join(errors.New("failed to extract symlink's file: "+path), err) + } + } else { + if op.UnbreakSymlink { + filTmp := f.GetSymlinkFile() + if filTmp == nil { + if op.Verbose { + log.Println("Failed to get symlink's file:", f.path()) + } + return errors.New("failed to get symlink's file") + } + extractLoc := filepath.Join(path, filepath.Dir(symPath)) + fil := filTmp.(*File) + err := fil.ExtractWithOptions(extractLoc, op) + if err != nil { + if op.Verbose { + log.Println("Error while extracting", fil.path(), "to make sure symlink at", f.path(), "is unbroken") + } + return errors.Join(errors.New("failed to extract symlink's file: "+extractLoc), err) + } + } + path = filepath.Join(path, f.b.Name) + err := os.Symlink(f.SymlinkPath(), path) + if err != nil { + if op.Verbose { + log.Println("Failed to create symlink:", path) + } + return errors.Join(errors.New("failed to create symlink: "+path), err) + } + } + case inode.Char, inode.EChar, inode.Block, inode.EBlock, inode.Fifo, inode.EFifo: + if runtime.GOOS == "windows" { + if op.Verbose { + log.Println(f.path(), "ignored. A device link and can't be created on Windows.") + } + return nil + } + _, err := exec.LookPath("mknod") + if err != nil { + if op.Verbose { + log.Println("mknot command not found, cannot create device link for", f.path()) + } + return errors.Join(errors.New("mknot command not found"), err) + } + path = filepath.Join(path, f.b.Name) + var typ string + if f.b.Inode.Type == inode.Char || f.b.Inode.Type == inode.EChar { + typ = "c" + } else if f.b.Inode.Type == inode.Block || f.b.Inode.Type == inode.EBlock { + typ = "b" + } else { //Fifo IPC + if runtime.GOOS == "darwin" { + if op.Verbose { + log.Println(f.path(), "ignored. A Fifo file and can't be created on Darwin.") + } + return nil + } + typ = "p" + } + cmd := exec.Command("mknod", path, typ) + if typ != "p" { + maj, min := f.deviceDevices() + cmd.Args = append(cmd.Args, strconv.Itoa(int(maj)), strconv.Itoa(int(min))) + } + if op.Verbose { + cmd.Stdout = op.LogOutput + cmd.Stderr = op.LogOutput + } + err = cmd.Run() + if err != nil { + if op.Verbose { + log.Println("Error while running mknod for", path) + } + return errors.Join(errors.New("error while running mknod for "+path), err) + } + case inode.Sock, inode.ESock: + if op.Verbose { + log.Println(f.path(), "ignored since it's a socket file.") + } + return nil + default: + return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.b.Inode.Type))) + } + if op.Verbose { + log.Println(f.path(), "extracted to", path) + } + if op.IgnorePerm { + return nil + } + uid, err := f.b.Uid(f.r.r) + if err != nil { + if op.Verbose { + log.Println("Failed to get uid for", path) + log.Println(err) + } + return nil + } + gid, err := f.b.Gid(f.r.r) + if err != nil { + if op.Verbose { + log.Println("Failed to get gid for", path) + log.Println(err) + } + return nil + } + os.Chmod(path, f.Mode()) + os.Chown(path, int(uid), int(gid)) + return nil } diff --git a/fs.go b/fs.go index 7f62f44..39f08cd 100644 --- a/fs.go +++ b/fs.go @@ -91,13 +91,20 @@ func (f *FS) Open(name string) (fs.File, error) { } } if name == "." || name == "" { - return &File{ - b: &f.d.Base, - r: f.r, - parent: f.parent, - }, nil + return f.File(), nil } split := strings.Split(name, "/") + if split[0] == ".." { + if f.parent == nil { // root directory + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrNotExist, + } + } + } else { + return f.parent.Open(strings.Join(split[1:], "/")) + } i, found := slices.BinarySearchFunc(f.d.Entries, split[0], func(e directory.Entry, name string) int { return strings.Compare(e.Name, name) }) @@ -116,7 +123,7 @@ func (f *FS) Open(name string) (fs.File, error) { return &File{ b: b, r: f.r, - parent: f.parent, + parent: f, }, nil } if !b.IsDir() { @@ -149,11 +156,7 @@ func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) { } } if name == "." || name == "" { - return (&File{ - b: &f.d.Base, - parent: f.parent, - r: f.r, - }).ReadDir(-1) + return f.File().ReadDir(-1) } fil, err := f.Open(name) if err != nil { @@ -196,11 +199,7 @@ func (f *FS) Stat(name string) (fs.FileInfo, error) { } } if name == "." || name == "" { - return (&File{ - b: &f.d.Base, - parent: f.parent, - r: f.r, - }).Stat() + return f.File().Stat() } fil, err := f.Open(name) if err != nil { @@ -236,6 +235,30 @@ func (f *FS) Sub(dir string) (fs.FS, error) { return fil.(*File).FS() } +// Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. +// Uses default extraction options. +func (f *FS) Extract(folder string) error { + return f.File().Extract(folder) +} + +// Extract the FS to the given folder. If the file is a folder, the folder's contents will be extracted to the folder. +// Allows setting various extraction options via ExtractionOptions. +func (f *FS) ExtractWithOptions(folder string, op *ExtractionOptions) error { + return f.File().ExtractWithOptions(folder, op) +} + +// Returns the FS as a *File +func (f *FS) File() *File { + return &File{ + b: &f.d.Base, + parent: f.parent, + r: f.r, + } +} + func (f *FS) path() string { + if f.parent == nil { + return f.d.Name + } return filepath.Join(f.parent.path(), f.d.Name) } diff --git a/internal/routinemanager/manager.go b/internal/routinemanager/manager.go new file mode 100644 index 0000000..4e47ed8 --- /dev/null +++ b/internal/routinemanager/manager.go @@ -0,0 +1,25 @@ +package routinemanager + +type Manager struct { + channel chan uint16 + maxRoutines uint16 +} + +func NewManager(maxRoutines uint16) *Manager { + m := &Manager{ + maxRoutines: maxRoutines, + channel: make(chan uint16, maxRoutines), + } + for i := uint16(0); i < maxRoutines; i++ { + m.channel <- i + } + return m +} + +func (m *Manager) Lock() uint16 { + return <-m.channel +} + +func (m *Manager) Unlock(i uint16) { + m.channel <- i +} diff --git a/squashfs/base.go b/squashfs/base.go index f799f1a..53c8207 100644 --- a/squashfs/base.go +++ b/squashfs/base.go @@ -129,3 +129,40 @@ func (b *Base) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, err } return outRdr, outFull, nil } + +func (b *Base) GetFullReader(r *Reader) (*data.FullReader, error) { + if !b.IsRegular() { + return nil, errors.New("not a regular file") + } + var blockStart uint64 + var fragIndex uint32 + var fragOffset uint32 + var fragSize uint64 + var sizes []uint32 + if b.Inode.Type == inode.Fil { + blockStart = uint64(b.Inode.Data.(inode.File).BlockStart) + fragIndex = b.Inode.Data.(inode.File).FragInd + fragOffset = b.Inode.Data.(inode.File).FragOffset + sizes = b.Inode.Data.(inode.File).BlockSizes + fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize) + } else { + blockStart = b.Inode.Data.(inode.EFile).BlockStart + fragIndex = b.Inode.Data.(inode.EFile).FragInd + fragOffset = b.Inode.Data.(inode.EFile).FragOffset + sizes = b.Inode.Data.(inode.EFile).BlockSizes + fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize) + } + outFull := data.NewFullReader(r.r, int64(blockStart), r.d, sizes, fragSize, r.Superblock.BlockSize) + if fragIndex != 0xffffffff { + outFull.AddFrag(func() (io.Reader, error) { + ent, err := r.fragEntry(fragIndex) + if err != nil { + return nil, err + } + frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize) + frag.Read(make([]byte, fragOffset)) + return io.LimitReader(frag, int64(fragSize)), nil + }) + } + return outFull, nil +} From 17d45eea505ef4a8191728ce4275bd0c816394a1 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Wed, 27 Dec 2023 21:35:40 -0600 Subject: [PATCH 08/11] Finishing touches Added FastOptions as an alternative to DefaultOptions A few performance improvements A few bug fixes --- README.md | 17 +++- extraction_options.go | 33 ++++-- file.go | 81 +++++++++------ fs.go | 19 ++-- reader.go | 12 ++- squashfs/data/fullreader.go | 3 +- squashfs/directory.go | 2 - squashfs/reader_test.go | 13 +-- squashfs_test.go | 193 ++++++++++++++++++++++++++++++++++++ 9 files changed, 310 insertions(+), 63 deletions(-) create mode 100644 squashfs_test.go diff --git a/README.md b/README.md index fcd6f07..92dd55a 100644 --- a/README.md +++ b/README.md @@ -11,18 +11,27 @@ Currently has support for reading squashfs files and extracting files and folder Special thanks to for some VERY important information in an easy to understand format. Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree/master/internal/squashfs) as I referenced it to figure some things out (and double check others). +## FUSE + +As of `v1.0`, FUSE capabilities has been moved to [a separate library](https://github.com/CalebQ42/squashfuse). + ## Limitations -* No Xattr parsing. This is simply because I haven't done any research on it and how to apply these in a pure go way. +* No Xattr parsing. * Socket files are not extracted. - * From my research, it seems like a socket file would be useless if it could be created. They are still exposed when fuse mounted. + * From my research, it seems like a socket file would be useless if it could be created. * Fifo files are ignored on `darwin` ## Issues -* Significantly slower then `unsquashfs` when extracting folders (about 5 ~ 7 times slower on a ~100MB archive using zstd compression) +* Significantly slower then `unsquashfs` when extracting folders * This seems to be related to above along with the general optimization of `unsquashfs` and it's compression libraries. - * The larger the file's tree, the slower the extraction will be. Arch Linux's Live USB's airootfs.sfs takes ~35x longer for a full extraction. + * Times seem to be largely dependent on file tree size and compression type. + * My main testing image (~100MB) using Zstd takes about 6x longer. + * An Arch Linux airootfs image (~780MB) using XZ compression with LZMA filters takes about 32x longer. + * A Tensorflow docker image (~3.3GB) using Zstd takes about 12x longer. + +Note: These numbers are using `FastOptions()`. `DefaultOptions()` takes about 2x longer. ## Recommendations on Usage diff --git a/extraction_options.go b/extraction_options.go index ad917de..c55a5ec 100644 --- a/extraction_options.go +++ b/extraction_options.go @@ -3,28 +3,47 @@ package squashfs import ( "io" "io/fs" - "os" + "runtime" "github.com/CalebQ42/squashfs/internal/routinemanager" ) type ExtractionOptions struct { manager *routinemanager.Manager - LogOutput io.Writer //Where the verbose log should write. Defaults to os.Stdout. + LogOutput io.Writer //Where the verbose log should write. DereferenceSymlink bool //Replace symlinks with the target file. UnbreakSymlink bool //Try to make sure symlinks remain unbroken when extracted, without changing the symlink. Verbose bool //Prints extra info to log on an error. IgnorePerm bool //Ignore file's permissions and instead use Perm. Perm fs.FileMode //Permission to use when IgnorePerm. Defaults to 0777. - SimultaneousFiles uint16 //Number of files to process in parallel. Defaults to 10. - ExtractionRoutines uint16 //Number of goroutines to use for each file's extraction. Only applies to regular files. Defaults to 10. + SimultaneousFiles uint16 //Number of files to process in parallel. Default set based on runtime.NumCPU(). + ExtractionRoutines uint16 //Number of goroutines to use for each file's extraction. Only applies to regular files. Default set based on runtime.NumCPU(). } +// The default extraction options. func DefaultOptions() *ExtractionOptions { + cores := uint16(runtime.NumCPU() / 2) + var files, routines uint16 + if cores <= 4 { + files = 1 + routines = cores + } else { + files = cores - 4 + routines = 4 + } return &ExtractionOptions{ - LogOutput: os.Stdout, Perm: 0777, - SimultaneousFiles: 10, - ExtractionRoutines: 10, + SimultaneousFiles: files, + ExtractionRoutines: routines, + } +} + +// Less limited default options. Can run up 2x faster than DefaultOptions. +// Tends to use all available CPU resources. +func FastOptions() *ExtractionOptions { + return &ExtractionOptions{ + Perm: 0777, + SimultaneousFiles: uint16(runtime.NumCPU()), + ExtractionRoutines: uint16(runtime.NumCPU()), } } diff --git a/file.go b/file.go index 2c51214..1b7cbd1 100644 --- a/file.go +++ b/file.go @@ -27,6 +27,15 @@ type File struct { dirsRead int } +// Creates a new *File from the given *squashfs.Base +func (r *Reader) FileFromBase(b *squashfs.Base, parent *FS) *File { + return &File{ + b: b, + parent: parent, + r: r, + } +} + func (f *File) FS() (*FS, error) { if !f.IsDir() { return nil, errors.New("not a directory") @@ -179,6 +188,9 @@ func (f *File) deviceDevices() (maj uint32, min uint32) { } func (f *File) path() string { + if f.parent == nil { + return f.b.Name + } return filepath.Join(f.parent.path(), f.b.Name) } @@ -193,7 +205,16 @@ func (f *File) Extract(folder string) error { func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { if op.manager == nil { op.manager = routinemanager.NewManager(op.SimultaneousFiles) - log.SetOutput(op.LogOutput) + if op.LogOutput != nil { + log.SetOutput(op.LogOutput) + } + err := os.MkdirAll(path, 0777) + if err != nil { + if op.Verbose { + log.Println("Failed to create initial directory", path) + } + return err + } } switch f.b.Inode.Type { case inode.Dir, inode.EDir: @@ -205,7 +226,6 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { return errors.Join(errors.New("failed to create squashfs.Directory: "+path), err) } errChan := make(chan error, len(d.Entries)) - files := len(d.Entries) for i := range d.Entries { b, err := f.r.r.BaseFromEntry(d.Entries[i]) if err != nil { @@ -214,37 +234,39 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { } return errors.Join(errors.New("failed to get base from entry: "+path), err) } - if b.IsDir() { - files-- - extDir := filepath.Join(path, b.Name) - err = os.Mkdir(extDir, 0777) - if err != nil { - if op.Verbose { - log.Println("Failed to create directory", path) + go func(b *squashfs.Base, path string) { + i := op.manager.Lock() + if b.IsDir() { + extDir := filepath.Join(path, b.Name) + err = os.Mkdir(extDir, 0777) + op.manager.Unlock(i) + if err != nil { + if op.Verbose { + log.Println("Failed to create directory", path) + } + errChan <- errors.Join(errors.New("failed to create directory: "+path), err) + return } - return errors.Join(errors.New("failed to create directory: "+path), err) - } - err = f.ExtractWithOptions(extDir, op) - if err != nil { - if op.Verbose { - log.Println("Failed to extract directory", path) + err = f.r.FileFromBase(b, f.r.FSFromDirectory(d, f.parent)).ExtractWithOptions(extDir, op) + if err != nil { + if op.Verbose { + log.Println("Failed to extract directory", path) + } + errChan <- errors.Join(errors.New("failed to extract directory: "+path), err) + return } - return errors.Join(errors.New("failed to extract directory: "+path), err) + errChan <- nil + } else { + fil := f.r.FileFromBase(b, f.r.FSFromDirectory(d, f.parent)) + err = fil.ExtractWithOptions(path, op) + op.manager.Unlock(i) + fil.Close() + errChan <- err } - } else { - fil := &File{ - b: b, - r: f.r, - } - go func(fil *File, folder string) { - i := op.manager.Lock() - defer op.manager.Unlock(i) - errChan <- fil.ExtractWithOptions(folder, op) - }(fil, path) - } + }(b, path) } var errCache []error - for i := 0; i < files; i++ { + for i := 0; i < len(d.Entries); i++ { err := <-errChan if err != nil { errCache = append(errCache, err) @@ -278,9 +300,6 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { } return errors.Join(errors.New("failed to write file: "+path), err) } - if op.Verbose { - log.Println(f.path(), "extracted to", path) - } case inode.Sym, inode.ESym: symPath := f.SymlinkPath() if op.DereferenceSymlink { diff --git a/fs.go b/fs.go index 39f08cd..e81acb0 100644 --- a/fs.go +++ b/fs.go @@ -20,6 +20,15 @@ type FS struct { parent *FS } +// Creates a new *FS from the given squashfs.directory +func (r *Reader) FSFromDirectory(d *squashfs.Directory, parent *FS) *FS { + return &FS{ + d: d, + r: r, + parent: parent, + } +} + // Glob returns the name of the files at the given pattern. // All paths are relative to the FS. // Uses filepath.Match to compare names. @@ -101,9 +110,9 @@ func (f *FS) Open(name string) (fs.File, error) { Path: name, Err: fs.ErrNotExist, } + } else { + return f.parent.Open(strings.Join(split[1:], "/")) } - } else { - return f.parent.Open(strings.Join(split[1:], "/")) } i, found := slices.BinarySearchFunc(f.d.Entries, split[0], func(e directory.Entry, name string) int { return strings.Compare(e.Name, name) @@ -137,11 +146,7 @@ func (f *FS) Open(name string) (fs.File, error) { if err != nil { return nil, err } - return (&FS{ - d: d, - r: f.r, - parent: f, - }).Open(strings.Join(split[1:], "/")) + return f.r.FSFromDirectory(d, f).Open(strings.Join(split[1:], "/")) } // Returns all DirEntry's for the directory at name. diff --git a/reader.go b/reader.go index 61946f4..43058c1 100644 --- a/reader.go +++ b/reader.go @@ -17,12 +17,14 @@ func NewReader(r io.ReaderAt) (*Reader, error) { if err != nil { return nil, err } - return &Reader{ + out := &Reader{ r: rdr, - FS: &FS{ - d: rdr.Root, - }, - }, nil + } + out.FS = &FS{ + d: rdr.Root, + r: out, + } + return out, nil } func (r *Reader) ModTime() time.Time { diff --git a/squashfs/data/fullreader.go b/squashfs/data/fullreader.go index d085e90..d31853d 100644 --- a/squashfs/data/fullreader.go +++ b/squashfs/data/fullreader.go @@ -5,6 +5,7 @@ import ( "errors" "io" "math" + "runtime" "sync" "github.com/CalebQ42/squashfs/internal/decompress" @@ -31,7 +32,7 @@ func NewFullReader(r io.ReaderAt, initialOffset int64, d decompress.Decompressor d: d, sizes: sizes, initialOffset: initialOffset, - goroutineLimit: 10, + goroutineLimit: uint16(runtime.NumCPU()), finalBlockSize: finalBlockSize, blockSize: blockSize, retPool: &sync.Pool{ diff --git a/squashfs/directory.go b/squashfs/directory.go index 07fcb82..1681e89 100644 --- a/squashfs/directory.go +++ b/squashfs/directory.go @@ -2,7 +2,6 @@ package squashfs import ( "errors" - "fmt" "io/fs" "path/filepath" "slices" @@ -22,7 +21,6 @@ type Directory struct { func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) { i, err := r.InodeFromRef(ref) if err != nil { - fmt.Println("yo") return nil, err } var blockStart uint32 diff --git a/squashfs/reader_test.go b/squashfs/reader_test.go index 38bd4ad..fa85bf9 100644 --- a/squashfs/reader_test.go +++ b/squashfs/reader_test.go @@ -1,4 +1,4 @@ -package squashfs +package squashfs_test import ( "fmt" @@ -8,6 +8,8 @@ import ( "os/exec" "path/filepath" "testing" + + "github.com/CalebQ42/squashfs/squashfs" ) const ( @@ -55,7 +57,7 @@ func TestReader(t *testing.T) { t.Fatal(err) } defer fil.Close() - rdr, err := NewReader(fil) + rdr, err := squashfs.NewReader(fil) if err != nil { t.Fatal(err) } @@ -75,7 +77,7 @@ func TestSingleFile(t *testing.T) { t.Fatal(err) } defer fil.Close() - rdr, err := NewReader(fil) + rdr, err := squashfs.NewReader(fil) if err != nil { t.Fatal(err) } @@ -90,7 +92,7 @@ func TestSingleFile(t *testing.T) { t.Fatal(err) } -func extractToDir(rdr *Reader, b *Base, folder string) error { +func extractToDir(rdr *squashfs.Reader, b *squashfs.Base, folder string) error { path := filepath.Join(folder, b.Name) if b.IsDir() { d, err := b.ToDir(rdr) @@ -101,7 +103,7 @@ func extractToDir(rdr *Reader, b *Base, folder string) error { if err != nil { return err } - var nestBast *Base + var nestBast *squashfs.Base for _, e := range d.Entries { nestBast, err = rdr.BaseFromEntry(e) if err != nil { @@ -115,7 +117,6 @@ func extractToDir(rdr *Reader, b *Base, folder string) error { } else if b.IsRegular() { _, full, err := b.GetRegFileReaders(rdr) if err != nil { - fmt.Println("yo", path) return err } fil, err := os.Create(path) diff --git a/squashfs_test.go b/squashfs_test.go new file mode 100644 index 0000000..0ae1658 --- /dev/null +++ b/squashfs_test.go @@ -0,0 +1,193 @@ +package squashfs_test + +//Actually proper tests go here. + +import ( + "errors" + "io" + "io/fs" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/CalebQ42/squashfs" +) + +const ( + squashfsURL = "https://darkstorm.tech/files/LinuxPATest.sfs" + squashfsName = "airootfs.sfs" +) + +func preTest(dir string) (fil *os.File, err error) { + fil, err = os.Open(filepath.Join(dir, squashfsName)) + if err != nil { + _, err = os.Open(dir) + if os.IsNotExist(err) { + err = os.Mkdir(dir, 0755) + } + if err != nil { + return + } + os.Remove(filepath.Join(dir, squashfsName)) + fil, err = os.Create(filepath.Join(dir, squashfsName)) + if err != nil { + return + } + var resp *http.Response + resp, err = http.DefaultClient.Get(squashfsURL) + if err != nil { + return + } + _, err = io.Copy(fil, resp.Body) + if err != nil { + return + } + } + _, err = exec.LookPath("unsquashfs") + if err != nil { + return + } + _, err = exec.LookPath("mksquashfs") + return +} + +func TestMisc(t *testing.T) { + tmpDir := "testing" + fil, err := preTest(tmpDir) + if err != nil { + t.Fatal(err) + } + rdr, err := squashfs.NewReader(fil) + if err != nil { + t.Fatal(err) + } + _ = rdr + // Put testing here + t.Fatal("UM") +} + +func BenchmarkRace(b *testing.B) { + tmpDir := "testing" + fil, err := preTest(tmpDir) + if err != nil { + b.Fatal(err) + } + libPath := filepath.Join(tmpDir, "ExtractLib") + unsquashPath := filepath.Join(tmpDir, "ExtractSquashfs") + os.RemoveAll(libPath) + os.RemoveAll(unsquashPath) + var libTime, unsquashTime time.Duration + op := squashfs.FastOptions() + start := time.Now() + rdr, err := squashfs.NewReader(fil) + if err != nil { + b.Fatal(err) + } + err = rdr.ExtractWithOptions(libPath, op) + if err != nil { + b.Fatal(err) + } + libTime = time.Since(start) + cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name()) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + start = time.Now() + err = cmd.Run() + if err != nil { + b.Log("Unsquashfs error:", err) + } + unsquashTime = time.Since(start) + b.Log("Library took:", libTime.Round(time.Millisecond)) + b.Log("unsquashfs took:", unsquashTime.Round(time.Millisecond)) + b.Log("unsquashfs is", strconv.FormatFloat(float64(libTime.Milliseconds())/float64(unsquashTime.Milliseconds()), 'f', 2, 64), "times faster") +} + +func TestExtractQuick(t *testing.T) { + //First, setup everything and extract the archive using the library and unsquashfs + + // tmpDir := b.TempDir() + tmpDir := "testing" + fil, err := preTest(tmpDir) + if err != nil { + t.Fatal(err) + } + libPath := filepath.Join(tmpDir, "ExtractLib") + unsquashPath := filepath.Join(tmpDir, "ExtractSquashfs") + os.RemoveAll(libPath) + os.RemoveAll(unsquashPath) + rdr, err := squashfs.NewReader(fil) + if err != nil { + t.Fatal(err) + } + os.RemoveAll(filepath.Join(tmpDir, "testLog.txt")) + logFil, _ := os.Create(filepath.Join(tmpDir, "testLog.txt")) + op := squashfs.DefaultOptions() + op.Verbose = true + op.IgnorePerm = true + op.LogOutput = logFil + err = rdr.ExtractWithOptions(libPath, op) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("unsquashfs", "-d", unsquashPath, fil.Name()) + err = cmd.Run() + if err != nil { + t.Fatal(err) + } + + //Then compare the sizes and existance between the two (using unsquashfs as a reference). + //If the file doesn't exist, or the size is different, we exit. + //TODO: Add long test that checks contents. + + squashFils := os.DirFS(unsquashPath) + err = fs.WalkDir(squashFils, ".", func(path string, _ fs.DirEntry, _ error) error { + libFil, e := os.Open(filepath.Join(libPath, path)) + if e != nil { + return e + } + sfsFile, e := os.Open(filepath.Join(unsquashPath, path)) + if e != nil { + return e + } + sfsStat, _ := sfsFile.Stat() + libStat, _ := libFil.Stat() + if sfsStat.Size() != libStat.Size() { + t.Log(libFil.Name(), "not the same size between library and unsquashfs") + t.Log("File is", libStat.Size()) + t.Log("Should be", sfsStat.Size()) + return errors.New("file not the correct size") + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +var filePath = "bin" + +func TestSingleFile(t *testing.T) { + tmpDir := "testing" + fil, err := preTest(tmpDir) + if err != nil { + t.Fatal(err) + } + os.Remove(filepath.Join(tmpDir, filePath)) + rdr, err := squashfs.NewReader(fil) + if err != nil { + t.Fatal(err) + } + f, err := rdr.Open(filePath) + if err != nil { + t.Fatal(err) + } + err = f.(*squashfs.File).ExtractWithOptions("testing", &squashfs.ExtractionOptions{Verbose: true}) + if err != nil { + t.Fatal(err) + } + t.Fatal("HI") +} From bfba5d5b60bc09d8457067a7812ae55746cc8cb0 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Wed, 27 Dec 2023 23:25:49 -0600 Subject: [PATCH 09/11] Rename squashfs/squashfs to squashfs/low squashfs/low library name is now squashfslow --- README.md | 2 +- file.go | 28 ++++++++++++------------ file_info.go | 6 ++--- fs.go | 12 +++++----- {squashfs => low}/README.md | 0 {squashfs => low}/base.go | 8 +++---- {squashfs => low}/data/fullreader.go | 0 {squashfs => low}/data/reader.go | 0 {squashfs => low}/directory.go | 6 ++--- {squashfs => low}/directory/directory.go | 0 {squashfs => low}/fragment.go | 2 +- {squashfs => low}/inode.go | 6 ++--- {squashfs => low}/inode/dir.go | 0 {squashfs => low}/inode/file.go | 0 {squashfs => low}/inode/inode.go | 0 {squashfs => low}/inode/misc.go | 0 {squashfs => low}/inode/sym.go | 0 {squashfs => low}/reader.go | 4 ++-- {squashfs => low}/reader_test.go | 12 +++++----- {squashfs => low}/superblock.go | 2 +- reader.go | 10 ++++----- 21 files changed, 49 insertions(+), 49 deletions(-) rename {squashfs => low}/README.md (100%) rename {squashfs => low}/base.go (96%) rename {squashfs => low}/data/fullreader.go (100%) rename {squashfs => low}/data/reader.go (100%) rename {squashfs => low}/directory.go (94%) rename {squashfs => low}/directory/directory.go (100%) rename {squashfs => low}/fragment.go (77%) rename {squashfs => low}/inode.go (86%) rename {squashfs => low}/inode/dir.go (100%) rename {squashfs => low}/inode/file.go (100%) rename {squashfs => low}/inode/inode.go (100%) rename {squashfs => low}/inode/misc.go (100%) rename {squashfs => low}/inode/sym.go (100%) rename {squashfs => low}/reader.go (98%) rename {squashfs => low}/reader_test.go (89%) rename {squashfs => low}/superblock.go (98%) diff --git a/README.md b/README.md index 92dd55a..da5e660 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A PURE Go library to read squashfs. There is currently no plans to add archive creation support as it will almost always be better to just call `mksquashfs`. I could see some possible use cases, but probably won't spend time on it unless it's requested (open a discussion if you want this feature). -The library has two parts with this `github.com/CalebQ42/squashfs` being easy to use as it implements `io/fs` interfaces and doesn't expose unnecessary information. 95% this is the library you want. If you need lower level access to the information, use `github.com/CalebQ42/squashfs/squashfs` where far more information is exposed. +The library has two parts with this `github.com/CalebQ42/squashfs` being easy to use as it implements `io/fs` interfaces and doesn't expose unnecessary information. 95% this is the library you want. If you need lower level access to the information, use `github.com/CalebQ42/squashfs/low` where far more information is exposed. Currently has support for reading squashfs files and extracting files and folders. diff --git a/file.go b/file.go index 1b7cbd1..142c7f8 100644 --- a/file.go +++ b/file.go @@ -12,14 +12,14 @@ import ( "strconv" "github.com/CalebQ42/squashfs/internal/routinemanager" - "github.com/CalebQ42/squashfs/squashfs" - "github.com/CalebQ42/squashfs/squashfs/data" - "github.com/CalebQ42/squashfs/squashfs/inode" + squashfslow "github.com/CalebQ42/squashfs/low" + "github.com/CalebQ42/squashfs/low/data" + "github.com/CalebQ42/squashfs/low/inode" ) // File represents a file inside a squashfs archive. type File struct { - b *squashfs.Base + b *squashfslow.Base full *data.FullReader rdr *data.Reader parent *FS @@ -28,7 +28,7 @@ type File struct { } // Creates a new *File from the given *squashfs.Base -func (r *Reader) FileFromBase(b *squashfs.Base, parent *FS) *File { +func (r *Reader) FileFromBase(b *squashfslow.Base, parent *FS) *File { return &File{ b: b, parent: parent, @@ -40,7 +40,7 @@ func (f *File) FS() (*FS, error) { if !f.IsDir() { return nil, errors.New("not a directory") } - d, err := f.b.ToDir(f.r.r) + d, err := f.b.ToDir(f.r.Low) if err != nil { return nil, err } @@ -114,7 +114,7 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { if !f.IsDir() { return nil, errors.New("file is not a directory") } - d, err := f.b.ToDir(f.r.r) + d, err := f.b.ToDir(f.r.Low) if err != nil { return nil, err } @@ -173,7 +173,7 @@ func (f *File) WriteTo(w io.Writer) (int64, error) { func (f *File) initializeReaders() error { var err error - f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.r) + f.rdr, f.full, err = f.b.GetRegFileReaders(f.r.Low) return err } @@ -218,7 +218,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { } switch f.b.Inode.Type { case inode.Dir, inode.EDir: - d, err := f.b.ToDir(f.r.r) + d, err := f.b.ToDir(f.r.Low) if err != nil { if op.Verbose { log.Println("Failed to create squashfs.Directory for", path) @@ -227,14 +227,14 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { } errChan := make(chan error, len(d.Entries)) for i := range d.Entries { - b, err := f.r.r.BaseFromEntry(d.Entries[i]) + b, err := f.r.Low.BaseFromEntry(d.Entries[i]) if err != nil { if op.Verbose { log.Println("Failed to get squashfs.Base from entry for", path) } return errors.Join(errors.New("failed to get base from entry: "+path), err) } - go func(b *squashfs.Base, path string) { + go func(b *squashfslow.Base, path string) { i := op.manager.Lock() if b.IsDir() { extDir := filepath.Join(path, b.Name) @@ -285,7 +285,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { return errors.Join(errors.New("failed to create file: "+path), err) } defer outFil.Close() - full, err := f.b.GetFullReader(f.r.r) + full, err := f.b.GetFullReader(f.r.Low) if err != nil { if op.Verbose { log.Println("Failed to create full reader for", path) @@ -406,7 +406,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { if op.IgnorePerm { return nil } - uid, err := f.b.Uid(f.r.r) + uid, err := f.b.Uid(f.r.Low) if err != nil { if op.Verbose { log.Println("Failed to get uid for", path) @@ -414,7 +414,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { } return nil } - gid, err := f.b.Gid(f.r.r) + gid, err := f.b.Gid(f.r.Low) if err != nil { if op.Verbose { log.Println("Failed to get gid for", path) diff --git a/file_info.go b/file_info.go index b4552a7..d7390a6 100644 --- a/file_info.go +++ b/file_info.go @@ -4,8 +4,8 @@ import ( "io/fs" "time" - "github.com/CalebQ42/squashfs/squashfs/directory" - "github.com/CalebQ42/squashfs/squashfs/inode" + "github.com/CalebQ42/squashfs/low/directory" + "github.com/CalebQ42/squashfs/low/inode" ) type fileInfo struct { @@ -17,7 +17,7 @@ type fileInfo struct { } func (r Reader) newFileInfo(e directory.Entry) (fileInfo, error) { - i, err := r.r.InodeFromEntry(e) + i, err := r.Low.InodeFromEntry(e) if err != nil { return fileInfo{}, err } diff --git a/fs.go b/fs.go index e81acb0..4986790 100644 --- a/fs.go +++ b/fs.go @@ -8,20 +8,20 @@ import ( "slices" "strings" - "github.com/CalebQ42/squashfs/squashfs" - "github.com/CalebQ42/squashfs/squashfs/directory" + squashfslow "github.com/CalebQ42/squashfs/low" + "github.com/CalebQ42/squashfs/low/directory" ) // FS is a fs.FS representation of a squashfs directory. // Implements fs.GlobFS, fs.ReadDirFS, fs.ReadFileFS, fs.StatFS, and fs.SubFS type FS struct { - d *squashfs.Directory + d *squashfslow.Directory r *Reader parent *FS } // Creates a new *FS from the given squashfs.directory -func (r *Reader) FSFromDirectory(d *squashfs.Directory, parent *FS) *FS { +func (r *Reader) FSFromDirectory(d *squashfslow.Directory, parent *FS) *FS { return &FS{ d: d, r: r, @@ -124,7 +124,7 @@ func (f *FS) Open(name string) (fs.File, error) { Err: fs.ErrNotExist, } } - b, err := f.r.r.BaseFromEntry(f.d.Entries[i]) + b, err := f.r.Low.BaseFromEntry(f.d.Entries[i]) if err != nil { return nil, err } @@ -142,7 +142,7 @@ func (f *FS) Open(name string) (fs.File, error) { Err: fs.ErrNotExist, } } - d, err := b.ToDir(f.r.r) + d, err := b.ToDir(f.r.Low) if err != nil { return nil, err } diff --git a/squashfs/README.md b/low/README.md similarity index 100% rename from squashfs/README.md rename to low/README.md diff --git a/squashfs/base.go b/low/base.go similarity index 96% rename from squashfs/base.go rename to low/base.go index 53c8207..caf43fb 100644 --- a/squashfs/base.go +++ b/low/base.go @@ -1,4 +1,4 @@ -package squashfs +package squashfslow import ( "errors" @@ -6,9 +6,9 @@ import ( "github.com/CalebQ42/squashfs/internal/metadata" "github.com/CalebQ42/squashfs/internal/toreader" - "github.com/CalebQ42/squashfs/squashfs/data" - "github.com/CalebQ42/squashfs/squashfs/directory" - "github.com/CalebQ42/squashfs/squashfs/inode" + "github.com/CalebQ42/squashfs/low/data" + "github.com/CalebQ42/squashfs/low/directory" + "github.com/CalebQ42/squashfs/low/inode" ) type Base struct { diff --git a/squashfs/data/fullreader.go b/low/data/fullreader.go similarity index 100% rename from squashfs/data/fullreader.go rename to low/data/fullreader.go diff --git a/squashfs/data/reader.go b/low/data/reader.go similarity index 100% rename from squashfs/data/reader.go rename to low/data/reader.go diff --git a/squashfs/directory.go b/low/directory.go similarity index 94% rename from squashfs/directory.go rename to low/directory.go index 1681e89..6bdc248 100644 --- a/squashfs/directory.go +++ b/low/directory.go @@ -1,4 +1,4 @@ -package squashfs +package squashfslow import ( "errors" @@ -9,8 +9,8 @@ import ( "github.com/CalebQ42/squashfs/internal/metadata" "github.com/CalebQ42/squashfs/internal/toreader" - "github.com/CalebQ42/squashfs/squashfs/directory" - "github.com/CalebQ42/squashfs/squashfs/inode" + "github.com/CalebQ42/squashfs/low/directory" + "github.com/CalebQ42/squashfs/low/inode" ) type Directory struct { diff --git a/squashfs/directory/directory.go b/low/directory/directory.go similarity index 100% rename from squashfs/directory/directory.go rename to low/directory/directory.go diff --git a/squashfs/fragment.go b/low/fragment.go similarity index 77% rename from squashfs/fragment.go rename to low/fragment.go index ad103d9..7230ef1 100644 --- a/squashfs/fragment.go +++ b/low/fragment.go @@ -1,4 +1,4 @@ -package squashfs +package squashfslow type fragEntry struct { Start uint64 diff --git a/squashfs/inode.go b/low/inode.go similarity index 86% rename from squashfs/inode.go rename to low/inode.go index d1cb5a2..a10a2d1 100644 --- a/squashfs/inode.go +++ b/low/inode.go @@ -1,10 +1,10 @@ -package squashfs +package squashfslow import ( "github.com/CalebQ42/squashfs/internal/metadata" "github.com/CalebQ42/squashfs/internal/toreader" - "github.com/CalebQ42/squashfs/squashfs/directory" - "github.com/CalebQ42/squashfs/squashfs/inode" + "github.com/CalebQ42/squashfs/low/directory" + "github.com/CalebQ42/squashfs/low/inode" ) func (r *Reader) InodeFromRef(ref uint64) (*inode.Inode, error) { diff --git a/squashfs/inode/dir.go b/low/inode/dir.go similarity index 100% rename from squashfs/inode/dir.go rename to low/inode/dir.go diff --git a/squashfs/inode/file.go b/low/inode/file.go similarity index 100% rename from squashfs/inode/file.go rename to low/inode/file.go diff --git a/squashfs/inode/inode.go b/low/inode/inode.go similarity index 100% rename from squashfs/inode/inode.go rename to low/inode/inode.go diff --git a/squashfs/inode/misc.go b/low/inode/misc.go similarity index 100% rename from squashfs/inode/misc.go rename to low/inode/misc.go diff --git a/squashfs/inode/sym.go b/low/inode/sym.go similarity index 100% rename from squashfs/inode/sym.go rename to low/inode/sym.go diff --git a/squashfs/reader.go b/low/reader.go similarity index 98% rename from squashfs/reader.go rename to low/reader.go index 302cbc9..d6d6bef 100644 --- a/squashfs/reader.go +++ b/low/reader.go @@ -1,4 +1,4 @@ -package squashfs +package squashfslow import ( "encoding/binary" @@ -9,7 +9,7 @@ import ( "github.com/CalebQ42/squashfs/internal/decompress" "github.com/CalebQ42/squashfs/internal/metadata" "github.com/CalebQ42/squashfs/internal/toreader" - "github.com/CalebQ42/squashfs/squashfs/inode" + "github.com/CalebQ42/squashfs/low/inode" ) // The types of compression supported by squashfs diff --git a/squashfs/reader_test.go b/low/reader_test.go similarity index 89% rename from squashfs/reader_test.go rename to low/reader_test.go index fa85bf9..1edf591 100644 --- a/squashfs/reader_test.go +++ b/low/reader_test.go @@ -1,4 +1,4 @@ -package squashfs_test +package squashfslow_test import ( "fmt" @@ -9,7 +9,7 @@ import ( "path/filepath" "testing" - "github.com/CalebQ42/squashfs/squashfs" + squashfslow "github.com/CalebQ42/squashfs/low" ) const ( @@ -57,7 +57,7 @@ func TestReader(t *testing.T) { t.Fatal(err) } defer fil.Close() - rdr, err := squashfs.NewReader(fil) + rdr, err := squashfslow.NewReader(fil) if err != nil { t.Fatal(err) } @@ -77,7 +77,7 @@ func TestSingleFile(t *testing.T) { t.Fatal(err) } defer fil.Close() - rdr, err := squashfs.NewReader(fil) + rdr, err := squashfslow.NewReader(fil) if err != nil { t.Fatal(err) } @@ -92,7 +92,7 @@ func TestSingleFile(t *testing.T) { t.Fatal(err) } -func extractToDir(rdr *squashfs.Reader, b *squashfs.Base, folder string) error { +func extractToDir(rdr *squashfslow.Reader, b *squashfslow.Base, folder string) error { path := filepath.Join(folder, b.Name) if b.IsDir() { d, err := b.ToDir(rdr) @@ -103,7 +103,7 @@ func extractToDir(rdr *squashfs.Reader, b *squashfs.Base, folder string) error { if err != nil { return err } - var nestBast *squashfs.Base + var nestBast *squashfslow.Base for _, e := range d.Entries { nestBast, err = rdr.BaseFromEntry(e) if err != nil { diff --git a/squashfs/superblock.go b/low/superblock.go similarity index 98% rename from squashfs/superblock.go rename to low/superblock.go index 0b19dd3..dca087a 100644 --- a/squashfs/superblock.go +++ b/low/superblock.go @@ -1,4 +1,4 @@ -package squashfs +package squashfslow import "math" diff --git a/reader.go b/reader.go index 43058c1..ce499f8 100644 --- a/reader.go +++ b/reader.go @@ -4,21 +4,21 @@ import ( "io" "time" - "github.com/CalebQ42/squashfs/squashfs" + squashfslow "github.com/CalebQ42/squashfs/low" ) type Reader struct { *FS - r *squashfs.Reader + Low *squashfslow.Reader } func NewReader(r io.ReaderAt) (*Reader, error) { - rdr, err := squashfs.NewReader(r) + rdr, err := squashfslow.NewReader(r) if err != nil { return nil, err } out := &Reader{ - r: rdr, + Low: rdr, } out.FS = &FS{ d: rdr.Root, @@ -28,5 +28,5 @@ func NewReader(r io.ReaderAt) (*Reader, error) { } func (r *Reader) ModTime() time.Time { - return time.Unix(int64(r.r.Superblock.ModTime), 0) + return time.Unix(int64(r.Low.Superblock.ModTime), 0) } From 144805e747018dad957d0c73f81658f9d5b1cbbe Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Wed, 27 Dec 2023 23:50:27 -0600 Subject: [PATCH 10/11] Rename squashfslow.Base to squashfslow.FileBase --- file.go | 6 ++-- fs.go | 2 +- low/directory.go | 10 +++--- low/{base.go => file_base.go} | 67 ++++++++++++++++++++++++++--------- low/reader_test.go | 6 ++-- 5 files changed, 63 insertions(+), 28 deletions(-) rename low/{base.go => file_base.go} (67%) diff --git a/file.go b/file.go index 142c7f8..7b2fcdf 100644 --- a/file.go +++ b/file.go @@ -19,7 +19,7 @@ import ( // File represents a file inside a squashfs archive. type File struct { - b *squashfslow.Base + b *squashfslow.FileBase full *data.FullReader rdr *data.Reader parent *FS @@ -28,7 +28,7 @@ type File struct { } // Creates a new *File from the given *squashfs.Base -func (r *Reader) FileFromBase(b *squashfslow.Base, parent *FS) *File { +func (r *Reader) FileFromBase(b *squashfslow.FileBase, parent *FS) *File { return &File{ b: b, parent: parent, @@ -234,7 +234,7 @@ func (f *File) ExtractWithOptions(path string, op *ExtractionOptions) error { } return errors.Join(errors.New("failed to get base from entry: "+path), err) } - go func(b *squashfslow.Base, path string) { + go func(b *squashfslow.FileBase, path string) { i := op.manager.Lock() if b.IsDir() { extDir := filepath.Join(path, b.Name) diff --git a/fs.go b/fs.go index 4986790..43d9e1e 100644 --- a/fs.go +++ b/fs.go @@ -255,7 +255,7 @@ func (f *FS) ExtractWithOptions(folder string, op *ExtractionOptions) error { // Returns the FS as a *File func (f *FS) File() *File { return &File{ - b: &f.d.Base, + b: &f.d.FileBase, parent: f.parent, r: f.r, } diff --git a/low/directory.go b/low/directory.go index 6bdc248..b219d77 100644 --- a/low/directory.go +++ b/low/directory.go @@ -14,7 +14,7 @@ import ( ) type Directory struct { - Base + FileBase Entries []directory.Entry } @@ -49,15 +49,15 @@ func (r *Reader) directoryFromRef(ref uint64, name string) (*Directory, error) { return nil, err } return &Directory{ - Base: *r.BaseFromInode(i, name), - Entries: entries, + FileBase: *r.BaseFromInode(i, name), + Entries: entries, }, nil } -func (d *Directory) Open(r *Reader, path string) (*Base, error) { +func (d *Directory) Open(r *Reader, path string) (*FileBase, error) { path = filepath.Clean(path) if path == "." || path == "" { - return &d.Base, nil + return &d.FileBase, nil } split := strings.Split(path, "/") i, found := slices.BinarySearchFunc(d.Entries, split[0], func(e directory.Entry, name string) int { diff --git a/low/base.go b/low/file_base.go similarity index 67% rename from low/base.go rename to low/file_base.go index caf43fb..e87bc9e 100644 --- a/low/base.go +++ b/low/file_base.go @@ -11,44 +11,44 @@ import ( "github.com/CalebQ42/squashfs/low/inode" ) -type Base struct { +type FileBase struct { Inode *inode.Inode Name string } -func (r *Reader) BaseFromInode(i *inode.Inode, name string) *Base { - return &Base{Inode: i, Name: name} +func (r *Reader) BaseFromInode(i *inode.Inode, name string) *FileBase { + return &FileBase{Inode: i, Name: name} } -func (r *Reader) BaseFromEntry(e directory.Entry) (*Base, error) { +func (r *Reader) BaseFromEntry(e directory.Entry) (*FileBase, error) { in, err := r.InodeFromEntry(e) if err != nil { return nil, err } - return &Base{Inode: in, Name: e.Name}, nil + return &FileBase{Inode: in, Name: e.Name}, nil } -func (r *Reader) BaseFromRef(ref uint64, name string) (*Base, error) { +func (r *Reader) BaseFromRef(ref uint64, name string) (*FileBase, error) { in, err := r.InodeFromRef(ref) if err != nil { return nil, err } - return &Base{Inode: in, Name: name}, nil + return &FileBase{Inode: in, Name: name}, nil } -func (b *Base) Uid(r *Reader) (uint32, error) { +func (b *FileBase) Uid(r *Reader) (uint32, error) { return r.Id(b.Inode.UidInd) } -func (b *Base) Gid(r *Reader) (uint32, error) { +func (b *FileBase) Gid(r *Reader) (uint32, error) { return r.Id(b.Inode.GidInd) } -func (b *Base) IsDir() bool { +func (b *FileBase) IsDir() bool { return b.Inode.Type == inode.Dir || b.Inode.Type == inode.EDir } -func (b *Base) ToDir(r *Reader) (*Directory, error) { +func (b *FileBase) ToDir(r *Reader) (*Directory, error) { var blockStart uint32 var size uint32 var offset uint16 @@ -75,16 +75,16 @@ func (b *Base) ToDir(r *Reader) (*Directory, error) { return nil, err } return &Directory{ - Base: *b, - Entries: entries, + FileBase: *b, + Entries: entries, }, nil } -func (b *Base) IsRegular() bool { +func (b *FileBase) IsRegular() bool { return b.Inode.Type == inode.Fil || b.Inode.Type == inode.EFil } -func (b *Base) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, error) { +func (b *FileBase) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, error) { if !b.IsRegular() { return nil, nil, errors.New("not a regular file") } @@ -130,7 +130,7 @@ func (b *Base) GetRegFileReaders(r *Reader) (*data.Reader, *data.FullReader, err return outRdr, outFull, nil } -func (b *Base) GetFullReader(r *Reader) (*data.FullReader, error) { +func (b *FileBase) GetFullReader(r *Reader) (*data.FullReader, error) { if !b.IsRegular() { return nil, errors.New("not a regular file") } @@ -166,3 +166,38 @@ func (b *Base) GetFullReader(r *Reader) (*data.FullReader, error) { } return outFull, nil } + +func (b *FileBase) GetReader(r *Reader) (*data.Reader, error) { + if !b.IsRegular() { + return nil, errors.New("not a regular file") + } + var blockStart uint64 + var fragIndex uint32 + var fragOffset uint32 + var fragSize uint64 + var sizes []uint32 + if b.Inode.Type == inode.Fil { + blockStart = uint64(b.Inode.Data.(inode.File).BlockStart) + fragIndex = b.Inode.Data.(inode.File).FragInd + fragOffset = b.Inode.Data.(inode.File).FragOffset + sizes = b.Inode.Data.(inode.File).BlockSizes + fragSize = uint64(b.Inode.Data.(inode.File).Size % r.Superblock.BlockSize) + } else { + blockStart = b.Inode.Data.(inode.EFile).BlockStart + fragIndex = b.Inode.Data.(inode.EFile).FragInd + fragOffset = b.Inode.Data.(inode.EFile).FragOffset + sizes = b.Inode.Data.(inode.EFile).BlockSizes + fragSize = b.Inode.Data.(inode.EFile).Size % uint64(r.Superblock.BlockSize) + } + outRdr := data.NewReader(toreader.NewReader(r.r, int64(blockStart)), r.d, sizes, fragSize, r.Superblock.BlockSize) + if fragIndex != 0xffffffff { + ent, err := r.fragEntry(fragIndex) + if err != nil { + return nil, err + } + frag := data.NewReader(toreader.NewReader(r.r, int64(ent.Start)), r.d, []uint32{ent.Size}, uint64(r.Superblock.BlockSize), r.Superblock.BlockSize) + frag.Read(make([]byte, fragOffset)) + outRdr.AddFrag(io.LimitReader(frag, int64(fragSize))) + } + return outRdr, nil +} diff --git a/low/reader_test.go b/low/reader_test.go index 1edf591..7fe237f 100644 --- a/low/reader_test.go +++ b/low/reader_test.go @@ -64,7 +64,7 @@ func TestReader(t *testing.T) { path := filepath.Join(tmpDir, "extractTest") os.RemoveAll(path) os.MkdirAll(path, 0777) - err = extractToDir(rdr, &rdr.Root.Base, path) + err = extractToDir(rdr, &rdr.Root.FileBase, path) t.Fatal(err) } @@ -92,7 +92,7 @@ func TestSingleFile(t *testing.T) { t.Fatal(err) } -func extractToDir(rdr *squashfslow.Reader, b *squashfslow.Base, folder string) error { +func extractToDir(rdr *squashfslow.Reader, b *squashfslow.FileBase, folder string) error { path := filepath.Join(folder, b.Name) if b.IsDir() { d, err := b.ToDir(rdr) @@ -103,7 +103,7 @@ func extractToDir(rdr *squashfslow.Reader, b *squashfslow.Base, folder string) e if err != nil { return err } - var nestBast *squashfslow.Base + var nestBast *squashfslow.FileBase for _, e := range d.Entries { nestBast, err = rdr.BaseFromEntry(e) if err != nil { From ef72408cd08fe44b353affcb8f9adf549e995a54 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Wed, 27 Dec 2023 23:55:57 -0600 Subject: [PATCH 11/11] Added inode number to directory.Entry --- low/directory/directory.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/low/directory/directory.go b/low/directory/directory.go index 1464335..be58199 100644 --- a/low/directory/directory.go +++ b/low/directory/directory.go @@ -24,6 +24,7 @@ type Entry struct { BlockStart uint32 Offset uint16 InodeType uint16 + Num uint32 } func ReadDirectory(r io.Reader, size uint32) (out []Entry, err error) { @@ -53,6 +54,7 @@ func ReadDirectory(r io.Reader, size uint32) (out []Entry, err error) { Offset: de.Offset, Name: string(nameTmp), InodeType: de.InodeType, + Num: h.Num + uint32(de.NumOffset), }) } }