From 40541575f876c65ddfb0181b4f237a67cb53830d Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Tue, 10 Nov 2020 03:48:03 -0600 Subject: [PATCH] Found some good squashfs documentation so I can start work --- reader_old.go | 618 +++++++++++++++++++++++++++++++++ reader_test.go | 373 ++++++++++++++++++++ superblock.go | 24 ++ unsquash.go | 9 + writer_old.go | 913 +++++++++++++++++++++++++++++++++++++++++++++++++ writer_test.go | 310 +++++++++++++++++ 6 files changed, 2247 insertions(+) create mode 100644 reader_old.go create mode 100644 reader_test.go create mode 100644 superblock.go create mode 100644 unsquash.go create mode 100644 writer_old.go create mode 100644 writer_test.go diff --git a/reader_old.go b/reader_old.go new file mode 100644 index 0000000..2187a66 --- /dev/null +++ b/reader_old.go @@ -0,0 +1,618 @@ +package squashfs + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "golang.org/x/xerrors" +) + +type Reader struct { + r io.ReaderAt + super superblock +} + +func NewReader(r io.ReaderAt) (*Reader, error) { + var sb superblock + + if err := binary.Read(io.NewSectionReader(r, 0, int64(binary.Size(sb))), binary.LittleEndian, &sb); err != nil { + return nil, fmt.Errorf("reading superblock: %v", err) + } + + if got, want := sb.Magic, uint32(magic); got != want { + return nil, fmt.Errorf("invalid magic (not a SquashFS image?): got %x, want %x", got, want) + } + + //log.Printf("superblock: %+v", sb) + return &Reader{ + r: r, + super: sb, + }, nil +} + +// TODO: maybe mmap instead of seeking? + +func (r *Reader) inode(i Inode) (blockoffset int64, offset int64) { + return int64(i >> 16), int64(i & 0xFFFF) +} + +type blockReader struct { + r io.ReadSeeker + lenBuf [2]byte + buf []byte + i int64 + + off int64 // TODO: remove this once using mmap +} + +func (br *blockReader) Read(p []byte) (n int, err error) { + if br.i >= int64(len(br.buf)) { + br.i = 0 + if _, err := io.ReadFull(br.r, br.lenBuf[:]); err != nil { + return 0, err + } + l := binary.LittleEndian.Uint16(br.lenBuf[:]) + //uncompressed := l&0x8000 > 0 + l &= 0x7FFF + //log.Printf("block of len %d, uncompressed: %v", l, uncompressed) + if int(l) > cap(br.buf) { + br.buf = make([]byte, int(l)) + } + br.buf = br.buf[:l] + if _, err := io.ReadFull(br.r, br.buf); err != nil { + return 0, err + } + //log.Printf("(retry) n = %v, err = %v", n, err) + } + n = copy(p, br.buf[br.i:]) + br.i += int64(n) + return n, err +} + +func (br *blockReader) Close() error { + blockReaderPool.Put(br) + return nil +} + +var blockReaderPool = sync.Pool{ + New: func() interface{} { + return &blockReader{ + buf: make([]byte, 0, metadataBlockSize), + } + }, +} + +func (r *Reader) blockReader(blockoffset, offset int64) (io.ReadCloser, error) { + //log.Printf("blockoffset %v (%x), offset %v (%x)", blockoffset, blockoffset, offset, offset) + br := blockReaderPool.Get().(*blockReader) + br.buf = br.buf[:0] + br.r = io.NewSectionReader(r.r, blockoffset, 5500*1024*1024) // TODO: correct limit? can we use IntMax + br.off = blockoffset + br.i = 0 + //log.Printf("discarding %d bytes", offset) + for n := int64(0); n < offset; { + remaining := offset - n + if remaining > metadataBlockSize { + remaining = metadataBlockSize + } + nn, err := br.Read(br.buf[:remaining]) + if err != nil { + return nil, err + } + n += int64(nn) + } + return br, nil +} + +// TODO: define an inode type to use instead of interface{}? +func (r *Reader) readInode(i Inode) (interface{}, error) { + blockoffset, offset := r.inode(i) + br, err := r.blockReader(r.super.InodeTableStart+blockoffset, offset) + if err != nil { + fmt.Println("oops! ", err) + return nil, err + } + defer br.Close() + fmt.Println("Hello There") + + // We need the inode type before we know which type to pass to binary.Read, + // so we need to read it twice: + var inodeType uint16 + typeBuf := bytes.NewBuffer(make([]byte, 0, binary.Size(inodeType))) + if err := binary.Read(io.TeeReader(br, typeBuf), binary.LittleEndian, &inodeType); err != nil { + return nil, err + } + br = ioutil.NopCloser(io.MultiReader(typeBuf, br)) + + // var ih inodeHeader + // if err := binary.Read(br, binary.LittleEndian, &ih); err != nil { + // return err + // } + // //log.Printf("ih: %+v", ih) + + //log.Printf("inode type: %v", inodeType) + switch inodeType { + case dirType: + var di dirInodeHeader + if err := binary.Read(br, binary.LittleEndian, &di); err != nil { + return nil, err + } + return di, nil + + case fileType: + var ri regInodeHeader + if err := binary.Read(br, binary.LittleEndian, &ri); err != nil { + return nil, err + } + return ri, nil + + case symlinkType: + var si symlinkInodeHeader + if err := binary.Read(br, binary.LittleEndian, &si); err != nil { + return nil, err + } + return si, nil + + case ldirType: + var di ldirInodeHeader + if err := binary.Read(br, binary.LittleEndian, &di); err != nil { + return nil, err + } + return di, nil + + case lregType: + var di lregInodeHeader + if err := binary.Read(br, binary.LittleEndian, &di); err != nil { + return nil, err + } + return di, nil + + // TODO: + // blkdevType + // chrdevType + // fifoType + // socketType + // // The larger types are used for e.g. sparse files, xattrs, etc. + // ldirType + // lsymlinkType + // lblkdevType + // lchrdevType + // lfifoType + // lsocketType + + } + return nil, fmt.Errorf("unknown inode type %d", inodeType) +} + +func (r *Reader) RootInode() Inode { + return r.super.RootInode +} + +func (r *Reader) Stat(name string, i Inode) (os.FileInfo, error) { + inode, err := r.readInode(i) + if err != nil { + return nil, err + } + //log.Printf("i %d, inode: %T, %+v", i, inode, inode) + switch x := inode.(type) { + case dirInodeHeader: + return &FileInfo{ + name: name, + size: int64(x.FileSize), + mode: os.ModeDir | os.FileMode(x.Mode), + modTime: time.Unix(int64(x.Mtime), 0), + Inode: i, + }, nil + + case ldirInodeHeader: + return &FileInfo{ + name: name, + size: int64(x.FileSize), + mode: os.ModeDir | os.FileMode(x.Mode), + modTime: time.Unix(int64(x.Mtime), 0), + Inode: i, + }, nil + + case regInodeHeader: + mode := os.FileMode(x.Mode & 0777) + if x.Mode&syscall.S_ISUID != 0 { + mode |= os.ModeSetuid + } + return &FileInfo{ + name: name, + size: int64(x.FileSize), + mode: mode, + modTime: time.Unix(int64(x.Mtime), 0), + Inode: i, + }, nil + + case lregInodeHeader: + mode := os.FileMode(x.Mode & 0777) + if x.Mode&syscall.S_ISUID != 0 { + mode |= os.ModeSetuid + } + return &FileInfo{ + name: name, + size: int64(x.FileSize), + mode: mode, + modTime: time.Unix(int64(x.Mtime), 0), + Inode: i, + }, nil + + case symlinkInodeHeader: + return &FileInfo{ + name: name, + size: int64(x.SymlinkSize), + mode: os.ModeSymlink | os.FileMode(x.Mode), + modTime: time.Unix(int64(x.Mtime), 0), + Inode: i, + }, nil + } + + return nil, fmt.Errorf("unknown inode type %T", inode) +} + +func (r *Reader) ReadLink(i Inode) (string, error) { + // TODO: reduce code duplication with readInode + blockoffset, offset := r.inode(i) + br, err := r.blockReader(r.super.InodeTableStart+blockoffset, offset) + if err != nil { + return "", err + } + defer br.Close() + + // We need the inode type before we know which type to pass to binary.Read, + // so we need to read it twice: + var inodeType uint16 + typeBuf := bytes.NewBuffer(make([]byte, 0, binary.Size(inodeType))) + if err := binary.Read(io.TeeReader(br, typeBuf), binary.LittleEndian, &inodeType); err != nil { + return "", err + } + br = ioutil.NopCloser(io.MultiReader(typeBuf, br)) + + if inodeType != symlinkType { + return "", fmt.Errorf("invalid inode type: got %d instead of symlink", inodeType) + } + var si symlinkInodeHeader + if err := binary.Read(br, binary.LittleEndian, &si); err != nil { + return "", err + } + + // Assumption: r.r is positioned right after the inode + buf := make([]byte, si.SymlinkSize) + if _, err := io.ReadFull(br, buf); err != nil { + return "", err + } + return string(buf), nil +} + +func (r *Reader) FileReader(inode Inode) (*io.SectionReader, error) { + //log.Printf("Readfile(%v)", inode) + i, err := r.readInode(inode) + if err != nil { + return nil, err + } + //log.Printf("i: %+v", i) + // TODO(compression): read the blocksizes to read compressed blocks + switch ri := i.(type) { + case regInodeHeader: + off := int64(ri.StartBlock) + int64(ri.Offset) + return io.NewSectionReader(r.r, off, int64(ri.FileSize)), nil + case lregInodeHeader: + off := int64(ri.StartBlock) + int64(ri.Offset) + return io.NewSectionReader(r.r, off, int64(ri.FileSize)), nil + default: + return nil, fmt.Errorf("BUG: non-file inode type") + } +} + +type FileNotFoundError struct { + path string +} + +func (e *FileNotFoundError) Error() string { + return fmt.Sprintf("%q not found", e.path) +} + +func (r *Reader) lookupComponent(parent Inode, component string) (Inode, error) { + rfis, err := r.readdir(parent, false) + if err != nil { + return 0, err + } + for _, rfi := range rfis { + if rfi.Name() == component { + return rfi.Sys().(*FileInfo).Inode, nil + } + } + return 0, &FileNotFoundError{path: component} +} + +func (r *Reader) lookupPath(path string, followSymlink bool) (Inode, error) { + inode := r.RootInode() + parts := strings.Split(path, "/") + for idx, part := range parts { + var err error + inode, err = r.lookupComponent(inode, part) + if err != nil { + if _, ok := err.(*FileNotFoundError); ok { + return 0, &FileNotFoundError{path: path} + } + return 0, err + } + if !followSymlink { + continue + } + i, err := r.readInode(inode) + if err != nil { + return 0, xerrors.Errorf("Stat(%d): %v", inode, err) + } + if _, ok := i.(symlinkInodeHeader); ok { + target, err := r.ReadLink(inode) + if err != nil { + return 0, err + } + //log.Printf("component %q (full: %q) resolved to %q", part, parts[:idx+1], target) + target = filepath.Clean(filepath.Join(append(parts[:idx] /* parent */, target)...)) + //log.Printf("-> %s", target) + i, err := r.LookupPath(target) + if err != nil { + return 0, err + } + inode = i + } + } + return inode, nil +} + +func (r *Reader) LookupPath(path string) (Inode, error) { + return r.lookupPath(path, true) +} + +// LlookupPath is like LookupPath, but does not follow symbolic links, i.e. will +// instead return the inode of the link itself. +func (r *Reader) LlookupPath(path string) (Inode, error) { + return r.lookupPath(path, false) +} + +func (r *Reader) Readdir(dirInode Inode) ([]os.FileInfo, error) { + return r.readdir(dirInode, true) +} + +// Like Readdir, but does not call Stat on each file. The returned FileInfo +// structs will still have a filled in Name, partly filled in Mode, and filled +// in Inode. +func (r *Reader) ReaddirNoStat(dirInode Inode) ([]os.FileInfo, error) { + return r.readdir(dirInode, false) +} + +var nameBufPool = sync.Pool{ + New: func() interface{} { + return &bytes.Buffer{} + }, +} + +func (r *Reader) readdir(dirInode Inode, stat bool) ([]os.FileInfo, error) { + //log.Printf("Readdir(%v (%x))", dirInode, dirInode) + i, err := r.readInode(dirInode) + fmt.Println("Yodle") + if err != nil { + return nil, err + } + var ( + startBlock int64 + fileSize int64 + offset int64 + ) + switch x := i.(type) { + case dirInodeHeader: + startBlock = int64(x.StartBlock) + fileSize = int64(x.FileSize) + offset = int64(x.Offset) + + case ldirInodeHeader: + startBlock = int64(x.StartBlock) + fileSize = int64(x.FileSize) + offset = int64(x.Offset) + + default: + return nil, fmt.Errorf("unknown directory inode type %T", i) + } + + br, err := r.blockReader(r.super.DirectoryTableStart+startBlock, offset) + if err != nil { + return nil, err + } + defer br.Close() + + // See also https://elixir.bootlin.com/linux/v4.18.9/source/fs/squashfs/dir.c#L63 + limit := fileSize - int64(len(".")) - int64(len("..")) + br = ioutil.NopCloser(io.LimitReader(br, limit)) + + var fis []os.FileInfo + var dh dirHeader + var de dirEntry + var dhBuf [12]byte + var deBuf [8]byte + nameBuf := nameBufPool.Get().(*bytes.Buffer) + defer nameBufPool.Put(nameBuf) + for { + if _, err := io.ReadFull(br, dhBuf[:]); err != nil { + if err == io.EOF { + return fis, nil + } + return nil, err + } + dh.Unmarshal(dhBuf[:]) + dh.Count++ // SquashFS stores count-1 + //log.Printf("dh: %+v", dh) + + for i := 0; i < int(dh.Count); i++ { + if _, err := io.ReadFull(br, deBuf[:]); err != nil { + return nil, err + } + de.Unmarshal(deBuf[:]) + de.Size++ // SquashFS stores size-1 + //log.Printf("de: %+v", de) + nameBuf.Reset() + nameBuf.Grow(int(de.Size)) + nb := nameBuf.Bytes()[:de.Size] + if _, err := io.ReadFull(br, nb); err != nil { + return nil, err + } + name := string(nb) + //log.Printf("name: %q", string(name)) + + var fi os.FileInfo + if stat { + var err error + fi, err = r.Stat(name, Inode(int64(dh.StartBlock)<<16|int64(de.Offset))) + if err != nil { + return nil, err + } + } else { + ffi := &FileInfo{ + name: name, + Inode: Inode(int64(dh.StartBlock)<<16 | int64(de.Offset)), + } + switch de.EntryType { + case dirType, ldirType: + ffi.mode |= os.ModeDir + case symlinkType, lsymlinkType: + ffi.mode |= os.ModeSymlink + } + fi = ffi + } + fis = append(fis, fi) + } + } + + return fis, nil +} + +type FileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time + Inode Inode +} + +func (fi *FileInfo) Name() string { return fi.name } +func (fi *FileInfo) Size() int64 { return fi.size } +func (fi *FileInfo) Mode() os.FileMode { return fi.mode } +func (fi *FileInfo) IsDir() bool { return fi.mode.IsDir() } +func (fi *FileInfo) ModTime() time.Time { return fi.modTime } +func (fi *FileInfo) Sys() interface{} { return fi } + +func (r *Reader) readXattr(tableHeader xattrTableHeader, id xattrId) (*Xattr, error) { + blockoffset, offset := r.inode(Inode(id.Xattr)) + br, err := r.blockReader(int64(tableHeader.XattrTableStart)+blockoffset, offset) + if err != nil { + return nil, err + } + defer br.Close() + var typ, nameSize uint16 + if err := binary.Read(br, binary.LittleEndian, &typ); err != nil { + return nil, err + } + if err := binary.Read(br, binary.LittleEndian, &nameSize); err != nil { + return nil, err + } + log.Printf("type = %v, nameSize = %v", typ, nameSize) + name := make([]byte, nameSize) + if _, err := io.ReadFull(br, name); err != nil { + return nil, err + } + log.Printf("name = %v", string(name)) + var valSize uint32 + if err := binary.Read(br, binary.LittleEndian, &valSize); err != nil { + return nil, err + } + val := make([]byte, valSize) + if _, err := io.ReadFull(br, val); err != nil { + return nil, err + } + log.Printf("val = %x", val) + return &Xattr{ + Type: typ, + FullName: xattrPrefix[int(typ)] + string(name), + Value: val, + }, nil +} + +func (r *Reader) ReadXattrs(inode Inode) ([]Xattr, error) { + //log.Printf("Readdir(%v (%x))", dirInode, dirInode) + i, err := r.readInode(inode) + if err != nil { + return nil, err + } + var xid uint32 + switch x := i.(type) { + case regInodeHeader, + dirInodeHeader, + ldirInodeHeader, + symlinkInodeHeader: + return nil, nil // no extended attributes + + case lregInodeHeader: + if x.Xattr == invalidXattr { + return nil, nil // file has no extended attributes + } + xid = x.Xattr + + default: + return nil, fmt.Errorf("unknown inode type %T", i) + } + + const idEntriesPerBlock = 512 // = 8192 / 16 /* sizeof(xattrId) */ + block := xid / idEntriesPerBlock + offset := (xid % idEntriesPerBlock) * 16 + log.Printf("xattr id %d, block %d, offset %d", xid, block, offset) + log.Printf("r.super.XattrIdTableStart = 0x%x, r.super.XattrIdTableStart = %v", r.super.XattrIdTableStart, r.super.XattrIdTableStart) + br := ioutil.NopCloser(io.NewSectionReader(r.r, r.super.XattrIdTableStart, int64(16 /* sizeof(xattrTableHeader) */ +(block+1)*4 /* sizeof(uint32) */))) + var tableHeader xattrTableHeader + if err := binary.Read(br, binary.LittleEndian, &tableHeader); err != nil { + return nil, err + } + // index starts here + if _, err := io.CopyN(ioutil.Discard, br, int64(block*4 /* sizeof(uint32) */)); err != nil { + return nil, err + } + var blockOffset uint32 + if err := binary.Read(br, binary.LittleEndian, &blockOffset); err != nil { + return nil, err + } + log.Printf("blockOffset = 0x%x (%d)", blockOffset, blockOffset) + br, err = r.blockReader(int64(blockOffset), int64(offset)) + if err != nil { + return nil, err + } + defer br.Close() + var id xattrId + if err := binary.Read(br, binary.LittleEndian, &id); err != nil { + return nil, err + } + log.Printf("id: %+v", id) + log.Printf("tableHeader: %+v (start 0x%x)", tableHeader, tableHeader.XattrTableStart) + + var xattrs []Xattr + for i := 0; i < int(id.Count); i++ { + xattr, err := r.readXattr(tableHeader, id) + if err != nil { + return nil, err + } + xattrs = append(xattrs, *xattr) + } + + return xattrs, nil +} diff --git a/reader_test.go b/reader_test.go new file mode 100644 index 0000000..500f5bb --- /dev/null +++ b/reader_test.go @@ -0,0 +1,373 @@ +package squashfs + +import ( + "crypto/md5" + "fmt" + "io" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" +) + +func cmpFileInfo(got os.FileInfo, want FileInfo) error { + if got, want := got.Name(), want.name; got != want { + return fmt.Errorf("unexpected file name: got %q, want %q", got, want) + } + if got, want := got.Size(), want.size; got != want { + return fmt.Errorf("unexpected size: got %d, want %d", got, want) + } + if got, want := got.IsDir(), want.mode.IsDir(); got != want { + return fmt.Errorf("IsDir: got %v, want %v", got, want) + } + // TODO: re-enable when it’s no longer just a change detector + // if got, want := got.ModTime(), want.modTime; !got.Equal(want) { + // return fmt.Errorf("IsDir: got %v, want %v", got, want) + // } + + return nil +} + +func TestReaddir(t *testing.T) { + t.Parallel() + // TODO: ship testdata files generated by mksquashfs + pwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + fmt.Println(pwd + "/testdata/testing.squashfs") + f, err := os.Open(pwd + "/testdata/testing.squashfs") + if err != nil { + t.Fatal(err) + } + defer f.Close() + rd, err := NewReader(f) + if err != nil { + t.Fatal(err) + } + fis, err := rd.Readdir(rd.RootInode()) + if err != nil { + t.Fatal(err) + } + for _, fi := range fis { + fmt.Println(fi.Name()) + } + rdr, err := rd.FileReader(fis[0].Sys().(*FileInfo).Inode) + if err != nil { + t.Fatal(err) + } + err = os.Remove(pwd + "/testdata/Magisk.zip") + if !os.IsNotExist(err) && err != nil { + t.Fatal(err) + } + // rdrzlib, err := zlib.NewReader(rdr) + magFil, err := os.Create(pwd + "/testdata/Magisk.zip") + io.Copy(magFil, rdr) + if got, want := len(fis), 3; got != want { + t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want) + } + + if err := cmpFileInfo(fis[0], FileInfo{ + name: "bin", + size: 26, + mode: 0555 | os.ModeDir, + modTime: time.Unix(1581275131, 0), // stat -c %Y /ro/ack-amd64-2.24/bin + }); err != nil { + t.Fatal(err) + } + + if err := cmpFileInfo(fis[1], FileInfo{ + name: "lib", + size: 3, + mode: 0555 | os.ModeDir, + modTime: time.Unix(1581275131, 0), // stat -c %Y /ro/ack-amd64-2.24/lib + }); err != nil { + t.Fatal(err) + } + + if err := cmpFileInfo(fis[2], FileInfo{ + name: "out", + size: 48, + mode: 0555 | os.ModeDir, + modTime: time.Unix(1581275130, 0), // stat -c %Y /ro/ack-amd64-2.24/out + }); err != nil { + t.Fatal(err) + } + + fis, err = rd.Readdir(fis[0].Sys().(*FileInfo).Inode) + if err != nil { + t.Fatal(err) + } + + if got, want := len(fis), 1; got != want { + t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want) + } + + if err := cmpFileInfo(fis[0], FileInfo{ + name: "ack", + size: 38400, + mode: 0755, + modTime: time.Unix(1581275131, 0), // stat -c %Y /ro/ack-amd64-2.24/bin/ack + }); err != nil { + t.Fatal(err) + } +} + +// TestReaddirSmoke is a smoke-test, reading the root directories of SquashFS +// images which are known to trigger code paths which were buggy. +func TestReaddirSmoke(t *testing.T) { + t.Parallel() + + for _, fn := range []string{ + // bash exercises the code path where an inode is split across metadata + // blocks. + "/home/michael/distri/_build/distri/pkg/bash-amd64-5.0-4.squashfs", + + // cmake exercises the code path where the root directory entries are + // located outside of the first block. + "/home/michael/distri/_build/distri/pkg/cmake-amd64-3.12.4-8.squashfs", + } { + // TODO: ship testdata files generated by mksquashfs + f, err := os.Open(fn) + if err != nil { + t.Fatal(err) + } + defer f.Close() + rd, err := NewReader(f) + if err != nil { + t.Fatal(err) + } + + fis, err := rd.Readdir(rd.RootInode()) + if err != nil { + t.Fatal(err) + } + + if got, want := len(fis), 4; got != want { + t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want) + } + } +} + +func TestReaddirEmpty(t *testing.T) { + t.Parallel() + // TODO: ship testdata files generated by mksquashfs + f, err := os.Open("/home/michael/distri/_build/distri/pkg/zlib-amd64-1.2.11-4.squashfs") + if err != nil { + t.Fatal(err) + } + defer f.Close() + rd, err := NewReader(f) + if err != nil { + t.Fatal(err) + } + + fis, err := rd.Readdir(rd.RootInode()) + if err != nil { + t.Fatal(err) + } + + if got, want := len(fis), 4; got != want { + t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want) + } + + if err := cmpFileInfo(fis[0], FileInfo{ + name: "bin", + size: 3, + mode: 0555 | os.ModeDir, + modTime: time.Unix(1583085224, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/bin + }); err != nil { + t.Fatal(err) + } + + fis, err = rd.Readdir(fis[0].Sys().(*FileInfo).Inode) + if err != nil { + t.Fatal(err) + } + + if got, want := len(fis), 0; got != want { + t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want) + } +} + +func TestReaddirSymlink(t *testing.T) { + t.Parallel() + // TODO: ship testdata files generated by mksquashfs + f, err := os.Open("/home/michael/distri/_build/distri/pkg/zlib-amd64-1.2.11-4.squashfs") + if err != nil { + t.Fatal(err) + } + defer f.Close() + rd, err := NewReader(f) + if err != nil { + t.Fatal(err) + } + + fis, err := rd.Readdir(rd.RootInode()) + if err != nil { + t.Fatal(err) + } + + if got, want := len(fis), 4; got != want { + t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want) + } + + if err := cmpFileInfo(fis[3], FileInfo{ + name: "out", + size: 54, + mode: 0555 | os.ModeDir, + modTime: time.Unix(1583085224, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/out + }); err != nil { + t.Fatal(err) + } + + fis, err = rd.Readdir(fis[3].Sys().(*FileInfo).Inode) + if err != nil { + t.Fatal(err) + } + + if got, want := len(fis), 3; got != want { + t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want) + } + + if err := cmpFileInfo(fis[1], FileInfo{ + name: "lib", + size: 83, + mode: 0555 | os.ModeDir, + modTime: time.Unix(1583085224, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/out/lib + }); err != nil { + t.Fatal(err) + } + + fis, err = rd.Readdir(fis[1].Sys().(*FileInfo).Inode) + if err != nil { + t.Fatal(err) + } + + if got, want := len(fis), 4; got != want { + t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want) + } + + if err := cmpFileInfo(fis[1], FileInfo{ + name: "libz.so", + size: 9, + mode: 0555 | os.ModeSymlink, + modTime: time.Unix(1583085223, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/out/lib/libz.so + }); err != nil { + t.Fatal(err) + } + + // TODO: readlink + target, err := rd.ReadLink(fis[1].Sys().(*FileInfo).Inode) + if err != nil { + t.Fatal(err) + } + if got, want := target, "libz.so.1"; got != want { + t.Fatalf("ReadLink(libz.so): got %q, want %q", got, want) + } +} + +func TestReadfile(t *testing.T) { + t.Parallel() + // TODO: ship testdata files generated by mksquashfs + f, err := os.Open("/home/michael/distri/_build/distri/pkg/ack-amd64-3.3.1-7.squashfs") + if err != nil { + t.Fatal(err) + } + defer f.Close() + rd, err := NewReader(f) + if err != nil { + t.Fatal(err) + } + + fis, err := rd.Readdir(rd.RootInode()) + if err != nil { + t.Fatal(err) + } + + if got, want := len(fis), 3; got != want { + t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want) + } + + fis, err = rd.Readdir(fis[0].Sys().(*FileInfo).Inode) + if err != nil { + t.Fatal(err) + } + + if got, want := len(fis), 1; got != want { + t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want) + } + + r, err := rd.FileReader(fis[0].Sys().(*FileInfo).Inode) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 2; i++ { + if _, err := r.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + h := md5.New() + if _, err := io.Copy(h, r); err != nil { + t.Fatal(err) + } + sum := fmt.Sprintf("%x", h.Sum(nil)) + if got, want := sum, "c6c9b5d4d2a49f1b8b5e501f0f827a5c"; got != want { + t.Fatalf("md5(bin/ack): got %s, want %s", got, want) + } + } +} + +// TODO: add test exercising ldirInodeHeader, e.g. '/mnt/loop/ca-certificates-3.39/buildoutput/etc/ssl' + +func TestReadXattr(t *testing.T) { + t.Parallel() + + // TODO: generate a smaller version of this file + f, err := os.Open("testdata/xattr.squashfs") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + rd, err := NewReader(f) + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + Path string + Want []Xattr + }{ + { + Path: "mtr-packet", + Want: []Xattr{ + { + Type: XattrTypeSecurity, + FullName: "security.capability", + Value: []byte{1, 0, 0, 2, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + }, + }, + { + Path: "gnome-keyring-daemon", + Want: []Xattr{ + { + Type: XattrTypeSecurity, + FullName: "security.capability", + Value: []byte{1, 0, 0, 2, 0, 0x40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + }, + }, + } { + inode, err := rd.LookupPath(tt.Path) + if err != nil { + t.Fatal(err) + } + xattrs, err := rd.ReadXattrs(inode) + if err != nil { + t.Fatalf("ReadXattrs(%v): %v", inode, err) + } + if diff := cmp.Diff(tt.Want, xattrs); diff != "" { + t.Fatalf("unexpected ReadXattrs result: diff (-want +got):\n%s", diff) + } + } +} diff --git a/superblock.go b/superblock.go new file mode 100644 index 0000000..e163c02 --- /dev/null +++ b/superblock.go @@ -0,0 +1,24 @@ +package squashfs + +//Descriptions provided by https://dr-emann.github.io/squashfs/ +type superblock struct { + Magic uint32 //Magic will be 0x73717368 if it's a legit Squashfs filesystem + Inodes uint32 //Inodes is the number of inodes in the inodes table + MkfsTime int32 //MkfsTime is when archive was created + BlockSize uint32 //BlockSize is the size of data blocks in bytes + Fragments uint32 //Fragments is the number of entries in fragment table + Compression uint16 //Compression is what type of compression is used + BlockLog uint16 //BlockLog should be log base 2 of BlockSize. If not then the squash might be corrupt + Flags uint16 //Flags are the superblock's flags + IDCount uint16 //IDCount is the number of IDs in the id lookup table + Major uint16 + Minor uint16 + RootInode Inode + BytesUsed int64 + IDTableStart int64 + XattrIDTableStart int64 + InodeTableStart int64 + DirectoryTableStart int64 + FragmentTableStart int64 + LookupTableStart int64 +} diff --git a/unsquash.go b/unsquash.go new file mode 100644 index 0000000..c6f4ba7 --- /dev/null +++ b/unsquash.go @@ -0,0 +1,9 @@ +package squashfs + +import "io" + +//Squashfs is a squashfs backed by a reader. +type Squashfs struct { + rdr *io.Reader //underlyting reader + super Superblock +} diff --git a/writer_old.go b/writer_old.go new file mode 100644 index 0000000..42b3720 --- /dev/null +++ b/writer_old.go @@ -0,0 +1,913 @@ +// Package squashfs implements writing SquashFS file system images using zlib +// compression for data blocks (inodes and directory entries are written +// uncompressed for simplicity). +// +// Note that SquashFS requires directory entries to be sorted, i.e. files and +// directories need to be added in the correct order. +// +// This package intentionally only implements a subset of SquashFS. Notably, +// block devices, character devices, FIFOs, sockets and xattrs are not +// supported. +package squashfs + +import ( + "bytes" + "compress/zlib" + "encoding/binary" + "io" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +// inode contains a block number + offset within that block. +type Inode int64 + +const ( + zlibCompression = 1 + iota + lzmaCompression + lzoCompression + xzCompression + lz4Compression + zstdCompression +) + +const ( + invalidFragment = 0xFFFFFFFF + invalidXattr = 0xFFFFFFFF +) + +const ( + dirType = 1 + iota + fileType + symlinkType + blkdevType + chrdevType + fifoType + socketType + // The larger types are used for e.g. sparse files, xattrs, etc. + ldirType + lregType + lsymlinkType + lblkdevType + lchrdevType + lfifoType + lsocketType +) + +type inodeHeader struct { + InodeType uint16 + Mode uint16 + Uid uint16 + Gid uint16 + Mtime int32 + InodeNumber uint32 +} + +// fileType +type regInodeHeader struct { + inodeHeader + + // full byte offset from the start of the file system, e.g. 96 for first + // file contents. Not using fragments limits us to 2^32-1-96 (≈ 4GiB) bytes + // of file contents. + StartBlock uint32 + Fragment uint32 + Offset uint32 + FileSize uint32 + + // Followed by a uint32 array of compressed block sizes. +} + +// lregType +type lregInodeHeader struct { + inodeHeader + + // full byte offset from the start of the file system, e.g. 96 for first + // file contents. Not using fragments limits us to 2^32-1-96 (≈ 4GiB) bytes + // of file contents. + StartBlock uint64 + FileSize uint64 + Sparse uint64 + Nlink uint32 + Fragment uint32 + Offset uint32 + Xattr uint32 + + // Followed by a uint32 array of compressed block sizes. +} + +// symlinkType +type symlinkInodeHeader struct { + inodeHeader + + Nlink uint32 + SymlinkSize uint32 + + // Followed by a byte array of SymlinkSize bytes. +} + +// chrdevType and blkdevType +type devInodeHeader struct { + inodeHeader + + Nlink uint32 + Rdev uint32 +} + +// fifoType and socketType +type ipcInodeHeader struct { + inodeHeader + + Nlink uint32 +} + +// dirType +type dirInodeHeader struct { + inodeHeader + + StartBlock uint32 + Nlink uint32 + FileSize uint16 + Offset uint16 + ParentInode uint32 +} + +// ldirType +type ldirInodeHeader struct { + inodeHeader + + Nlink uint32 + FileSize uint32 + StartBlock uint32 + ParentInode uint32 + Icount uint16 + Offset uint16 + Xattr uint32 +} + +type dirHeader struct { + Count uint32 + StartBlock uint32 + InodeOffset uint32 +} + +func (d *dirHeader) Unmarshal(b []byte) { + _ = b[11] + e := binary.LittleEndian + d.Count = e.Uint32(b) + d.StartBlock = e.Uint32(b[4:]) + d.InodeOffset = e.Uint32(b[8:]) +} + +type dirEntry struct { + Offset uint16 + InodeNumber int16 + EntryType uint16 + Size uint16 + + // Followed by a byte array of Size bytes. +} + +func (d *dirEntry) Unmarshal(b []byte) { + _ = b[7] + e := binary.LittleEndian + d.Offset = e.Uint16(b) + d.InodeNumber = int16(e.Uint16(b[2:])) + d.EntryType = e.Uint16(b[4:]) + d.Size = e.Uint16(b[6:]) +} + +// xattr types +const ( + XattrTypeUser = iota + XattrTypeTrusted + XattrTypeSecurity +) + +var xattrPrefix = map[int]string{ + XattrTypeUser: "user.", + XattrTypeTrusted: "trusted.", + XattrTypeSecurity: "security.", +} + +type Xattr struct { + Type uint16 + FullName string + Value []byte +} + +func XattrFromAttr(attr string, val []byte) Xattr { + for typ, prefix := range xattrPrefix { + if !strings.HasPrefix(attr, prefix) { + continue + } + return Xattr{ + Type: uint16(typ), + FullName: strings.TrimPrefix(attr, prefix), + Value: val, + } + } + return Xattr{} +} + +type xattrId struct { + Xattr uint64 + Count uint32 + Size uint32 +} + +func writeIdTable(w io.WriteSeeker, ids []uint32) (start int64, err error) { + metaOff, err := w.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err + } + var buf bytes.Buffer + if err := binary.Write(&buf, binary.LittleEndian, ids); err != nil { + return 0, err + } + + if err := binary.Write(w, binary.LittleEndian, uint16(buf.Len())|0x8000); err != nil { + return 0, err + } + if _, err := io.Copy(w, &buf); err != nil { + return 0, err + } + off, err := w.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err + } + return off, binary.Write(w, binary.LittleEndian, metaOff) +} + +type fullDirEntry struct { + startBlock uint32 + offset uint16 + inodeNumber uint32 + entryType uint16 + name string +} + +const ( + magic = 0x73717368 + dataBlockSize = 131072 + metadataBlockSize = 8192 + majorVersion = 4 + minorVersion = 0 +) + +type Writer struct { + // Root represents the file system root. Like all directories, Flush must be + // called precisely once. + Root *Directory + + xattrs []Xattr + xattrIds []xattrId + + w io.WriteSeeker + + sb superblock + inodeBuf bytes.Buffer + dirBuf bytes.Buffer + + writeInodeNumTo map[string][]int64 +} + +// TODO: document what this is doing and what it is used for +func slog(block uint32) uint16 { + for i := uint16(12); i <= 20; i++ { + if block == (1 << i) { + return i + } + } + return 0 +} + +// filesystemFlags returns flags for a SquashFS file system created by this +// package (disabling most features for now). +func filesystemFlags() uint16 { + const ( + noI = 1 << iota // uncompressed metadata + noD // uncompressed data + _ + noF // uncompressed fragments + noFrag // never use fragments + alwaysFrag // always use fragments + duplicateChecking // de-duplication + exportable // exportable via NFS + noX // uncompressed xattrs + noXattr // no xattrs + compopt // compressor-specific options present? + ) + return noI | noF | noFrag | noX | noXattr +} + +// NewWriter returns a Writer which will write a SquashFS file system image to w +// once Flush is called. +// +// Create new files and directories with the corresponding methods on the Root +// directory of the Writer. +// +// File data is written to w even before Flush is called. +func NewWriter(w io.WriteSeeker, mkfsTime time.Time) (*Writer, error) { + // Skip over superblock to the data area, we come back to the superblock + // when flushing. + if _, err := w.Seek(96, io.SeekStart); err != nil { + return nil, err + } + wr := &Writer{ + w: w, + sb: superblock{ + Magic: magic, + MkfsTime: int32(mkfsTime.Unix()), + BlockSize: dataBlockSize, + Fragments: 0, + Compression: zlibCompression, + BlockLog: slog(dataBlockSize), + Flags: filesystemFlags(), + NoIds: 1, // just one uid/gid mapping (for root) + Major: majorVersion, + Minor: minorVersion, + XattrIdTableStart: -1, // not present + LookupTableStart: -1, // not present + }, + writeInodeNumTo: make(map[string][]int64), + } + wr.Root = &Directory{ + w: wr, + name: "", // root + modTime: mkfsTime, + } + return wr, nil +} + +// Directory represents a SquashFS directory. +type Directory struct { + w *Writer + name string + modTime time.Time + dirEntries []fullDirEntry + parent *Directory +} + +func (d *Directory) path() string { + if d.parent == nil { + return d.name + } + return filepath.Join(d.parent.path(), d.name) +} + +type file struct { + w *Writer + d *Directory + off int64 + size uint32 + name string + modTime time.Time + mode uint16 + + // buf accumulates at least dataBlockSize bytes, at which point a new block + // is being written. + buf bytes.Buffer + + // blocksizes stores, for each block of dataBlockSize bytes (uncompressed), + // the number of bytes the block compressed down to. + blocksizes []uint32 + + // compBuf is used for holding a block during compression to avoid memory + // allocations. + compBuf *bytes.Buffer + // zlibWriter is re-used for each compressed block + zlibWriter *zlib.Writer + + xattrRef uint32 +} + +// Directory creates a new directory with the specified name and modTime. +func (d *Directory) Directory(name string, modTime time.Time) *Directory { + return &Directory{ + w: d.w, + name: name, + modTime: modTime, + parent: d, + } +} + +// File creates a file with the specified name, modTime and mode. The returned +// io.WriterCloser must be closed after writing the file. +func (d *Directory) File(name string, modTime time.Time, mode uint16, xattrs []Xattr) (io.WriteCloser, error) { + off, err := d.w.w.Seek(0, io.SeekCurrent) + if err != nil { + return nil, err + } + + // zlib.BestSpeed results in only a 2x slow-down over no compression + // (compared to >4x slow-down with DefaultCompression), but generates + // results which are in the same ball park (10% larger). + zw, err := zlib.NewWriterLevel(nil, zlib.BestSpeed) + if err != nil { + return nil, err + } + + xattrRef := uint32(invalidXattr) + if len(xattrs) > 0 { + xattrRef = uint32(len(d.w.xattrs)) + d.w.xattrs = append(d.w.xattrs, xattrs[0]) // TODO: support multiple + size := len(xattrs[0].FullName) + len(xattrs[0].Value) + d.w.xattrIds = append(d.w.xattrIds, xattrId{ + // Xattr is populated in writeXattrTables + Count: 1, // TODO: support multiple + Size: uint32(size), + }) + } + return &file{ + w: d.w, + d: d, + off: off, + name: name, + modTime: modTime, + mode: mode, + compBuf: bytes.NewBuffer(make([]byte, dataBlockSize)), + zlibWriter: zw, + xattrRef: xattrRef, + }, nil +} + +// Symlink creates a symbolic link from newname to oldname with the specified +// modTime and mode. +func (d *Directory) Symlink(oldname, newname string, modTime time.Time, mode os.FileMode) error { + startBlock := d.w.inodeBuf.Len() / metadataBlockSize + offset := d.w.inodeBuf.Len() - startBlock*metadataBlockSize + + if err := binary.Write(&d.w.inodeBuf, binary.LittleEndian, symlinkInodeHeader{ + inodeHeader: inodeHeader{ + InodeType: symlinkType, + Mode: uint16(mode), + Uid: 0, + Gid: 0, + Mtime: int32(modTime.Unix()), + InodeNumber: d.w.sb.Inodes + 1, + }, + Nlink: 1, // TODO(later): when is this not 1? + SymlinkSize: uint32(len(oldname)), + }); err != nil { + return err + } + if _, err := d.w.inodeBuf.Write([]byte(oldname)); err != nil { + return err + } + + d.dirEntries = append(d.dirEntries, fullDirEntry{ + startBlock: uint32(startBlock), + offset: uint16(offset), + inodeNumber: d.w.sb.Inodes + 1, + entryType: symlinkType, + name: newname, + }) + + d.w.sb.Inodes++ + return nil +} + +// Flush writes directory entries and creates inodes for the directory. +func (d *Directory) Flush() error { + countByStartBlock := make(map[uint32]uint32) + for _, de := range d.dirEntries { + countByStartBlock[de.startBlock]++ + } + + dirBufStartBlock := d.w.dirBuf.Len() / metadataBlockSize + dirBufOffset := d.w.dirBuf.Len() + + currentBlock := int64(-1) + currentInodeOffset := int64(-1) + var subdirs int + for _, de := range d.dirEntries { + if de.entryType == dirType { + subdirs++ + } + if int64(de.startBlock) != currentBlock { + dh := dirHeader{ + Count: countByStartBlock[de.startBlock] - 1, + StartBlock: de.startBlock * (metadataBlockSize + 2), + InodeOffset: de.inodeNumber, + } + if err := binary.Write(&d.w.dirBuf, binary.LittleEndian, &dh); err != nil { + return err + } + + currentBlock = int64(de.startBlock) + currentInodeOffset = int64(de.inodeNumber) + } + if err := binary.Write(&d.w.dirBuf, binary.LittleEndian, &dirEntry{ + Offset: de.offset, + InodeNumber: int16(de.inodeNumber - uint32(currentInodeOffset)), + EntryType: de.entryType, + Size: uint16(len(de.name) - 1), + }); err != nil { + return err + } + if _, err := d.w.dirBuf.Write([]byte(de.name)); err != nil { + return err + } + } + + startBlock := d.w.inodeBuf.Len() / metadataBlockSize + offset := d.w.inodeBuf.Len() - startBlock*metadataBlockSize + inodeBufOffset := d.w.inodeBuf.Len() + + // parentInodeOffset is the offset (in bytes) of the ParentInode field + // within a dirInodeHeader or ldirInodeHeader + var parentInodeOffset int64 + + if len(d.dirEntries) > 256 || + d.w.dirBuf.Len()-dirBufOffset > metadataBlockSize { + parentInodeOffset = (2 + 2 + 2 + 2 + 4 + 4) + 4 + 4 + 4 + if err := binary.Write(&d.w.inodeBuf, binary.LittleEndian, ldirInodeHeader{ + inodeHeader: inodeHeader{ + InodeType: ldirType, + Mode: unix.S_IRUSR | unix.S_IWUSR | unix.S_IXUSR | + unix.S_IRGRP | unix.S_IXGRP | + unix.S_IROTH | unix.S_IXOTH, + Uid: 0, + Gid: 0, + Mtime: int32(d.modTime.Unix()), + InodeNumber: d.w.sb.Inodes + 1, + }, + + Nlink: uint32(subdirs + 2 - 1), // + 2 for . and .. + FileSize: uint32(d.w.dirBuf.Len()-dirBufOffset) + 3, + StartBlock: uint32(dirBufStartBlock * (metadataBlockSize + 2)), + ParentInode: d.w.sb.Inodes + 2, // invalid + Icount: 0, // no directory index + Offset: uint16(dirBufOffset - dirBufStartBlock*metadataBlockSize), + Xattr: invalidXattr, + }); err != nil { + return err + } + } else { + parentInodeOffset = (2 + 2 + 2 + 2 + 4 + 4) + 4 + 4 + 2 + 2 + if err := binary.Write(&d.w.inodeBuf, binary.LittleEndian, dirInodeHeader{ + inodeHeader: inodeHeader{ + InodeType: dirType, + Mode: unix.S_IRUSR | unix.S_IWUSR | unix.S_IXUSR | + unix.S_IRGRP | unix.S_IXGRP | + unix.S_IROTH | unix.S_IXOTH, + Uid: 0, + Gid: 0, + Mtime: int32(d.modTime.Unix()), + InodeNumber: d.w.sb.Inodes + 1, + }, + StartBlock: uint32(dirBufStartBlock * (metadataBlockSize + 2)), + Nlink: uint32(subdirs + 2 - 1), // + 2 for . and .. + FileSize: uint16(d.w.dirBuf.Len()-dirBufOffset) + 3, + Offset: uint16(dirBufOffset - dirBufStartBlock*metadataBlockSize), + ParentInode: d.w.sb.Inodes + 2, // invalid + }); err != nil { + return err + } + } + + path := d.path() + for _, offset := range d.w.writeInodeNumTo[path] { + // Directly manipulating unread data in bytes.Buffer via Bytes(), as per + // https://groups.google.com/d/msg/golang-nuts/1ON9XVQ1jXE/8j9RaeSYxuEJ + b := d.w.inodeBuf.Bytes() + binary.LittleEndian.PutUint32(b[offset:offset+4], d.w.sb.Inodes+1) + } + + if d.parent != nil { + parentPath := filepath.Dir(d.path()) + if parentPath == "." { + parentPath = "" + } + d.w.writeInodeNumTo[parentPath] = append(d.w.writeInodeNumTo[parentPath], int64(inodeBufOffset)+parentInodeOffset) + d.parent.dirEntries = append(d.parent.dirEntries, fullDirEntry{ + startBlock: uint32(startBlock), + offset: uint16(offset), + inodeNumber: d.w.sb.Inodes + 1, + entryType: dirType, + name: d.name, + }) + } else { // root + d.w.sb.RootInode = Inode((startBlock*(metadataBlockSize+2))<<16 | offset) + } + + d.w.sb.Inodes++ + + return nil +} + +// Write implements io.Writer +func (f *file) Write(p []byte) (n int, err error) { + n, err = f.buf.Write(p) + if n > 0 { + // Keep track of the uncompressed file size. + f.size += uint32(n) + for f.buf.Len() >= dataBlockSize { + if err := f.writeBlock(); err != nil { + return 0, err + } + } + } + return n, err +} + +func (f *file) writeBlock() error { + n := f.buf.Len() + if n > dataBlockSize { + n = dataBlockSize + } + // Feed dataBlockSize bytes to the compressor + b := f.buf.Bytes() + block := b[:n] + rest := b[n:] + /* + f.compBuf.Reset() + f.zlibWriter.Reset(f.compBuf) + if _, err := f.zlibWriter.Write(block); err != nil { + return err + } + if err := f.zlibWriter.Close(); err != nil { + return err + } + + size := f.compBuf.Len() + if size > len(block) { + // Copy uncompressed data: Linux returns i/o errors when it encounters a + // compressed block which is larger than the uncompressed data: + // https://github.com/torvalds/linux/blob/3ca24ce9ff764bc27bceb9b2fd8ece74846c3fd3/fs/squashfs/block.c#L150 + size = len(block) | (1 << 24) // SQUASHFS_COMPRESSED_BIT_BLOCK + if _, err := f.w.w.Write(block); err != nil { + return err + } + } else { + if _, err := io.Copy(f.w.w, f.compBuf); err != nil { + return err + } + } + */ + // Copy uncompressed data: Linux returns i/o errors when it encounters a + // compressed block which is larger than the uncompressed data: + // https://github.com/torvalds/linux/blob/3ca24ce9ff764bc27bceb9b2fd8ece74846c3fd3/fs/squashfs/block.c#L150 + size := len(block) | (1 << 24) // SQUASHFS_COMPRESSED_BIT_BLOCK + if _, err := f.w.w.Write(block); err != nil { + return err + } + + f.blocksizes = append(f.blocksizes, uint32(size)) + + // Keep the rest in f.buf for the next write + copy(b, rest) + f.buf.Truncate(len(rest)) + return nil +} + +// Close implements io.Closer +func (f *file) Close() error { + for f.buf.Len() > 0 { + if err := f.writeBlock(); err != nil { + return err + } + } + + startBlock := f.w.inodeBuf.Len() / metadataBlockSize + offset := f.w.inodeBuf.Len() - startBlock*metadataBlockSize + + if err := binary.Write(&f.w.inodeBuf, binary.LittleEndian, lregInodeHeader{ + inodeHeader: inodeHeader{ + InodeType: lregType, + Mode: f.mode, + Uid: 0, + Gid: 0, + Mtime: int32(f.modTime.Unix()), + InodeNumber: f.w.sb.Inodes + 1, + }, + StartBlock: uint64(f.off), + FileSize: uint64(f.size), + Nlink: 1, + Fragment: invalidFragment, + Offset: 0, + Xattr: f.xattrRef, + }); err != nil { + return err + } + + if err := binary.Write(&f.w.inodeBuf, binary.LittleEndian, f.blocksizes); err != nil { + return err + } + + f.d.dirEntries = append(f.d.dirEntries, fullDirEntry{ + startBlock: uint32(startBlock), + offset: uint16(offset), + inodeNumber: f.w.sb.Inodes + 1, + entryType: fileType, + name: f.name, + }) + + f.w.sb.Inodes++ + + return nil +} + +func writeXattr(w io.Writer, xattrs []Xattr) error { + for _, attr := range xattrs { + if err := binary.Write(w, binary.LittleEndian, struct { + Type uint16 + NameSize uint16 + }{ + Type: attr.Type, + NameSize: uint16(len(attr.FullName)), + }); err != nil { + return err + } + if _, err := w.Write([]byte(attr.FullName)); err != nil { + return err + } + + if err := binary.Write(w, binary.LittleEndian, struct { + ValSize uint32 + }{ + ValSize: uint32(len(attr.Value)), + }); err != nil { + return err + } + + if _, err := w.Write(attr.Value); err != nil { + return err + } + } + return nil +} + +type xattrTableHeader struct { + XattrTableStart uint64 + XattrIds uint32 + Unused uint32 +} + +func (w *Writer) writeXattrTables() (int64, error) { + if len(w.xattrs) == 0 { + return -1, nil + } + off, err := w.w.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err + } + xattrTableStart := uint64(off) + + var xattrBuf bytes.Buffer + if err := writeXattr(&xattrBuf, w.xattrs); err != nil { + return 0, err + } + xattrBlocks := (xattrBuf.Len() + (metadataBlockSize - 1)) / metadataBlockSize + + if err := w.writeMetadataChunks(&xattrBuf); err != nil { + return 0, err + } + + // write xattr id table + off, err = w.w.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err + } + idTableOff := uint64(off) + var xattrIdBuf bytes.Buffer + size := uint64(0) + for _, id := range w.xattrIds { + id.Xattr = uint64(size) + size += uint64(id.Size) + 8 /* sizeof(Type+NameSize+ValSize) */ + if err := binary.Write(&xattrIdBuf, binary.LittleEndian, id); err != nil { + return 0, err + } + } + if err := w.writeMetadataChunks(&xattrIdBuf); err != nil { + return 0, err + } + + // xattr table header + off, err = w.w.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err + } + if err := binary.Write(w.w, binary.LittleEndian, xattrTableHeader{ + XattrTableStart: xattrTableStart, + XattrIds: uint32(len(w.xattrs)), + }); err != nil { + return 0, err + } + // write block index + for i := 0; i < xattrBlocks; i++ { + if err := binary.Write(w.w, binary.LittleEndian, struct { + BlockOffset uint64 + }{ + BlockOffset: idTableOff + (uint64(i) * (8192 + 2 /* sizeof(uint16) */)), + }); err != nil { + return 0, err + } + } + return off, nil +} + +// writeMetadataChunks copies from r to w in blocks of metadataBlockSize bytes +// each, prefixing each block with a uint16 length header, setting the +// uncompressed bit. +func (w *Writer) writeMetadataChunks(r io.Reader) error { + buf := make([]byte, metadataBlockSize) + for { + buf = buf[:metadataBlockSize] + n, err := r.Read(buf) + if err != nil { + if err == io.EOF { // done + return nil + } + return err + } + buf = buf[:n] + if err := binary.Write(w.w, binary.LittleEndian, uint16(len(buf))|0x8000); err != nil { + return err + } + if _, err := w.w.Write(buf); err != nil { + return err + } + } +} + +// Flush writes the SquashFS file system. The Writer must not be used after +// calling Flush. +func (w *Writer) Flush() error { + // (1) superblock will be written later + + // (2) compressor-specific options omitted + + // (3) data has already been written + + // (4) write inode table + off, err := w.w.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + w.sb.InodeTableStart = off + + if err := w.writeMetadataChunks(&w.inodeBuf); err != nil { + return err + } + + // (5) write directory table + off, err = w.w.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + w.sb.DirectoryTableStart = off + + if err := w.writeMetadataChunks(&w.dirBuf); err != nil { + return err + } + + // (6) fragment table omitted + off, err = w.w.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + w.sb.FragmentTableStart = off + + // (7) export table omitted + + // (8) write uid/gid lookup table + idTableStart, err := writeIdTable(w.w, []uint32{0}) + if err != nil { + return err + } + w.sb.IdTableStart = idTableStart + + // (9) xattr table + off, err = w.writeXattrTables() + if err != nil { + return err + } + w.sb.XattrIdTableStart = off + + off, err = w.w.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + w.sb.BytesUsed = off + + // Pad to 4096, required for the kernel to be able to access all pages + if pad := off % 4096; pad > 0 { + padding := make([]byte, 4096-pad) + if _, err := w.w.Write(padding); err != nil { + return err + } + } + + // (1) Write superblock + if _, err := w.w.Seek(0, io.SeekStart); err != nil { + return err + } + + return binary.Write(w.w, binary.LittleEndian, &w.sb) +} diff --git a/writer_test.go b/writer_test.go new file mode 100644 index 0000000..a42b51b --- /dev/null +++ b/writer_test.go @@ -0,0 +1,310 @@ +package squashfs + +import ( + "bytes" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/distr1/distri" + // "github.com/distr1/distri/internal/distritest" + "github.com/google/go-cmp/cmp" + "github.com/orcaman/writerseeker" + "golang.org/x/sys/unix" +) + +var fsImagePath = flag.String("fs_image_path", "", "Store the SquashFS test file system in the specified path for manual inspection") + +func writeTestImage(iow io.WriteSeeker, xattr bool) error { + w, err := NewWriter(iow, time.Now()) + if err != nil { + return err + } + + var xattrs []Xattr + if xattr { + xattrs = append(xattrs, Xattr{ + Type: 2, + FullName: "capability", + Value: []byte{1, 0, 0, 2, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }) + } + ff, err := w.Root.File("hellö wörld", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, xattrs) + if err != nil { + return err + } + if _, err := ff.Write([]byte("hello world!")); err != nil { + return err + } + if err := ff.Close(); err != nil { + return err + } + + ff, err = w.Root.File("leer", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, nil) + if err != nil { + return err + } + if err := ff.Close(); err != nil { + return err + } + + ff, err = w.Root.File("second file", time.Now(), unix.S_IRUSR|unix.S_IXUSR| + unix.S_IRGRP|unix.S_IXGRP| + unix.S_IROTH|unix.S_IXOTH, + nil) + if err != nil { + return err + } + if _, err := ff.Write([]byte("NON.\n")); err != nil { + return err + } + if err := ff.Close(); err != nil { + return err + } + + if err := w.Root.Symlink("second file", "second link", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH); err != nil { + return err + } + + subdir := w.Root.Directory("subdir", time.Now()) + + subsubdir := subdir.Directory("deep", time.Now()) + ff, err = subsubdir.File("yo", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, nil) + if err != nil { + return err + } + if _, err := ff.Write([]byte("foo\n")); err != nil { + return err + } + if err := ff.Close(); err != nil { + return err + } + if err := subsubdir.Flush(); err != nil { + return err + } + + // TODO: write another file in subdir now, will result in invalid parent inode + + ff, err = subdir.File("third file (in subdir)", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, nil) + if err != nil { + return err + } + if _, err := ff.Write([]byte("contents\n")); err != nil { + return err + } + if err := ff.Close(); err != nil { + return err + } + + if err := subdir.Flush(); err != nil { + return err + } + ff, err = w.Root.File("testbin", time.Now(), unix.S_IRUSR|unix.S_IXUSR| + unix.S_IRGRP|unix.S_IXGRP| + unix.S_IROTH|unix.S_IXOTH, + nil) + if err != nil { + return err + } + zf, err := os.Open(os.Args[0]) + if err != nil { + return err + } + defer zf.Close() + if _, err := io.Copy(ff, zf); err != nil { + return err + } + if err := ff.Close(); err != nil { + return err + } + + if err := w.Root.Flush(); err != nil { + return err + } + if err := w.Flush(); err != nil { + return err + } + return nil +} + +func TestUnsquashfs(t *testing.T) { + t.Parallel() + + ctx, canc := distri.InterruptibleContext() + defer canc() + + if _, err := exec.LookPath("unsquashfs"); err != nil { + t.Skip("unsquashfs not found in $PATH") + } + + for _, xattr := range []bool{false, true} { + t.Run(fmt.Sprintf("xattr %v", xattr), func(t *testing.T) { + var ( + f *os.File + err error + ) + if *fsImagePath != "" { + f, err = os.Create(*fsImagePath + fmt.Sprintf("-xattr-%v", xattr)) + } else { + f, err = ioutil.TempFile("", fmt.Sprintf("squashfs-xattr-%v", xattr)) + if err == nil { + defer os.Remove(f.Name()) + } + } + if err != nil { + t.Fatal(err) + } + + if err := writeTestImage(f, xattr); err != nil { + t.Fatal(err) + } + + if err := f.Close(); err != nil { + t.Fatal(err) + } + + // Extract our generated file system using unsquashfs(1) + out, err := ioutil.TempDir("", fmt.Sprintf("unsquashfs-xattr-%v", xattr)) + if err != nil { + t.Fatal(err) + } + // defer distritest.RemoveAll(t, out) + cmd := exec.CommandContext(ctx, "unsquashfs", "-no-xattrs", "-d", filepath.Join(out, "x"), f.Name()) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + t.Fatal(err) + } + + fbin, err := os.Open(os.Args[0]) + if err != nil { + t.Fatal(err) + } + + // Verify the extracted files match our expectations. + for _, entry := range []struct { + path string + contents io.Reader + }{ + {"leer", strings.NewReader("")}, + {"hellö wörld", strings.NewReader("hello world!")}, + {"testbin", fbin}, + {"subdir/third file (in subdir)", strings.NewReader("contents\n")}, + } { + entry := entry // copy + t.Run(entry.path, func(t *testing.T) { + in, err := os.Open(filepath.Join(out, "x", entry.path)) + if err != nil { + t.Fatal(err) + } + got, err := ioutil.ReadAll(in) + if err != nil { + t.Fatal(err) + } + want, err := ioutil.ReadAll(entry.contents) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("path %q differs", entry.path) + } + }) + } + }) + } +} + +func TestReader(t *testing.T) { + t.Parallel() + + for _, xattr := range []bool{false, true} { + t.Run(fmt.Sprintf("xattr %v", xattr), func(t *testing.T) { + var err error + buf := &writerseeker.WriterSeeker{} + if err := writeTestImage(buf, xattr); err != nil { + t.Fatal(err) + } + + if _, err := buf.Seek(0, io.SeekCurrent); err != nil { + t.Fatal(err) + } + + rd, err := NewReader(buf.BytesReader()) + if err != nil { + t.Fatal(err) + } + + fbin, err := os.Open(os.Args[0]) + if err != nil { + t.Fatal(err) + } + + // Verify the extracted files match our expectations. + for _, entry := range []struct { + path string + contents io.Reader + }{ + {"leer", strings.NewReader("")}, + {"hellö wörld", strings.NewReader("hello world!")}, + {"testbin", fbin}, + {"subdir/third file (in subdir)", strings.NewReader("contents\n")}, + } { + entry := entry // copy + t.Run(entry.path, func(t *testing.T) { + // TODO: is this t.Parallel()-safe? + inode, err := rd.LookupPath(entry.path) + if err != nil { + t.Fatal(err) + } + in, err := rd.FileReader(inode) + if err != nil { + t.Fatal(err) + } + got, err := ioutil.ReadAll(in) + if err != nil { + t.Fatal(err) + } + want, err := ioutil.ReadAll(entry.contents) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("path %q differs", entry.path) + } + }) + } + + if xattr { + t.Run("xattrs", func(t *testing.T) { + inode, err := rd.LookupPath("hellö wörld") + if err != nil { + t.Fatal(err) + } + + xattrs, err := rd.ReadXattrs(inode) + if err != nil { + t.Fatal(err) + } + + if got, want := len(xattrs), 1; got != want { + t.Fatalf("unexpected number of extended attributes: got %d, want %d", got, want) + } + wantXattr := Xattr{ + Type: 2, + FullName: "security.capability", + Value: []byte{1, 0, 0, 2, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + } + if diff := cmp.Diff(wantXattr, xattrs[0]); diff != "" { + t.Errorf("unexpected extended attribute: diff (-want +got):\n%s", diff) + } + }) + } + }) + } +}