Found some good squashfs documentation so I can start work
This commit is contained in:
Caleb Gardner
+618
@@ -0,0 +1,618 @@
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type Reader struct {
|
||||
r io.ReaderAt
|
||||
super superblock
|
||||
}
|
||||
|
||||
func NewReader(r io.ReaderAt) (*Reader, error) {
|
||||
var sb superblock
|
||||
|
||||
if err := binary.Read(io.NewSectionReader(r, 0, int64(binary.Size(sb))), binary.LittleEndian, &sb); err != nil {
|
||||
return nil, fmt.Errorf("reading superblock: %v", err)
|
||||
}
|
||||
|
||||
if got, want := sb.Magic, uint32(magic); got != want {
|
||||
return nil, fmt.Errorf("invalid magic (not a SquashFS image?): got %x, want %x", got, want)
|
||||
}
|
||||
|
||||
//log.Printf("superblock: %+v", sb)
|
||||
return &Reader{
|
||||
r: r,
|
||||
super: sb,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TODO: maybe mmap instead of seeking?
|
||||
|
||||
func (r *Reader) inode(i Inode) (blockoffset int64, offset int64) {
|
||||
return int64(i >> 16), int64(i & 0xFFFF)
|
||||
}
|
||||
|
||||
type blockReader struct {
|
||||
r io.ReadSeeker
|
||||
lenBuf [2]byte
|
||||
buf []byte
|
||||
i int64
|
||||
|
||||
off int64 // TODO: remove this once using mmap
|
||||
}
|
||||
|
||||
func (br *blockReader) Read(p []byte) (n int, err error) {
|
||||
if br.i >= int64(len(br.buf)) {
|
||||
br.i = 0
|
||||
if _, err := io.ReadFull(br.r, br.lenBuf[:]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
l := binary.LittleEndian.Uint16(br.lenBuf[:])
|
||||
//uncompressed := l&0x8000 > 0
|
||||
l &= 0x7FFF
|
||||
//log.Printf("block of len %d, uncompressed: %v", l, uncompressed)
|
||||
if int(l) > cap(br.buf) {
|
||||
br.buf = make([]byte, int(l))
|
||||
}
|
||||
br.buf = br.buf[:l]
|
||||
if _, err := io.ReadFull(br.r, br.buf); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
//log.Printf("(retry) n = %v, err = %v", n, err)
|
||||
}
|
||||
n = copy(p, br.buf[br.i:])
|
||||
br.i += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (br *blockReader) Close() error {
|
||||
blockReaderPool.Put(br)
|
||||
return nil
|
||||
}
|
||||
|
||||
var blockReaderPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &blockReader{
|
||||
buf: make([]byte, 0, metadataBlockSize),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
func (r *Reader) blockReader(blockoffset, offset int64) (io.ReadCloser, error) {
|
||||
//log.Printf("blockoffset %v (%x), offset %v (%x)", blockoffset, blockoffset, offset, offset)
|
||||
br := blockReaderPool.Get().(*blockReader)
|
||||
br.buf = br.buf[:0]
|
||||
br.r = io.NewSectionReader(r.r, blockoffset, 5500*1024*1024) // TODO: correct limit? can we use IntMax
|
||||
br.off = blockoffset
|
||||
br.i = 0
|
||||
//log.Printf("discarding %d bytes", offset)
|
||||
for n := int64(0); n < offset; {
|
||||
remaining := offset - n
|
||||
if remaining > metadataBlockSize {
|
||||
remaining = metadataBlockSize
|
||||
}
|
||||
nn, err := br.Read(br.buf[:remaining])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n += int64(nn)
|
||||
}
|
||||
return br, nil
|
||||
}
|
||||
|
||||
// TODO: define an inode type to use instead of interface{}?
|
||||
func (r *Reader) readInode(i Inode) (interface{}, error) {
|
||||
blockoffset, offset := r.inode(i)
|
||||
br, err := r.blockReader(r.super.InodeTableStart+blockoffset, offset)
|
||||
if err != nil {
|
||||
fmt.Println("oops! ", err)
|
||||
return nil, err
|
||||
}
|
||||
defer br.Close()
|
||||
fmt.Println("Hello There")
|
||||
|
||||
// We need the inode type before we know which type to pass to binary.Read,
|
||||
// so we need to read it twice:
|
||||
var inodeType uint16
|
||||
typeBuf := bytes.NewBuffer(make([]byte, 0, binary.Size(inodeType)))
|
||||
if err := binary.Read(io.TeeReader(br, typeBuf), binary.LittleEndian, &inodeType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
br = ioutil.NopCloser(io.MultiReader(typeBuf, br))
|
||||
|
||||
// var ih inodeHeader
|
||||
// if err := binary.Read(br, binary.LittleEndian, &ih); err != nil {
|
||||
// return err
|
||||
// }
|
||||
// //log.Printf("ih: %+v", ih)
|
||||
|
||||
//log.Printf("inode type: %v", inodeType)
|
||||
switch inodeType {
|
||||
case dirType:
|
||||
var di dirInodeHeader
|
||||
if err := binary.Read(br, binary.LittleEndian, &di); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return di, nil
|
||||
|
||||
case fileType:
|
||||
var ri regInodeHeader
|
||||
if err := binary.Read(br, binary.LittleEndian, &ri); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ri, nil
|
||||
|
||||
case symlinkType:
|
||||
var si symlinkInodeHeader
|
||||
if err := binary.Read(br, binary.LittleEndian, &si); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return si, nil
|
||||
|
||||
case ldirType:
|
||||
var di ldirInodeHeader
|
||||
if err := binary.Read(br, binary.LittleEndian, &di); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return di, nil
|
||||
|
||||
case lregType:
|
||||
var di lregInodeHeader
|
||||
if err := binary.Read(br, binary.LittleEndian, &di); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return di, nil
|
||||
|
||||
// TODO:
|
||||
// blkdevType
|
||||
// chrdevType
|
||||
// fifoType
|
||||
// socketType
|
||||
// // The larger types are used for e.g. sparse files, xattrs, etc.
|
||||
// ldirType
|
||||
// lsymlinkType
|
||||
// lblkdevType
|
||||
// lchrdevType
|
||||
// lfifoType
|
||||
// lsocketType
|
||||
|
||||
}
|
||||
return nil, fmt.Errorf("unknown inode type %d", inodeType)
|
||||
}
|
||||
|
||||
func (r *Reader) RootInode() Inode {
|
||||
return r.super.RootInode
|
||||
}
|
||||
|
||||
func (r *Reader) Stat(name string, i Inode) (os.FileInfo, error) {
|
||||
inode, err := r.readInode(i)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
//log.Printf("i %d, inode: %T, %+v", i, inode, inode)
|
||||
switch x := inode.(type) {
|
||||
case dirInodeHeader:
|
||||
return &FileInfo{
|
||||
name: name,
|
||||
size: int64(x.FileSize),
|
||||
mode: os.ModeDir | os.FileMode(x.Mode),
|
||||
modTime: time.Unix(int64(x.Mtime), 0),
|
||||
Inode: i,
|
||||
}, nil
|
||||
|
||||
case ldirInodeHeader:
|
||||
return &FileInfo{
|
||||
name: name,
|
||||
size: int64(x.FileSize),
|
||||
mode: os.ModeDir | os.FileMode(x.Mode),
|
||||
modTime: time.Unix(int64(x.Mtime), 0),
|
||||
Inode: i,
|
||||
}, nil
|
||||
|
||||
case regInodeHeader:
|
||||
mode := os.FileMode(x.Mode & 0777)
|
||||
if x.Mode&syscall.S_ISUID != 0 {
|
||||
mode |= os.ModeSetuid
|
||||
}
|
||||
return &FileInfo{
|
||||
name: name,
|
||||
size: int64(x.FileSize),
|
||||
mode: mode,
|
||||
modTime: time.Unix(int64(x.Mtime), 0),
|
||||
Inode: i,
|
||||
}, nil
|
||||
|
||||
case lregInodeHeader:
|
||||
mode := os.FileMode(x.Mode & 0777)
|
||||
if x.Mode&syscall.S_ISUID != 0 {
|
||||
mode |= os.ModeSetuid
|
||||
}
|
||||
return &FileInfo{
|
||||
name: name,
|
||||
size: int64(x.FileSize),
|
||||
mode: mode,
|
||||
modTime: time.Unix(int64(x.Mtime), 0),
|
||||
Inode: i,
|
||||
}, nil
|
||||
|
||||
case symlinkInodeHeader:
|
||||
return &FileInfo{
|
||||
name: name,
|
||||
size: int64(x.SymlinkSize),
|
||||
mode: os.ModeSymlink | os.FileMode(x.Mode),
|
||||
modTime: time.Unix(int64(x.Mtime), 0),
|
||||
Inode: i,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unknown inode type %T", inode)
|
||||
}
|
||||
|
||||
func (r *Reader) ReadLink(i Inode) (string, error) {
|
||||
// TODO: reduce code duplication with readInode
|
||||
blockoffset, offset := r.inode(i)
|
||||
br, err := r.blockReader(r.super.InodeTableStart+blockoffset, offset)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer br.Close()
|
||||
|
||||
// We need the inode type before we know which type to pass to binary.Read,
|
||||
// so we need to read it twice:
|
||||
var inodeType uint16
|
||||
typeBuf := bytes.NewBuffer(make([]byte, 0, binary.Size(inodeType)))
|
||||
if err := binary.Read(io.TeeReader(br, typeBuf), binary.LittleEndian, &inodeType); err != nil {
|
||||
return "", err
|
||||
}
|
||||
br = ioutil.NopCloser(io.MultiReader(typeBuf, br))
|
||||
|
||||
if inodeType != symlinkType {
|
||||
return "", fmt.Errorf("invalid inode type: got %d instead of symlink", inodeType)
|
||||
}
|
||||
var si symlinkInodeHeader
|
||||
if err := binary.Read(br, binary.LittleEndian, &si); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Assumption: r.r is positioned right after the inode
|
||||
buf := make([]byte, si.SymlinkSize)
|
||||
if _, err := io.ReadFull(br, buf); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(buf), nil
|
||||
}
|
||||
|
||||
func (r *Reader) FileReader(inode Inode) (*io.SectionReader, error) {
|
||||
//log.Printf("Readfile(%v)", inode)
|
||||
i, err := r.readInode(inode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
//log.Printf("i: %+v", i)
|
||||
// TODO(compression): read the blocksizes to read compressed blocks
|
||||
switch ri := i.(type) {
|
||||
case regInodeHeader:
|
||||
off := int64(ri.StartBlock) + int64(ri.Offset)
|
||||
return io.NewSectionReader(r.r, off, int64(ri.FileSize)), nil
|
||||
case lregInodeHeader:
|
||||
off := int64(ri.StartBlock) + int64(ri.Offset)
|
||||
return io.NewSectionReader(r.r, off, int64(ri.FileSize)), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("BUG: non-file inode type")
|
||||
}
|
||||
}
|
||||
|
||||
type FileNotFoundError struct {
|
||||
path string
|
||||
}
|
||||
|
||||
func (e *FileNotFoundError) Error() string {
|
||||
return fmt.Sprintf("%q not found", e.path)
|
||||
}
|
||||
|
||||
func (r *Reader) lookupComponent(parent Inode, component string) (Inode, error) {
|
||||
rfis, err := r.readdir(parent, false)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, rfi := range rfis {
|
||||
if rfi.Name() == component {
|
||||
return rfi.Sys().(*FileInfo).Inode, nil
|
||||
}
|
||||
}
|
||||
return 0, &FileNotFoundError{path: component}
|
||||
}
|
||||
|
||||
func (r *Reader) lookupPath(path string, followSymlink bool) (Inode, error) {
|
||||
inode := r.RootInode()
|
||||
parts := strings.Split(path, "/")
|
||||
for idx, part := range parts {
|
||||
var err error
|
||||
inode, err = r.lookupComponent(inode, part)
|
||||
if err != nil {
|
||||
if _, ok := err.(*FileNotFoundError); ok {
|
||||
return 0, &FileNotFoundError{path: path}
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
if !followSymlink {
|
||||
continue
|
||||
}
|
||||
i, err := r.readInode(inode)
|
||||
if err != nil {
|
||||
return 0, xerrors.Errorf("Stat(%d): %v", inode, err)
|
||||
}
|
||||
if _, ok := i.(symlinkInodeHeader); ok {
|
||||
target, err := r.ReadLink(inode)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
//log.Printf("component %q (full: %q) resolved to %q", part, parts[:idx+1], target)
|
||||
target = filepath.Clean(filepath.Join(append(parts[:idx] /* parent */, target)...))
|
||||
//log.Printf("-> %s", target)
|
||||
i, err := r.LookupPath(target)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
inode = i
|
||||
}
|
||||
}
|
||||
return inode, nil
|
||||
}
|
||||
|
||||
func (r *Reader) LookupPath(path string) (Inode, error) {
|
||||
return r.lookupPath(path, true)
|
||||
}
|
||||
|
||||
// LlookupPath is like LookupPath, but does not follow symbolic links, i.e. will
|
||||
// instead return the inode of the link itself.
|
||||
func (r *Reader) LlookupPath(path string) (Inode, error) {
|
||||
return r.lookupPath(path, false)
|
||||
}
|
||||
|
||||
func (r *Reader) Readdir(dirInode Inode) ([]os.FileInfo, error) {
|
||||
return r.readdir(dirInode, true)
|
||||
}
|
||||
|
||||
// Like Readdir, but does not call Stat on each file. The returned FileInfo
|
||||
// structs will still have a filled in Name, partly filled in Mode, and filled
|
||||
// in Inode.
|
||||
func (r *Reader) ReaddirNoStat(dirInode Inode) ([]os.FileInfo, error) {
|
||||
return r.readdir(dirInode, false)
|
||||
}
|
||||
|
||||
var nameBufPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &bytes.Buffer{}
|
||||
},
|
||||
}
|
||||
|
||||
func (r *Reader) readdir(dirInode Inode, stat bool) ([]os.FileInfo, error) {
|
||||
//log.Printf("Readdir(%v (%x))", dirInode, dirInode)
|
||||
i, err := r.readInode(dirInode)
|
||||
fmt.Println("Yodle")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var (
|
||||
startBlock int64
|
||||
fileSize int64
|
||||
offset int64
|
||||
)
|
||||
switch x := i.(type) {
|
||||
case dirInodeHeader:
|
||||
startBlock = int64(x.StartBlock)
|
||||
fileSize = int64(x.FileSize)
|
||||
offset = int64(x.Offset)
|
||||
|
||||
case ldirInodeHeader:
|
||||
startBlock = int64(x.StartBlock)
|
||||
fileSize = int64(x.FileSize)
|
||||
offset = int64(x.Offset)
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown directory inode type %T", i)
|
||||
}
|
||||
|
||||
br, err := r.blockReader(r.super.DirectoryTableStart+startBlock, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer br.Close()
|
||||
|
||||
// See also https://elixir.bootlin.com/linux/v4.18.9/source/fs/squashfs/dir.c#L63
|
||||
limit := fileSize - int64(len(".")) - int64(len(".."))
|
||||
br = ioutil.NopCloser(io.LimitReader(br, limit))
|
||||
|
||||
var fis []os.FileInfo
|
||||
var dh dirHeader
|
||||
var de dirEntry
|
||||
var dhBuf [12]byte
|
||||
var deBuf [8]byte
|
||||
nameBuf := nameBufPool.Get().(*bytes.Buffer)
|
||||
defer nameBufPool.Put(nameBuf)
|
||||
for {
|
||||
if _, err := io.ReadFull(br, dhBuf[:]); err != nil {
|
||||
if err == io.EOF {
|
||||
return fis, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
dh.Unmarshal(dhBuf[:])
|
||||
dh.Count++ // SquashFS stores count-1
|
||||
//log.Printf("dh: %+v", dh)
|
||||
|
||||
for i := 0; i < int(dh.Count); i++ {
|
||||
if _, err := io.ReadFull(br, deBuf[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
de.Unmarshal(deBuf[:])
|
||||
de.Size++ // SquashFS stores size-1
|
||||
//log.Printf("de: %+v", de)
|
||||
nameBuf.Reset()
|
||||
nameBuf.Grow(int(de.Size))
|
||||
nb := nameBuf.Bytes()[:de.Size]
|
||||
if _, err := io.ReadFull(br, nb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name := string(nb)
|
||||
//log.Printf("name: %q", string(name))
|
||||
|
||||
var fi os.FileInfo
|
||||
if stat {
|
||||
var err error
|
||||
fi, err = r.Stat(name, Inode(int64(dh.StartBlock)<<16|int64(de.Offset)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
ffi := &FileInfo{
|
||||
name: name,
|
||||
Inode: Inode(int64(dh.StartBlock)<<16 | int64(de.Offset)),
|
||||
}
|
||||
switch de.EntryType {
|
||||
case dirType, ldirType:
|
||||
ffi.mode |= os.ModeDir
|
||||
case symlinkType, lsymlinkType:
|
||||
ffi.mode |= os.ModeSymlink
|
||||
}
|
||||
fi = ffi
|
||||
}
|
||||
fis = append(fis, fi)
|
||||
}
|
||||
}
|
||||
|
||||
return fis, nil
|
||||
}
|
||||
|
||||
type FileInfo struct {
|
||||
name string
|
||||
size int64
|
||||
mode os.FileMode
|
||||
modTime time.Time
|
||||
Inode Inode
|
||||
}
|
||||
|
||||
func (fi *FileInfo) Name() string { return fi.name }
|
||||
func (fi *FileInfo) Size() int64 { return fi.size }
|
||||
func (fi *FileInfo) Mode() os.FileMode { return fi.mode }
|
||||
func (fi *FileInfo) IsDir() bool { return fi.mode.IsDir() }
|
||||
func (fi *FileInfo) ModTime() time.Time { return fi.modTime }
|
||||
func (fi *FileInfo) Sys() interface{} { return fi }
|
||||
|
||||
func (r *Reader) readXattr(tableHeader xattrTableHeader, id xattrId) (*Xattr, error) {
|
||||
blockoffset, offset := r.inode(Inode(id.Xattr))
|
||||
br, err := r.blockReader(int64(tableHeader.XattrTableStart)+blockoffset, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer br.Close()
|
||||
var typ, nameSize uint16
|
||||
if err := binary.Read(br, binary.LittleEndian, &typ); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := binary.Read(br, binary.LittleEndian, &nameSize); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Printf("type = %v, nameSize = %v", typ, nameSize)
|
||||
name := make([]byte, nameSize)
|
||||
if _, err := io.ReadFull(br, name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Printf("name = %v", string(name))
|
||||
var valSize uint32
|
||||
if err := binary.Read(br, binary.LittleEndian, &valSize); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
val := make([]byte, valSize)
|
||||
if _, err := io.ReadFull(br, val); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Printf("val = %x", val)
|
||||
return &Xattr{
|
||||
Type: typ,
|
||||
FullName: xattrPrefix[int(typ)] + string(name),
|
||||
Value: val,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Reader) ReadXattrs(inode Inode) ([]Xattr, error) {
|
||||
//log.Printf("Readdir(%v (%x))", dirInode, dirInode)
|
||||
i, err := r.readInode(inode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var xid uint32
|
||||
switch x := i.(type) {
|
||||
case regInodeHeader,
|
||||
dirInodeHeader,
|
||||
ldirInodeHeader,
|
||||
symlinkInodeHeader:
|
||||
return nil, nil // no extended attributes
|
||||
|
||||
case lregInodeHeader:
|
||||
if x.Xattr == invalidXattr {
|
||||
return nil, nil // file has no extended attributes
|
||||
}
|
||||
xid = x.Xattr
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown inode type %T", i)
|
||||
}
|
||||
|
||||
const idEntriesPerBlock = 512 // = 8192 / 16 /* sizeof(xattrId) */
|
||||
block := xid / idEntriesPerBlock
|
||||
offset := (xid % idEntriesPerBlock) * 16
|
||||
log.Printf("xattr id %d, block %d, offset %d", xid, block, offset)
|
||||
log.Printf("r.super.XattrIdTableStart = 0x%x, r.super.XattrIdTableStart = %v", r.super.XattrIdTableStart, r.super.XattrIdTableStart)
|
||||
br := ioutil.NopCloser(io.NewSectionReader(r.r, r.super.XattrIdTableStart, int64(16 /* sizeof(xattrTableHeader) */ +(block+1)*4 /* sizeof(uint32) */)))
|
||||
var tableHeader xattrTableHeader
|
||||
if err := binary.Read(br, binary.LittleEndian, &tableHeader); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// index starts here
|
||||
if _, err := io.CopyN(ioutil.Discard, br, int64(block*4 /* sizeof(uint32) */)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var blockOffset uint32
|
||||
if err := binary.Read(br, binary.LittleEndian, &blockOffset); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Printf("blockOffset = 0x%x (%d)", blockOffset, blockOffset)
|
||||
br, err = r.blockReader(int64(blockOffset), int64(offset))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer br.Close()
|
||||
var id xattrId
|
||||
if err := binary.Read(br, binary.LittleEndian, &id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Printf("id: %+v", id)
|
||||
log.Printf("tableHeader: %+v (start 0x%x)", tableHeader, tableHeader.XattrTableStart)
|
||||
|
||||
var xattrs []Xattr
|
||||
for i := 0; i < int(id.Count); i++ {
|
||||
xattr, err := r.readXattr(tableHeader, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
xattrs = append(xattrs, *xattr)
|
||||
}
|
||||
|
||||
return xattrs, nil
|
||||
}
|
||||
+373
@@ -0,0 +1,373 @@
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
)
|
||||
|
||||
func cmpFileInfo(got os.FileInfo, want FileInfo) error {
|
||||
if got, want := got.Name(), want.name; got != want {
|
||||
return fmt.Errorf("unexpected file name: got %q, want %q", got, want)
|
||||
}
|
||||
if got, want := got.Size(), want.size; got != want {
|
||||
return fmt.Errorf("unexpected size: got %d, want %d", got, want)
|
||||
}
|
||||
if got, want := got.IsDir(), want.mode.IsDir(); got != want {
|
||||
return fmt.Errorf("IsDir: got %v, want %v", got, want)
|
||||
}
|
||||
// TODO: re-enable when it’s no longer just a change detector
|
||||
// if got, want := got.ModTime(), want.modTime; !got.Equal(want) {
|
||||
// return fmt.Errorf("IsDir: got %v, want %v", got, want)
|
||||
// }
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestReaddir(t *testing.T) {
|
||||
t.Parallel()
|
||||
// TODO: ship testdata files generated by mksquashfs
|
||||
pwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fmt.Println(pwd + "/testdata/testing.squashfs")
|
||||
f, err := os.Open(pwd + "/testdata/testing.squashfs")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
rd, err := NewReader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fis, err := rd.Readdir(rd.RootInode())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, fi := range fis {
|
||||
fmt.Println(fi.Name())
|
||||
}
|
||||
rdr, err := rd.FileReader(fis[0].Sys().(*FileInfo).Inode)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = os.Remove(pwd + "/testdata/Magisk.zip")
|
||||
if !os.IsNotExist(err) && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// rdrzlib, err := zlib.NewReader(rdr)
|
||||
magFil, err := os.Create(pwd + "/testdata/Magisk.zip")
|
||||
io.Copy(magFil, rdr)
|
||||
if got, want := len(fis), 3; got != want {
|
||||
t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
if err := cmpFileInfo(fis[0], FileInfo{
|
||||
name: "bin",
|
||||
size: 26,
|
||||
mode: 0555 | os.ModeDir,
|
||||
modTime: time.Unix(1581275131, 0), // stat -c %Y /ro/ack-amd64-2.24/bin
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cmpFileInfo(fis[1], FileInfo{
|
||||
name: "lib",
|
||||
size: 3,
|
||||
mode: 0555 | os.ModeDir,
|
||||
modTime: time.Unix(1581275131, 0), // stat -c %Y /ro/ack-amd64-2.24/lib
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cmpFileInfo(fis[2], FileInfo{
|
||||
name: "out",
|
||||
size: 48,
|
||||
mode: 0555 | os.ModeDir,
|
||||
modTime: time.Unix(1581275130, 0), // stat -c %Y /ro/ack-amd64-2.24/out
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fis, err = rd.Readdir(fis[0].Sys().(*FileInfo).Inode)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(fis), 1; got != want {
|
||||
t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
if err := cmpFileInfo(fis[0], FileInfo{
|
||||
name: "ack",
|
||||
size: 38400,
|
||||
mode: 0755,
|
||||
modTime: time.Unix(1581275131, 0), // stat -c %Y /ro/ack-amd64-2.24/bin/ack
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaddirSmoke is a smoke-test, reading the root directories of SquashFS
|
||||
// images which are known to trigger code paths which were buggy.
|
||||
func TestReaddirSmoke(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, fn := range []string{
|
||||
// bash exercises the code path where an inode is split across metadata
|
||||
// blocks.
|
||||
"/home/michael/distri/_build/distri/pkg/bash-amd64-5.0-4.squashfs",
|
||||
|
||||
// cmake exercises the code path where the root directory entries are
|
||||
// located outside of the first block.
|
||||
"/home/michael/distri/_build/distri/pkg/cmake-amd64-3.12.4-8.squashfs",
|
||||
} {
|
||||
// TODO: ship testdata files generated by mksquashfs
|
||||
f, err := os.Open(fn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
rd, err := NewReader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fis, err := rd.Readdir(rd.RootInode())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(fis), 4; got != want {
|
||||
t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaddirEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
// TODO: ship testdata files generated by mksquashfs
|
||||
f, err := os.Open("/home/michael/distri/_build/distri/pkg/zlib-amd64-1.2.11-4.squashfs")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
rd, err := NewReader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fis, err := rd.Readdir(rd.RootInode())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(fis), 4; got != want {
|
||||
t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
if err := cmpFileInfo(fis[0], FileInfo{
|
||||
name: "bin",
|
||||
size: 3,
|
||||
mode: 0555 | os.ModeDir,
|
||||
modTime: time.Unix(1583085224, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/bin
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fis, err = rd.Readdir(fis[0].Sys().(*FileInfo).Inode)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(fis), 0; got != want {
|
||||
t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaddirSymlink(t *testing.T) {
|
||||
t.Parallel()
|
||||
// TODO: ship testdata files generated by mksquashfs
|
||||
f, err := os.Open("/home/michael/distri/_build/distri/pkg/zlib-amd64-1.2.11-4.squashfs")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
rd, err := NewReader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fis, err := rd.Readdir(rd.RootInode())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(fis), 4; got != want {
|
||||
t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
if err := cmpFileInfo(fis[3], FileInfo{
|
||||
name: "out",
|
||||
size: 54,
|
||||
mode: 0555 | os.ModeDir,
|
||||
modTime: time.Unix(1583085224, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/out
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fis, err = rd.Readdir(fis[3].Sys().(*FileInfo).Inode)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(fis), 3; got != want {
|
||||
t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
if err := cmpFileInfo(fis[1], FileInfo{
|
||||
name: "lib",
|
||||
size: 83,
|
||||
mode: 0555 | os.ModeDir,
|
||||
modTime: time.Unix(1583085224, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/out/lib
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fis, err = rd.Readdir(fis[1].Sys().(*FileInfo).Inode)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(fis), 4; got != want {
|
||||
t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
if err := cmpFileInfo(fis[1], FileInfo{
|
||||
name: "libz.so",
|
||||
size: 9,
|
||||
mode: 0555 | os.ModeSymlink,
|
||||
modTime: time.Unix(1583085223, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/out/lib/libz.so
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// TODO: readlink
|
||||
target, err := rd.ReadLink(fis[1].Sys().(*FileInfo).Inode)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := target, "libz.so.1"; got != want {
|
||||
t.Fatalf("ReadLink(libz.so): got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadfile(t *testing.T) {
|
||||
t.Parallel()
|
||||
// TODO: ship testdata files generated by mksquashfs
|
||||
f, err := os.Open("/home/michael/distri/_build/distri/pkg/ack-amd64-3.3.1-7.squashfs")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
rd, err := NewReader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fis, err := rd.Readdir(rd.RootInode())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(fis), 3; got != want {
|
||||
t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
fis, err = rd.Readdir(fis[0].Sys().(*FileInfo).Inode)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(fis), 1; got != want {
|
||||
t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
r, err := rd.FileReader(fis[0].Sys().(*FileInfo).Inode)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := r.Seek(0, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := md5.New()
|
||||
if _, err := io.Copy(h, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := fmt.Sprintf("%x", h.Sum(nil))
|
||||
if got, want := sum, "c6c9b5d4d2a49f1b8b5e501f0f827a5c"; got != want {
|
||||
t.Fatalf("md5(bin/ack): got %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: add test exercising ldirInodeHeader, e.g. '/mnt/loop/ca-certificates-3.39/buildoutput/etc/ssl'
|
||||
|
||||
func TestReadXattr(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// TODO: generate a smaller version of this file
|
||||
f, err := os.Open("testdata/xattr.squashfs")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
rd, err := NewReader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tt := range []struct {
|
||||
Path string
|
||||
Want []Xattr
|
||||
}{
|
||||
{
|
||||
Path: "mtr-packet",
|
||||
Want: []Xattr{
|
||||
{
|
||||
Type: XattrTypeSecurity,
|
||||
FullName: "security.capability",
|
||||
Value: []byte{1, 0, 0, 2, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Path: "gnome-keyring-daemon",
|
||||
Want: []Xattr{
|
||||
{
|
||||
Type: XattrTypeSecurity,
|
||||
FullName: "security.capability",
|
||||
Value: []byte{1, 0, 0, 2, 0, 0x40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
|
||||
},
|
||||
},
|
||||
} {
|
||||
inode, err := rd.LookupPath(tt.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xattrs, err := rd.ReadXattrs(inode)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadXattrs(%v): %v", inode, err)
|
||||
}
|
||||
if diff := cmp.Diff(tt.Want, xattrs); diff != "" {
|
||||
t.Fatalf("unexpected ReadXattrs result: diff (-want +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package squashfs
|
||||
|
||||
//Descriptions provided by https://dr-emann.github.io/squashfs/
|
||||
type superblock struct {
|
||||
Magic uint32 //Magic will be 0x73717368 if it's a legit Squashfs filesystem
|
||||
Inodes uint32 //Inodes is the number of inodes in the inodes table
|
||||
MkfsTime int32 //MkfsTime is when archive was created
|
||||
BlockSize uint32 //BlockSize is the size of data blocks in bytes
|
||||
Fragments uint32 //Fragments is the number of entries in fragment table
|
||||
Compression uint16 //Compression is what type of compression is used
|
||||
BlockLog uint16 //BlockLog should be log base 2 of BlockSize. If not then the squash might be corrupt
|
||||
Flags uint16 //Flags are the superblock's flags
|
||||
IDCount uint16 //IDCount is the number of IDs in the id lookup table
|
||||
Major uint16
|
||||
Minor uint16
|
||||
RootInode Inode
|
||||
BytesUsed int64
|
||||
IDTableStart int64
|
||||
XattrIDTableStart int64
|
||||
InodeTableStart int64
|
||||
DirectoryTableStart int64
|
||||
FragmentTableStart int64
|
||||
LookupTableStart int64
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package squashfs
|
||||
|
||||
import "io"
|
||||
|
||||
//Squashfs is a squashfs backed by a reader.
|
||||
type Squashfs struct {
|
||||
rdr *io.Reader //underlyting reader
|
||||
super Superblock
|
||||
}
|
||||
+913
@@ -0,0 +1,913 @@
|
||||
// Package squashfs implements writing SquashFS file system images using zlib
|
||||
// compression for data blocks (inodes and directory entries are written
|
||||
// uncompressed for simplicity).
|
||||
//
|
||||
// Note that SquashFS requires directory entries to be sorted, i.e. files and
|
||||
// directories need to be added in the correct order.
|
||||
//
|
||||
// This package intentionally only implements a subset of SquashFS. Notably,
|
||||
// block devices, character devices, FIFOs, sockets and xattrs are not
|
||||
// supported.
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/zlib"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// inode contains a block number + offset within that block.
|
||||
type Inode int64
|
||||
|
||||
const (
|
||||
zlibCompression = 1 + iota
|
||||
lzmaCompression
|
||||
lzoCompression
|
||||
xzCompression
|
||||
lz4Compression
|
||||
zstdCompression
|
||||
)
|
||||
|
||||
const (
|
||||
invalidFragment = 0xFFFFFFFF
|
||||
invalidXattr = 0xFFFFFFFF
|
||||
)
|
||||
|
||||
const (
|
||||
dirType = 1 + iota
|
||||
fileType
|
||||
symlinkType
|
||||
blkdevType
|
||||
chrdevType
|
||||
fifoType
|
||||
socketType
|
||||
// The larger types are used for e.g. sparse files, xattrs, etc.
|
||||
ldirType
|
||||
lregType
|
||||
lsymlinkType
|
||||
lblkdevType
|
||||
lchrdevType
|
||||
lfifoType
|
||||
lsocketType
|
||||
)
|
||||
|
||||
type inodeHeader struct {
|
||||
InodeType uint16
|
||||
Mode uint16
|
||||
Uid uint16
|
||||
Gid uint16
|
||||
Mtime int32
|
||||
InodeNumber uint32
|
||||
}
|
||||
|
||||
// fileType
|
||||
type regInodeHeader struct {
|
||||
inodeHeader
|
||||
|
||||
// full byte offset from the start of the file system, e.g. 96 for first
|
||||
// file contents. Not using fragments limits us to 2^32-1-96 (≈ 4GiB) bytes
|
||||
// of file contents.
|
||||
StartBlock uint32
|
||||
Fragment uint32
|
||||
Offset uint32
|
||||
FileSize uint32
|
||||
|
||||
// Followed by a uint32 array of compressed block sizes.
|
||||
}
|
||||
|
||||
// lregType
|
||||
type lregInodeHeader struct {
|
||||
inodeHeader
|
||||
|
||||
// full byte offset from the start of the file system, e.g. 96 for first
|
||||
// file contents. Not using fragments limits us to 2^32-1-96 (≈ 4GiB) bytes
|
||||
// of file contents.
|
||||
StartBlock uint64
|
||||
FileSize uint64
|
||||
Sparse uint64
|
||||
Nlink uint32
|
||||
Fragment uint32
|
||||
Offset uint32
|
||||
Xattr uint32
|
||||
|
||||
// Followed by a uint32 array of compressed block sizes.
|
||||
}
|
||||
|
||||
// symlinkType
|
||||
type symlinkInodeHeader struct {
|
||||
inodeHeader
|
||||
|
||||
Nlink uint32
|
||||
SymlinkSize uint32
|
||||
|
||||
// Followed by a byte array of SymlinkSize bytes.
|
||||
}
|
||||
|
||||
// chrdevType and blkdevType
|
||||
type devInodeHeader struct {
|
||||
inodeHeader
|
||||
|
||||
Nlink uint32
|
||||
Rdev uint32
|
||||
}
|
||||
|
||||
// fifoType and socketType
|
||||
type ipcInodeHeader struct {
|
||||
inodeHeader
|
||||
|
||||
Nlink uint32
|
||||
}
|
||||
|
||||
// dirType
|
||||
type dirInodeHeader struct {
|
||||
inodeHeader
|
||||
|
||||
StartBlock uint32
|
||||
Nlink uint32
|
||||
FileSize uint16
|
||||
Offset uint16
|
||||
ParentInode uint32
|
||||
}
|
||||
|
||||
// ldirType
|
||||
type ldirInodeHeader struct {
|
||||
inodeHeader
|
||||
|
||||
Nlink uint32
|
||||
FileSize uint32
|
||||
StartBlock uint32
|
||||
ParentInode uint32
|
||||
Icount uint16
|
||||
Offset uint16
|
||||
Xattr uint32
|
||||
}
|
||||
|
||||
type dirHeader struct {
|
||||
Count uint32
|
||||
StartBlock uint32
|
||||
InodeOffset uint32
|
||||
}
|
||||
|
||||
func (d *dirHeader) Unmarshal(b []byte) {
|
||||
_ = b[11]
|
||||
e := binary.LittleEndian
|
||||
d.Count = e.Uint32(b)
|
||||
d.StartBlock = e.Uint32(b[4:])
|
||||
d.InodeOffset = e.Uint32(b[8:])
|
||||
}
|
||||
|
||||
type dirEntry struct {
|
||||
Offset uint16
|
||||
InodeNumber int16
|
||||
EntryType uint16
|
||||
Size uint16
|
||||
|
||||
// Followed by a byte array of Size bytes.
|
||||
}
|
||||
|
||||
func (d *dirEntry) Unmarshal(b []byte) {
|
||||
_ = b[7]
|
||||
e := binary.LittleEndian
|
||||
d.Offset = e.Uint16(b)
|
||||
d.InodeNumber = int16(e.Uint16(b[2:]))
|
||||
d.EntryType = e.Uint16(b[4:])
|
||||
d.Size = e.Uint16(b[6:])
|
||||
}
|
||||
|
||||
// xattr types
|
||||
const (
|
||||
XattrTypeUser = iota
|
||||
XattrTypeTrusted
|
||||
XattrTypeSecurity
|
||||
)
|
||||
|
||||
var xattrPrefix = map[int]string{
|
||||
XattrTypeUser: "user.",
|
||||
XattrTypeTrusted: "trusted.",
|
||||
XattrTypeSecurity: "security.",
|
||||
}
|
||||
|
||||
type Xattr struct {
|
||||
Type uint16
|
||||
FullName string
|
||||
Value []byte
|
||||
}
|
||||
|
||||
func XattrFromAttr(attr string, val []byte) Xattr {
|
||||
for typ, prefix := range xattrPrefix {
|
||||
if !strings.HasPrefix(attr, prefix) {
|
||||
continue
|
||||
}
|
||||
return Xattr{
|
||||
Type: uint16(typ),
|
||||
FullName: strings.TrimPrefix(attr, prefix),
|
||||
Value: val,
|
||||
}
|
||||
}
|
||||
return Xattr{}
|
||||
}
|
||||
|
||||
type xattrId struct {
|
||||
Xattr uint64
|
||||
Count uint32
|
||||
Size uint32
|
||||
}
|
||||
|
||||
func writeIdTable(w io.WriteSeeker, ids []uint32) (start int64, err error) {
|
||||
metaOff, err := w.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := binary.Write(&buf, binary.LittleEndian, ids); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := binary.Write(w, binary.LittleEndian, uint16(buf.Len())|0x8000); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := io.Copy(w, &buf); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
off, err := w.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return off, binary.Write(w, binary.LittleEndian, metaOff)
|
||||
}
|
||||
|
||||
type fullDirEntry struct {
|
||||
startBlock uint32
|
||||
offset uint16
|
||||
inodeNumber uint32
|
||||
entryType uint16
|
||||
name string
|
||||
}
|
||||
|
||||
const (
|
||||
magic = 0x73717368
|
||||
dataBlockSize = 131072
|
||||
metadataBlockSize = 8192
|
||||
majorVersion = 4
|
||||
minorVersion = 0
|
||||
)
|
||||
|
||||
type Writer struct {
|
||||
// Root represents the file system root. Like all directories, Flush must be
|
||||
// called precisely once.
|
||||
Root *Directory
|
||||
|
||||
xattrs []Xattr
|
||||
xattrIds []xattrId
|
||||
|
||||
w io.WriteSeeker
|
||||
|
||||
sb superblock
|
||||
inodeBuf bytes.Buffer
|
||||
dirBuf bytes.Buffer
|
||||
|
||||
writeInodeNumTo map[string][]int64
|
||||
}
|
||||
|
||||
// TODO: document what this is doing and what it is used for
|
||||
func slog(block uint32) uint16 {
|
||||
for i := uint16(12); i <= 20; i++ {
|
||||
if block == (1 << i) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// filesystemFlags returns flags for a SquashFS file system created by this
|
||||
// package (disabling most features for now).
|
||||
func filesystemFlags() uint16 {
|
||||
const (
|
||||
noI = 1 << iota // uncompressed metadata
|
||||
noD // uncompressed data
|
||||
_
|
||||
noF // uncompressed fragments
|
||||
noFrag // never use fragments
|
||||
alwaysFrag // always use fragments
|
||||
duplicateChecking // de-duplication
|
||||
exportable // exportable via NFS
|
||||
noX // uncompressed xattrs
|
||||
noXattr // no xattrs
|
||||
compopt // compressor-specific options present?
|
||||
)
|
||||
return noI | noF | noFrag | noX | noXattr
|
||||
}
|
||||
|
||||
// NewWriter returns a Writer which will write a SquashFS file system image to w
|
||||
// once Flush is called.
|
||||
//
|
||||
// Create new files and directories with the corresponding methods on the Root
|
||||
// directory of the Writer.
|
||||
//
|
||||
// File data is written to w even before Flush is called.
|
||||
func NewWriter(w io.WriteSeeker, mkfsTime time.Time) (*Writer, error) {
|
||||
// Skip over superblock to the data area, we come back to the superblock
|
||||
// when flushing.
|
||||
if _, err := w.Seek(96, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wr := &Writer{
|
||||
w: w,
|
||||
sb: superblock{
|
||||
Magic: magic,
|
||||
MkfsTime: int32(mkfsTime.Unix()),
|
||||
BlockSize: dataBlockSize,
|
||||
Fragments: 0,
|
||||
Compression: zlibCompression,
|
||||
BlockLog: slog(dataBlockSize),
|
||||
Flags: filesystemFlags(),
|
||||
NoIds: 1, // just one uid/gid mapping (for root)
|
||||
Major: majorVersion,
|
||||
Minor: minorVersion,
|
||||
XattrIdTableStart: -1, // not present
|
||||
LookupTableStart: -1, // not present
|
||||
},
|
||||
writeInodeNumTo: make(map[string][]int64),
|
||||
}
|
||||
wr.Root = &Directory{
|
||||
w: wr,
|
||||
name: "", // root
|
||||
modTime: mkfsTime,
|
||||
}
|
||||
return wr, nil
|
||||
}
|
||||
|
||||
// Directory represents a SquashFS directory.
|
||||
type Directory struct {
|
||||
w *Writer
|
||||
name string
|
||||
modTime time.Time
|
||||
dirEntries []fullDirEntry
|
||||
parent *Directory
|
||||
}
|
||||
|
||||
func (d *Directory) path() string {
|
||||
if d.parent == nil {
|
||||
return d.name
|
||||
}
|
||||
return filepath.Join(d.parent.path(), d.name)
|
||||
}
|
||||
|
||||
type file struct {
|
||||
w *Writer
|
||||
d *Directory
|
||||
off int64
|
||||
size uint32
|
||||
name string
|
||||
modTime time.Time
|
||||
mode uint16
|
||||
|
||||
// buf accumulates at least dataBlockSize bytes, at which point a new block
|
||||
// is being written.
|
||||
buf bytes.Buffer
|
||||
|
||||
// blocksizes stores, for each block of dataBlockSize bytes (uncompressed),
|
||||
// the number of bytes the block compressed down to.
|
||||
blocksizes []uint32
|
||||
|
||||
// compBuf is used for holding a block during compression to avoid memory
|
||||
// allocations.
|
||||
compBuf *bytes.Buffer
|
||||
// zlibWriter is re-used for each compressed block
|
||||
zlibWriter *zlib.Writer
|
||||
|
||||
xattrRef uint32
|
||||
}
|
||||
|
||||
// Directory creates a new directory with the specified name and modTime.
|
||||
func (d *Directory) Directory(name string, modTime time.Time) *Directory {
|
||||
return &Directory{
|
||||
w: d.w,
|
||||
name: name,
|
||||
modTime: modTime,
|
||||
parent: d,
|
||||
}
|
||||
}
|
||||
|
||||
// File creates a file with the specified name, modTime and mode. The returned
|
||||
// io.WriterCloser must be closed after writing the file.
|
||||
func (d *Directory) File(name string, modTime time.Time, mode uint16, xattrs []Xattr) (io.WriteCloser, error) {
|
||||
off, err := d.w.w.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// zlib.BestSpeed results in only a 2x slow-down over no compression
|
||||
// (compared to >4x slow-down with DefaultCompression), but generates
|
||||
// results which are in the same ball park (10% larger).
|
||||
zw, err := zlib.NewWriterLevel(nil, zlib.BestSpeed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
xattrRef := uint32(invalidXattr)
|
||||
if len(xattrs) > 0 {
|
||||
xattrRef = uint32(len(d.w.xattrs))
|
||||
d.w.xattrs = append(d.w.xattrs, xattrs[0]) // TODO: support multiple
|
||||
size := len(xattrs[0].FullName) + len(xattrs[0].Value)
|
||||
d.w.xattrIds = append(d.w.xattrIds, xattrId{
|
||||
// Xattr is populated in writeXattrTables
|
||||
Count: 1, // TODO: support multiple
|
||||
Size: uint32(size),
|
||||
})
|
||||
}
|
||||
return &file{
|
||||
w: d.w,
|
||||
d: d,
|
||||
off: off,
|
||||
name: name,
|
||||
modTime: modTime,
|
||||
mode: mode,
|
||||
compBuf: bytes.NewBuffer(make([]byte, dataBlockSize)),
|
||||
zlibWriter: zw,
|
||||
xattrRef: xattrRef,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Symlink creates a symbolic link from newname to oldname with the specified
|
||||
// modTime and mode.
|
||||
func (d *Directory) Symlink(oldname, newname string, modTime time.Time, mode os.FileMode) error {
|
||||
startBlock := d.w.inodeBuf.Len() / metadataBlockSize
|
||||
offset := d.w.inodeBuf.Len() - startBlock*metadataBlockSize
|
||||
|
||||
if err := binary.Write(&d.w.inodeBuf, binary.LittleEndian, symlinkInodeHeader{
|
||||
inodeHeader: inodeHeader{
|
||||
InodeType: symlinkType,
|
||||
Mode: uint16(mode),
|
||||
Uid: 0,
|
||||
Gid: 0,
|
||||
Mtime: int32(modTime.Unix()),
|
||||
InodeNumber: d.w.sb.Inodes + 1,
|
||||
},
|
||||
Nlink: 1, // TODO(later): when is this not 1?
|
||||
SymlinkSize: uint32(len(oldname)),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.w.inodeBuf.Write([]byte(oldname)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.dirEntries = append(d.dirEntries, fullDirEntry{
|
||||
startBlock: uint32(startBlock),
|
||||
offset: uint16(offset),
|
||||
inodeNumber: d.w.sb.Inodes + 1,
|
||||
entryType: symlinkType,
|
||||
name: newname,
|
||||
})
|
||||
|
||||
d.w.sb.Inodes++
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush writes directory entries and creates inodes for the directory.
|
||||
func (d *Directory) Flush() error {
|
||||
countByStartBlock := make(map[uint32]uint32)
|
||||
for _, de := range d.dirEntries {
|
||||
countByStartBlock[de.startBlock]++
|
||||
}
|
||||
|
||||
dirBufStartBlock := d.w.dirBuf.Len() / metadataBlockSize
|
||||
dirBufOffset := d.w.dirBuf.Len()
|
||||
|
||||
currentBlock := int64(-1)
|
||||
currentInodeOffset := int64(-1)
|
||||
var subdirs int
|
||||
for _, de := range d.dirEntries {
|
||||
if de.entryType == dirType {
|
||||
subdirs++
|
||||
}
|
||||
if int64(de.startBlock) != currentBlock {
|
||||
dh := dirHeader{
|
||||
Count: countByStartBlock[de.startBlock] - 1,
|
||||
StartBlock: de.startBlock * (metadataBlockSize + 2),
|
||||
InodeOffset: de.inodeNumber,
|
||||
}
|
||||
if err := binary.Write(&d.w.dirBuf, binary.LittleEndian, &dh); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
currentBlock = int64(de.startBlock)
|
||||
currentInodeOffset = int64(de.inodeNumber)
|
||||
}
|
||||
if err := binary.Write(&d.w.dirBuf, binary.LittleEndian, &dirEntry{
|
||||
Offset: de.offset,
|
||||
InodeNumber: int16(de.inodeNumber - uint32(currentInodeOffset)),
|
||||
EntryType: de.entryType,
|
||||
Size: uint16(len(de.name) - 1),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.w.dirBuf.Write([]byte(de.name)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
startBlock := d.w.inodeBuf.Len() / metadataBlockSize
|
||||
offset := d.w.inodeBuf.Len() - startBlock*metadataBlockSize
|
||||
inodeBufOffset := d.w.inodeBuf.Len()
|
||||
|
||||
// parentInodeOffset is the offset (in bytes) of the ParentInode field
|
||||
// within a dirInodeHeader or ldirInodeHeader
|
||||
var parentInodeOffset int64
|
||||
|
||||
if len(d.dirEntries) > 256 ||
|
||||
d.w.dirBuf.Len()-dirBufOffset > metadataBlockSize {
|
||||
parentInodeOffset = (2 + 2 + 2 + 2 + 4 + 4) + 4 + 4 + 4
|
||||
if err := binary.Write(&d.w.inodeBuf, binary.LittleEndian, ldirInodeHeader{
|
||||
inodeHeader: inodeHeader{
|
||||
InodeType: ldirType,
|
||||
Mode: unix.S_IRUSR | unix.S_IWUSR | unix.S_IXUSR |
|
||||
unix.S_IRGRP | unix.S_IXGRP |
|
||||
unix.S_IROTH | unix.S_IXOTH,
|
||||
Uid: 0,
|
||||
Gid: 0,
|
||||
Mtime: int32(d.modTime.Unix()),
|
||||
InodeNumber: d.w.sb.Inodes + 1,
|
||||
},
|
||||
|
||||
Nlink: uint32(subdirs + 2 - 1), // + 2 for . and ..
|
||||
FileSize: uint32(d.w.dirBuf.Len()-dirBufOffset) + 3,
|
||||
StartBlock: uint32(dirBufStartBlock * (metadataBlockSize + 2)),
|
||||
ParentInode: d.w.sb.Inodes + 2, // invalid
|
||||
Icount: 0, // no directory index
|
||||
Offset: uint16(dirBufOffset - dirBufStartBlock*metadataBlockSize),
|
||||
Xattr: invalidXattr,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
parentInodeOffset = (2 + 2 + 2 + 2 + 4 + 4) + 4 + 4 + 2 + 2
|
||||
if err := binary.Write(&d.w.inodeBuf, binary.LittleEndian, dirInodeHeader{
|
||||
inodeHeader: inodeHeader{
|
||||
InodeType: dirType,
|
||||
Mode: unix.S_IRUSR | unix.S_IWUSR | unix.S_IXUSR |
|
||||
unix.S_IRGRP | unix.S_IXGRP |
|
||||
unix.S_IROTH | unix.S_IXOTH,
|
||||
Uid: 0,
|
||||
Gid: 0,
|
||||
Mtime: int32(d.modTime.Unix()),
|
||||
InodeNumber: d.w.sb.Inodes + 1,
|
||||
},
|
||||
StartBlock: uint32(dirBufStartBlock * (metadataBlockSize + 2)),
|
||||
Nlink: uint32(subdirs + 2 - 1), // + 2 for . and ..
|
||||
FileSize: uint16(d.w.dirBuf.Len()-dirBufOffset) + 3,
|
||||
Offset: uint16(dirBufOffset - dirBufStartBlock*metadataBlockSize),
|
||||
ParentInode: d.w.sb.Inodes + 2, // invalid
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
path := d.path()
|
||||
for _, offset := range d.w.writeInodeNumTo[path] {
|
||||
// Directly manipulating unread data in bytes.Buffer via Bytes(), as per
|
||||
// https://groups.google.com/d/msg/golang-nuts/1ON9XVQ1jXE/8j9RaeSYxuEJ
|
||||
b := d.w.inodeBuf.Bytes()
|
||||
binary.LittleEndian.PutUint32(b[offset:offset+4], d.w.sb.Inodes+1)
|
||||
}
|
||||
|
||||
if d.parent != nil {
|
||||
parentPath := filepath.Dir(d.path())
|
||||
if parentPath == "." {
|
||||
parentPath = ""
|
||||
}
|
||||
d.w.writeInodeNumTo[parentPath] = append(d.w.writeInodeNumTo[parentPath], int64(inodeBufOffset)+parentInodeOffset)
|
||||
d.parent.dirEntries = append(d.parent.dirEntries, fullDirEntry{
|
||||
startBlock: uint32(startBlock),
|
||||
offset: uint16(offset),
|
||||
inodeNumber: d.w.sb.Inodes + 1,
|
||||
entryType: dirType,
|
||||
name: d.name,
|
||||
})
|
||||
} else { // root
|
||||
d.w.sb.RootInode = Inode((startBlock*(metadataBlockSize+2))<<16 | offset)
|
||||
}
|
||||
|
||||
d.w.sb.Inodes++
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write implements io.Writer
|
||||
func (f *file) Write(p []byte) (n int, err error) {
|
||||
n, err = f.buf.Write(p)
|
||||
if n > 0 {
|
||||
// Keep track of the uncompressed file size.
|
||||
f.size += uint32(n)
|
||||
for f.buf.Len() >= dataBlockSize {
|
||||
if err := f.writeBlock(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (f *file) writeBlock() error {
|
||||
n := f.buf.Len()
|
||||
if n > dataBlockSize {
|
||||
n = dataBlockSize
|
||||
}
|
||||
// Feed dataBlockSize bytes to the compressor
|
||||
b := f.buf.Bytes()
|
||||
block := b[:n]
|
||||
rest := b[n:]
|
||||
/*
|
||||
f.compBuf.Reset()
|
||||
f.zlibWriter.Reset(f.compBuf)
|
||||
if _, err := f.zlibWriter.Write(block); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.zlibWriter.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
size := f.compBuf.Len()
|
||||
if size > len(block) {
|
||||
// Copy uncompressed data: Linux returns i/o errors when it encounters a
|
||||
// compressed block which is larger than the uncompressed data:
|
||||
// https://github.com/torvalds/linux/blob/3ca24ce9ff764bc27bceb9b2fd8ece74846c3fd3/fs/squashfs/block.c#L150
|
||||
size = len(block) | (1 << 24) // SQUASHFS_COMPRESSED_BIT_BLOCK
|
||||
if _, err := f.w.w.Write(block); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if _, err := io.Copy(f.w.w, f.compBuf); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
*/
|
||||
// Copy uncompressed data: Linux returns i/o errors when it encounters a
|
||||
// compressed block which is larger than the uncompressed data:
|
||||
// https://github.com/torvalds/linux/blob/3ca24ce9ff764bc27bceb9b2fd8ece74846c3fd3/fs/squashfs/block.c#L150
|
||||
size := len(block) | (1 << 24) // SQUASHFS_COMPRESSED_BIT_BLOCK
|
||||
if _, err := f.w.w.Write(block); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.blocksizes = append(f.blocksizes, uint32(size))
|
||||
|
||||
// Keep the rest in f.buf for the next write
|
||||
copy(b, rest)
|
||||
f.buf.Truncate(len(rest))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close implements io.Closer
|
||||
func (f *file) Close() error {
|
||||
for f.buf.Len() > 0 {
|
||||
if err := f.writeBlock(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
startBlock := f.w.inodeBuf.Len() / metadataBlockSize
|
||||
offset := f.w.inodeBuf.Len() - startBlock*metadataBlockSize
|
||||
|
||||
if err := binary.Write(&f.w.inodeBuf, binary.LittleEndian, lregInodeHeader{
|
||||
inodeHeader: inodeHeader{
|
||||
InodeType: lregType,
|
||||
Mode: f.mode,
|
||||
Uid: 0,
|
||||
Gid: 0,
|
||||
Mtime: int32(f.modTime.Unix()),
|
||||
InodeNumber: f.w.sb.Inodes + 1,
|
||||
},
|
||||
StartBlock: uint64(f.off),
|
||||
FileSize: uint64(f.size),
|
||||
Nlink: 1,
|
||||
Fragment: invalidFragment,
|
||||
Offset: 0,
|
||||
Xattr: f.xattrRef,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := binary.Write(&f.w.inodeBuf, binary.LittleEndian, f.blocksizes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.d.dirEntries = append(f.d.dirEntries, fullDirEntry{
|
||||
startBlock: uint32(startBlock),
|
||||
offset: uint16(offset),
|
||||
inodeNumber: f.w.sb.Inodes + 1,
|
||||
entryType: fileType,
|
||||
name: f.name,
|
||||
})
|
||||
|
||||
f.w.sb.Inodes++
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeXattr(w io.Writer, xattrs []Xattr) error {
|
||||
for _, attr := range xattrs {
|
||||
if err := binary.Write(w, binary.LittleEndian, struct {
|
||||
Type uint16
|
||||
NameSize uint16
|
||||
}{
|
||||
Type: attr.Type,
|
||||
NameSize: uint16(len(attr.FullName)),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write([]byte(attr.FullName)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := binary.Write(w, binary.LittleEndian, struct {
|
||||
ValSize uint32
|
||||
}{
|
||||
ValSize: uint32(len(attr.Value)),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := w.Write(attr.Value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type xattrTableHeader struct {
|
||||
XattrTableStart uint64
|
||||
XattrIds uint32
|
||||
Unused uint32
|
||||
}
|
||||
|
||||
func (w *Writer) writeXattrTables() (int64, error) {
|
||||
if len(w.xattrs) == 0 {
|
||||
return -1, nil
|
||||
}
|
||||
off, err := w.w.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
xattrTableStart := uint64(off)
|
||||
|
||||
var xattrBuf bytes.Buffer
|
||||
if err := writeXattr(&xattrBuf, w.xattrs); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
xattrBlocks := (xattrBuf.Len() + (metadataBlockSize - 1)) / metadataBlockSize
|
||||
|
||||
if err := w.writeMetadataChunks(&xattrBuf); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// write xattr id table
|
||||
off, err = w.w.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
idTableOff := uint64(off)
|
||||
var xattrIdBuf bytes.Buffer
|
||||
size := uint64(0)
|
||||
for _, id := range w.xattrIds {
|
||||
id.Xattr = uint64(size)
|
||||
size += uint64(id.Size) + 8 /* sizeof(Type+NameSize+ValSize) */
|
||||
if err := binary.Write(&xattrIdBuf, binary.LittleEndian, id); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := w.writeMetadataChunks(&xattrIdBuf); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// xattr table header
|
||||
off, err = w.w.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := binary.Write(w.w, binary.LittleEndian, xattrTableHeader{
|
||||
XattrTableStart: xattrTableStart,
|
||||
XattrIds: uint32(len(w.xattrs)),
|
||||
}); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// write block index
|
||||
for i := 0; i < xattrBlocks; i++ {
|
||||
if err := binary.Write(w.w, binary.LittleEndian, struct {
|
||||
BlockOffset uint64
|
||||
}{
|
||||
BlockOffset: idTableOff + (uint64(i) * (8192 + 2 /* sizeof(uint16) */)),
|
||||
}); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return off, nil
|
||||
}
|
||||
|
||||
// writeMetadataChunks copies from r to w in blocks of metadataBlockSize bytes
|
||||
// each, prefixing each block with a uint16 length header, setting the
|
||||
// uncompressed bit.
|
||||
func (w *Writer) writeMetadataChunks(r io.Reader) error {
|
||||
buf := make([]byte, metadataBlockSize)
|
||||
for {
|
||||
buf = buf[:metadataBlockSize]
|
||||
n, err := r.Read(buf)
|
||||
if err != nil {
|
||||
if err == io.EOF { // done
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
buf = buf[:n]
|
||||
if err := binary.Write(w.w, binary.LittleEndian, uint16(len(buf))|0x8000); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.w.Write(buf); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush writes the SquashFS file system. The Writer must not be used after
|
||||
// calling Flush.
|
||||
func (w *Writer) Flush() error {
|
||||
// (1) superblock will be written later
|
||||
|
||||
// (2) compressor-specific options omitted
|
||||
|
||||
// (3) data has already been written
|
||||
|
||||
// (4) write inode table
|
||||
off, err := w.w.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.sb.InodeTableStart = off
|
||||
|
||||
if err := w.writeMetadataChunks(&w.inodeBuf); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// (5) write directory table
|
||||
off, err = w.w.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.sb.DirectoryTableStart = off
|
||||
|
||||
if err := w.writeMetadataChunks(&w.dirBuf); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// (6) fragment table omitted
|
||||
off, err = w.w.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.sb.FragmentTableStart = off
|
||||
|
||||
// (7) export table omitted
|
||||
|
||||
// (8) write uid/gid lookup table
|
||||
idTableStart, err := writeIdTable(w.w, []uint32{0})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.sb.IdTableStart = idTableStart
|
||||
|
||||
// (9) xattr table
|
||||
off, err = w.writeXattrTables()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.sb.XattrIdTableStart = off
|
||||
|
||||
off, err = w.w.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.sb.BytesUsed = off
|
||||
|
||||
// Pad to 4096, required for the kernel to be able to access all pages
|
||||
if pad := off % 4096; pad > 0 {
|
||||
padding := make([]byte, 4096-pad)
|
||||
if _, err := w.w.Write(padding); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// (1) Write superblock
|
||||
if _, err := w.w.Seek(0, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return binary.Write(w.w, binary.LittleEndian, &w.sb)
|
||||
}
|
||||
+310
@@ -0,0 +1,310 @@
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/distr1/distri"
|
||||
// "github.com/distr1/distri/internal/distritest"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/orcaman/writerseeker"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
var fsImagePath = flag.String("fs_image_path", "", "Store the SquashFS test file system in the specified path for manual inspection")
|
||||
|
||||
func writeTestImage(iow io.WriteSeeker, xattr bool) error {
|
||||
w, err := NewWriter(iow, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var xattrs []Xattr
|
||||
if xattr {
|
||||
xattrs = append(xattrs, Xattr{
|
||||
Type: 2,
|
||||
FullName: "capability",
|
||||
Value: []byte{1, 0, 0, 2, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
|
||||
})
|
||||
}
|
||||
ff, err := w.Root.File("hellö wörld", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, xattrs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ff.Write([]byte("hello world!")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ff.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ff, err = w.Root.File("leer", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ff.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ff, err = w.Root.File("second file", time.Now(), unix.S_IRUSR|unix.S_IXUSR|
|
||||
unix.S_IRGRP|unix.S_IXGRP|
|
||||
unix.S_IROTH|unix.S_IXOTH,
|
||||
nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ff.Write([]byte("NON.\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ff.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := w.Root.Symlink("second file", "second link", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
subdir := w.Root.Directory("subdir", time.Now())
|
||||
|
||||
subsubdir := subdir.Directory("deep", time.Now())
|
||||
ff, err = subsubdir.File("yo", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ff.Write([]byte("foo\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ff.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := subsubdir.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: write another file in subdir now, will result in invalid parent inode
|
||||
|
||||
ff, err = subdir.File("third file (in subdir)", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ff.Write([]byte("contents\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ff.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := subdir.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
ff, err = w.Root.File("testbin", time.Now(), unix.S_IRUSR|unix.S_IXUSR|
|
||||
unix.S_IRGRP|unix.S_IXGRP|
|
||||
unix.S_IROTH|unix.S_IXOTH,
|
||||
nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
zf, err := os.Open(os.Args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zf.Close()
|
||||
if _, err := io.Copy(ff, zf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ff.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := w.Root.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestUnsquashfs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, canc := distri.InterruptibleContext()
|
||||
defer canc()
|
||||
|
||||
if _, err := exec.LookPath("unsquashfs"); err != nil {
|
||||
t.Skip("unsquashfs not found in $PATH")
|
||||
}
|
||||
|
||||
for _, xattr := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("xattr %v", xattr), func(t *testing.T) {
|
||||
var (
|
||||
f *os.File
|
||||
err error
|
||||
)
|
||||
if *fsImagePath != "" {
|
||||
f, err = os.Create(*fsImagePath + fmt.Sprintf("-xattr-%v", xattr))
|
||||
} else {
|
||||
f, err = ioutil.TempFile("", fmt.Sprintf("squashfs-xattr-%v", xattr))
|
||||
if err == nil {
|
||||
defer os.Remove(f.Name())
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := writeTestImage(f, xattr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := f.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Extract our generated file system using unsquashfs(1)
|
||||
out, err := ioutil.TempDir("", fmt.Sprintf("unsquashfs-xattr-%v", xattr))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// defer distritest.RemoveAll(t, out)
|
||||
cmd := exec.CommandContext(ctx, "unsquashfs", "-no-xattrs", "-d", filepath.Join(out, "x"), f.Name())
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fbin, err := os.Open(os.Args[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify the extracted files match our expectations.
|
||||
for _, entry := range []struct {
|
||||
path string
|
||||
contents io.Reader
|
||||
}{
|
||||
{"leer", strings.NewReader("")},
|
||||
{"hellö wörld", strings.NewReader("hello world!")},
|
||||
{"testbin", fbin},
|
||||
{"subdir/third file (in subdir)", strings.NewReader("contents\n")},
|
||||
} {
|
||||
entry := entry // copy
|
||||
t.Run(entry.path, func(t *testing.T) {
|
||||
in, err := os.Open(filepath.Join(out, "x", entry.path))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := ioutil.ReadAll(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, err := ioutil.ReadAll(entry.contents)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("path %q differs", entry.path)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReader(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, xattr := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("xattr %v", xattr), func(t *testing.T) {
|
||||
var err error
|
||||
buf := &writerseeker.WriterSeeker{}
|
||||
if err := writeTestImage(buf, xattr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := buf.Seek(0, io.SeekCurrent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rd, err := NewReader(buf.BytesReader())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fbin, err := os.Open(os.Args[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify the extracted files match our expectations.
|
||||
for _, entry := range []struct {
|
||||
path string
|
||||
contents io.Reader
|
||||
}{
|
||||
{"leer", strings.NewReader("")},
|
||||
{"hellö wörld", strings.NewReader("hello world!")},
|
||||
{"testbin", fbin},
|
||||
{"subdir/third file (in subdir)", strings.NewReader("contents\n")},
|
||||
} {
|
||||
entry := entry // copy
|
||||
t.Run(entry.path, func(t *testing.T) {
|
||||
// TODO: is this t.Parallel()-safe?
|
||||
inode, err := rd.LookupPath(entry.path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in, err := rd.FileReader(inode)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := ioutil.ReadAll(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, err := ioutil.ReadAll(entry.contents)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("path %q differs", entry.path)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if xattr {
|
||||
t.Run("xattrs", func(t *testing.T) {
|
||||
inode, err := rd.LookupPath("hellö wörld")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
xattrs, err := rd.ReadXattrs(inode)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(xattrs), 1; got != want {
|
||||
t.Fatalf("unexpected number of extended attributes: got %d, want %d", got, want)
|
||||
}
|
||||
wantXattr := Xattr{
|
||||
Type: 2,
|
||||
FullName: "security.capability",
|
||||
Value: []byte{1, 0, 0, 2, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
|
||||
}
|
||||
if diff := cmp.Diff(wantXattr, xattrs[0]); diff != "" {
|
||||
t.Errorf("unexpected extended attribute: diff (-want +got):\n%s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user