Restarted some stuff so I can do it better.

Made a reader that can reade across data blocks if necessary Still can't get things to read right
2020-11-13 16:11:44 -06:00
parent dbf7e9465a
commit 06b188d53c
20 changed files with 802 additions and 248 deletions
@@ -0,0 +1,618 @@
+package squashfs
+
+// import (
+// 	"bytes"
+// 	"encoding/binary"
+// 	"fmt"
+// 	"io"
+// 	"io/ioutil"
+// 	"log"
+// 	"os"
+// 	"path/filepath"
+// 	"strings"
+// 	"sync"
+// 	"syscall"
+// 	"time"
+
+// 	"golang.org/x/xerrors"
+// )
+
+// type Reader struct {
+// 	r     io.ReaderAt
+// 	super superblock
+// }
+
+// func NewReader(r io.ReaderAt) (*Reader, error) {
+// 	var sb superblock
+
+// 	if err := binary.Read(io.NewSectionReader(r, 0, int64(binary.Size(sb))), binary.LittleEndian, &sb); err != nil {
+// 		return nil, fmt.Errorf("reading superblock: %v", err)
+// 	}
+
+// 	if got, want := sb.Magic, uint32(magic); got != want {
+// 		return nil, fmt.Errorf("invalid magic (not a SquashFS image?): got %x, want %x", got, want)
+// 	}
+
+// 	//log.Printf("superblock: %+v", sb)
+// 	return &Reader{
+// 		r:     r,
+// 		super: sb,
+// 	}, nil
+// }
+
+// // TODO: maybe mmap instead of seeking?
+
+// func (r *Reader) inode(i Inode) (blockoffset int64, offset int64) {
+// 	return int64(i >> 16), int64(i & 0xFFFF)
+// }
+
+// type blockReader struct {
+// 	r      io.ReadSeeker
+// 	lenBuf [2]byte
+// 	buf    []byte
+// 	i      int64
+
+// 	off int64 // TODO: remove this once using mmap
+// }
+
+// func (br *blockReader) Read(p []byte) (n int, err error) {
+// 	if br.i >= int64(len(br.buf)) {
+// 		br.i = 0
+// 		if _, err := io.ReadFull(br.r, br.lenBuf[:]); err != nil {
+// 			return 0, err
+// 		}
+// 		l := binary.LittleEndian.Uint16(br.lenBuf[:])
+// 		//uncompressed := l&0x8000 > 0
+// 		l &= 0x7FFF
+// 		//log.Printf("block of len %d, uncompressed: %v", l, uncompressed)
+// 		if int(l) > cap(br.buf) {
+// 			br.buf = make([]byte, int(l))
+// 		}
+// 		br.buf = br.buf[:l]
+// 		if _, err := io.ReadFull(br.r, br.buf); err != nil {
+// 			return 0, err
+// 		}
+// 		//log.Printf("(retry) n = %v, err = %v", n, err)
+// 	}
+// 	n = copy(p, br.buf[br.i:])
+// 	br.i += int64(n)
+// 	return n, err
+// }
+
+// func (br *blockReader) Close() error {
+// 	blockReaderPool.Put(br)
+// 	return nil
+// }
+
+// var blockReaderPool = sync.Pool{
+// 	New: func() interface{} {
+// 		return &blockReader{
+// 			buf: make([]byte, 0, metadataBlockSize),
+// 		}
+// 	},
+// }
+
+// func (r *Reader) blockReader(blockoffset, offset int64) (io.ReadCloser, error) {
+// 	//log.Printf("blockoffset %v (%x), offset %v (%x)", blockoffset, blockoffset, offset, offset)
+// 	br := blockReaderPool.Get().(*blockReader)
+// 	br.buf = br.buf[:0]
+// 	br.r = io.NewSectionReader(r.r, blockoffset, 5500*1024*1024) // TODO: correct limit? can we use IntMax
+// 	br.off = blockoffset
+// 	br.i = 0
+// 	//log.Printf("discarding %d bytes", offset)
+// 	for n := int64(0); n < offset; {
+// 		remaining := offset - n
+// 		if remaining > metadataBlockSize {
+// 			remaining = metadataBlockSize
+// 		}
+// 		nn, err := br.Read(br.buf[:remaining])
+// 		if err != nil {
+// 			return nil, err
+// 		}
+// 		n += int64(nn)
+// 	}
+// 	return br, nil
+// }
+
+// // TODO: define an inode type to use instead of interface{}?
+// func (r *Reader) readInode(i Inode) (interface{}, error) {
+// 	blockoffset, offset := r.inode(i)
+// 	br, err := r.blockReader(r.super.InodeTableStart+blockoffset, offset)
+// 	if err != nil {
+// 		fmt.Println("oops! ", err)
+// 		return nil, err
+// 	}
+// 	defer br.Close()
+// 	fmt.Println("Hello There")
+
+// 	// We need the inode type before we know which type to pass to binary.Read,
+// 	// so we need to read it twice:
+// 	var inodeType uint16
+// 	typeBuf := bytes.NewBuffer(make([]byte, 0, binary.Size(inodeType)))
+// 	if err := binary.Read(io.TeeReader(br, typeBuf), binary.LittleEndian, &inodeType); err != nil {
+// 		return nil, err
+// 	}
+// 	br = ioutil.NopCloser(io.MultiReader(typeBuf, br))
+
+// 	// var ih inodeHeader
+// 	// if err := binary.Read(br, binary.LittleEndian, &ih); err != nil {
+// 	// 	return err
+// 	// }
+// 	// //log.Printf("ih: %+v", ih)
+
+// 	//log.Printf("inode type: %v", inodeType)
+// 	switch inodeType {
+// 	case dirType:
+// 		var di dirInodeHeader
+// 		if err := binary.Read(br, binary.LittleEndian, &di); err != nil {
+// 			return nil, err
+// 		}
+// 		return di, nil
+
+// 	case fileType:
+// 		var ri regInodeHeader
+// 		if err := binary.Read(br, binary.LittleEndian, &ri); err != nil {
+// 			return nil, err
+// 		}
+// 		return ri, nil
+
+// 	case symlinkType:
+// 		var si symlinkInodeHeader
+// 		if err := binary.Read(br, binary.LittleEndian, &si); err != nil {
+// 			return nil, err
+// 		}
+// 		return si, nil
+
+// 	case ldirType:
+// 		var di ldirInodeHeader
+// 		if err := binary.Read(br, binary.LittleEndian, &di); err != nil {
+// 			return nil, err
+// 		}
+// 		return di, nil
+
+// 	case lregType:
+// 		var di lregInodeHeader
+// 		if err := binary.Read(br, binary.LittleEndian, &di); err != nil {
+// 			return nil, err
+// 		}
+// 		return di, nil
+
+// 		// TODO:
+// 		// blkdevType
+// 		// chrdevType
+// 		// fifoType
+// 		// socketType
+// 		// // The larger types are used for e.g. sparse files, xattrs, etc.
+// 		// ldirType
+// 		// lsymlinkType
+// 		// lblkdevType
+// 		// lchrdevType
+// 		// lfifoType
+// 		// lsocketType
+
+// 	}
+// 	return nil, fmt.Errorf("unknown inode type %d", inodeType)
+// }
+
+// func (r *Reader) RootInode() Inode {
+// 	return r.super.RootInode
+// }
+
+// func (r *Reader) Stat(name string, i Inode) (os.FileInfo, error) {
+// 	inode, err := r.readInode(i)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	//log.Printf("i %d, inode: %T, %+v", i, inode, inode)
+// 	switch x := inode.(type) {
+// 	case dirInodeHeader:
+// 		return &FileInfo{
+// 			name:    name,
+// 			size:    int64(x.FileSize),
+// 			mode:    os.ModeDir | os.FileMode(x.Mode),
+// 			modTime: time.Unix(int64(x.Mtime), 0),
+// 			Inode:   i,
+// 		}, nil
+
+// 	case ldirInodeHeader:
+// 		return &FileInfo{
+// 			name:    name,
+// 			size:    int64(x.FileSize),
+// 			mode:    os.ModeDir | os.FileMode(x.Mode),
+// 			modTime: time.Unix(int64(x.Mtime), 0),
+// 			Inode:   i,
+// 		}, nil
+
+// 	case regInodeHeader:
+// 		mode := os.FileMode(x.Mode & 0777)
+// 		if x.Mode&syscall.S_ISUID != 0 {
+// 			mode |= os.ModeSetuid
+// 		}
+// 		return &FileInfo{
+// 			name:    name,
+// 			size:    int64(x.FileSize),
+// 			mode:    mode,
+// 			modTime: time.Unix(int64(x.Mtime), 0),
+// 			Inode:   i,
+// 		}, nil
+
+// 	case lregInodeHeader:
+// 		mode := os.FileMode(x.Mode & 0777)
+// 		if x.Mode&syscall.S_ISUID != 0 {
+// 			mode |= os.ModeSetuid
+// 		}
+// 		return &FileInfo{
+// 			name:    name,
+// 			size:    int64(x.FileSize),
+// 			mode:    mode,
+// 			modTime: time.Unix(int64(x.Mtime), 0),
+// 			Inode:   i,
+// 		}, nil
+
+// 	case symlinkInodeHeader:
+// 		return &FileInfo{
+// 			name:    name,
+// 			size:    int64(x.SymlinkSize),
+// 			mode:    os.ModeSymlink | os.FileMode(x.Mode),
+// 			modTime: time.Unix(int64(x.Mtime), 0),
+// 			Inode:   i,
+// 		}, nil
+// 	}
+
+// 	return nil, fmt.Errorf("unknown inode type %T", inode)
+// }
+
+// func (r *Reader) ReadLink(i Inode) (string, error) {
+// 	// TODO: reduce code duplication with readInode
+// 	blockoffset, offset := r.inode(i)
+// 	br, err := r.blockReader(r.super.InodeTableStart+blockoffset, offset)
+// 	if err != nil {
+// 		return "", err
+// 	}
+// 	defer br.Close()
+
+// 	// We need the inode type before we know which type to pass to binary.Read,
+// 	// so we need to read it twice:
+// 	var inodeType uint16
+// 	typeBuf := bytes.NewBuffer(make([]byte, 0, binary.Size(inodeType)))
+// 	if err := binary.Read(io.TeeReader(br, typeBuf), binary.LittleEndian, &inodeType); err != nil {
+// 		return "", err
+// 	}
+// 	br = ioutil.NopCloser(io.MultiReader(typeBuf, br))
+
+// 	if inodeType != symlinkType {
+// 		return "", fmt.Errorf("invalid inode type: got %d instead of symlink", inodeType)
+// 	}
+// 	var si symlinkInodeHeader
+// 	if err := binary.Read(br, binary.LittleEndian, &si); err != nil {
+// 		return "", err
+// 	}
+
+// 	// Assumption: r.r is positioned right after the inode
+// 	buf := make([]byte, si.SymlinkSize)
+// 	if _, err := io.ReadFull(br, buf); err != nil {
+// 		return "", err
+// 	}
+// 	return string(buf), nil
+// }
+
+// func (r *Reader) FileReader(inode Inode) (*io.SectionReader, error) {
+// 	//log.Printf("Readfile(%v)", inode)
+// 	i, err := r.readInode(inode)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	//log.Printf("i: %+v", i)
+// 	// TODO(compression): read the blocksizes to read compressed blocks
+// 	switch ri := i.(type) {
+// 	case regInodeHeader:
+// 		off := int64(ri.StartBlock) + int64(ri.Offset)
+// 		return io.NewSectionReader(r.r, off, int64(ri.FileSize)), nil
+// 	case lregInodeHeader:
+// 		off := int64(ri.StartBlock) + int64(ri.Offset)
+// 		return io.NewSectionReader(r.r, off, int64(ri.FileSize)), nil
+// 	default:
+// 		return nil, fmt.Errorf("BUG: non-file inode type")
+// 	}
+// }
+
+// type FileNotFoundError struct {
+// 	path string
+// }
+
+// func (e *FileNotFoundError) Error() string {
+// 	return fmt.Sprintf("%q not found", e.path)
+// }
+
+// func (r *Reader) lookupComponent(parent Inode, component string) (Inode, error) {
+// 	rfis, err := r.readdir(parent, false)
+// 	if err != nil {
+// 		return 0, err
+// 	}
+// 	for _, rfi := range rfis {
+// 		if rfi.Name() == component {
+// 			return rfi.Sys().(*FileInfo).Inode, nil
+// 		}
+// 	}
+// 	return 0, &FileNotFoundError{path: component}
+// }
+
+// func (r *Reader) lookupPath(path string, followSymlink bool) (Inode, error) {
+// 	inode := r.RootInode()
+// 	parts := strings.Split(path, "/")
+// 	for idx, part := range parts {
+// 		var err error
+// 		inode, err = r.lookupComponent(inode, part)
+// 		if err != nil {
+// 			if _, ok := err.(*FileNotFoundError); ok {
+// 				return 0, &FileNotFoundError{path: path}
+// 			}
+// 			return 0, err
+// 		}
+// 		if !followSymlink {
+// 			continue
+// 		}
+// 		i, err := r.readInode(inode)
+// 		if err != nil {
+// 			return 0, xerrors.Errorf("Stat(%d): %v", inode, err)
+// 		}
+// 		if _, ok := i.(symlinkInodeHeader); ok {
+// 			target, err := r.ReadLink(inode)
+// 			if err != nil {
+// 				return 0, err
+// 			}
+// 			//log.Printf("component %q (full: %q) resolved to %q", part, parts[:idx+1], target)
+// 			target = filepath.Clean(filepath.Join(append(parts[:idx] /* parent */, target)...))
+// 			//log.Printf("-> %s", target)
+// 			i, err := r.LookupPath(target)
+// 			if err != nil {
+// 				return 0, err
+// 			}
+// 			inode = i
+// 		}
+// 	}
+// 	return inode, nil
+// }
+
+// func (r *Reader) LookupPath(path string) (Inode, error) {
+// 	return r.lookupPath(path, true)
+// }
+
+// // LlookupPath is like LookupPath, but does not follow symbolic links, i.e. will
+// // instead return the inode of the link itself.
+// func (r *Reader) LlookupPath(path string) (Inode, error) {
+// 	return r.lookupPath(path, false)
+// }
+
+// func (r *Reader) Readdir(dirInode Inode) ([]os.FileInfo, error) {
+// 	return r.readdir(dirInode, true)
+// }
+
+// // Like Readdir, but does not call Stat on each file. The returned FileInfo
+// // structs will still have a filled in Name, partly filled in Mode, and filled
+// // in Inode.
+// func (r *Reader) ReaddirNoStat(dirInode Inode) ([]os.FileInfo, error) {
+// 	return r.readdir(dirInode, false)
+// }
+
+// var nameBufPool = sync.Pool{
+// 	New: func() interface{} {
+// 		return &bytes.Buffer{}
+// 	},
+// }
+
+// func (r *Reader) readdir(dirInode Inode, stat bool) ([]os.FileInfo, error) {
+// 	//log.Printf("Readdir(%v (%x))", dirInode, dirInode)
+// 	i, err := r.readInode(dirInode)
+// 	fmt.Println("Yodle")
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	var (
+// 		startBlock int64
+// 		fileSize   int64
+// 		offset     int64
+// 	)
+// 	switch x := i.(type) {
+// 	case dirInodeHeader:
+// 		startBlock = int64(x.StartBlock)
+// 		fileSize = int64(x.FileSize)
+// 		offset = int64(x.Offset)
+
+// 	case ldirInodeHeader:
+// 		startBlock = int64(x.StartBlock)
+// 		fileSize = int64(x.FileSize)
+// 		offset = int64(x.Offset)
+
+// 	default:
+// 		return nil, fmt.Errorf("unknown directory inode type %T", i)
+// 	}
+
+// 	br, err := r.blockReader(r.super.DirectoryTableStart+startBlock, offset)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	defer br.Close()
+
+// 	// See also https://elixir.bootlin.com/linux/v4.18.9/source/fs/squashfs/dir.c#L63
+// 	limit := fileSize - int64(len(".")) - int64(len(".."))
+// 	br = ioutil.NopCloser(io.LimitReader(br, limit))
+
+// 	var fis []os.FileInfo
+// 	var dh dirHeader
+// 	var de dirEntry
+// 	var dhBuf [12]byte
+// 	var deBuf [8]byte
+// 	nameBuf := nameBufPool.Get().(*bytes.Buffer)
+// 	defer nameBufPool.Put(nameBuf)
+// 	for {
+// 		if _, err := io.ReadFull(br, dhBuf[:]); err != nil {
+// 			if err == io.EOF {
+// 				return fis, nil
+// 			}
+// 			return nil, err
+// 		}
+// 		dh.Unmarshal(dhBuf[:])
+// 		dh.Count++ // SquashFS stores count-1
+// 		//log.Printf("dh: %+v", dh)
+
+// 		for i := 0; i < int(dh.Count); i++ {
+// 			if _, err := io.ReadFull(br, deBuf[:]); err != nil {
+// 				return nil, err
+// 			}
+// 			de.Unmarshal(deBuf[:])
+// 			de.Size++ // SquashFS stores size-1
+// 			//log.Printf("de: %+v", de)
+// 			nameBuf.Reset()
+// 			nameBuf.Grow(int(de.Size))
+// 			nb := nameBuf.Bytes()[:de.Size]
+// 			if _, err := io.ReadFull(br, nb); err != nil {
+// 				return nil, err
+// 			}
+// 			name := string(nb)
+// 			//log.Printf("name: %q", string(name))
+
+// 			var fi os.FileInfo
+// 			if stat {
+// 				var err error
+// 				fi, err = r.Stat(name, Inode(int64(dh.StartBlock)<<16|int64(de.Offset)))
+// 				if err != nil {
+// 					return nil, err
+// 				}
+// 			} else {
+// 				ffi := &FileInfo{
+// 					name:  name,
+// 					Inode: Inode(int64(dh.StartBlock)<<16 | int64(de.Offset)),
+// 				}
+// 				switch de.EntryType {
+// 				case dirType, ldirType:
+// 					ffi.mode |= os.ModeDir
+// 				case symlinkType, lsymlinkType:
+// 					ffi.mode |= os.ModeSymlink
+// 				}
+// 				fi = ffi
+// 			}
+// 			fis = append(fis, fi)
+// 		}
+// 	}
+
+// 	return fis, nil
+// }
+
+// type FileInfo struct {
+// 	name    string
+// 	size    int64
+// 	mode    os.FileMode
+// 	modTime time.Time
+// 	Inode   Inode
+// }
+
+// func (fi *FileInfo) Name() string       { return fi.name }
+// func (fi *FileInfo) Size() int64        { return fi.size }
+// func (fi *FileInfo) Mode() os.FileMode  { return fi.mode }
+// func (fi *FileInfo) IsDir() bool        { return fi.mode.IsDir() }
+// func (fi *FileInfo) ModTime() time.Time { return fi.modTime }
+// func (fi *FileInfo) Sys() interface{}   { return fi }
+
+// func (r *Reader) readXattr(tableHeader xattrTableHeader, id xattrId) (*Xattr, error) {
+// 	blockoffset, offset := r.inode(Inode(id.Xattr))
+// 	br, err := r.blockReader(int64(tableHeader.XattrTableStart)+blockoffset, offset)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	defer br.Close()
+// 	var typ, nameSize uint16
+// 	if err := binary.Read(br, binary.LittleEndian, &typ); err != nil {
+// 		return nil, err
+// 	}
+// 	if err := binary.Read(br, binary.LittleEndian, &nameSize); err != nil {
+// 		return nil, err
+// 	}
+// 	log.Printf("type = %v, nameSize = %v", typ, nameSize)
+// 	name := make([]byte, nameSize)
+// 	if _, err := io.ReadFull(br, name); err != nil {
+// 		return nil, err
+// 	}
+// 	log.Printf("name = %v", string(name))
+// 	var valSize uint32
+// 	if err := binary.Read(br, binary.LittleEndian, &valSize); err != nil {
+// 		return nil, err
+// 	}
+// 	val := make([]byte, valSize)
+// 	if _, err := io.ReadFull(br, val); err != nil {
+// 		return nil, err
+// 	}
+// 	log.Printf("val = %x", val)
+// 	return &Xattr{
+// 		Type:     typ,
+// 		FullName: xattrPrefix[int(typ)] + string(name),
+// 		Value:    val,
+// 	}, nil
+// }
+
+// func (r *Reader) ReadXattrs(inode Inode) ([]Xattr, error) {
+// 	//log.Printf("Readdir(%v (%x))", dirInode, dirInode)
+// 	i, err := r.readInode(inode)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	var xid uint32
+// 	switch x := i.(type) {
+// 	case regInodeHeader,
+// 		dirInodeHeader,
+// 		ldirInodeHeader,
+// 		symlinkInodeHeader:
+// 		return nil, nil // no extended attributes
+
+// 	case lregInodeHeader:
+// 		if x.Xattr == invalidXattr {
+// 			return nil, nil // file has no extended attributes
+// 		}
+// 		xid = x.Xattr
+
+// 	default:
+// 		return nil, fmt.Errorf("unknown inode type %T", i)
+// 	}
+
+// 	const idEntriesPerBlock = 512 // = 8192 / 16 /* sizeof(xattrId) */
+// 	block := xid / idEntriesPerBlock
+// 	offset := (xid % idEntriesPerBlock) * 16
+// 	log.Printf("xattr id %d, block %d, offset %d", xid, block, offset)
+// 	log.Printf("r.super.XattrIdTableStart = 0x%x, r.super.XattrIdTableStart = %v", r.super.XattrIdTableStart, r.super.XattrIdTableStart)
+// 	br := ioutil.NopCloser(io.NewSectionReader(r.r, r.super.XattrIdTableStart, int64(16 /* sizeof(xattrTableHeader) */ +(block+1)*4 /* sizeof(uint32) */)))
+// 	var tableHeader xattrTableHeader
+// 	if err := binary.Read(br, binary.LittleEndian, &tableHeader); err != nil {
+// 		return nil, err
+// 	}
+// 	// index starts here
+// 	if _, err := io.CopyN(ioutil.Discard, br, int64(block*4 /* sizeof(uint32) */)); err != nil {
+// 		return nil, err
+// 	}
+// 	var blockOffset uint32
+// 	if err := binary.Read(br, binary.LittleEndian, &blockOffset); err != nil {
+// 		return nil, err
+// 	}
+// 	log.Printf("blockOffset = 0x%x (%d)", blockOffset, blockOffset)
+// 	br, err = r.blockReader(int64(blockOffset), int64(offset))
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	defer br.Close()
+// 	var id xattrId
+// 	if err := binary.Read(br, binary.LittleEndian, &id); err != nil {
+// 		return nil, err
+// 	}
+// 	log.Printf("id: %+v", id)
+// 	log.Printf("tableHeader: %+v (start 0x%x)", tableHeader, tableHeader.XattrTableStart)
+
+// 	var xattrs []Xattr
+// 	for i := 0; i < int(id.Count); i++ {
+// 		xattr, err := r.readXattr(tableHeader, id)
+// 		if err != nil {
+// 			return nil, err
+// 		}
+// 		xattrs = append(xattrs, *xattr)
+// 	}
+
+// 	return xattrs, nil
+// }