Belac/squashfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 55 Wiki Activity

Re-wrote a bunch to try to figure out why things weren't working.

Browse Source 
Turned out I was reading if a block was compressed exactly opposite.
Started to work more on looking up dirs.
This commit is contained in:
Caleb Gardner
2020-11-16 14:56:19 -06:00
parent 06b188d53c
commit 7a2f9a87ba
25 changed files with 504 additions and 3363 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+7 -5
View File
@@ -10,16 +10,18 @@ I am focusing purely on unsquashing before squashing.
# Working
* Reading the header
* Reading data (slightly important :P)
* Reading inodes
* Reading directories
* Basic gzip compression (Shouldn't be too hard to implement other, but for right now, this works)
# Not Working (Yet). Roughly in order.
* Actually reading the compressed data
* Reading Inodes
* Reading the Directory structure
* Understanding the directory table. It's a bit weird TBH.
* Reading the UID, GUID, Xatt, Compression Options, Export, and Fragment tables.
* Implement other compression types
* Squashing
# Where I'm at
* Redid a bunch. Implemented a custom reader that can read across blocks.
* As of yet, doesn't seem to be reading things quite right (seems to be issue with encryption reading)
* Re-redid a bunch to try to make sure I wasn't durping. After that didn't work, I tried to figure out why things wheren't working, then realized HOW I was durping.
blockreader.go
+167
View File
@@ -0,0 +1,167 @@
package squashfs
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
)
type metadata struct {
raw uint16
size uint16
compressed bool
}
type BlockReader struct {
s *Reader
offset int64
headers []*metadata
data []byte
readOffset int
}
func (s *Reader) NewBlockReader(offset int64) (*BlockReader, error) {
fmt.Println("Offset", offset)
var br BlockReader
br.s = s
br.offset = offset
err := br.parseMetadata()
if err != nil {
return nil, err
}
err = br.readNextDataBlock()
if err != nil {
return nil, err
}
return &br, nil
}
func (br *BlockReader) parseMetadata() error {
fmt.Println("meta offset", br.offset)
var raw uint16
err := binary.Read(io.NewSectionReader(br.s.r, br.offset, 2), binary.LittleEndian, &raw)
if err != nil {
return err
}
br.offset += 2
compressed := !(raw&0x8000 == 0x8000)
size := raw &^ 0x8000
br.headers = append(br.headers, &metadata{
raw: raw,
size: size,
compressed: compressed,
})
fmt.Println("compression", compressed)
return nil
}
func (br *BlockReader) readNextDataBlock() error {
meta := br.headers[len(br.headers)-1]
r := io.NewSectionReader(br.s.r, br.offset, int64(meta.size))
if meta.compressed {
byts, err := br.s.decompressor.Decompress(r)
if err != nil {
return err
}
br.offset += int64(meta.size)
br.data = append(br.data, byts...)
return nil
}
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
if err != nil {
return err
}
br.offset += int64(meta.size)
br.data = append(br.data, buf.Bytes()...)
return nil
}
func (br *BlockReader) Read(p []byte) (int, error) {
if br.readOffset+len(p) < len(br.data) {
for i := 0; i < len(p); i++ {
p[i] = br.data[br.readOffset+i]
}
br.readOffset += len(p)
return len(p), nil
}
read := 0
for read < len(p) {
if read+br.readOffset == len(br.data) {
err := br.parseMetadata()
if err != nil {
br.readOffset += read
return read, err
}
err = br.readNextDataBlock()
if err != nil {
br.readOffset += read
return read, err
}
}
for ; read < len(p); read++ {
if br.readOffset+read < len(br.data) {
p[read] = br.data[br.readOffset+read]
} else {
break
}
}
}
br.readOffset += read
if read != len(p) {
return read, errors.New("Didn't read enough data")
}
fmt.Println("Read", p)
return read, nil
}
//Seek will seek to the specified location (if possible).
//When io.SeekCurrent or io.SeekStart is set, if seeking would put the offset beyond the current cached data, it will try to read the next data blocks to accomodate. On a failure it will seek to the end of the data.
//When io.SeekEnd is set, it wil seek reletive to the currently cached data.
func (br *BlockReader) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekCurrent:
br.readOffset += int(offset)
for {
if br.readOffset < len(br.data) {
break
}
err := br.parseMetadata()
if err != nil {
br.readOffset = len(br.data)
return int64(br.readOffset), err
}
err = br.readNextDataBlock()
if err != nil {
br.readOffset = len(br.data)
return int64(br.readOffset), err
}
}
case io.SeekStart:
br.readOffset = int(offset)
for {
if br.readOffset < len(br.data) {
break
}
err := br.parseMetadata()
if err != nil {
br.readOffset = len(br.data)
return int64(br.readOffset), err
}
err = br.readNextDataBlock()
if err != nil {
br.readOffset = len(br.data)
return int64(br.readOffset), err
}
}
case io.SeekEnd:
br.readOffset = len(br.data) - int(offset)
if br.readOffset < 0 {
br.readOffset = 0
return int64(br.readOffset), errors.New("Trying to seek to a negative value")
}
}
return int64(br.readOffset), nil
}
blocks.go
-117
View File
@@ -1,117 +0,0 @@
package squashfs
import (
"encoding/binary"
"fmt"
"io"
"github.com/CalebQ42/GoSquashfs/bytereadwrite"
)
type MetadataHeader struct {
rawHeader uint16
Compressed bool
Size uint16
}
type BlockReader struct {
initalOffset int64
offset int64
squash *Squashfs
headers []MetadataHeader
dataCache []byte
dataOffset int64
}
//NewBlockReader creates a new BlockReader from a squashfs.Reader. Reads the first header and caches the first set of data.
func (s *Squashfs) NewBlockReader(offset int64) (*BlockReader, error) {
var br BlockReader
br.squash = s
br.initalOffset = offset
br.offset = offset
br.headers = make([]MetadataHeader, 0)
br.dataCache = make([]byte, 0)
err := br.parseNewBlock()
if err != nil {
fmt.Println("Problem creating BlockReader")
return nil, err
}
return &br, nil
}
func (br *BlockReader) parseNewBlock() error {
var header MetadataHeader
err := binary.Read(io.NewSectionReader(&br.squash.r, br.offset, 2), binary.LittleEndian, &header.rawHeader)
if err != nil {
fmt.Println("Error while reading the header ", len(br.headers), " in BlockReader")
return err
}
header.Compressed = (header.rawHeader&0x8000 == 0x8000)
header.Size = header.rawHeader &^ 0x8000
br.headers = append(br.headers, header)
br.offset += 2
sectionReader := io.NewSectionReader(&br.squash.r, br.offset, br.offset+int64(header.Size))
dataWriter := bytereadwrite.NewByteReaderWriter()
_, err = io.Copy(dataWriter, sectionReader)
br.offset += int64(header.Size)
if header.Compressed {
data, err := br.squash.compression.Decompress(dataWriter)
if err != nil {
fmt.Println("Error while reading the encrypted data block in header", len(br.headers))
return err
}
br.dataCache = append(br.dataCache, data...)
return nil
}
if err != nil {
fmt.Println("Error while reading uncompressed data in header", len(br.headers))
return err
}
br.dataCache = append(br.dataCache, dataWriter.GetBytes()...)
return nil
}
//Read reads data into p. If it reaches EOF, tries to read a new block and add it's data to the cache
func (br *BlockReader) Read(p []byte) (n int, err error) {
read := 0
for {
byter := bytereadwrite.NewByteReaderWriterFromBytes(br.dataCache[br.dataOffset:])
temp, err := byter.Read(p[read:])
br.dataOffset += int64(temp)
read += temp
if err == nil {
return read, nil
} else if err != io.EOF {
return read, err
}
if err == io.EOF {
err = br.parseNewBlock()
if err != nil {
fmt.Println("Error while reading a new block")
return read, err
}
}
}
}
//ReadAt reads data into p from the offset. If it reaches EOF, tries to read a new block and add it's data to the cache.
//
//Offset is reletive to the offset set on creation.
func (br *BlockReader) ReadAt(p []byte, offset int) (n int, err error) {
read := 0
for err == io.EOF {
byter := bytereadwrite.NewByteReaderWriterFromBytes(br.dataCache[offset+read:])
temp, inErr := byter.Read(p[read:])
err = inErr
read += temp
br.dataOffset += int64(temp)
if err == io.EOF {
inErr = br.parseNewBlock()
if inErr != nil {
fmt.Println("Error while reading a new block")
return read, inErr
}
}
}
return
}
bytereadwrite/bytereadwrite.go
-60
View File
@@ -1,60 +0,0 @@
package bytereadwrite
import "io"
//ByteReaderWriter allows you to read to and from a byte slice. When writing, it expands the slice to accomodate any data.
type ByteReaderWriter struct {
byts []byte
offset int
}
//NewByteReaderWriter creates a ByteReaderWriter with an internal byte slice of the given length.
func NewByteReaderWriter() *ByteReaderWriter {
return &ByteReaderWriter{
byts: make([]byte, 0),
offset: 0,
}
}
//NewByteReaderWriter creates a ByteReaderWriter with an internal byte slice of the given length.
func NewByteReaderWriterWithLength(length int) *ByteReaderWriter {
return &ByteReaderWriter{
byts: make([]byte, length),
offset: 0,
}
}
//NewByteReaderWriterFromBytes creates a new ByteReaderWriter initialized with the given bytes
func NewByteReaderWriterFromBytes(byts []byte) *ByteReaderWriter {
return &ByteReaderWriter{
byts: byts,
offset: 0,
}
}
//GetBytes return the underlyting byte slice of the readerwriter
func (bwr *ByteReaderWriter) GetBytes() []byte {
return bwr.byts
}
//Read reads the bytes.
func (bwr *ByteReaderWriter) Read(byt []byte) (int, error) {
if len(bwr.byts) < bwr.offset+len(byt) {
bytesWritten := len(bwr.byts) - bwr.offset
for i := 0; i < bytesWritten; i++ {
byt[i] = bwr.byts[i+bwr.offset]
}
return bytesWritten, io.EOF
}
for i := 0; i < len(byt); i++ {
byt[i] = bwr.byts[bwr.offset+i]
}
bwr.offset += len(byt)
return len(byt), nil
}
//Write writes to the end of the bytes. WILL expand to accept the incoming bytes.
func (bwr *ByteReaderWriter) Write(byts []byte) (int, error) {
bwr.byts = append(bwr.byts, byts...)
return len(byts), nil
}
compression.go
+39
View File
@@ -0,0 +1,39 @@
package squashfs
import (
"bytes"
"compress/zlib"
"io"
)
type Decompressor interface {
Decompress(io.Reader) ([]byte, error)
DecompressCopy(*io.Writer, *io.SectionReader) error
}
type ZlibDecompressor struct{}
func (z *ZlibDecompressor) Decompress(r io.Reader) ([]byte, error) {
rdr, err := zlib.NewReader(r)
if err != nil {
return nil, err
}
var data bytes.Buffer
_, err = io.Copy(&data, rdr)
if err != nil {
return nil, err
}
return data.Bytes(), nil
}
func (z *ZlibDecompressor) DecompressCopy(w *io.Writer, r *io.SectionReader) error {
rdr, err := zlib.NewReader(r)
if err != nil {
return err
}
_, err = io.Copy(*w, rdr)
if err != nil {
return err
}
return err
}
compressionoptions.go
-136
View File
@@ -1,136 +0,0 @@
package squashfs
import (
"compress/gzip"
"io"
"github.com/CalebQ42/GoSquashfs/bytereadwrite"
)
const (
gzipCompression = 1 + iota
lzmaCompression
lzoCompression
xzCompression
lz4Compression
zstdCompression
)
//TODO: implement for each type of Options
type CompressionOptions interface {
Decompress(io.Reader) ([]byte, error)
DecompressCopy(*io.Reader, *io.Writer) (int, error)
Compress(*io.Reader) ([]byte, error)
CompressCopy(*io.Reader, *io.Writer) (int, error)
}
//TODO: Allow creation of options for compression.
type gzipOptionsRaw struct {
CompressionLevel int32
WindowSize int16
Strategies int16
}
//GzipOptions is the options used for gzip compression. Backed by the raw format, with strategies parsed.
type GzipOptions struct {
CompressionOptions
raw *gzipOptionsRaw
DefaultStrategy bool
FilteredStrategy bool
HuffmanOnlyStrategy bool
RunLengthEncodedStrategy bool
FixedStretegy bool
}
func NewGzipOptions(raw gzipOptionsRaw) *GzipOptions {
//TODO: parse strategies
return &GzipOptions{
raw: &raw,
}
}
func (gzipOp *GzipOptions) Decompress(rdr io.Reader) ([]byte, error) {
gzipRdr, err := gzip.NewReader(rdr)
// defer gzipRdr.Close()
if err != nil {
return nil, err
}
bytrw := bytereadwrite.NewByteReaderWriter()
_, err = io.Copy(bytrw, gzipRdr)
if err != nil {
return nil, err
}
return bytrw.GetBytes(), nil
}
func (gzipOp *GzipOptions) DecompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) {
gzipRdr, err := gzip.NewReader(*rdr)
defer gzipRdr.Close()
if err != nil {
return 0, err
}
n, err := io.Copy(*wrt, gzipRdr)
return int(n), err
}
func (gzipOp *GzipOptions) Compress(rdr *io.Reader) ([]byte, error) {
bytWrt := bytereadwrite.NewByteReaderWriter()
gzipWrt := gzip.NewWriter(bytWrt) //TODO: allow setting level
defer gzipWrt.Close()
_, err := io.Copy(gzipWrt, *rdr)
if err != nil {
return bytWrt.GetBytes(), err
}
return bytWrt.GetBytes(), nil
}
func (gzipOp *GzipOptions) CompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) {
gzipWrt := gzip.NewWriter(*wrt) //TODO: allow setting level
defer gzipWrt.Close()
n, err := io.Copy(gzipWrt, *rdr)
return int(n), err
}
type xzOptionsRaw struct {
DictionarySize int32
ExecutableFilters int32
}
type XzOptions struct {
CompressionOptions //TODO: Remove
raw *xzOptionsRaw
Execx86 bool
ExecPower bool
Execa64 bool
ExecArm bool
ExecArmThumb bool
ExecSparc bool
}
func NewXzOption(raw xzOptionsRaw) XzOptions {
return XzOptions{
raw: &raw,
Execx86: raw.ExecutableFilters&0x1 == 0x1,
ExecPower: raw.ExecutableFilters&0x2 == 0x2,
Execa64: raw.ExecutableFilters&0x4 == 0x4,
ExecArm: raw.ExecutableFilters&0x8 == 0x8,
ExecArmThumb: raw.ExecutableFilters&0x10 == 0x10,
ExecSparc: raw.ExecutableFilters&0x20 == 0x20,
}
}
type lz4OptionsRaw struct {
Version int32
Flags int32
}
//ZstdOptions is the options set for zstdOptions
type ZstdOptions struct {
CompressionLevel int32 //CompressionLevel should be between 1 and 22
}
type lzoOptionsRaw struct {
Algorithm int32
CompressionLevel int32
}
internal/directory/directory.go
+5 -5
View File
@@ -30,12 +30,12 @@ type Entry struct {
//NewEntry creates a new directory entry
func NewEntry(rdr io.Reader) (Entry, error) {
var entry Entry
err := binary.Read(rdr, binary.LittleEndian, entry.Init)
err := binary.Read(rdr, binary.LittleEndian, &entry.Init)
if err != nil {
return entry, err
}
entry.Name = make([]byte, entry.Init.NameSize, entry.Init.NameSize)
err = binary.Read(rdr, binary.LittleEndian, entry.Name)
entry.Name = make([]byte, entry.Init.NameSize+1, entry.Init.NameSize+1)
err = binary.Read(rdr, binary.LittleEndian, &entry.Name)
return entry, err
}
@@ -54,9 +54,9 @@ func NewDirectory(rdr io.Reader) (*Directory, error) {
if err != nil {
return nil, err
}
fmt.Println(hdr)
hdr.Count++
headers := hdr.Count / 256
if headers%256 > 0 {
if hdr.Count%256 > 0 {
headers++
}
headersRead := 1
internal/inode/inode.go
+36 -19
View File
@@ -6,8 +6,25 @@ import (
"io"
)
//Common is the comon header for all inodes
type Common struct {
const (
BasicDirectoryType = iota + 1
BasicFileType
BasicSymlinkType
BasicBlockDeviceType
BasicCharDeviceType
BasicFifoType
BasicSocketType
ExtDirType
ExtFileType
ExtSymlinkType
ExtBlockDeviceType
ExtCharDeviceType
ExtFifoType
ExtSocketType
)
//Header is the common header for all inodes
type Header struct {
InodeType uint16
Permissions uint16
UID uint16
@@ -43,9 +60,9 @@ type ExtendedDirectory struct {
}
//NewExtendedDirectory creates a new ExtendedDirectory
func NewExtendedDirectory(rdr *io.Reader) (*ExtendedDirectory, error) {
func NewExtendedDirectory(rdr io.Reader) (*ExtendedDirectory, error) {
var inode ExtendedDirectory
err := binary.Read(*rdr, binary.LittleEndian, inode.Init)
err := binary.Read(rdr, binary.LittleEndian, &inode.Init)
if err != nil {
return &inode, err
}
@@ -76,14 +93,14 @@ type DirectoryIndex struct {
}
//NewDirectoryIndex return a new DirectoryIndex
func NewDirectoryIndex(rdr *io.Reader) (DirectoryIndex, error) {
func NewDirectoryIndex(rdr io.Reader) (DirectoryIndex, error) {
var index DirectoryIndex
err := binary.Read(*rdr, binary.LittleEndian, index.Init)
err := binary.Read(rdr, binary.LittleEndian, &index.Init)
if err != nil {
return index, err
}
index.Name = make([]byte, index.Init.NameSize, index.Init.NameSize)
err = binary.Read(*rdr, binary.LittleEndian, index.Name)
err = binary.Read(rdr, binary.LittleEndian, &index.Name)
return index, err
}
@@ -102,9 +119,9 @@ type BasicFile struct {
}
//NewBasicFile creates a new BasicFile
func NewBasicFile(rdr *io.Reader, blockSize uint32) (*BasicFile, error) {
func NewBasicFile(rdr io.Reader, blockSize uint32) (*BasicFile, error) {
var inode BasicFile
err := binary.Read(*rdr, binary.LittleEndian, inode.Init)
err := binary.Read(rdr, binary.LittleEndian, &inode.Init)
if err != nil {
return &inode, err
}
@@ -113,7 +130,7 @@ func NewBasicFile(rdr *io.Reader, blockSize uint32) (*BasicFile, error) {
blocks++
}
inode.BlockSizes = make([]uint32, blocks, blocks)
err = binary.Read(*rdr, binary.LittleEndian, inode.BlockSizes)
err = binary.Read(rdr, binary.LittleEndian, &inode.BlockSizes)
return &inode, err
}
@@ -135,9 +152,9 @@ type ExtendedFile struct {
}
//NewExtendedFile creates a new ExtendedFile
func NewExtendedFile(rdr *io.Reader, blockSize uint32) (ExtendedFile, error) {
func NewExtendedFile(rdr io.Reader, blockSize uint32) (ExtendedFile, error) {
var inode ExtendedFile
err := binary.Read(*rdr, binary.LittleEndian, inode.Init)
err := binary.Read(rdr, binary.LittleEndian, &inode.Init)
if err != nil {
return inode, err
}
@@ -146,7 +163,7 @@ func NewExtendedFile(rdr *io.Reader, blockSize uint32) (ExtendedFile, error) {
blocks++
}
inode.BlockSizes = make([]uint32, blocks, blocks)
err = binary.Read(*rdr, binary.LittleEndian, inode.BlockSizes)
err = binary.Read(rdr, binary.LittleEndian, &inode.BlockSizes)
return inode, err
}
@@ -163,14 +180,14 @@ type BasicSymlink struct {
}
//NewBasicSymlink creates a new BasicSymlink
func NewBasicSymlink(rdr *io.Reader) (BasicSymlink, error) {
func NewBasicSymlink(rdr io.Reader) (BasicSymlink, error) {
var inode BasicSymlink
err := binary.Read(*rdr, binary.LittleEndian, inode.Init)
err := binary.Read(rdr, binary.LittleEndian, &inode.Init)
if err != nil {
return inode, err
}
inode.targetPath = make([]byte, inode.Init.TargetPathSize, inode.Init.TargetPathSize)
err = binary.Read(*rdr, binary.LittleEndian, inode.targetPath)
err = binary.Read(rdr, binary.LittleEndian, &inode.targetPath)
return inode, err
}
@@ -188,14 +205,14 @@ type ExtendedSymlink struct {
}
//NewExtendedSymlink creates a new ExtendedSymlink
func NewExtendedSymlink(rdr *io.Reader) (ExtendedSymlink, error) {
func NewExtendedSymlink(rdr io.Reader) (ExtendedSymlink, error) {
var inode ExtendedSymlink
err := binary.Read(*rdr, binary.LittleEndian, inode.Init)
err := binary.Read(rdr, binary.LittleEndian, &inode.Init)
if err != nil {
return inode, err
}
inode.TargetPath = make([]uint8, inode.Init.TargetPathSize, inode.Init.TargetPathSize)
err = binary.Read(*rdr, binary.LittleEndian, &inode.XattrIndex)
err = binary.Read(rdr, binary.LittleEndian, &inode.XattrIndex)
return inode, err
}
internal/inode/process.go
+125
View File
@@ -0,0 +1,125 @@
package inode
import (
"encoding/binary"
"io"
)
//Inode holds an inode. Header is the header that's common for all inodes.
//
//Info holds the actual Inode. Due to each inode type being a different type, it's store as an interface{}
type Inode struct {
Header Header
Type int //Type the inode type defined in the header. Here so it's easy to access
Info interface{} //Info is the parsed specific data. It's type is defined by Type.
}
//ProcessInode tries to read an inode from the BlockReader
func ProcessInode(br io.Reader, blockSize uint32) (Inode, error) {
var head Header
err := binary.Read(br, binary.LittleEndian, &head)
if err != nil {
return Inode{}, err
}
var info interface{}
switch head.InodeType {
case BasicDirectoryType:
var inode BasicDirectory
err = binary.Read(br, binary.LittleEndian, &inode)
if err != nil {
return Inode{}, err
}
info = inode
case BasicFileType:
inode, err := NewBasicFile(br, blockSize)
if err != nil {
return Inode{}, err
}
info = inode
case BasicSymlinkType:
inode, err := NewBasicSymlink(br)
if err != nil {
return Inode{}, err
}
info = inode
case BasicBlockDeviceType:
var inode BasicDevice
err = binary.Read(br, binary.LittleEndian, &inode)
if err != nil {
return Inode{}, err
}
info = inode
case BasicCharDeviceType:
var inode BasicDevice
err = binary.Read(br, binary.LittleEndian, &inode)
if err != nil {
return Inode{}, err
}
info = inode
case BasicFifoType:
var inode BasicIPC
err = binary.Read(br, binary.LittleEndian, &inode)
if err != nil {
return Inode{}, err
}
info = inode
case BasicSocketType:
var inode BasicIPC
err = binary.Read(br, binary.LittleEndian, &inode)
if err != nil {
return Inode{}, err
}
info = inode
case ExtDirType:
inode, err := NewExtendedDirectory(br)
if err != nil {
return Inode{}, err
}
info = inode
case ExtFileType:
inode, err := NewExtendedFile(br, blockSize)
if err != nil {
return Inode{}, err
}
info = inode
case ExtSymlinkType:
inode, err := NewExtendedSymlink(br)
if err != nil {
return Inode{}, err
}
info = inode
case ExtBlockDeviceType:
var inode ExtendedDevice
err = binary.Read(br, binary.LittleEndian, &inode)
if err != nil {
return Inode{}, err
}
info = inode
case ExtCharDeviceType:
var inode ExtendedDevice
err = binary.Read(br, binary.LittleEndian, &inode)
if err != nil {
return Inode{}, err
}
info = inode
case ExtFifoType:
var inode ExtendedIPC
err = binary.Read(br, binary.LittleEndian, &inode)
if err != nil {
return Inode{}, err
}
info = inode
case ExtSocketType:
var inode ExtendedIPC
err = binary.Read(br, binary.LittleEndian, &inode)
if err != nil {
return Inode{}, err
}
info = inode
}
return Inode{
Type: int(head.InodeType),
Header: head,
Info: info,
}, nil
}
internal/inode/processInode.go
-91
View File
@@ -1,91 +0,0 @@
package inode
import (
"encoding/binary"
"errors"
"io"
"strconv"
)
const (
//The inode type from inode.Common.InodeType
BasicDirectoryType = iota
BasicFileType
BasicSymlinkType
BasicBlockDeviceType
BasicCharDeviceType
BasicFifoType
BasicSocketType
ExtendedDirectoryType
ExtendedFileType
ExtendedSymlinkType
ExtendedBlockDeviceType
ExtendedCharDeviceType
ExtendedFifoType
ExtendedSocketType
)
//ProcessInode processes the next inode in the given reader
func ProcessInode(rdr io.Reader, blockSize uint32) (*Common, interface{}, error) {
var inodeHeader Common
err := binary.Read(rdr, binary.LittleEndian, &inodeHeader)
if err != nil {
return nil, nil, err
}
switch inodeHeader.InodeType {
case BasicDirectoryType:
var inode BasicDirectory
err = binary.Read(rdr, binary.LittleEndian, &inode)
return &inodeHeader, &inode, err
case BasicFileType:
inode, err := NewBasicFile(&rdr, blockSize)
return &inodeHeader, inode, err
case BasicSymlinkType:
inode, err := NewBasicSymlink(&rdr)
return &inodeHeader, inode, err
case BasicBlockDeviceType:
var inode BasicDevice
err = binary.Read(rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case BasicCharDeviceType:
var inode BasicDevice
err = binary.Read(rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case BasicFifoType:
var inode BasicIPC
err = binary.Read(rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case BasicSocketType:
var inode BasicIPC
err = binary.Read(rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case ExtendedDirectoryType:
inode, err := NewExtendedDirectory(&rdr)
return &inodeHeader, inode, err
case ExtendedFileType:
inode, err := NewExtendedFile(&rdr, blockSize)
return &inodeHeader, inode, err
case ExtendedSymlinkType:
inode, err := NewExtendedSymlink(&rdr)
return &inodeHeader, inode, err
case ExtendedBlockDeviceType:
var inode ExtendedDevice
err = binary.Read(rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case ExtendedCharDeviceType:
var inode ExtendedDevice
err = binary.Read(rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case ExtendedFifoType:
var inode ExtendedIPC
err = binary.Read(rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case ExtendedSocketType:
var inode ExtendedIPC
err = binary.Read(rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
default:
return nil, nil, errors.New("Inode type is unrecognized: " + strconv.FormatInt(int64(inodeHeader.InodeType), 2))
}
}
old/bytereadwrite.go
-48
View File
@@ -1,48 +0,0 @@
package squashfs
import "io"
type byteWriterReader struct {
byts []byte
offset int
}
func newByteReadWrite(length int) *byteWriterReader {
return &byteWriterReader{
byts: make([]byte, length),
offset: 0,
}
}
func newByteReadWriteFromBytes(byts []byte) *byteWriterReader {
return &byteWriterReader{
byts: byts,
offset: 0,
}
}
func (bwr *byteWriterReader) getBytes() []byte {
return bwr.byts
}
//Read reads the bytes.
func (bwr *byteWriterReader) Read(byt []byte) (int, error) {
if len(bwr.byts) < bwr.offset+len(byt) {
bytesWritten := len(bwr.byts) - bwr.offset
for i := 0; i < bytesWritten; i++ {
byt[i] = bwr.byts[i+bwr.offset]
}
return bytesWritten, io.EOF
}
for i := 0; i < len(byt); i++ {
byt[i] = bwr.byts[bwr.offset+i]
}
bwr.offset += len(byt)
return len(byt), nil
}
//Write writes to the bytes. WILL expand to accept the incoming bytes.
func (bwr *byteWriterReader) Write(byts []byte) (int, error) {
bwr.byts = append(bwr.byts, byts...)
return len(byts), nil
}
old/compressionoptions.go
-143
View File
@@ -1,143 +0,0 @@
package squashfs
import (
"compress/zlib"
"io"
"gopkg.in/src-d/go-git.v4/utils/ioutil"
)
const (
zlibCompression = 1 + iota
lzmaCompression
lzoCompression
xzCompression
lz4Compression
zstdCompression
)
//TODO: implement for each type of Options
type CompressionOptions interface {
Decompress(*io.SectionReader) ([]byte, error)
DecompressCopy(*io.Reader, *io.Writer) (int, error)
Compress(*io.SectionReader) ([]byte, error)
CompressCopy(*io.Reader, *io.Writer) (int, error)
Reader(io.Reader) (*io.ReadCloser, error)
}
//TODO: Allow creation of options for compression.
type zlibOptionsRaw struct {
compressionLevel int32
windowSize int16
strategies int16
}
//ZlibOptions is the options used for zlib compression. Backed by the raw format, with strategies parsed.
type ZlibOptions struct {
CompressionOptions
raw *zlibOptionsRaw
DefaultStrategy bool
FilteredStrategy bool
HuffmanOnlyStrategy bool
RunLengthEncodedStrategy bool
FixedStretegy bool
}
func NewZlibOptions(raw zlibOptionsRaw) *ZlibOptions {
//TODO: parse strategies
return &ZlibOptions{
raw: &raw,
}
}
func (zlibOp *ZlibOptions) Decompress(rdr *io.SectionReader) ([]byte, error) {
zlibRdr, err := zlib.NewReader(rdr)
defer zlibRdr.Close()
if err != nil {
return nil, err
}
bytrw := newByteReadWrite(0)
_, err = io.Copy(bytrw, zlibRdr)
if err != nil {
return bytrw.byts, err
}
return bytrw.byts, nil
}
func (zlibOp *ZlibOptions) DecompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) {
zlibRdr, err := zlib.NewReader(*rdr)
defer zlibRdr.Close()
if err != nil {
return 0, err
}
n, err := io.Copy(*wrt, zlibRdr)
return int(n), err
}
func (zlibOp *ZlibOptions) Compress(rdr *io.SectionReader) ([]byte, error) {
bytWrt := newByteReadWrite(0)
zlibWrt := zlib.NewWriter(bytWrt) //TODO: allow setting level
defer zlibWrt.Close()
_, err := io.Copy(zlibWrt, rdr)
if err != nil {
return bytWrt.byts, err
}
return bytWrt.byts, nil
}
func (zlibOp *ZlibOptions) CompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) {
zlibWrt := zlib.NewWriter(*wrt) //TODO: allow setting level
defer zlibWrt.Close()
n, err := io.Copy(zlibWrt, *rdr)
return int(n), err
}
func (zlibOp *ZlibOptions) Reader(rdr io.Reader) (*io.ReadCloser, error) {
read, err := zlib.NewReader(rdr)
redClo := ioutil.NewReadCloser(NewByteBufferedReader(read), read)
return &redClo, err
}
type xzOptionsRaw struct {
dictionarySize int32
executableFilters int32
}
type XzOptions struct {
CompressionOptions //TODO: Remove
raw *xzOptionsRaw
Execx86 bool
ExecPower bool
Execa64 bool
ExecArm bool
ExecArmThumb bool
ExecSparc bool
}
func NewXzOption(raw xzOptionsRaw) XzOptions {
return XzOptions{
raw: &raw,
Execx86: raw.executableFilters&0x1 == 0x1,
ExecPower: raw.executableFilters&0x2 == 0x2,
Execa64: raw.executableFilters&0x4 == 0x4,
ExecArm: raw.executableFilters&0x8 == 0x8,
ExecArmThumb: raw.executableFilters&0x10 == 0x10,
ExecSparc: raw.executableFilters&0x20 == 0x20,
}
}
type lz4OptionsRaw struct {
version int32
flags int32
}
//ZstdOptions is the options set for zstdOptions
type ZstdOptions struct {
CompressionLevel int32 //CompressionLevel should be between 1 and 22
}
type lzoOptionsRaw struct {
algorithm int32
compressionLevel int32
}
old/reader.go
-59
View File
@@ -1,59 +0,0 @@
package squashfs
import (
"errors"
"io"
)
//TODO: possible custom reader because I'm havng some issuse...
//Reader is a reader which implements Reader, ReaderAt, and Seeker, all with an accesible offset (for reasons)
type Reader struct {
rdr io.ReaderAt
offset int64
}
//NewReader creates a squashfs.Reader from a io.ReaderAt
func NewReader(baseReader io.ReaderAt) Reader {
return Reader{
rdr: baseReader,
offset: 0,
}
}
//Read reads len(byt) into byt. Advances the internal offset
func (r *Reader) Read(byt []byte) (int, error) {
n, err := r.rdr.ReadAt(byt, r.offset)
r.offset += int64(n)
return n, err
}
//ReadAt wraps the internal io.ReadAt's function. DOES NOT advance the internal offset for Read function.
//Returns how many bytes were read.
func (r *Reader) ReadAt(byt []byte, offset int64) (int, error) {
return r.rdr.ReadAt(byt, offset)
}
//ReadAtFromOffset is the same as ReadAt, but the given offset is offset by the internal offset. DOES NOT advance the internal offset.
//Returns how many bytes were read.
func (r *Reader) ReadAtFromOffset(byt []byte, offset int64) (int, error) {
offset += r.offset
return r.rdr.ReadAt(byt, offset)
}
//Seek advances the internal offset. SeekEnd DOES NOT work
//Might not be necessary, but here just in case
func (r *Reader) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekCurrent:
n, err := r.Read(make([]byte, offset))
return int64(n), err
case io.SeekStart:
r.offset = 0
n, err := r.Read(make([]byte, offset))
return int64(n), err
case io.SeekEnd:
return 0, errors.New("SeekEnd is NOT supported")
}
return 0, errors.New("incorrect whence")
}
old/reader_old.go
-618
View File
@@ -1,618 +0,0 @@
package squashfs
// import (
// "bytes"
// "encoding/binary"
// "fmt"
// "io"
// "io/ioutil"
// "log"
// "os"
// "path/filepath"
// "strings"
// "sync"
// "syscall"
// "time"
// "golang.org/x/xerrors"
// )
// type Reader struct {
// r io.ReaderAt
// super superblock
// }
// func NewReader(r io.ReaderAt) (*Reader, error) {
// var sb superblock
// if err := binary.Read(io.NewSectionReader(r, 0, int64(binary.Size(sb))), binary.LittleEndian, &sb); err != nil {
// return nil, fmt.Errorf("reading superblock: %v", err)
// }
// if got, want := sb.Magic, uint32(magic); got != want {
// return nil, fmt.Errorf("invalid magic (not a SquashFS image?): got %x, want %x", got, want)
// }
// //log.Printf("superblock: %+v", sb)
// return &Reader{
// r: r,
// super: sb,
// }, nil
// }
// // TODO: maybe mmap instead of seeking?
// func (r *Reader) inode(i Inode) (blockoffset int64, offset int64) {
// return int64(i >> 16), int64(i & 0xFFFF)
// }
// type blockReader struct {
// r io.ReadSeeker
// lenBuf [2]byte
// buf []byte
// i int64
// off int64 // TODO: remove this once using mmap
// }
// func (br *blockReader) Read(p []byte) (n int, err error) {
// if br.i >= int64(len(br.buf)) {
// br.i = 0
// if _, err := io.ReadFull(br.r, br.lenBuf[:]); err != nil {
// return 0, err
// }
// l := binary.LittleEndian.Uint16(br.lenBuf[:])
// //uncompressed := l&0x8000 > 0
// l &= 0x7FFF
// //log.Printf("block of len %d, uncompressed: %v", l, uncompressed)
// if int(l) > cap(br.buf) {
// br.buf = make([]byte, int(l))
// }
// br.buf = br.buf[:l]
// if _, err := io.ReadFull(br.r, br.buf); err != nil {
// return 0, err
// }
// //log.Printf("(retry) n = %v, err = %v", n, err)
// }
// n = copy(p, br.buf[br.i:])
// br.i += int64(n)
// return n, err
// }
// func (br *blockReader) Close() error {
// blockReaderPool.Put(br)
// return nil
// }
// var blockReaderPool = sync.Pool{
// New: func() interface{} {
// return &blockReader{
// buf: make([]byte, 0, metadataBlockSize),
// }
// },
// }
// func (r *Reader) blockReader(blockoffset, offset int64) (io.ReadCloser, error) {
// //log.Printf("blockoffset %v (%x), offset %v (%x)", blockoffset, blockoffset, offset, offset)
// br := blockReaderPool.Get().(*blockReader)
// br.buf = br.buf[:0]
// br.r = io.NewSectionReader(r.r, blockoffset, 5500*1024*1024) // TODO: correct limit? can we use IntMax
// br.off = blockoffset
// br.i = 0
// //log.Printf("discarding %d bytes", offset)
// for n := int64(0); n < offset; {
// remaining := offset - n
// if remaining > metadataBlockSize {
// remaining = metadataBlockSize
// }
// nn, err := br.Read(br.buf[:remaining])
// if err != nil {
// return nil, err
// }
// n += int64(nn)
// }
// return br, nil
// }
// // TODO: define an inode type to use instead of interface{}?
// func (r *Reader) readInode(i Inode) (interface{}, error) {
// blockoffset, offset := r.inode(i)
// br, err := r.blockReader(r.super.InodeTableStart+blockoffset, offset)
// if err != nil {
// fmt.Println("oops! ", err)
// return nil, err
// }
// defer br.Close()
// fmt.Println("Hello There")
// // We need the inode type before we know which type to pass to binary.Read,
// // so we need to read it twice:
// var inodeType uint16
// typeBuf := bytes.NewBuffer(make([]byte, 0, binary.Size(inodeType)))
// if err := binary.Read(io.TeeReader(br, typeBuf), binary.LittleEndian, &inodeType); err != nil {
// return nil, err
// }
// br = ioutil.NopCloser(io.MultiReader(typeBuf, br))
// // var ih inodeHeader
// // if err := binary.Read(br, binary.LittleEndian, &ih); err != nil {
// // return err
// // }
// // //log.Printf("ih: %+v", ih)
// //log.Printf("inode type: %v", inodeType)
// switch inodeType {
// case dirType:
// var di dirInodeHeader
// if err := binary.Read(br, binary.LittleEndian, &di); err != nil {
// return nil, err
// }
// return di, nil
// case fileType:
// var ri regInodeHeader
// if err := binary.Read(br, binary.LittleEndian, &ri); err != nil {
// return nil, err
// }
// return ri, nil
// case symlinkType:
// var si symlinkInodeHeader
// if err := binary.Read(br, binary.LittleEndian, &si); err != nil {
// return nil, err
// }
// return si, nil
// case ldirType:
// var di ldirInodeHeader
// if err := binary.Read(br, binary.LittleEndian, &di); err != nil {
// return nil, err
// }
// return di, nil
// case lregType:
// var di lregInodeHeader
// if err := binary.Read(br, binary.LittleEndian, &di); err != nil {
// return nil, err
// }
// return di, nil
// // TODO:
// // blkdevType
// // chrdevType
// // fifoType
// // socketType
// // // The larger types are used for e.g. sparse files, xattrs, etc.
// // ldirType
// // lsymlinkType
// // lblkdevType
// // lchrdevType
// // lfifoType
// // lsocketType
// }
// return nil, fmt.Errorf("unknown inode type %d", inodeType)
// }
// func (r *Reader) RootInode() Inode {
// return r.super.RootInode
// }
// func (r *Reader) Stat(name string, i Inode) (os.FileInfo, error) {
// inode, err := r.readInode(i)
// if err != nil {
// return nil, err
// }
// //log.Printf("i %d, inode: %T, %+v", i, inode, inode)
// switch x := inode.(type) {
// case dirInodeHeader:
// return &FileInfo{
// name: name,
// size: int64(x.FileSize),
// mode: os.ModeDir | os.FileMode(x.Mode),
// modTime: time.Unix(int64(x.Mtime), 0),
// Inode: i,
// }, nil
// case ldirInodeHeader:
// return &FileInfo{
// name: name,
// size: int64(x.FileSize),
// mode: os.ModeDir | os.FileMode(x.Mode),
// modTime: time.Unix(int64(x.Mtime), 0),
// Inode: i,
// }, nil
// case regInodeHeader:
// mode := os.FileMode(x.Mode & 0777)
// if x.Mode&syscall.S_ISUID != 0 {
// mode |= os.ModeSetuid
// }
// return &FileInfo{
// name: name,
// size: int64(x.FileSize),
// mode: mode,
// modTime: time.Unix(int64(x.Mtime), 0),
// Inode: i,
// }, nil
// case lregInodeHeader:
// mode := os.FileMode(x.Mode & 0777)
// if x.Mode&syscall.S_ISUID != 0 {
// mode |= os.ModeSetuid
// }
// return &FileInfo{
// name: name,
// size: int64(x.FileSize),
// mode: mode,
// modTime: time.Unix(int64(x.Mtime), 0),
// Inode: i,
// }, nil
// case symlinkInodeHeader:
// return &FileInfo{
// name: name,
// size: int64(x.SymlinkSize),
// mode: os.ModeSymlink | os.FileMode(x.Mode),
// modTime: time.Unix(int64(x.Mtime), 0),
// Inode: i,
// }, nil
// }
// return nil, fmt.Errorf("unknown inode type %T", inode)
// }
// func (r *Reader) ReadLink(i Inode) (string, error) {
// // TODO: reduce code duplication with readInode
// blockoffset, offset := r.inode(i)
// br, err := r.blockReader(r.super.InodeTableStart+blockoffset, offset)
// if err != nil {
// return "", err
// }
// defer br.Close()
// // We need the inode type before we know which type to pass to binary.Read,
// // so we need to read it twice:
// var inodeType uint16
// typeBuf := bytes.NewBuffer(make([]byte, 0, binary.Size(inodeType)))
// if err := binary.Read(io.TeeReader(br, typeBuf), binary.LittleEndian, &inodeType); err != nil {
// return "", err
// }
// br = ioutil.NopCloser(io.MultiReader(typeBuf, br))
// if inodeType != symlinkType {
// return "", fmt.Errorf("invalid inode type: got %d instead of symlink", inodeType)
// }
// var si symlinkInodeHeader
// if err := binary.Read(br, binary.LittleEndian, &si); err != nil {
// return "", err
// }
// // Assumption: r.r is positioned right after the inode
// buf := make([]byte, si.SymlinkSize)
// if _, err := io.ReadFull(br, buf); err != nil {
// return "", err
// }
// return string(buf), nil
// }
// func (r *Reader) FileReader(inode Inode) (*io.SectionReader, error) {
// //log.Printf("Readfile(%v)", inode)
// i, err := r.readInode(inode)
// if err != nil {
// return nil, err
// }
// //log.Printf("i: %+v", i)
// // TODO(compression): read the blocksizes to read compressed blocks
// switch ri := i.(type) {
// case regInodeHeader:
// off := int64(ri.StartBlock) + int64(ri.Offset)
// return io.NewSectionReader(r.r, off, int64(ri.FileSize)), nil
// case lregInodeHeader:
// off := int64(ri.StartBlock) + int64(ri.Offset)
// return io.NewSectionReader(r.r, off, int64(ri.FileSize)), nil
// default:
// return nil, fmt.Errorf("BUG: non-file inode type")
// }
// }
// type FileNotFoundError struct {
// path string
// }
// func (e *FileNotFoundError) Error() string {
// return fmt.Sprintf("%q not found", e.path)
// }
// func (r *Reader) lookupComponent(parent Inode, component string) (Inode, error) {
// rfis, err := r.readdir(parent, false)
// if err != nil {
// return 0, err
// }
// for _, rfi := range rfis {
// if rfi.Name() == component {
// return rfi.Sys().(*FileInfo).Inode, nil
// }
// }
// return 0, &FileNotFoundError{path: component}
// }
// func (r *Reader) lookupPath(path string, followSymlink bool) (Inode, error) {
// inode := r.RootInode()
// parts := strings.Split(path, "/")
// for idx, part := range parts {
// var err error
// inode, err = r.lookupComponent(inode, part)
// if err != nil {
// if _, ok := err.(*FileNotFoundError); ok {
// return 0, &FileNotFoundError{path: path}
// }
// return 0, err
// }
// if !followSymlink {
// continue
// }
// i, err := r.readInode(inode)
// if err != nil {
// return 0, xerrors.Errorf("Stat(%d): %v", inode, err)
// }
// if _, ok := i.(symlinkInodeHeader); ok {
// target, err := r.ReadLink(inode)
// if err != nil {
// return 0, err
// }
// //log.Printf("component %q (full: %q) resolved to %q", part, parts[:idx+1], target)
// target = filepath.Clean(filepath.Join(append(parts[:idx] /* parent */, target)...))
// //log.Printf("-> %s", target)
// i, err := r.LookupPath(target)
// if err != nil {
// return 0, err
// }
// inode = i
// }
// }
// return inode, nil
// }
// func (r *Reader) LookupPath(path string) (Inode, error) {
// return r.lookupPath(path, true)
// }
// // LlookupPath is like LookupPath, but does not follow symbolic links, i.e. will
// // instead return the inode of the link itself.
// func (r *Reader) LlookupPath(path string) (Inode, error) {
// return r.lookupPath(path, false)
// }
// func (r *Reader) Readdir(dirInode Inode) ([]os.FileInfo, error) {
// return r.readdir(dirInode, true)
// }
// // Like Readdir, but does not call Stat on each file. The returned FileInfo
// // structs will still have a filled in Name, partly filled in Mode, and filled
// // in Inode.
// func (r *Reader) ReaddirNoStat(dirInode Inode) ([]os.FileInfo, error) {
// return r.readdir(dirInode, false)
// }
// var nameBufPool = sync.Pool{
// New: func() interface{} {
// return &bytes.Buffer{}
// },
// }
// func (r *Reader) readdir(dirInode Inode, stat bool) ([]os.FileInfo, error) {
// //log.Printf("Readdir(%v (%x))", dirInode, dirInode)
// i, err := r.readInode(dirInode)
// fmt.Println("Yodle")
// if err != nil {
// return nil, err
// }
// var (
// startBlock int64
// fileSize int64
// offset int64
// )
// switch x := i.(type) {
// case dirInodeHeader:
// startBlock = int64(x.StartBlock)
// fileSize = int64(x.FileSize)
// offset = int64(x.Offset)
// case ldirInodeHeader:
// startBlock = int64(x.StartBlock)
// fileSize = int64(x.FileSize)
// offset = int64(x.Offset)
// default:
// return nil, fmt.Errorf("unknown directory inode type %T", i)
// }
// br, err := r.blockReader(r.super.DirectoryTableStart+startBlock, offset)
// if err != nil {
// return nil, err
// }
// defer br.Close()
// // See also https://elixir.bootlin.com/linux/v4.18.9/source/fs/squashfs/dir.c#L63
// limit := fileSize - int64(len(".")) - int64(len(".."))
// br = ioutil.NopCloser(io.LimitReader(br, limit))
// var fis []os.FileInfo
// var dh dirHeader
// var de dirEntry
// var dhBuf [12]byte
// var deBuf [8]byte
// nameBuf := nameBufPool.Get().(*bytes.Buffer)
// defer nameBufPool.Put(nameBuf)
// for {
// if _, err := io.ReadFull(br, dhBuf[:]); err != nil {
// if err == io.EOF {
// return fis, nil
// }
// return nil, err
// }
// dh.Unmarshal(dhBuf[:])
// dh.Count++ // SquashFS stores count-1
// //log.Printf("dh: %+v", dh)
// for i := 0; i < int(dh.Count); i++ {
// if _, err := io.ReadFull(br, deBuf[:]); err != nil {
// return nil, err
// }
// de.Unmarshal(deBuf[:])
// de.Size++ // SquashFS stores size-1
// //log.Printf("de: %+v", de)
// nameBuf.Reset()
// nameBuf.Grow(int(de.Size))
// nb := nameBuf.Bytes()[:de.Size]
// if _, err := io.ReadFull(br, nb); err != nil {
// return nil, err
// }
// name := string(nb)
// //log.Printf("name: %q", string(name))
// var fi os.FileInfo
// if stat {
// var err error
// fi, err = r.Stat(name, Inode(int64(dh.StartBlock)<<16|int64(de.Offset)))
// if err != nil {
// return nil, err
// }
// } else {
// ffi := &FileInfo{
// name: name,
// Inode: Inode(int64(dh.StartBlock)<<16 | int64(de.Offset)),
// }
// switch de.EntryType {
// case dirType, ldirType:
// ffi.mode |= os.ModeDir
// case symlinkType, lsymlinkType:
// ffi.mode |= os.ModeSymlink
// }
// fi = ffi
// }
// fis = append(fis, fi)
// }
// }
// return fis, nil
// }
// type FileInfo struct {
// name string
// size int64
// mode os.FileMode
// modTime time.Time
// Inode Inode
// }
// func (fi *FileInfo) Name() string { return fi.name }
// func (fi *FileInfo) Size() int64 { return fi.size }
// func (fi *FileInfo) Mode() os.FileMode { return fi.mode }
// func (fi *FileInfo) IsDir() bool { return fi.mode.IsDir() }
// func (fi *FileInfo) ModTime() time.Time { return fi.modTime }
// func (fi *FileInfo) Sys() interface{} { return fi }
// func (r *Reader) readXattr(tableHeader xattrTableHeader, id xattrId) (*Xattr, error) {
// blockoffset, offset := r.inode(Inode(id.Xattr))
// br, err := r.blockReader(int64(tableHeader.XattrTableStart)+blockoffset, offset)
// if err != nil {
// return nil, err
// }
// defer br.Close()
// var typ, nameSize uint16
// if err := binary.Read(br, binary.LittleEndian, &typ); err != nil {
// return nil, err
// }
// if err := binary.Read(br, binary.LittleEndian, &nameSize); err != nil {
// return nil, err
// }
// log.Printf("type = %v, nameSize = %v", typ, nameSize)
// name := make([]byte, nameSize)
// if _, err := io.ReadFull(br, name); err != nil {
// return nil, err
// }
// log.Printf("name = %v", string(name))
// var valSize uint32
// if err := binary.Read(br, binary.LittleEndian, &valSize); err != nil {
// return nil, err
// }
// val := make([]byte, valSize)
// if _, err := io.ReadFull(br, val); err != nil {
// return nil, err
// }
// log.Printf("val = %x", val)
// return &Xattr{
// Type: typ,
// FullName: xattrPrefix[int(typ)] + string(name),
// Value: val,
// }, nil
// }
// func (r *Reader) ReadXattrs(inode Inode) ([]Xattr, error) {
// //log.Printf("Readdir(%v (%x))", dirInode, dirInode)
// i, err := r.readInode(inode)
// if err != nil {
// return nil, err
// }
// var xid uint32
// switch x := i.(type) {
// case regInodeHeader,
// dirInodeHeader,
// ldirInodeHeader,
// symlinkInodeHeader:
// return nil, nil // no extended attributes
// case lregInodeHeader:
// if x.Xattr == invalidXattr {
// return nil, nil // file has no extended attributes
// }
// xid = x.Xattr
// default:
// return nil, fmt.Errorf("unknown inode type %T", i)
// }
// const idEntriesPerBlock = 512 // = 8192 / 16 /* sizeof(xattrId) */
// block := xid / idEntriesPerBlock
// offset := (xid % idEntriesPerBlock) * 16
// log.Printf("xattr id %d, block %d, offset %d", xid, block, offset)
// log.Printf("r.super.XattrIdTableStart = 0x%x, r.super.XattrIdTableStart = %v", r.super.XattrIdTableStart, r.super.XattrIdTableStart)
// br := ioutil.NopCloser(io.NewSectionReader(r.r, r.super.XattrIdTableStart, int64(16 /* sizeof(xattrTableHeader) */ +(block+1)*4 /* sizeof(uint32) */)))
// var tableHeader xattrTableHeader
// if err := binary.Read(br, binary.LittleEndian, &tableHeader); err != nil {
// return nil, err
// }
// // index starts here
// if _, err := io.CopyN(ioutil.Discard, br, int64(block*4 /* sizeof(uint32) */)); err != nil {
// return nil, err
// }
// var blockOffset uint32
// if err := binary.Read(br, binary.LittleEndian, &blockOffset); err != nil {
// return nil, err
// }
// log.Printf("blockOffset = 0x%x (%d)", blockOffset, blockOffset)
// br, err = r.blockReader(int64(blockOffset), int64(offset))
// if err != nil {
// return nil, err
// }
// defer br.Close()
// var id xattrId
// if err := binary.Read(br, binary.LittleEndian, &id); err != nil {
// return nil, err
// }
// log.Printf("id: %+v", id)
// log.Printf("tableHeader: %+v (start 0x%x)", tableHeader, tableHeader.XattrTableStart)
// var xattrs []Xattr
// for i := 0; i < int(id.Count); i++ {
// xattr, err := r.readXattr(tableHeader, id)
// if err != nil {
// return nil, err
// }
// xattrs = append(xattrs, *xattr)
// }
// return xattrs, nil
// }
old/reader_test.go
-373
View File
@@ -1,373 +0,0 @@
package squashfs
// import (
// "crypto/md5"
// "fmt"
// "io"
// "os"
// "testing"
// "time"
// "github.com/google/go-cmp/cmp"
// )
// func cmpFileInfo(got os.FileInfo, want FileInfo) error {
// if got, want := got.Name(), want.name; got != want {
// return fmt.Errorf("unexpected file name: got %q, want %q", got, want)
// }
// if got, want := got.Size(), want.size; got != want {
// return fmt.Errorf("unexpected size: got %d, want %d", got, want)
// }
// if got, want := got.IsDir(), want.mode.IsDir(); got != want {
// return fmt.Errorf("IsDir: got %v, want %v", got, want)
// }
// // TODO: re-enable when it’s no longer just a change detector
// // if got, want := got.ModTime(), want.modTime; !got.Equal(want) {
// // return fmt.Errorf("IsDir: got %v, want %v", got, want)
// // }
// return nil
// }
// func TestReaddir(t *testing.T) {
// t.Parallel()
// // TODO: ship testdata files generated by mksquashfs
// pwd, err := os.Getwd()
// if err != nil {
// t.Fatal(err)
// }
// fmt.Println(pwd + "/testdata/testing.squashfs")
// f, err := os.Open(pwd + "/testdata/testing.squashfs")
// if err != nil {
// t.Fatal(err)
// }
// defer f.Close()
// rd, err := NewReader(f)
// if err != nil {
// t.Fatal(err)
// }
// fis, err := rd.Readdir(rd.RootInode())
// if err != nil {
// t.Fatal(err)
// }
// for _, fi := range fis {
// fmt.Println(fi.Name())
// }
// rdr, err := rd.FileReader(fis[0].Sys().(*FileInfo).Inode)
// if err != nil {
// t.Fatal(err)
// }
// err = os.Remove(pwd + "/testdata/Magisk.zip")
// if !os.IsNotExist(err) && err != nil {
// t.Fatal(err)
// }
// // rdrzlib, err := zlib.NewReader(rdr)
// magFil, err := os.Create(pwd + "/testdata/Magisk.zip")
// io.Copy(magFil, rdr)
// if got, want := len(fis), 3; got != want {
// t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
// }
// if err := cmpFileInfo(fis[0], FileInfo{
// name: "bin",
// size: 26,
// mode: 0555 | os.ModeDir,
// modTime: time.Unix(1581275131, 0), // stat -c %Y /ro/ack-amd64-2.24/bin
// }); err != nil {
// t.Fatal(err)
// }
// if err := cmpFileInfo(fis[1], FileInfo{
// name: "lib",
// size: 3,
// mode: 0555 | os.ModeDir,
// modTime: time.Unix(1581275131, 0), // stat -c %Y /ro/ack-amd64-2.24/lib
// }); err != nil {
// t.Fatal(err)
// }
// if err := cmpFileInfo(fis[2], FileInfo{
// name: "out",
// size: 48,
// mode: 0555 | os.ModeDir,
// modTime: time.Unix(1581275130, 0), // stat -c %Y /ro/ack-amd64-2.24/out
// }); err != nil {
// t.Fatal(err)
// }
// fis, err = rd.Readdir(fis[0].Sys().(*FileInfo).Inode)
// if err != nil {
// t.Fatal(err)
// }
// if got, want := len(fis), 1; got != want {
// t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
// }
// if err := cmpFileInfo(fis[0], FileInfo{
// name: "ack",
// size: 38400,
// mode: 0755,
// modTime: time.Unix(1581275131, 0), // stat -c %Y /ro/ack-amd64-2.24/bin/ack
// }); err != nil {
// t.Fatal(err)
// }
// }
// // TestReaddirSmoke is a smoke-test, reading the root directories of SquashFS
// // images which are known to trigger code paths which were buggy.
// func TestReaddirSmoke(t *testing.T) {
// t.Parallel()
// for _, fn := range []string{
// // bash exercises the code path where an inode is split across metadata
// // blocks.
// "/home/michael/distri/_build/distri/pkg/bash-amd64-5.0-4.squashfs",
// // cmake exercises the code path where the root directory entries are
// // located outside of the first block.
// "/home/michael/distri/_build/distri/pkg/cmake-amd64-3.12.4-8.squashfs",
// } {
// // TODO: ship testdata files generated by mksquashfs
// f, err := os.Open(fn)
// if err != nil {
// t.Fatal(err)
// }
// defer f.Close()
// rd, err := NewReader(f)
// if err != nil {
// t.Fatal(err)
// }
// fis, err := rd.Readdir(rd.RootInode())
// if err != nil {
// t.Fatal(err)
// }
// if got, want := len(fis), 4; got != want {
// t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
// }
// }
// }
// func TestReaddirEmpty(t *testing.T) {
// t.Parallel()
// // TODO: ship testdata files generated by mksquashfs
// f, err := os.Open("/home/michael/distri/_build/distri/pkg/zlib-amd64-1.2.11-4.squashfs")
// if err != nil {
// t.Fatal(err)
// }
// defer f.Close()
// rd, err := NewReader(f)
// if err != nil {
// t.Fatal(err)
// }
// fis, err := rd.Readdir(rd.RootInode())
// if err != nil {
// t.Fatal(err)
// }
// if got, want := len(fis), 4; got != want {
// t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
// }
// if err := cmpFileInfo(fis[0], FileInfo{
// name: "bin",
// size: 3,
// mode: 0555 | os.ModeDir,
// modTime: time.Unix(1583085224, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/bin
// }); err != nil {
// t.Fatal(err)
// }
// fis, err = rd.Readdir(fis[0].Sys().(*FileInfo).Inode)
// if err != nil {
// t.Fatal(err)
// }
// if got, want := len(fis), 0; got != want {
// t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
// }
// }
// func TestReaddirSymlink(t *testing.T) {
// t.Parallel()
// // TODO: ship testdata files generated by mksquashfs
// f, err := os.Open("/home/michael/distri/_build/distri/pkg/zlib-amd64-1.2.11-4.squashfs")
// if err != nil {
// t.Fatal(err)
// }
// defer f.Close()
// rd, err := NewReader(f)
// if err != nil {
// t.Fatal(err)
// }
// fis, err := rd.Readdir(rd.RootInode())
// if err != nil {
// t.Fatal(err)
// }
// if got, want := len(fis), 4; got != want {
// t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
// }
// if err := cmpFileInfo(fis[3], FileInfo{
// name: "out",
// size: 54,
// mode: 0555 | os.ModeDir,
// modTime: time.Unix(1583085224, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/out
// }); err != nil {
// t.Fatal(err)
// }
// fis, err = rd.Readdir(fis[3].Sys().(*FileInfo).Inode)
// if err != nil {
// t.Fatal(err)
// }
// if got, want := len(fis), 3; got != want {
// t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
// }
// if err := cmpFileInfo(fis[1], FileInfo{
// name: "lib",
// size: 83,
// mode: 0555 | os.ModeDir,
// modTime: time.Unix(1583085224, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/out/lib
// }); err != nil {
// t.Fatal(err)
// }
// fis, err = rd.Readdir(fis[1].Sys().(*FileInfo).Inode)
// if err != nil {
// t.Fatal(err)
// }
// if got, want := len(fis), 4; got != want {
// t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
// }
// if err := cmpFileInfo(fis[1], FileInfo{
// name: "libz.so",
// size: 9,
// mode: 0555 | os.ModeSymlink,
// modTime: time.Unix(1583085223, 0), // stat -c %Y /ro/zlib-amd64-1.2.11/out/lib/libz.so
// }); err != nil {
// t.Fatal(err)
// }
// // TODO: readlink
// target, err := rd.ReadLink(fis[1].Sys().(*FileInfo).Inode)
// if err != nil {
// t.Fatal(err)
// }
// if got, want := target, "libz.so.1"; got != want {
// t.Fatalf("ReadLink(libz.so): got %q, want %q", got, want)
// }
// }
// func TestReadfile(t *testing.T) {
// t.Parallel()
// // TODO: ship testdata files generated by mksquashfs
// f, err := os.Open("/home/michael/distri/_build/distri/pkg/ack-amd64-3.3.1-7.squashfs")
// if err != nil {
// t.Fatal(err)
// }
// defer f.Close()
// rd, err := NewReader(f)
// if err != nil {
// t.Fatal(err)
// }
// fis, err := rd.Readdir(rd.RootInode())
// if err != nil {
// t.Fatal(err)
// }
// if got, want := len(fis), 3; got != want {
// t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
// }
// fis, err = rd.Readdir(fis[0].Sys().(*FileInfo).Inode)
// if err != nil {
// t.Fatal(err)
// }
// if got, want := len(fis), 1; got != want {
// t.Fatalf("unexpected number of directory entries: got %d, want %d", got, want)
// }
// r, err := rd.FileReader(fis[0].Sys().(*FileInfo).Inode)
// if err != nil {
// t.Fatal(err)
// }
// for i := 0; i < 2; i++ {
// if _, err := r.Seek(0, io.SeekStart); err != nil {
// t.Fatal(err)
// }
// h := md5.New()
// if _, err := io.Copy(h, r); err != nil {
// t.Fatal(err)
// }
// sum := fmt.Sprintf("%x", h.Sum(nil))
// if got, want := sum, "c6c9b5d4d2a49f1b8b5e501f0f827a5c"; got != want {
// t.Fatalf("md5(bin/ack): got %s, want %s", got, want)
// }
// }
// }
// // TODO: add test exercising ldirInodeHeader, e.g. '/mnt/loop/ca-certificates-3.39/buildoutput/etc/ssl'
// func TestReadXattr(t *testing.T) {
// t.Parallel()
// // TODO: generate a smaller version of this file
// f, err := os.Open("testdata/xattr.squashfs")
// if err != nil {
// t.Fatal(err)
// }
// defer f.Close()
// rd, err := NewReader(f)
// if err != nil {
// t.Fatal(err)
// }
// for _, tt := range []struct {
// Path string
// Want []Xattr
// }{
// {
// Path: "mtr-packet",
// Want: []Xattr{
// {
// Type: XattrTypeSecurity,
// FullName: "security.capability",
// Value: []byte{1, 0, 0, 2, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
// },
// },
// {
// Path: "gnome-keyring-daemon",
// Want: []Xattr{
// {
// Type: XattrTypeSecurity,
// FullName: "security.capability",
// Value: []byte{1, 0, 0, 2, 0, 0x40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
// },
// },
// } {
// inode, err := rd.LookupPath(tt.Path)
// if err != nil {
// t.Fatal(err)
// }
// xattrs, err := rd.ReadXattrs(inode)
// if err != nil {
// t.Fatalf("ReadXattrs(%v): %v", inode, err)
// }
// if diff := cmp.Diff(tt.Want, xattrs); diff != "" {
// t.Fatalf("unexpected ReadXattrs result: diff (-want +got):\n%s", diff)
// }
// }
// }
old/squashfs.go
-181
View File
@@ -1,181 +0,0 @@
package squashfs
import (
"encoding/binary"
"errors"
"fmt"
"io"
"github.com/CalebQ42/GoSquashfs/old/internal/inode"
)
var (
//ErrNotMagical happens when making a new Squashfs and it doesn't have the magic number
ErrNotMagical = errors.New("Not Magical")
)
//Squashfs is a squashfs backed by a ReadSeeker.
type Squashfs struct {
rdr *Reader //underlying reader
offset int
super Superblock
flags SuperblockFlags
compressionOptions CompressionOptions
}
//NewSquashfs creates a new Squashfs backed by the given reader
func NewSquashfs(reader io.ReaderAt) (*Squashfs, error) {
rdr := NewReader(reader)
var superblock Superblock
err := binary.Read(&rdr, binary.LittleEndian, &superblock)
if err != nil {
return nil, err
}
if superblock.Magic != squashfsMagic {
return nil, ErrNotMagical
}
flags := superblock.GetFlags()
var compressionOptions CompressionOptions
switch superblock.Compression {
case zlibCompression:
if flags.CompressorOptions {
var zlibOpRaw zlibOptionsRaw
err = binary.Read(&rdr, binary.LittleEndian, &zlibOpRaw)
if err != nil {
return nil, err
}
compressionOptions = NewZlibOptions(zlibOpRaw)
} else {
compressionOptions = NewZlibOptions(zlibOptionsRaw{})
}
case xzCompression:
if flags.CompressorOptions {
var xzOpRaw xzOptionsRaw
err = binary.Read(&rdr, binary.LittleEndian, xzOpRaw)
if err != nil {
return nil, err
}
compressionOptions = NewXzOption(xzOpRaw)
} else {
compressionOptions = NewXzOption(xzOptionsRaw{})
}
default:
//TODO: all the compression options
return nil, errors.New("This type of compression isn't supported yet")
}
//TODO: parse more info
return &Squashfs{
rdr: &rdr,
super: superblock,
flags: flags,
compressionOptions: compressionOptions,
}, nil
}
func (s *Squashfs) readRootDirectoryTable() error {
offset, metaOffset := inode.ProcessInodeRef(s.super.RootInode)
meta, err := s.parseMetadataAt(int64(s.super.InodeTableOffset) + int64(offset))
if err != nil {
fmt.Println("Error processing metadata")
return err
}
_, err = meta.Data.Read(make([]byte, metaOffset))
if err != nil {
fmt.Println("error reading forward to offset")
return err
}
common, _, err := inode.ProcessInode(&meta.Data, s.super.BlockSize)
if err != nil {
fmt.Println("Error reading inode")
return err
}
if common.InodeType != inode.BasicDirectoryType {
return errors.New("Not a basic directory")
}
// rootDir := inodeInterface.(inode.BasicDirectory)
// dirTable, err := directory.NewDirectory(meta.Data)
// if err != nil {
// return err
// }
// for _, entry := range dirTable.Entries {
// fmt.Println(entry.Name)
// }
return nil
}
//GetFlags return the SuperblockFlags of the Squashfs
func (s *Squashfs) GetFlags() SuperblockFlags {
return s.flags
}
//metadata is a parsed metadata block
type metadata struct {
Compressed bool
Size uint16
Data io.Reader
}
func (m *metadata) close() {
switch m.Data.(type) {
case io.ReadCloser:
m.Data.(io.ReadCloser).Close()
}
}
func (s *Squashfs) parseNextMetadata() (*metadata, error) {
var metaHeader uint16
err := binary.Read(s.rdr, binary.LittleEndian, &metaHeader)
if err != nil {
return nil, err
}
reader := io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(metaHeader&^0x8000))
if metaHeader&0x8000 == 0x8000 {
metaHeader = metaHeader &^ 0x8000
compressRead, err := s.compressionOptions.Reader(reader)
return &metadata{
Compressed: true,
Size: metaHeader,
Data: *compressRead,
}, err
}
return &metadata{
Compressed: false,
Size: metaHeader,
Data: reader,
}, nil
}
func (s *Squashfs) parseMetadataAt(offset int64) (*metadata, error) {
// tmp := s.rdr.offset
// meta, err := s.parseNextMetadata()
// s.rdr.offset = tmp
// return meta, err
var metaHeader uint16
var headerBytes []byte
headerBytes = make([]byte, 2)
s.rdr.ReadAt(headerBytes, offset)
metaHeader = binary.LittleEndian.Uint16(headerBytes)
if metaHeader&0x8000 == 0x8000 {
fmt.Println("Compressed")
metaHeader = metaHeader &^ 0x8000
compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)))
return &metadata{
Compressed: true,
Size: metaHeader,
Data: *compressRead,
}, err
}
return &metadata{
Compressed: false,
Size: metaHeader,
Data: io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)),
}, nil
//TODO: figure out why things break when I use a straight zlib reader, but not when I use a byte reader writer
// byts, err := s.compressionOptions.Decompress(io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)))
// return &metadata{
// Compressed: false,
// Size: metaHeader,
// Data: newByteReadWriteFromBytes(byts),
// }, err
}
old/squashfs_test.go
-106
View File
@@ -1,106 +0,0 @@
package squashfs
import (
"io"
"net/http"
"os"
"testing"
appimage "github.com/CalebQ42/GoAppImage"
)
const (
downloadURL = "https://github.com/zilti/code-oss.AppImage/releases/download/continuous/Code_OSS-x86_64.AppImage"
appImageName = "Code_OSS.AppImage"
squashfsName = "Code_OSS.Squashfs"
)
func TestAppImageSquash(t *testing.T) {
t.Parallel()
wd, err := os.Getwd()
if err != nil {
t.Error(err)
}
squashFil, err := os.Open(wd + "/testing/" + squashfsName)
if os.IsNotExist(err) {
TestCreateSquashFromAppImage(t)
squashFil, err = os.Open(wd + "/testing/" + squashfsName)
if err != nil {
t.Error(err)
}
}
defer squashFil.Close()
stat, _ := squashFil.Stat()
squash, err := NewSquashfs(io.NewSectionReader(squashFil, 0, stat.Size()))
if err != nil {
t.Error(err)
}
err = squash.readRootDirectoryTable()
t.Fatal(err)
}
func TestCreateSquashFromAppImage(t *testing.T) {
wd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
err = os.Mkdir(wd+"/testing", 0777)
if err != nil && !os.IsExist(err) {
t.Fatal(err)
}
_, err = os.Open(wd + "/testing/" + appImageName)
if os.IsNotExist(err) {
downloadTestAppImage(t, wd+"/testing")
_, err = os.Open(wd + "/testing/" + appImageName)
if err != nil {
t.Fatal(err)
}
} else if err != nil {
t.Fatal(err)
}
ai := appimage.NewAppImage(wd + "/testing/" + appImageName)
aiFil, err := os.Open(wd + "/testing/" + appImageName)
if err != nil {
t.Fatal(err)
}
defer aiFil.Close()
aiFil.Seek(ai.Offset, 0)
os.Remove(wd + "/testing/" + squashfsName)
aiSquash, err := os.Create(wd + "/testing/" + squashfsName)
if err != nil {
t.Fatal(err)
}
_, err = io.Copy(aiSquash, aiFil)
if err != nil {
t.Fatal(err)
}
}
func downloadTestAppImage(t *testing.T, dir string) {
//seems to time out. Need to fix that at some point
appImage, err := os.Create(dir + "/" + appImageName)
if err != nil {
t.Fatal(err)
}
defer appImage.Close()
check := http.Client{
CheckRedirect: func(r *http.Request, via []*http.Request) error {
r.URL.Opaque = r.URL.Path
return nil
},
}
resp, err := check.Get(downloadURL)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
_, err = io.Copy(appImage, resp.Body)
if err != nil {
t.Fatal(err)
}
}
func TestLookInsideSquash(t *testing.T) {
t.Parallel()
//TODO
}
old/writer_old.go
-913
View File
@@ -1,913 +0,0 @@
// // Package squashfs implements writing SquashFS file system images using zlib
// // compression for data blocks (inodes and directory entries are written
// // uncompressed for simplicity).
// //
// // Note that SquashFS requires directory entries to be sorted, i.e. files and
// // directories need to be added in the correct order.
// //
// // This package intentionally only implements a subset of SquashFS. Notably,
// // block devices, character devices, FIFOs, sockets and xattrs are not
// // supported.
package squashfs
// import (
// "bytes"
// "compress/zlib"
// "encoding/binary"
// "io"
// "os"
// "path/filepath"
// "strings"
// "time"
// "golang.org/x/sys/unix"
// )
// // inode contains a block number + offset within that block.
// type Inode int64
// const (
// zlibCompression = 1 + iota
// lzmaCompression
// lzoCompression
// xzCompression
// lz4Compression
// zstdCompression
// )
// const (
// invalidFragment = 0xFFFFFFFF
// invalidXattr = 0xFFFFFFFF
// )
// const (
// dirType = 1 + iota
// fileType
// symlinkType
// blkdevType
// chrdevType
// fifoType
// socketType
// // The larger types are used for e.g. sparse files, xattrs, etc.
// ldirType
// lregType
// lsymlinkType
// lblkdevType
// lchrdevType
// lfifoType
// lsocketType
// )
// type inodeHeader struct {
// InodeType uint16
// Mode uint16
// Uid uint16
// Gid uint16
// Mtime int32
// InodeNumber uint32
// }
// // fileType
// type regInodeHeader struct {
// inodeHeader
// // full byte offset from the start of the file system, e.g. 96 for first
// // file contents. Not using fragments limits us to 2^32-1-96 (≈ 4GiB) bytes
// // of file contents.
// StartBlock uint32
// Fragment uint32
// Offset uint32
// FileSize uint32
// // Followed by a uint32 array of compressed block sizes.
// }
// // lregType
// type lregInodeHeader struct {
// inodeHeader
// // full byte offset from the start of the file system, e.g. 96 for first
// // file contents. Not using fragments limits us to 2^32-1-96 (≈ 4GiB) bytes
// // of file contents.
// StartBlock uint64
// FileSize uint64
// Sparse uint64
// Nlink uint32
// Fragment uint32
// Offset uint32
// Xattr uint32
// // Followed by a uint32 array of compressed block sizes.
// }
// // symlinkType
// type symlinkInodeHeader struct {
// inodeHeader
// Nlink uint32
// SymlinkSize uint32
// // Followed by a byte array of SymlinkSize bytes.
// }
// // chrdevType and blkdevType
// type devInodeHeader struct {
// inodeHeader
// Nlink uint32
// Rdev uint32
// }
// // fifoType and socketType
// type ipcInodeHeader struct {
// inodeHeader
// Nlink uint32
// }
// // dirType
// type dirInodeHeader struct {
// inodeHeader
// StartBlock uint32
// Nlink uint32
// FileSize uint16
// Offset uint16
// ParentInode uint32
// }
// // ldirType
// type ldirInodeHeader struct {
// inodeHeader
// Nlink uint32
// FileSize uint32
// StartBlock uint32
// ParentInode uint32
// Icount uint16
// Offset uint16
// Xattr uint32
// }
// type dirHeader struct {
// Count uint32
// StartBlock uint32
// InodeOffset uint32
// }
// func (d *dirHeader) Unmarshal(b []byte) {
// _ = b[11]
// e := binary.LittleEndian
// d.Count = e.Uint32(b)
// d.StartBlock = e.Uint32(b[4:])
// d.InodeOffset = e.Uint32(b[8:])
// }
// type dirEntry struct {
// Offset uint16
// InodeNumber int16
// EntryType uint16
// Size uint16
// // Followed by a byte array of Size bytes.
// }
// func (d *dirEntry) Unmarshal(b []byte) {
// _ = b[7]
// e := binary.LittleEndian
// d.Offset = e.Uint16(b)
// d.InodeNumber = int16(e.Uint16(b[2:]))
// d.EntryType = e.Uint16(b[4:])
// d.Size = e.Uint16(b[6:])
// }
// // xattr types
// const (
// XattrTypeUser = iota
// XattrTypeTrusted
// XattrTypeSecurity
// )
// var xattrPrefix = map[int]string{
// XattrTypeUser: "user.",
// XattrTypeTrusted: "trusted.",
// XattrTypeSecurity: "security.",
// }
// type Xattr struct {
// Type uint16
// FullName string
// Value []byte
// }
// func XattrFromAttr(attr string, val []byte) Xattr {
// for typ, prefix := range xattrPrefix {
// if !strings.HasPrefix(attr, prefix) {
// continue
// }
// return Xattr{
// Type: uint16(typ),
// FullName: strings.TrimPrefix(attr, prefix),
// Value: val,
// }
// }
// return Xattr{}
// }
// type xattrId struct {
// Xattr uint64
// Count uint32
// Size uint32
// }
// func writeIdTable(w io.WriteSeeker, ids []uint32) (start int64, err error) {
// metaOff, err := w.Seek(0, io.SeekCurrent)
// if err != nil {
// return 0, err
// }
// var buf bytes.Buffer
// if err := binary.Write(&buf, binary.LittleEndian, ids); err != nil {
// return 0, err
// }
// if err := binary.Write(w, binary.LittleEndian, uint16(buf.Len())|0x8000); err != nil {
// return 0, err
// }
// if _, err := io.Copy(w, &buf); err != nil {
// return 0, err
// }
// off, err := w.Seek(0, io.SeekCurrent)
// if err != nil {
// return 0, err
// }
// return off, binary.Write(w, binary.LittleEndian, metaOff)
// }
// type fullDirEntry struct {
// startBlock uint32
// offset uint16
// inodeNumber uint32
// entryType uint16
// name string
// }
// const (
// magic = 0x73717368
// dataBlockSize = 131072
// metadataBlockSize = 8192
// majorVersion = 4
// minorVersion = 0
// )
// type Writer struct {
// // Root represents the file system root. Like all directories, Flush must be
// // called precisely once.
// Root *Directory
// xattrs []Xattr
// xattrIds []xattrId
// w io.WriteSeeker
// sb superblock
// inodeBuf bytes.Buffer
// dirBuf bytes.Buffer
// writeInodeNumTo map[string][]int64
// }
// // TODO: document what this is doing and what it is used for
// func slog(block uint32) uint16 {
// for i := uint16(12); i <= 20; i++ {
// if block == (1 << i) {
// return i
// }
// }
// return 0
// }
// // filesystemFlags returns flags for a SquashFS file system created by this
// // package (disabling most features for now).
// func filesystemFlags() uint16 {
// const (
// noI = 1 << iota // uncompressed metadata
// noD // uncompressed data
// _
// noF // uncompressed fragments
// noFrag // never use fragments
// alwaysFrag // always use fragments
// duplicateChecking // de-duplication
// exportable // exportable via NFS
// noX // uncompressed xattrs
// noXattr // no xattrs
// compopt // compressor-specific options present?
// )
// return noI | noF | noFrag | noX | noXattr
// }
// // NewWriter returns a Writer which will write a SquashFS file system image to w
// // once Flush is called.
// //
// // Create new files and directories with the corresponding methods on the Root
// // directory of the Writer.
// //
// // File data is written to w even before Flush is called.
// func NewWriter(w io.WriteSeeker, mkfsTime time.Time) (*Writer, error) {
// // Skip over superblock to the data area, we come back to the superblock
// // when flushing.
// if _, err := w.Seek(96, io.SeekStart); err != nil {
// return nil, err
// }
// wr := &Writer{
// w: w,
// sb: superblock{
// Magic: magic,
// MkfsTime: int32(mkfsTime.Unix()),
// BlockSize: dataBlockSize,
// Fragments: 0,
// Compression: zlibCompression,
// BlockLog: slog(dataBlockSize),
// Flags: filesystemFlags(),
// NoIds: 1, // just one uid/gid mapping (for root)
// Major: majorVersion,
// Minor: minorVersion,
// XattrIdTableStart: -1, // not present
// LookupTableStart: -1, // not present
// },
// writeInodeNumTo: make(map[string][]int64),
// }
// wr.Root = &Directory{
// w: wr,
// name: "", // root
// modTime: mkfsTime,
// }
// return wr, nil
// }
// // Directory represents a SquashFS directory.
// type Directory struct {
// w *Writer
// name string
// modTime time.Time
// dirEntries []fullDirEntry
// parent *Directory
// }
// func (d *Directory) path() string {
// if d.parent == nil {
// return d.name
// }
// return filepath.Join(d.parent.path(), d.name)
// }
// type file struct {
// w *Writer
// d *Directory
// off int64
// size uint32
// name string
// modTime time.Time
// mode uint16
// // buf accumulates at least dataBlockSize bytes, at which point a new block
// // is being written.
// buf bytes.Buffer
// // blocksizes stores, for each block of dataBlockSize bytes (uncompressed),
// // the number of bytes the block compressed down to.
// blocksizes []uint32
// // compBuf is used for holding a block during compression to avoid memory
// // allocations.
// compBuf *bytes.Buffer
// // zlibWriter is re-used for each compressed block
// zlibWriter *zlib.Writer
// xattrRef uint32
// }
// // Directory creates a new directory with the specified name and modTime.
// func (d *Directory) Directory(name string, modTime time.Time) *Directory {
// return &Directory{
// w: d.w,
// name: name,
// modTime: modTime,
// parent: d,
// }
// }
// // File creates a file with the specified name, modTime and mode. The returned
// // io.WriterCloser must be closed after writing the file.
// func (d *Directory) File(name string, modTime time.Time, mode uint16, xattrs []Xattr) (io.WriteCloser, error) {
// off, err := d.w.w.Seek(0, io.SeekCurrent)
// if err != nil {
// return nil, err
// }
// // zlib.BestSpeed results in only a 2x slow-down over no compression
// // (compared to >4x slow-down with DefaultCompression), but generates
// // results which are in the same ball park (10% larger).
// zw, err := zlib.NewWriterLevel(nil, zlib.BestSpeed)
// if err != nil {
// return nil, err
// }
// xattrRef := uint32(invalidXattr)
// if len(xattrs) > 0 {
// xattrRef = uint32(len(d.w.xattrs))
// d.w.xattrs = append(d.w.xattrs, xattrs[0]) // TODO: support multiple
// size := len(xattrs[0].FullName) + len(xattrs[0].Value)
// d.w.xattrIds = append(d.w.xattrIds, xattrId{
// // Xattr is populated in writeXattrTables
// Count: 1, // TODO: support multiple
// Size: uint32(size),
// })
// }
// return &file{
// w: d.w,
// d: d,
// off: off,
// name: name,
// modTime: modTime,
// mode: mode,
// compBuf: bytes.NewBuffer(make([]byte, dataBlockSize)),
// zlibWriter: zw,
// xattrRef: xattrRef,
// }, nil
// }
// // Symlink creates a symbolic link from newname to oldname with the specified
// // modTime and mode.
// func (d *Directory) Symlink(oldname, newname string, modTime time.Time, mode os.FileMode) error {
// startBlock := d.w.inodeBuf.Len() / metadataBlockSize
// offset := d.w.inodeBuf.Len() - startBlock*metadataBlockSize
// if err := binary.Write(&d.w.inodeBuf, binary.LittleEndian, symlinkInodeHeader{
// inodeHeader: inodeHeader{
// InodeType: symlinkType,
// Mode: uint16(mode),
// Uid: 0,
// Gid: 0,
// Mtime: int32(modTime.Unix()),
// InodeNumber: d.w.sb.Inodes + 1,
// },
// Nlink: 1, // TODO(later): when is this not 1?
// SymlinkSize: uint32(len(oldname)),
// }); err != nil {
// return err
// }
// if _, err := d.w.inodeBuf.Write([]byte(oldname)); err != nil {
// return err
// }
// d.dirEntries = append(d.dirEntries, fullDirEntry{
// startBlock: uint32(startBlock),
// offset: uint16(offset),
// inodeNumber: d.w.sb.Inodes + 1,
// entryType: symlinkType,
// name: newname,
// })
// d.w.sb.Inodes++
// return nil
// }
// // Flush writes directory entries and creates inodes for the directory.
// func (d *Directory) Flush() error {
// countByStartBlock := make(map[uint32]uint32)
// for _, de := range d.dirEntries {
// countByStartBlock[de.startBlock]++
// }
// dirBufStartBlock := d.w.dirBuf.Len() / metadataBlockSize
// dirBufOffset := d.w.dirBuf.Len()
// currentBlock := int64(-1)
// currentInodeOffset := int64(-1)
// var subdirs int
// for _, de := range d.dirEntries {
// if de.entryType == dirType {
// subdirs++
// }
// if int64(de.startBlock) != currentBlock {
// dh := dirHeader{
// Count: countByStartBlock[de.startBlock] - 1,
// StartBlock: de.startBlock * (metadataBlockSize + 2),
// InodeOffset: de.inodeNumber,
// }
// if err := binary.Write(&d.w.dirBuf, binary.LittleEndian, &dh); err != nil {
// return err
// }
// currentBlock = int64(de.startBlock)
// currentInodeOffset = int64(de.inodeNumber)
// }
// if err := binary.Write(&d.w.dirBuf, binary.LittleEndian, &dirEntry{
// Offset: de.offset,
// InodeNumber: int16(de.inodeNumber - uint32(currentInodeOffset)),
// EntryType: de.entryType,
// Size: uint16(len(de.name) - 1),
// }); err != nil {
// return err
// }
// if _, err := d.w.dirBuf.Write([]byte(de.name)); err != nil {
// return err
// }
// }
// startBlock := d.w.inodeBuf.Len() / metadataBlockSize
// offset := d.w.inodeBuf.Len() - startBlock*metadataBlockSize
// inodeBufOffset := d.w.inodeBuf.Len()
// // parentInodeOffset is the offset (in bytes) of the ParentInode field
// // within a dirInodeHeader or ldirInodeHeader
// var parentInodeOffset int64
// if len(d.dirEntries) > 256 ||
// d.w.dirBuf.Len()-dirBufOffset > metadataBlockSize {
// parentInodeOffset = (2 + 2 + 2 + 2 + 4 + 4) + 4 + 4 + 4
// if err := binary.Write(&d.w.inodeBuf, binary.LittleEndian, ldirInodeHeader{
// inodeHeader: inodeHeader{
// InodeType: ldirType,
// Mode: unix.S_IRUSR | unix.S_IWUSR | unix.S_IXUSR |
// unix.S_IRGRP | unix.S_IXGRP |
// unix.S_IROTH | unix.S_IXOTH,
// Uid: 0,
// Gid: 0,
// Mtime: int32(d.modTime.Unix()),
// InodeNumber: d.w.sb.Inodes + 1,
// },
// Nlink: uint32(subdirs + 2 - 1), // + 2 for . and ..
// FileSize: uint32(d.w.dirBuf.Len()-dirBufOffset) + 3,
// StartBlock: uint32(dirBufStartBlock * (metadataBlockSize + 2)),
// ParentInode: d.w.sb.Inodes + 2, // invalid
// Icount: 0, // no directory index
// Offset: uint16(dirBufOffset - dirBufStartBlock*metadataBlockSize),
// Xattr: invalidXattr,
// }); err != nil {
// return err
// }
// } else {
// parentInodeOffset = (2 + 2 + 2 + 2 + 4 + 4) + 4 + 4 + 2 + 2
// if err := binary.Write(&d.w.inodeBuf, binary.LittleEndian, dirInodeHeader{
// inodeHeader: inodeHeader{
// InodeType: dirType,
// Mode: unix.S_IRUSR | unix.S_IWUSR | unix.S_IXUSR |
// unix.S_IRGRP | unix.S_IXGRP |
// unix.S_IROTH | unix.S_IXOTH,
// Uid: 0,
// Gid: 0,
// Mtime: int32(d.modTime.Unix()),
// InodeNumber: d.w.sb.Inodes + 1,
// },
// StartBlock: uint32(dirBufStartBlock * (metadataBlockSize + 2)),
// Nlink: uint32(subdirs + 2 - 1), // + 2 for . and ..
// FileSize: uint16(d.w.dirBuf.Len()-dirBufOffset) + 3,
// Offset: uint16(dirBufOffset - dirBufStartBlock*metadataBlockSize),
// ParentInode: d.w.sb.Inodes + 2, // invalid
// }); err != nil {
// return err
// }
// }
// path := d.path()
// for _, offset := range d.w.writeInodeNumTo[path] {
// // Directly manipulating unread data in bytes.Buffer via Bytes(), as per
// // https://groups.google.com/d/msg/golang-nuts/1ON9XVQ1jXE/8j9RaeSYxuEJ
// b := d.w.inodeBuf.Bytes()
// binary.LittleEndian.PutUint32(b[offset:offset+4], d.w.sb.Inodes+1)
// }
// if d.parent != nil {
// parentPath := filepath.Dir(d.path())
// if parentPath == "." {
// parentPath = ""
// }
// d.w.writeInodeNumTo[parentPath] = append(d.w.writeInodeNumTo[parentPath], int64(inodeBufOffset)+parentInodeOffset)
// d.parent.dirEntries = append(d.parent.dirEntries, fullDirEntry{
// startBlock: uint32(startBlock),
// offset: uint16(offset),
// inodeNumber: d.w.sb.Inodes + 1,
// entryType: dirType,
// name: d.name,
// })
// } else { // root
// d.w.sb.RootInode = Inode((startBlock*(metadataBlockSize+2))<<16 | offset)
// }
// d.w.sb.Inodes++
// return nil
// }
// // Write implements io.Writer
// func (f *file) Write(p []byte) (n int, err error) {
// n, err = f.buf.Write(p)
// if n > 0 {
// // Keep track of the uncompressed file size.
// f.size += uint32(n)
// for f.buf.Len() >= dataBlockSize {
// if err := f.writeBlock(); err != nil {
// return 0, err
// }
// }
// }
// return n, err
// }
// func (f *file) writeBlock() error {
// n := f.buf.Len()
// if n > dataBlockSize {
// n = dataBlockSize
// }
// // Feed dataBlockSize bytes to the compressor
// b := f.buf.Bytes()
// block := b[:n]
// rest := b[n:]
// /*
// f.compBuf.Reset()
// f.zlibWriter.Reset(f.compBuf)
// if _, err := f.zlibWriter.Write(block); err != nil {
// return err
// }
// if err := f.zlibWriter.Close(); err != nil {
// return err
// }
// size := f.compBuf.Len()
// if size > len(block) {
// // Copy uncompressed data: Linux returns i/o errors when it encounters a
// // compressed block which is larger than the uncompressed data:
// // https://github.com/torvalds/linux/blob/3ca24ce9ff764bc27bceb9b2fd8ece74846c3fd3/fs/squashfs/block.c#L150
// size = len(block) | (1 << 24) // SQUASHFS_COMPRESSED_BIT_BLOCK
// if _, err := f.w.w.Write(block); err != nil {
// return err
// }
// } else {
// if _, err := io.Copy(f.w.w, f.compBuf); err != nil {
// return err
// }
// }
// */
// // Copy uncompressed data: Linux returns i/o errors when it encounters a
// // compressed block which is larger than the uncompressed data:
// // https://github.com/torvalds/linux/blob/3ca24ce9ff764bc27bceb9b2fd8ece74846c3fd3/fs/squashfs/block.c#L150
// size := len(block) | (1 << 24) // SQUASHFS_COMPRESSED_BIT_BLOCK
// if _, err := f.w.w.Write(block); err != nil {
// return err
// }
// f.blocksizes = append(f.blocksizes, uint32(size))
// // Keep the rest in f.buf for the next write
// copy(b, rest)
// f.buf.Truncate(len(rest))
// return nil
// }
// // Close implements io.Closer
// func (f *file) Close() error {
// for f.buf.Len() > 0 {
// if err := f.writeBlock(); err != nil {
// return err
// }
// }
// startBlock := f.w.inodeBuf.Len() / metadataBlockSize
// offset := f.w.inodeBuf.Len() - startBlock*metadataBlockSize
// if err := binary.Write(&f.w.inodeBuf, binary.LittleEndian, lregInodeHeader{
// inodeHeader: inodeHeader{
// InodeType: lregType,
// Mode: f.mode,
// Uid: 0,
// Gid: 0,
// Mtime: int32(f.modTime.Unix()),
// InodeNumber: f.w.sb.Inodes + 1,
// },
// StartBlock: uint64(f.off),
// FileSize: uint64(f.size),
// Nlink: 1,
// Fragment: invalidFragment,
// Offset: 0,
// Xattr: f.xattrRef,
// }); err != nil {
// return err
// }
// if err := binary.Write(&f.w.inodeBuf, binary.LittleEndian, f.blocksizes); err != nil {
// return err
// }
// f.d.dirEntries = append(f.d.dirEntries, fullDirEntry{
// startBlock: uint32(startBlock),
// offset: uint16(offset),
// inodeNumber: f.w.sb.Inodes + 1,
// entryType: fileType,
// name: f.name,
// })
// f.w.sb.Inodes++
// return nil
// }
// func writeXattr(w io.Writer, xattrs []Xattr) error {
// for _, attr := range xattrs {
// if err := binary.Write(w, binary.LittleEndian, struct {
// Type uint16
// NameSize uint16
// }{
// Type: attr.Type,
// NameSize: uint16(len(attr.FullName)),
// }); err != nil {
// return err
// }
// if _, err := w.Write([]byte(attr.FullName)); err != nil {
// return err
// }
// if err := binary.Write(w, binary.LittleEndian, struct {
// ValSize uint32
// }{
// ValSize: uint32(len(attr.Value)),
// }); err != nil {
// return err
// }
// if _, err := w.Write(attr.Value); err != nil {
// return err
// }
// }
// return nil
// }
// type xattrTableHeader struct {
// XattrTableStart uint64
// XattrIds uint32
// Unused uint32
// }
// func (w *Writer) writeXattrTables() (int64, error) {
// if len(w.xattrs) == 0 {
// return -1, nil
// }
// off, err := w.w.Seek(0, io.SeekCurrent)
// if err != nil {
// return 0, err
// }
// xattrTableStart := uint64(off)
// var xattrBuf bytes.Buffer
// if err := writeXattr(&xattrBuf, w.xattrs); err != nil {
// return 0, err
// }
// xattrBlocks := (xattrBuf.Len() + (metadataBlockSize - 1)) / metadataBlockSize
// if err := w.writeMetadataChunks(&xattrBuf); err != nil {
// return 0, err
// }
// // write xattr id table
// off, err = w.w.Seek(0, io.SeekCurrent)
// if err != nil {
// return 0, err
// }
// idTableOff := uint64(off)
// var xattrIdBuf bytes.Buffer
// size := uint64(0)
// for _, id := range w.xattrIds {
// id.Xattr = uint64(size)
// size += uint64(id.Size) + 8 /* sizeof(Type+NameSize+ValSize) */
// if err := binary.Write(&xattrIdBuf, binary.LittleEndian, id); err != nil {
// return 0, err
// }
// }
// if err := w.writeMetadataChunks(&xattrIdBuf); err != nil {
// return 0, err
// }
// // xattr table header
// off, err = w.w.Seek(0, io.SeekCurrent)
// if err != nil {
// return 0, err
// }
// if err := binary.Write(w.w, binary.LittleEndian, xattrTableHeader{
// XattrTableStart: xattrTableStart,
// XattrIds: uint32(len(w.xattrs)),
// }); err != nil {
// return 0, err
// }
// // write block index
// for i := 0; i < xattrBlocks; i++ {
// if err := binary.Write(w.w, binary.LittleEndian, struct {
// BlockOffset uint64
// }{
// BlockOffset: idTableOff + (uint64(i) * (8192 + 2 /* sizeof(uint16) */)),
// }); err != nil {
// return 0, err
// }
// }
// return off, nil
// }
// // writeMetadataChunks copies from r to w in blocks of metadataBlockSize bytes
// // each, prefixing each block with a uint16 length header, setting the
// // uncompressed bit.
// func (w *Writer) writeMetadataChunks(r io.Reader) error {
// buf := make([]byte, metadataBlockSize)
// for {
// buf = buf[:metadataBlockSize]
// n, err := r.Read(buf)
// if err != nil {
// if err == io.EOF { // done
// return nil
// }
// return err
// }
// buf = buf[:n]
// if err := binary.Write(w.w, binary.LittleEndian, uint16(len(buf))|0x8000); err != nil {
// return err
// }
// if _, err := w.w.Write(buf); err != nil {
// return err
// }
// }
// }
// // Flush writes the SquashFS file system. The Writer must not be used after
// // calling Flush.
// func (w *Writer) Flush() error {
// // (1) superblock will be written later
// // (2) compressor-specific options omitted
// // (3) data has already been written
// // (4) write inode table
// off, err := w.w.Seek(0, io.SeekCurrent)
// if err != nil {
// return err
// }
// w.sb.InodeTableStart = off
// if err := w.writeMetadataChunks(&w.inodeBuf); err != nil {
// return err
// }
// // (5) write directory table
// off, err = w.w.Seek(0, io.SeekCurrent)
// if err != nil {
// return err
// }
// w.sb.DirectoryTableStart = off
// if err := w.writeMetadataChunks(&w.dirBuf); err != nil {
// return err
// }
// // (6) fragment table omitted
// off, err = w.w.Seek(0, io.SeekCurrent)
// if err != nil {
// return err
// }
// w.sb.FragmentTableStart = off
// // (7) export table omitted
// // (8) write uid/gid lookup table
// idTableStart, err := writeIdTable(w.w, []uint32{0})
// if err != nil {
// return err
// }
// w.sb.IdTableStart = idTableStart
// // (9) xattr table
// off, err = w.writeXattrTables()
// if err != nil {
// return err
// }
// w.sb.XattrIdTableStart = off
// off, err = w.w.Seek(0, io.SeekCurrent)
// if err != nil {
// return err
// }
// w.sb.BytesUsed = off
// // Pad to 4096, required for the kernel to be able to access all pages
// if pad := off % 4096; pad > 0 {
// padding := make([]byte, 4096-pad)
// if _, err := w.w.Write(padding); err != nil {
// return err
// }
// }
// // (1) Write superblock
// if _, err := w.w.Seek(0, io.SeekStart); err != nil {
// return err
// }
// return binary.Write(w.w, binary.LittleEndian, &w.sb)
// }
old/writer_test.go
-310
View File
@@ -1,310 +0,0 @@
package squashfs
// import (
// "bytes"
// "flag"
// "fmt"
// "io"
// "io/ioutil"
// "os"
// "os/exec"
// "path/filepath"
// "strings"
// "testing"
// "time"
// "github.com/distr1/distri"
// // "github.com/distr1/distri/internal/distritest"
// "github.com/google/go-cmp/cmp"
// "github.com/orcaman/writerseeker"
// "golang.org/x/sys/unix"
// )
// var fsImagePath = flag.String("fs_image_path", "", "Store the SquashFS test file system in the specified path for manual inspection")
// func writeTestImage(iow io.WriteSeeker, xattr bool) error {
// w, err := NewWriter(iow, time.Now())
// if err != nil {
// return err
// }
// var xattrs []Xattr
// if xattr {
// xattrs = append(xattrs, Xattr{
// Type: 2,
// FullName: "capability",
// Value: []byte{1, 0, 0, 2, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
// })
// }
// ff, err := w.Root.File("hellö wörld", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, xattrs)
// if err != nil {
// return err
// }
// if _, err := ff.Write([]byte("hello world!")); err != nil {
// return err
// }
// if err := ff.Close(); err != nil {
// return err
// }
// ff, err = w.Root.File("leer", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, nil)
// if err != nil {
// return err
// }
// if err := ff.Close(); err != nil {
// return err
// }
// ff, err = w.Root.File("second file", time.Now(), unix.S_IRUSR|unix.S_IXUSR|
// unix.S_IRGRP|unix.S_IXGRP|
// unix.S_IROTH|unix.S_IXOTH,
// nil)
// if err != nil {
// return err
// }
// if _, err := ff.Write([]byte("NON.\n")); err != nil {
// return err
// }
// if err := ff.Close(); err != nil {
// return err
// }
// if err := w.Root.Symlink("second file", "second link", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH); err != nil {
// return err
// }
// subdir := w.Root.Directory("subdir", time.Now())
// subsubdir := subdir.Directory("deep", time.Now())
// ff, err = subsubdir.File("yo", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, nil)
// if err != nil {
// return err
// }
// if _, err := ff.Write([]byte("foo\n")); err != nil {
// return err
// }
// if err := ff.Close(); err != nil {
// return err
// }
// if err := subsubdir.Flush(); err != nil {
// return err
// }
// // TODO: write another file in subdir now, will result in invalid parent inode
// ff, err = subdir.File("third file (in subdir)", time.Now(), unix.S_IRUSR|unix.S_IRGRP|unix.S_IROTH, nil)
// if err != nil {
// return err
// }
// if _, err := ff.Write([]byte("contents\n")); err != nil {
// return err
// }
// if err := ff.Close(); err != nil {
// return err
// }
// if err := subdir.Flush(); err != nil {
// return err
// }
// ff, err = w.Root.File("testbin", time.Now(), unix.S_IRUSR|unix.S_IXUSR|
// unix.S_IRGRP|unix.S_IXGRP|
// unix.S_IROTH|unix.S_IXOTH,
// nil)
// if err != nil {
// return err
// }
// zf, err := os.Open(os.Args[0])
// if err != nil {
// return err
// }
// defer zf.Close()
// if _, err := io.Copy(ff, zf); err != nil {
// return err
// }
// if err := ff.Close(); err != nil {
// return err
// }
// if err := w.Root.Flush(); err != nil {
// return err
// }
// if err := w.Flush(); err != nil {
// return err
// }
// return nil
// }
// func TestUnsquashfs(t *testing.T) {
// t.Parallel()
// ctx, canc := distri.InterruptibleContext()
// defer canc()
// if _, err := exec.LookPath("unsquashfs"); err != nil {
// t.Skip("unsquashfs not found in $PATH")
// }
// for _, xattr := range []bool{false, true} {
// t.Run(fmt.Sprintf("xattr %v", xattr), func(t *testing.T) {
// var (
// f *os.File
// err error
// )
// if *fsImagePath != "" {
// f, err = os.Create(*fsImagePath + fmt.Sprintf("-xattr-%v", xattr))
// } else {
// f, err = ioutil.TempFile("", fmt.Sprintf("squashfs-xattr-%v", xattr))
// if err == nil {
// defer os.Remove(f.Name())
// }
// }
// if err != nil {
// t.Fatal(err)
// }
// if err := writeTestImage(f, xattr); err != nil {
// t.Fatal(err)
// }
// if err := f.Close(); err != nil {
// t.Fatal(err)
// }
// // Extract our generated file system using unsquashfs(1)
// out, err := ioutil.TempDir("", fmt.Sprintf("unsquashfs-xattr-%v", xattr))
// if err != nil {
// t.Fatal(err)
// }
// // defer distritest.RemoveAll(t, out)
// cmd := exec.CommandContext(ctx, "unsquashfs", "-no-xattrs", "-d", filepath.Join(out, "x"), f.Name())
// cmd.Stderr = os.Stderr
// if err := cmd.Run(); err != nil {
// t.Fatal(err)
// }
// fbin, err := os.Open(os.Args[0])
// if err != nil {
// t.Fatal(err)
// }
// // Verify the extracted files match our expectations.
// for _, entry := range []struct {
// path string
// contents io.Reader
// }{
// {"leer", strings.NewReader("")},
// {"hellö wörld", strings.NewReader("hello world!")},
// {"testbin", fbin},
// {"subdir/third file (in subdir)", strings.NewReader("contents\n")},
// } {
// entry := entry // copy
// t.Run(entry.path, func(t *testing.T) {
// in, err := os.Open(filepath.Join(out, "x", entry.path))
// if err != nil {
// t.Fatal(err)
// }
// got, err := ioutil.ReadAll(in)
// if err != nil {
// t.Fatal(err)
// }
// want, err := ioutil.ReadAll(entry.contents)
// if err != nil {
// t.Fatal(err)
// }
// if !bytes.Equal(got, want) {
// t.Fatalf("path %q differs", entry.path)
// }
// })
// }
// })
// }
// }
// func TestReader(t *testing.T) {
// t.Parallel()
// for _, xattr := range []bool{false, true} {
// t.Run(fmt.Sprintf("xattr %v", xattr), func(t *testing.T) {
// var err error
// buf := &writerseeker.WriterSeeker{}
// if err := writeTestImage(buf, xattr); err != nil {
// t.Fatal(err)
// }
// if _, err := buf.Seek(0, io.SeekCurrent); err != nil {
// t.Fatal(err)
// }
// rd, err := NewReader(buf.BytesReader())
// if err != nil {
// t.Fatal(err)
// }
// fbin, err := os.Open(os.Args[0])
// if err != nil {
// t.Fatal(err)
// }
// // Verify the extracted files match our expectations.
// for _, entry := range []struct {
// path string
// contents io.Reader
// }{
// {"leer", strings.NewReader("")},
// {"hellö wörld", strings.NewReader("hello world!")},
// {"testbin", fbin},
// {"subdir/third file (in subdir)", strings.NewReader("contents\n")},
// } {
// entry := entry // copy
// t.Run(entry.path, func(t *testing.T) {
// // TODO: is this t.Parallel()-safe?
// inode, err := rd.LookupPath(entry.path)
// if err != nil {
// t.Fatal(err)
// }
// in, err := rd.FileReader(inode)
// if err != nil {
// t.Fatal(err)
// }
// got, err := ioutil.ReadAll(in)
// if err != nil {
// t.Fatal(err)
// }
// want, err := ioutil.ReadAll(entry.contents)
// if err != nil {
// t.Fatal(err)
// }
// if !bytes.Equal(got, want) {
// t.Fatalf("path %q differs", entry.path)
// }
// })
// }
// if xattr {
// t.Run("xattrs", func(t *testing.T) {
// inode, err := rd.LookupPath("hellö wörld")
// if err != nil {
// t.Fatal(err)
// }
// xattrs, err := rd.ReadXattrs(inode)
// if err != nil {
// t.Fatal(err)
// }
// if got, want := len(xattrs), 1; got != want {
// t.Fatalf("unexpected number of extended attributes: got %d, want %d", got, want)
// }
// wantXattr := Xattr{
// Type: 2,
// FullName: "security.capability",
// Value: []byte{1, 0, 0, 2, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
// }
// if diff := cmp.Diff(wantXattr, xattrs[0]); diff != "" {
// t.Errorf("unexpected extended attribute: diff (-want +got):\n%s", diff)
// }
// })
// }
// })
// }
// }
reader.go
-59
View File
@@ -1,59 +0,0 @@
package squashfs
import (
"errors"
"io"
)
//TODO: possible custom reader because I'm havng some issuse...
//Reader is a reader which implements Reader, ReaderAt, and Seeker, all with an accesible offset (for reasons)
type Reader struct {
rdr io.ReaderAt
offset int64
}
//NewReader creates a squashfs.Reader from a io.ReaderAt
func NewReader(baseReader io.ReaderAt) Reader {
return Reader{
rdr: baseReader,
offset: 0,
}
}
//Read reads len(byt) into byt. Advances the internal offset
func (r *Reader) Read(byt []byte) (int, error) {
n, err := r.rdr.ReadAt(byt, r.offset)
r.offset += int64(n)
return n, err
}
//ReadAt wraps the internal io.ReadAt's function. DOES NOT advance the internal offset for Read function.
//Returns how many bytes were read.
func (r *Reader) ReadAt(byt []byte, offset int64) (int, error) {
return r.rdr.ReadAt(byt, offset)
}
//ReadAtFromOffset is the same as ReadAt, but the given offset is offset by the internal offset. DOES NOT advance the internal offset.
//Returns how many bytes were read.
func (r *Reader) ReadAtFromOffset(byt []byte, offset int64) (int, error) {
offset += r.offset
return r.rdr.ReadAt(byt, offset)
}
//Seek advances the internal offset. SeekEnd DOES NOT work
//Might not be necessary, but here just in case
func (r *Reader) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekCurrent:
n, err := r.Read(make([]byte, offset))
return int64(n), err
case io.SeekStart:
r.offset = 0
n, err := r.Read(make([]byte, offset))
return int64(n), err
case io.SeekEnd:
return 0, errors.New("SeekEnd is NOT supported")
}
return 0, errors.New("incorrect whence")
}
squashfs_test.go → squash_test.go
+11 -13
View File
@@ -6,7 +6,7 @@ import (
"os"
"testing"
appimage "github.com/CalebQ42/GoAppImage"
goappimage "github.com/CalebQ42/GoAppImage"
)
const (
@@ -15,27 +15,30 @@ const (
squashfsName = "Code_OSS.Squashfs"
)
func TestAppImageSquash(t *testing.T) {
func TestMain(t *testing.T) {
t.Parallel()
wd, err := os.Getwd()
if err != nil {
t.Error(err)
t.Fatal(err)
}
squashFil, err := os.Open(wd + "/testing/" + squashfsName)
if os.IsNotExist(err) {
TestCreateSquashFromAppImage(t)
squashFil, err = os.Open(wd + "/testing/" + squashfsName)
if err != nil {
t.Error(err)
t.Fatal(err)
}
}
defer squashFil.Close()
stat, _ := squashFil.Stat()
squash, err := NewSquashfs(io.NewSectionReader(squashFil, 0, stat.Size()))
rdr, err := NewSquashfsReader(io.NewSectionReader(squashFil, 0, stat.Size()))
if err != nil {
t.Error(err)
t.Fatal(err)
}
err = rdr.readRootDirTable()
if err != nil {
t.Fatal(err)
}
err = squash.printDirTable()
t.Fatal(err)
}
@@ -58,7 +61,7 @@ func TestCreateSquashFromAppImage(t *testing.T) {
} else if err != nil {
t.Fatal(err)
}
ai := appimage.NewAppImage(wd + "/testing/" + appImageName)
ai := goappimage.NewAppImage(wd + "/testing/" + appImageName)
aiFil, err := os.Open(wd + "/testing/" + appImageName)
if err != nil {
t.Fatal(err)
@@ -99,8 +102,3 @@ func downloadTestAppImage(t *testing.T, dir string) {
t.Fatal(err)
}
}
func TestLookInsideSquash(t *testing.T) {
t.Parallel()
//TODO
}
squashfs.go
-74
View File
@@ -1,74 +0,0 @@
package squashfs
import (
"encoding/binary"
"fmt"
"io"
"github.com/CalebQ42/GoSquashfs/internal/directory"
"github.com/CalebQ42/GoSquashfs/internal/inode"
)
type Squashfs struct {
r Reader
super Superblock
flags SuperblockFlags
compression CompressionOptions
}
func NewSquashfs(rdr io.ReaderAt) (*Squashfs, error) {
var squash Squashfs
squash.r = NewReader(rdr)
err := binary.Read(&squash.r, binary.LittleEndian, &squash.super)
if err != nil {
return nil, err
}
squash.flags = squash.super.getFlags()
switch squash.super.Compression {
case gzipCompression:
var raw gzipOptionsRaw
err := binary.Read(&squash.r, binary.LittleEndian, &raw)
if err != nil {
return nil, err
}
squash.compression = NewGzipOptions(raw)
default:
fmt.Println("Other compression options are not currently supported")
return nil, err
}
return &squash, nil
}
func (s *Squashfs) printDirTable() error {
offset, metaOffset := inode.ProcessInodeRef(s.super.RootInode)
br, err := s.NewBlockReader(int64(offset))
if err != nil {
return err
}
fmt.Println(offset, metaOffset)
br.dataOffset = int64(metaOffset)
_, inodeType, err := inode.ProcessInode(br, s.super.BlockSize)
if err != nil {
return err
}
rootDir := inodeType.(*inode.BasicDirectory)
fmt.Println(*rootDir)
br, err = s.NewBlockReader(int64(s.super.DirectoryTableOffset) + int64(rootDir.DirectoryIndex))
if err != nil {
return err
}
br.dataOffset = int64(rootDir.DirectoryOffset)
dir, err := directory.NewDirectory(br)
if err != nil {
return err
}
for _, entry := range dir.Entries {
fmt.Println(entry.Name)
}
return nil
}
//GetFlags returns the SuperblockFlags from the Superblock
func (s *Squashfs) GetFlags() SuperblockFlags {
return s.flags
}
squashfsreader.go
+75
View File
@@ -0,0 +1,75 @@
package squashfs
import (
"encoding/binary"
"errors"
"fmt"
"io"
"github.com/CalebQ42/GoSquashfs/internal/directory"
"github.com/CalebQ42/GoSquashfs/internal/inode"
)
const (
magic = 0x73717368
)
type Reader struct {
r io.ReaderAt
super Superblock
flags SuperblockFlags
decompressor Decompressor
}
func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
var rdr Reader
rdr.r = r
err := binary.Read(io.NewSectionReader(rdr.r, 0, int64(binary.Size(rdr.super))), binary.LittleEndian, &rdr.super)
if err != nil {
return nil, err
}
if rdr.super.Magic != magic {
return nil, errors.New("doesn't have magic number, probably isn't a squashfs")
}
rdr.flags = rdr.super.GetFlags()
switch rdr.super.CompressionType {
case gzipCompression:
rdr.decompressor = &ZlibDecompressor{}
default:
return nil, errors.New("Unsupported compression type")
}
if rdr.flags.CompressorOptions {
//TODO: parse compressor options
fmt.Println("Compressor options is NOT currently supported")
return nil, errors.New("Has compressor options")
}
return &rdr, nil
}
func (r *Reader) readRootDirTable() error {
offset, blockOffset := processInodeRef(r.super.RootInodeRef)
br, err := r.NewBlockReader(int64(r.super.InodeTableStart + offset))
if err != nil {
return err
}
_, err = br.Seek(int64(blockOffset), io.SeekStart)
if err != nil {
return err
}
i, err := inode.ProcessInode(br, r.super.BlockSize)
if err != nil {
return err
}
dirRdr, err := r.NewBlockReader(int64(r.super.DirTableStart + uint64(i.Info.(inode.BasicDirectory).DirectoryIndex)))
if err != nil {
return err
}
dir, err := directory.NewDirectory(dirRdr)
if err != nil {
return err
}
for _, entry := range dir.Entries {
fmt.Println(string(entry.Name))
}
return nil
}
superblock.go
+34 -30
View File
@@ -1,48 +1,52 @@
package squashfs
const squashfsMagic = 0x73717368
const (
gzipCompression = 1 + iota
lzmaCompression
lzoCompression
xzCompression
lz4Compression
zstdCompression
)
//Superblock is a raw representation of a squashfs
//Descriptions provided by https://dr-emann.github.io/squashfs/
type Superblock struct {
Magic uint32 //Magic will be 0x73717368 if it's a legit Squashfs filesystem
Inodes uint32 //Inodes is the number of inodes in the inodes table
MkfsTime uint32 //MkfsTime is when archive was created
BlockSize uint32 //BlockSize is the size of data blocks in bytes
Fragments uint32 //Fragments is the number of entries in fragment table
Compression uint16 //Compression is what type of compression is used
BlockLog uint16 //BlockLog should be log base 2 of BlockSize. If not then the squash might be corrupt
Flags uint16 //Flags are the superblock's flags
IDCount uint16 //IDCount is the number of IDs in the id lookup table
Major uint16 //Major version of squashfs format
Minor uint16 //Minor version of squashfs format
RootInode uint64 //RootInode is a reference to the root of the squashfs
BytesUsed uint64 //BytesUsed is how many bytes the archive is. squashfs archives are often padded to 4KB.
IDTableOffset uint64 //IDTableOff is the byte offset of the IDTable
XattrIDTableOffset uint64 //XattrIDTableOffset is the byte offset of the xattr id table
InodeTableOffset uint64 //InodeTableOffset is the byte offset of the inode table
DirectoryTableOffset uint64 //DirectoryTableOffset is the byte offset of the directory table
FragmentTableOffset uint64 //FragmentTableOffset is the byte offset of the fragment table
ExportTableOffset uint64 //ExportTableOffset is the byte offset of the export table
Magic uint32
InodeCount uint32
CreationTime uint32
BlockSize uint32
FragCount uint32
CompressionType uint16
BlockLog uint16
Flags uint16
IDCount uint16
MajorVersion uint16
MinorVersion uint16
RootInodeRef uint64
BytesUsed uint64
IDTableStart uint64
XattrTableStart uint64
InodeTableStart uint64
DirTableStart uint64
FragTableStart uint64
ExportTableStart uint64
}
//SuperblockFlags is a parsed list of options set in Superblock.Flags
type SuperblockFlags struct {
UncompressedInodes bool
UncompressedData bool
Check bool //Check is unused in current versions of squashfs
Check bool
UncompressedFragments bool
NoFragments bool
AlwaysFragments bool
Duplicates bool //Identical files are stored only once
Duplicates bool
Exportable bool
UncompressedXattrs bool
NoXattrs bool
UncompressedXattr bool
NoXattr bool
CompressorOptions bool
UncompressedIDs bool
}
func (s *Superblock) getFlags() SuperblockFlags {
func (s *Superblock) GetFlags() SuperblockFlags {
return SuperblockFlags{
UncompressedInodes: s.Flags&0x1 == 0x1,
UncompressedData: s.Flags&0x2 == 0x2,
@@ -52,8 +56,8 @@ func (s *Superblock) getFlags() SuperblockFlags {
AlwaysFragments: s.Flags&0x20 == 0x20,
Duplicates: s.Flags&0x40 == 0x40,
Exportable: s.Flags&0x80 == 0x80,
UncompressedXattrs: s.Flags&0x100 == 0x100,
NoXattrs: s.Flags&0x200 == 0x200,
UncompressedXattr: s.Flags&0x100 == 0x100,
NoXattr: s.Flags&0x200 == 0x200,
CompressorOptions: s.Flags&0x400 == 0x400,
UncompressedIDs: s.Flags&0x800 == 0x800,
}
internal/inode/util.go → utils.go
+5 -3
View File
@@ -1,9 +1,11 @@
package inode
package squashfs
//ProcessInodeRef processes an inode reference and returns two values
//The first value is the inode table offset. AKA, it's where the metadata block of the inode STARTS.
//
//The first value is the inode table offset. AKA, it's where the metadata block of the inode STARTS relative to the inode table.
//
//The second value is the offset of the inode, INSIDE of the metadata.
func ProcessInodeRef(inodeRef uint64) (tableOffset uint64, metaOffset uint64) {
func processInodeRef(inodeRef uint64) (tableOffset uint64, metaOffset uint64) {
tableOffset = inodeRef >> 16
metaOffset = inodeRef &^ 0xFFFFFFFF0000
return