Belac/squashfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 55 Wiki Activity

Directorty table info parsing.

Browse Source 
Futher work on reading data.
This commit is contained in:
Caleb Gardner
2020-11-13 04:47:54 -06:00
parent f135457443
commit dbf7e9465a
9 changed files with 288 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+5 -1
View File
@@ -18,4 +18,8 @@ I am focusing purely on unsquashing before squashing.
* Reading Inodes
* Reading the Directory structure
* Implement other compression types
* Squashing
* Squashing
# Where I'm at
* I can read the metadata, but can't read inodes just yet.
compressionoptions.go
+19 -16
View File
@@ -1,12 +1,14 @@
package squashfs
import (
"compress/zlib"
"compress/gzip"
"io"
"gopkg.in/src-d/go-git.v4/utils/ioutil"
)
const (
zlibCompression = 1 + iota
gzipCompression = 1 + iota
lzmaCompression
lzoCompression
xzCompression
@@ -95,13 +97,13 @@ func NewGzipOptions(raw gzipOptionsRaw) *GzipOptions {
}
func (gzipOp *GzipOptions) Decompress(rdr *io.SectionReader, blockSize int) ([]byte, error) {
zlibRdr, err := zlib.NewReader(rdr)
defer zlibRdr.Close()
gzipRdr, err := gzip.NewReader(rdr)
defer gzipRdr.Close()
if err != nil {
return nil, err
}
bytrw := newByteReadWrite(0)
_, err = io.Copy(bytrw, zlibRdr)
_, err = io.Copy(bytrw, gzipRdr)
if err != nil {
return bytrw.byts, err
}
@@ -109,20 +111,20 @@ func (gzipOp *GzipOptions) Decompress(rdr *io.SectionReader, blockSize int) ([]b
}
func (gzipOp *GzipOptions) DecompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) {
zlibRdr, err := zlib.NewReader(*rdr)
defer zlibRdr.Close()
gzipRdr, err := gzip.NewReader(*rdr)
defer gzipRdr.Close()
if err != nil {
return 0, err
}
n, err := io.Copy(*wrt, zlibRdr)
n, err := io.Copy(*wrt, gzipRdr)
return int(n), err
}
func (gzipOp *GzipOptions) Compress(rdr *io.SectionReader, blockSize int) ([]byte, error) {
bytWrt := newByteReadWrite(0)
zlibWrt := zlib.NewWriter(bytWrt) //TODO: allow setting level
defer zlibWrt.Close()
_, err := io.Copy(zlibWrt, rdr)
gzipWrt := gzip.NewWriter(bytWrt) //TODO: allow setting level
defer gzipWrt.Close()
_, err := io.Copy(gzipWrt, rdr)
if err != nil {
return bytWrt.byts, err
}
@@ -130,15 +132,16 @@ func (gzipOp *GzipOptions) Compress(rdr *io.SectionReader, blockSize int) ([]byt
}
func (gzipOp *GzipOptions) CompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) {
zlibWrt := zlib.NewWriter(*wrt) //TODO: allow setting level
defer zlibWrt.Close()
n, err := io.Copy(zlibWrt, *rdr)
gzipWrt := gzip.NewWriter(*wrt) //TODO: allow setting level
defer gzipWrt.Close()
n, err := io.Copy(gzipWrt, *rdr)
return int(n), err
}
func (gzipOp *GzipOptions) Reader(rdr io.Reader) (*io.ReadCloser, error) {
read, err := zlib.NewReader(rdr)
return &read, err
read, err := gzip.NewReader(rdr)
redClo := ioutil.NewReadCloser(read, read)
return &redClo, err
}
type xzOptionsRaw struct {
internal/directory/directory.go
+83
View File
@@ -0,0 +1,83 @@
package directory
import (
"encoding/binary"
"fmt"
"io"
)
//Header is the header for a directory in the directory table
type Header struct {
Count uint32
InodeOffset uint32
InodeNumber uint32
}
//EntryInit is the values that can be easily decoded
type EntryInit struct {
Offset uint16
InodeOffset int16
Type uint16
NameSize uint16
}
//Entry is an entry in a directory.
type Entry struct {
Init EntryInit
Name []byte
}
//NewEntry creates a new directory entry
func NewEntry(rdr io.Reader) (Entry, error) {
var entry Entry
err := binary.Read(rdr, binary.LittleEndian, entry.Init)
if err != nil {
return entry, err
}
entry.Name = make([]byte, entry.Init.NameSize, entry.Init.NameSize)
err = binary.Read(rdr, binary.LittleEndian, entry.Name)
return entry, err
}
//Directory is an entry in the directory table of a squashfs.
//Will only have multiple headers if there are more then 256 entries
type Directory struct {
Headers []Header
Entries []Entry
}
//NewDirectory reads the directory from rdr
func NewDirectory(rdr io.Reader) (*Directory, error) {
var dir Directory
var hdr Header
err := binary.Read(rdr, binary.LittleEndian, &hdr)
if err != nil {
return nil, err
}
headers := hdr.Count / 256
if headers%256 > 0 {
headers++
}
headersRead := 1
dir.Headers = make([]Header, headers)
dir.Headers[0] = hdr
for i := uint32(0); i < hdr.Count; i++ {
if i != 0 && i%256 == 0 {
var newHdr Header
err = binary.Read(rdr, binary.LittleEndian, &newHdr)
if err != nil {
fmt.Println("Error processing header ", headersRead)
return &dir, err
}
dir.Headers[headersRead] = newHdr
headersRead++
}
ent, err := NewEntry(rdr)
if err != nil {
fmt.Println("Error processing entry ", len(dir.Entries))
return &dir, err
}
dir.Entries = append(dir.Entries, ent)
}
return &dir, nil
}
internal/inode/inode.go
+58 -18
View File
@@ -2,11 +2,12 @@ package inode
import (
"encoding/binary"
"fmt"
"io"
)
//InodeCommon is the comon header for all inodes
type InodeCommon struct {
//Common is the comon header for all inodes
type Common struct {
InodeType uint16
Permissions uint16
UID uint16
@@ -37,25 +38,61 @@ type ExtendedDirectoryInit struct {
//ExtendedDirectory is a directory with extra info
type ExtendedDirectory struct {
Init ExtendedDirectoryInit
//TODO: indexes []DirectoryIndex
Init ExtendedDirectoryInit
Indexes []DirectoryIndex
}
//NewExtendedDirectory creates a new ExtendedDirectory
func NewExtendedDirectory(rdr *io.Reader) (*ExtendedDirectory, error) {
var inode ExtendedDirectory
err := binary.Read(*rdr, binary.LittleEndian, inode.Init)
//TODO: Read directory indexes
if err != nil {
return &inode, err
}
if inode.Init.IndexCount > 0 {
inode.Indexes = make([]DirectoryIndex, inode.Init.IndexCount)
for i := uint16(0); i < inode.Init.IndexCount; i++ {
inode.Indexes[i], err = NewDirectoryIndex(rdr)
if err != nil {
fmt.Println("Error while reading Directory Index ", i)
return &inode, err
}
}
}
return &inode, err
}
//DirectoryIndexInit holds the values that can be easily decoded
type DirectoryIndexInit struct {
Offset uint32
DirTableOffset uint32
NameSize uint32
}
//DirectoryIndex is a quick lookup provided by an ExtendedDirectory
type DirectoryIndex struct {
Init DirectoryIndexInit
Name []byte
}
//NewDirectoryIndex return a new DirectoryIndex
func NewDirectoryIndex(rdr *io.Reader) (DirectoryIndex, error) {
var index DirectoryIndex
err := binary.Read(*rdr, binary.LittleEndian, index.Init)
if err != nil {
return index, err
}
index.Name = make([]byte, index.Init.NameSize, index.Init.NameSize)
err = binary.Read(*rdr, binary.LittleEndian, index.Name)
return index, err
}
//BasicFileInit is the information that can be directoy decoded
type BasicFileInit struct {
BlockStart uint32
FragmentIndex uint32
FragmentOffset uint32
Size uint32
//TODO: possibly fix BlockSizes
}
//BasicFile is self explainatory
@@ -89,7 +126,6 @@ type ExtendedFileInit struct {
FragmentIndex uint32
FragmentOffset uint32
XattrIndex uint32
//TODO: possibly fix BlockSizes
}
//ExtendedFile is a file with more information
@@ -99,11 +135,11 @@ type ExtendedFile struct {
}
//NewExtendedFile creates a new ExtendedFile
func NewExtendedFile(rdr *io.Reader, blockSize uint32) (*ExtendedFile, error) {
func NewExtendedFile(rdr *io.Reader, blockSize uint32) (ExtendedFile, error) {
var inode ExtendedFile
err := binary.Read(*rdr, binary.LittleEndian, inode.Init)
if err != nil {
return &inode, err
return inode, err
}
blocks := inode.Init.Size / blockSize
if inode.Init.Size%blockSize > 0 {
@@ -111,7 +147,7 @@ func NewExtendedFile(rdr *io.Reader, blockSize uint32) (*ExtendedFile, error) {
}
inode.BlockSizes = make([]uint32, blocks, blocks)
err = binary.Read(*rdr, binary.LittleEndian, inode.BlockSizes)
return &inode, err
return inode, err
}
//BasicSymlinkInit is all the values that can be directly decoded
@@ -123,19 +159,19 @@ type BasicSymlinkInit struct {
//BasicSymlink is a symlink
type BasicSymlink struct {
Init BasicSymlinkInit
targetPath []uint8 //len is TargetPathSize
targetPath []byte //len is TargetPathSize
}
//NewBasicSymlink creates a new BasicSymlink
func NewBasicSymlink(rdr *io.Reader) (*BasicSymlink, error) {
func NewBasicSymlink(rdr *io.Reader) (BasicSymlink, error) {
var inode BasicSymlink
err := binary.Read(*rdr, binary.LittleEndian, inode.Init)
if err != nil {
return nil, err
return inode, err
}
inode.targetPath = make([]uint8, inode.Init.TargetPathSize, inode.Init.TargetPathSize)
inode.targetPath = make([]byte, inode.Init.TargetPathSize, inode.Init.TargetPathSize)
err = binary.Read(*rdr, binary.LittleEndian, inode.targetPath)
return &inode, err
return inode, err
}
//ExtendedSymlinkInit is all the values that can be directly decoded
@@ -152,31 +188,35 @@ type ExtendedSymlink struct {
}
//NewExtendedSymlink creates a new ExtendedSymlink
func NewExtendedSymlink(rdr *io.Reader) (*ExtendedSymlink, error) {
func NewExtendedSymlink(rdr *io.Reader) (ExtendedSymlink, error) {
var inode ExtendedSymlink
err := binary.Read(*rdr, binary.LittleEndian, inode.Init)
if err != nil {
return &inode, err
return inode, err
}
inode.TargetPath = make([]uint8, inode.Init.TargetPathSize, inode.Init.TargetPathSize)
err = binary.Read(*rdr, binary.LittleEndian, &inode.XattrIndex)
return &inode, err
return inode, err
}
//BasicDevice is a device
type BasicDevice struct {
HardLinks uint32
Device uint32
}
//ExtendedDevice is a device with more info
type ExtendedDevice struct {
BasicDevice
XattrIndex uint32
}
//BasicIPC is a Fifo or Socket device
type BasicIPC struct {
HardLink uint32
}
//ExtendedIPC is a IPC device with extra info
type ExtendedIPC struct {
BasicIPC
XattrIndex uint32
internal/inode/processInode.go
+65 -62
View File
@@ -8,80 +8,83 @@ import (
)
const (
basicDirectory = iota + 1
basicFile
basicSymlink
basicBlockDevice
basicCharDevice
basicFifo
basicSocket
extendedDirectory
extendedFile
extendedSymlink
extendedBlockDevice
extendedCharDevice
extendedFifo
extendedSocket
//The inode type from inode.Common.InodeType
BasicDirectoryType = iota + 1
BasicFileType
BasicSymlinkType
BasicBlockDeviceType
BasicCharDeviceType
BasicFifoType
BasicSocketType
ExtendedDirectoryType
ExtendedFileType
ExtendedSymlinkType
ExtendedBlockDeviceType
ExtendedCharDeviceType
ExtendedFifoType
ExtendedSocketType
)
func ProcessInode(rdr *io.Reader) (*InodeCommon, interface{}, error) {
var inodeHeader InodeCommon
//ProcessInode processes the next inode in the given reader
func ProcessInode(rdr *io.Reader, blockSize uint32) (*Common, interface{}, error) {
var inodeHeader Common
err := binary.Read(*rdr, binary.LittleEndian, &inodeHeader)
if err != nil {
return nil, nil, err
}
switch inodeHeader.InodeType {
case basicDirectory:
case BasicDirectoryType:
var inode BasicDirectory
err = binary.Read(*rdr, binary.LittleEndian, &inode)
return &inodeHeader, &inode, err
case BasicFileType:
inode, err := NewBasicFile(rdr, blockSize)
return &inodeHeader, inode, err
case basicFile:
var inode BasicFile
err = binary.Read(*rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case basicSymlink:
case BasicSymlinkType:
inode, err := NewBasicSymlink(rdr)
return &inodeHeader, inode, err
// case basicFile:
// var inode BasicFile
// err = binary.Read(*rdr, binary.LittleEndian, &inode)
// return &inodeHeader, inode, err
// case basicFile:
// var inode BasicFile
// err = binary.Read(*rdr, binary.LittleEndian, &inode)
// return &inodeHeader, inode, err
// case basicFile:
// var inode BasicFile
// err = binary.Read(*rdr, binary.LittleEndian, &inode)
// return &inodeHeader, inode, err
// case basicFile:
// var inode BasicFile
// err = binary.Read(*rdr, binary.LittleEndian, &inode)
// return &inodeHeader, inode, err
// case basicFile:
// var inode BasicFile
// err = binary.Read(*rdr, binary.LittleEndian, &inode)
// return &inodeHeader, inode, err
// case basicFile:
// var inode BasicFile
// err = binary.Read(*rdr, binary.LittleEndian, &inode)
// return &inodeHeader, inode, err
// case basicFile:
// var inode BasicFile
// err = binary.Read(*rdr, binary.LittleEndian, &inode)
// return &inodeHeader, inode, err
// case basicFile:
// var inode BasicFile
// err = binary.Read(*rdr, binary.LittleEndian, &inode)
// return &inodeHeader, inode, err
// case basicFile:
// var inode BasicFile
// err = binary.Read(*rdr, binary.LittleEndian, &inode)
// return &inodeHeader, inode, err
// case basicFile:
// var inode BasicFile
// err = binary.Read(*rdr, binary.LittleEndian, &inode)
// return &inodeHeader, inode, err
case BasicBlockDeviceType:
var inode BasicDevice
err = binary.Read(*rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case BasicCharDeviceType:
var inode BasicDevice
err = binary.Read(*rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case BasicFifoType:
var inode BasicIPC
err = binary.Read(*rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case BasicSocketType:
var inode BasicIPC
err = binary.Read(*rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case ExtendedDirectoryType:
inode, err := NewExtendedDirectory(rdr)
return &inodeHeader, inode, err
case ExtendedFileType:
inode, err := NewExtendedFile(rdr, blockSize)
return &inodeHeader, inode, err
case ExtendedSymlinkType:
inode, err := NewExtendedSymlink(rdr)
return &inodeHeader, inode, err
case ExtendedBlockDeviceType:
var inode ExtendedDevice
err = binary.Read(*rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case ExtendedCharDeviceType:
var inode ExtendedDevice
err = binary.Read(*rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case ExtendedFifoType:
var inode ExtendedIPC
err = binary.Read(*rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
case ExtendedSocketType:
var inode ExtendedIPC
err = binary.Read(*rdr, binary.LittleEndian, &inode)
return &inodeHeader, inode, err
//TODO: implement ALL cases
default:
return nil, nil, errors.New("Inode type is unrecognized: " + strconv.FormatInt(int64(inodeHeader.InodeType), 2))
internal/inode/util.go
+10
View File
@@ -0,0 +1,10 @@
package inode
//ProcessInodeRef processes an inode reference and returns two values
//The first value is the inode table offset. AKA, it's where the metadata block of the inode STARTS.
//The second value is the offset of the inode, INSIDE of the metadata.
func ProcessInodeRef(inodeRef uint64) (tableOffset uint32, metaOffset uint16) {
tableOffset = uint32(inodeRef >> 16)
metaOffset = uint16(inodeRef &^ 0xFFFFFFFF0000)
return
}
reader.go
-3
View File
@@ -9,9 +9,6 @@ import (
//Reader is a reader which implements Reader, ReaderAt, and Seeker, all with an accesible offset (for reasons)
type Reader struct {
io.Reader
io.ReaderAt
io.Seeker
rdr io.ReaderAt
offset int64
}
squashfs.go
+45 -11
View File
@@ -3,7 +3,10 @@ package squashfs
import (
"encoding/binary"
"errors"
"fmt"
"io"
"github.com/CalebQ42/GoSquashfs/internal/inode"
)
var (
@@ -34,7 +37,7 @@ func NewSquashfs(reader io.ReaderAt) (*Squashfs, error) {
flags := superblock.GetFlags()
var compressionOptions CompressionOptions
switch superblock.Compression {
case zlibCompression:
case gzipCompression:
if flags.CompressorOptions {
var gzipOpRaw gzipOptionsRaw
err = binary.Read(&rdr, binary.LittleEndian, &gzipOpRaw)
@@ -69,9 +72,39 @@ func NewSquashfs(reader io.ReaderAt) (*Squashfs, error) {
}, nil
}
func (s *Squashfs) readRootDirectoryTable() error {
offset, metaOffset := inode.ProcessInodeRef(s.super.RootInode)
meta, err := s.parseMetadataAt(int64(s.super.InodeTableOffset) + int64(offset))
if err != nil {
fmt.Println("Error processing metadata")
return err
}
_, err = meta.Data.Read(make([]byte, metaOffset))
if err != nil {
fmt.Println("error reading forward to offset")
return err
}
common, _, err := inode.ProcessInode(&meta.Data, s.super.BlockSize)
if err != nil {
fmt.Println("Error reading inode")
return err
}
if common.InodeType != inode.BasicDirectoryType {
return errors.New("Not a basic directory")
}
// dirTable, err := directory.NewDirectory(meta.Data)
// if err != nil {
// return err
// }
// for _, entry := range dirTable.Entries {
// fmt.Println(entry.Name)
// }
return nil
}
//GetFlags return the SuperblockFlags of the Squashfs
func (s *Squashfs) GetFlags() SuperblockFlags {
return s.super.GetFlags()
return s.flags
}
//metadata is a parsed metadata block
@@ -90,13 +123,14 @@ func (m *metadata) close() {
func (s *Squashfs) parseNextMetadata() (*metadata, error) {
var metaHeader uint16
err := binary.Read(s.rdr, binary.LittleEndian, metaHeader)
err := binary.Read(s.rdr, binary.LittleEndian, &metaHeader)
if err != nil {
return nil, err
}
reader := io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(metaHeader))
if metaHeader&0x8000 == 0x8000 {
metaHeader = metaHeader &^ 0x8000
compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, s.rdr.offset, int64(metaHeader)))
compressRead, err := s.compressionOptions.Reader(reader)
return &metadata{
Compressed: true,
Size: metaHeader,
@@ -106,19 +140,19 @@ func (s *Squashfs) parseNextMetadata() (*metadata, error) {
return &metadata{
Compressed: false,
Size: metaHeader,
Data: io.NewSectionReader(s.rdr, s.rdr.offset, int64(metaHeader)),
Data: reader,
}, nil
}
func (s *Squashfs) parseMetadataAt(offset int64) (*metadata, error) {
var metaHeader uint16
err := binary.Read(s.rdr, binary.LittleEndian, metaHeader)
if err != nil {
return nil, err
}
var headerBytes []byte
headerBytes = make([]byte, 2)
s.rdr.ReadAt(headerBytes, offset)
metaHeader = binary.LittleEndian.Uint16(headerBytes)
if metaHeader&0x8000 == 0x8000 {
metaHeader = metaHeader &^ 0x8000
compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, offset, int64(metaHeader)))
compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)))
return &metadata{
Compressed: true,
Size: metaHeader,
@@ -128,6 +162,6 @@ func (s *Squashfs) parseMetadataAt(offset int64) (*metadata, error) {
return &metadata{
Compressed: false,
Size: metaHeader,
Data: io.NewSectionReader(s.rdr, offset, int64(metaHeader)),
Data: io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)),
}, nil
}
squashfs_test.go
+3 -3
View File
@@ -1,7 +1,6 @@
package squashfs
import (
"fmt"
"io"
"net/http"
"os"
@@ -36,8 +35,8 @@ func TestAppImageSquash(t *testing.T) {
if err != nil {
t.Error(err)
}
fmt.Println(squash.GetFlags())
t.Fatal("Testing")
err = squash.readRootDirectoryTable()
t.Fatal(err)
}
func TestCreateSquashFromAppImage(t *testing.T) {
@@ -78,6 +77,7 @@ func TestCreateSquashFromAppImage(t *testing.T) {
}
func downloadTestAppImage(t *testing.T, dir string) {
//seems to time out. Need to fix that at some point
appImage, err := os.Create(dir + "/" + appImageName)
if err != nil {
t.Fatal(err)