From dbf7e9465aa0f59d67f89c8a9882a0cb975aa17f Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Fri, 13 Nov 2020 04:47:54 -0600 Subject: [PATCH] Directorty table info parsing. Futher work on reading data. --- README.md | 6 +- compressionoptions.go | 35 +++++---- internal/directory/directory.go | 83 +++++++++++++++++++++ internal/inode/inode.go | 76 ++++++++++++++----- internal/inode/processInode.go | 127 ++++++++++++++++---------------- internal/inode/util.go | 10 +++ reader.go | 3 - squashfs.go | 56 +++++++++++--- squashfs_test.go | 6 +- 9 files changed, 288 insertions(+), 114 deletions(-) create mode 100644 internal/directory/directory.go create mode 100644 internal/inode/util.go diff --git a/README.md b/README.md index 73efa1a..d9b2340 100644 --- a/README.md +++ b/README.md @@ -18,4 +18,8 @@ I am focusing purely on unsquashing before squashing. * Reading Inodes * Reading the Directory structure * Implement other compression types -* Squashing \ No newline at end of file +* Squashing + +# Where I'm at + +* I can read the metadata, but can't read inodes just yet. \ No newline at end of file diff --git a/compressionoptions.go b/compressionoptions.go index b85b8f1..069ea6d 100644 --- a/compressionoptions.go +++ b/compressionoptions.go @@ -1,12 +1,14 @@ package squashfs import ( - "compress/zlib" + "compress/gzip" "io" + + "gopkg.in/src-d/go-git.v4/utils/ioutil" ) const ( - zlibCompression = 1 + iota + gzipCompression = 1 + iota lzmaCompression lzoCompression xzCompression @@ -95,13 +97,13 @@ func NewGzipOptions(raw gzipOptionsRaw) *GzipOptions { } func (gzipOp *GzipOptions) Decompress(rdr *io.SectionReader, blockSize int) ([]byte, error) { - zlibRdr, err := zlib.NewReader(rdr) - defer zlibRdr.Close() + gzipRdr, err := gzip.NewReader(rdr) + defer gzipRdr.Close() if err != nil { return nil, err } bytrw := newByteReadWrite(0) - _, err = io.Copy(bytrw, zlibRdr) + _, err = io.Copy(bytrw, gzipRdr) if err != nil { return bytrw.byts, err } @@ -109,20 +111,20 @@ func (gzipOp *GzipOptions) Decompress(rdr *io.SectionReader, blockSize int) ([]b } func (gzipOp *GzipOptions) DecompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) { - zlibRdr, err := zlib.NewReader(*rdr) - defer zlibRdr.Close() + gzipRdr, err := gzip.NewReader(*rdr) + defer gzipRdr.Close() if err != nil { return 0, err } - n, err := io.Copy(*wrt, zlibRdr) + n, err := io.Copy(*wrt, gzipRdr) return int(n), err } func (gzipOp *GzipOptions) Compress(rdr *io.SectionReader, blockSize int) ([]byte, error) { bytWrt := newByteReadWrite(0) - zlibWrt := zlib.NewWriter(bytWrt) //TODO: allow setting level - defer zlibWrt.Close() - _, err := io.Copy(zlibWrt, rdr) + gzipWrt := gzip.NewWriter(bytWrt) //TODO: allow setting level + defer gzipWrt.Close() + _, err := io.Copy(gzipWrt, rdr) if err != nil { return bytWrt.byts, err } @@ -130,15 +132,16 @@ func (gzipOp *GzipOptions) Compress(rdr *io.SectionReader, blockSize int) ([]byt } func (gzipOp *GzipOptions) CompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) { - zlibWrt := zlib.NewWriter(*wrt) //TODO: allow setting level - defer zlibWrt.Close() - n, err := io.Copy(zlibWrt, *rdr) + gzipWrt := gzip.NewWriter(*wrt) //TODO: allow setting level + defer gzipWrt.Close() + n, err := io.Copy(gzipWrt, *rdr) return int(n), err } func (gzipOp *GzipOptions) Reader(rdr io.Reader) (*io.ReadCloser, error) { - read, err := zlib.NewReader(rdr) - return &read, err + read, err := gzip.NewReader(rdr) + redClo := ioutil.NewReadCloser(read, read) + return &redClo, err } type xzOptionsRaw struct { diff --git a/internal/directory/directory.go b/internal/directory/directory.go new file mode 100644 index 0000000..4194d82 --- /dev/null +++ b/internal/directory/directory.go @@ -0,0 +1,83 @@ +package directory + +import ( + "encoding/binary" + "fmt" + "io" +) + +//Header is the header for a directory in the directory table +type Header struct { + Count uint32 + InodeOffset uint32 + InodeNumber uint32 +} + +//EntryInit is the values that can be easily decoded +type EntryInit struct { + Offset uint16 + InodeOffset int16 + Type uint16 + NameSize uint16 +} + +//Entry is an entry in a directory. +type Entry struct { + Init EntryInit + Name []byte +} + +//NewEntry creates a new directory entry +func NewEntry(rdr io.Reader) (Entry, error) { + var entry Entry + err := binary.Read(rdr, binary.LittleEndian, entry.Init) + if err != nil { + return entry, err + } + entry.Name = make([]byte, entry.Init.NameSize, entry.Init.NameSize) + err = binary.Read(rdr, binary.LittleEndian, entry.Name) + return entry, err +} + +//Directory is an entry in the directory table of a squashfs. +//Will only have multiple headers if there are more then 256 entries +type Directory struct { + Headers []Header + Entries []Entry +} + +//NewDirectory reads the directory from rdr +func NewDirectory(rdr io.Reader) (*Directory, error) { + var dir Directory + var hdr Header + err := binary.Read(rdr, binary.LittleEndian, &hdr) + if err != nil { + return nil, err + } + headers := hdr.Count / 256 + if headers%256 > 0 { + headers++ + } + headersRead := 1 + dir.Headers = make([]Header, headers) + dir.Headers[0] = hdr + for i := uint32(0); i < hdr.Count; i++ { + if i != 0 && i%256 == 0 { + var newHdr Header + err = binary.Read(rdr, binary.LittleEndian, &newHdr) + if err != nil { + fmt.Println("Error processing header ", headersRead) + return &dir, err + } + dir.Headers[headersRead] = newHdr + headersRead++ + } + ent, err := NewEntry(rdr) + if err != nil { + fmt.Println("Error processing entry ", len(dir.Entries)) + return &dir, err + } + dir.Entries = append(dir.Entries, ent) + } + return &dir, nil +} diff --git a/internal/inode/inode.go b/internal/inode/inode.go index 81a9c49..61caa65 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -2,11 +2,12 @@ package inode import ( "encoding/binary" + "fmt" "io" ) -//InodeCommon is the comon header for all inodes -type InodeCommon struct { +//Common is the comon header for all inodes +type Common struct { InodeType uint16 Permissions uint16 UID uint16 @@ -37,25 +38,61 @@ type ExtendedDirectoryInit struct { //ExtendedDirectory is a directory with extra info type ExtendedDirectory struct { - Init ExtendedDirectoryInit - //TODO: indexes []DirectoryIndex + Init ExtendedDirectoryInit + Indexes []DirectoryIndex } //NewExtendedDirectory creates a new ExtendedDirectory func NewExtendedDirectory(rdr *io.Reader) (*ExtendedDirectory, error) { var inode ExtendedDirectory err := binary.Read(*rdr, binary.LittleEndian, inode.Init) - //TODO: Read directory indexes + if err != nil { + return &inode, err + } + if inode.Init.IndexCount > 0 { + inode.Indexes = make([]DirectoryIndex, inode.Init.IndexCount) + for i := uint16(0); i < inode.Init.IndexCount; i++ { + inode.Indexes[i], err = NewDirectoryIndex(rdr) + if err != nil { + fmt.Println("Error while reading Directory Index ", i) + return &inode, err + } + } + } return &inode, err } +//DirectoryIndexInit holds the values that can be easily decoded +type DirectoryIndexInit struct { + Offset uint32 + DirTableOffset uint32 + NameSize uint32 +} + +//DirectoryIndex is a quick lookup provided by an ExtendedDirectory +type DirectoryIndex struct { + Init DirectoryIndexInit + Name []byte +} + +//NewDirectoryIndex return a new DirectoryIndex +func NewDirectoryIndex(rdr *io.Reader) (DirectoryIndex, error) { + var index DirectoryIndex + err := binary.Read(*rdr, binary.LittleEndian, index.Init) + if err != nil { + return index, err + } + index.Name = make([]byte, index.Init.NameSize, index.Init.NameSize) + err = binary.Read(*rdr, binary.LittleEndian, index.Name) + return index, err +} + //BasicFileInit is the information that can be directoy decoded type BasicFileInit struct { BlockStart uint32 FragmentIndex uint32 FragmentOffset uint32 Size uint32 - //TODO: possibly fix BlockSizes } //BasicFile is self explainatory @@ -89,7 +126,6 @@ type ExtendedFileInit struct { FragmentIndex uint32 FragmentOffset uint32 XattrIndex uint32 - //TODO: possibly fix BlockSizes } //ExtendedFile is a file with more information @@ -99,11 +135,11 @@ type ExtendedFile struct { } //NewExtendedFile creates a new ExtendedFile -func NewExtendedFile(rdr *io.Reader, blockSize uint32) (*ExtendedFile, error) { +func NewExtendedFile(rdr *io.Reader, blockSize uint32) (ExtendedFile, error) { var inode ExtendedFile err := binary.Read(*rdr, binary.LittleEndian, inode.Init) if err != nil { - return &inode, err + return inode, err } blocks := inode.Init.Size / blockSize if inode.Init.Size%blockSize > 0 { @@ -111,7 +147,7 @@ func NewExtendedFile(rdr *io.Reader, blockSize uint32) (*ExtendedFile, error) { } inode.BlockSizes = make([]uint32, blocks, blocks) err = binary.Read(*rdr, binary.LittleEndian, inode.BlockSizes) - return &inode, err + return inode, err } //BasicSymlinkInit is all the values that can be directly decoded @@ -123,19 +159,19 @@ type BasicSymlinkInit struct { //BasicSymlink is a symlink type BasicSymlink struct { Init BasicSymlinkInit - targetPath []uint8 //len is TargetPathSize + targetPath []byte //len is TargetPathSize } //NewBasicSymlink creates a new BasicSymlink -func NewBasicSymlink(rdr *io.Reader) (*BasicSymlink, error) { +func NewBasicSymlink(rdr *io.Reader) (BasicSymlink, error) { var inode BasicSymlink err := binary.Read(*rdr, binary.LittleEndian, inode.Init) if err != nil { - return nil, err + return inode, err } - inode.targetPath = make([]uint8, inode.Init.TargetPathSize, inode.Init.TargetPathSize) + inode.targetPath = make([]byte, inode.Init.TargetPathSize, inode.Init.TargetPathSize) err = binary.Read(*rdr, binary.LittleEndian, inode.targetPath) - return &inode, err + return inode, err } //ExtendedSymlinkInit is all the values that can be directly decoded @@ -152,31 +188,35 @@ type ExtendedSymlink struct { } //NewExtendedSymlink creates a new ExtendedSymlink -func NewExtendedSymlink(rdr *io.Reader) (*ExtendedSymlink, error) { +func NewExtendedSymlink(rdr *io.Reader) (ExtendedSymlink, error) { var inode ExtendedSymlink err := binary.Read(*rdr, binary.LittleEndian, inode.Init) if err != nil { - return &inode, err + return inode, err } inode.TargetPath = make([]uint8, inode.Init.TargetPathSize, inode.Init.TargetPathSize) err = binary.Read(*rdr, binary.LittleEndian, &inode.XattrIndex) - return &inode, err + return inode, err } +//BasicDevice is a device type BasicDevice struct { HardLinks uint32 Device uint32 } +//ExtendedDevice is a device with more info type ExtendedDevice struct { BasicDevice XattrIndex uint32 } +//BasicIPC is a Fifo or Socket device type BasicIPC struct { HardLink uint32 } +//ExtendedIPC is a IPC device with extra info type ExtendedIPC struct { BasicIPC XattrIndex uint32 diff --git a/internal/inode/processInode.go b/internal/inode/processInode.go index 522753a..7ada4c9 100644 --- a/internal/inode/processInode.go +++ b/internal/inode/processInode.go @@ -8,80 +8,83 @@ import ( ) const ( - basicDirectory = iota + 1 - basicFile - basicSymlink - basicBlockDevice - basicCharDevice - basicFifo - basicSocket - extendedDirectory - extendedFile - extendedSymlink - extendedBlockDevice - extendedCharDevice - extendedFifo - extendedSocket + //The inode type from inode.Common.InodeType + + BasicDirectoryType = iota + 1 + BasicFileType + BasicSymlinkType + BasicBlockDeviceType + BasicCharDeviceType + BasicFifoType + BasicSocketType + ExtendedDirectoryType + ExtendedFileType + ExtendedSymlinkType + ExtendedBlockDeviceType + ExtendedCharDeviceType + ExtendedFifoType + ExtendedSocketType ) -func ProcessInode(rdr *io.Reader) (*InodeCommon, interface{}, error) { - var inodeHeader InodeCommon +//ProcessInode processes the next inode in the given reader +func ProcessInode(rdr *io.Reader, blockSize uint32) (*Common, interface{}, error) { + var inodeHeader Common err := binary.Read(*rdr, binary.LittleEndian, &inodeHeader) if err != nil { return nil, nil, err } switch inodeHeader.InodeType { - case basicDirectory: + case BasicDirectoryType: var inode BasicDirectory err = binary.Read(*rdr, binary.LittleEndian, &inode) + return &inodeHeader, &inode, err + case BasicFileType: + inode, err := NewBasicFile(rdr, blockSize) return &inodeHeader, inode, err - case basicFile: - var inode BasicFile - err = binary.Read(*rdr, binary.LittleEndian, &inode) - return &inodeHeader, inode, err - case basicSymlink: + case BasicSymlinkType: inode, err := NewBasicSymlink(rdr) return &inodeHeader, inode, err - // case basicFile: - // var inode BasicFile - // err = binary.Read(*rdr, binary.LittleEndian, &inode) - // return &inodeHeader, inode, err - // case basicFile: - // var inode BasicFile - // err = binary.Read(*rdr, binary.LittleEndian, &inode) - // return &inodeHeader, inode, err - // case basicFile: - // var inode BasicFile - // err = binary.Read(*rdr, binary.LittleEndian, &inode) - // return &inodeHeader, inode, err - // case basicFile: - // var inode BasicFile - // err = binary.Read(*rdr, binary.LittleEndian, &inode) - // return &inodeHeader, inode, err - // case basicFile: - // var inode BasicFile - // err = binary.Read(*rdr, binary.LittleEndian, &inode) - // return &inodeHeader, inode, err - // case basicFile: - // var inode BasicFile - // err = binary.Read(*rdr, binary.LittleEndian, &inode) - // return &inodeHeader, inode, err - // case basicFile: - // var inode BasicFile - // err = binary.Read(*rdr, binary.LittleEndian, &inode) - // return &inodeHeader, inode, err - // case basicFile: - // var inode BasicFile - // err = binary.Read(*rdr, binary.LittleEndian, &inode) - // return &inodeHeader, inode, err - // case basicFile: - // var inode BasicFile - // err = binary.Read(*rdr, binary.LittleEndian, &inode) - // return &inodeHeader, inode, err - // case basicFile: - // var inode BasicFile - // err = binary.Read(*rdr, binary.LittleEndian, &inode) - // return &inodeHeader, inode, err + case BasicBlockDeviceType: + var inode BasicDevice + err = binary.Read(*rdr, binary.LittleEndian, &inode) + return &inodeHeader, inode, err + case BasicCharDeviceType: + var inode BasicDevice + err = binary.Read(*rdr, binary.LittleEndian, &inode) + return &inodeHeader, inode, err + case BasicFifoType: + var inode BasicIPC + err = binary.Read(*rdr, binary.LittleEndian, &inode) + return &inodeHeader, inode, err + case BasicSocketType: + var inode BasicIPC + err = binary.Read(*rdr, binary.LittleEndian, &inode) + return &inodeHeader, inode, err + case ExtendedDirectoryType: + inode, err := NewExtendedDirectory(rdr) + return &inodeHeader, inode, err + case ExtendedFileType: + inode, err := NewExtendedFile(rdr, blockSize) + return &inodeHeader, inode, err + case ExtendedSymlinkType: + inode, err := NewExtendedSymlink(rdr) + return &inodeHeader, inode, err + case ExtendedBlockDeviceType: + var inode ExtendedDevice + err = binary.Read(*rdr, binary.LittleEndian, &inode) + return &inodeHeader, inode, err + case ExtendedCharDeviceType: + var inode ExtendedDevice + err = binary.Read(*rdr, binary.LittleEndian, &inode) + return &inodeHeader, inode, err + case ExtendedFifoType: + var inode ExtendedIPC + err = binary.Read(*rdr, binary.LittleEndian, &inode) + return &inodeHeader, inode, err + case ExtendedSocketType: + var inode ExtendedIPC + err = binary.Read(*rdr, binary.LittleEndian, &inode) + return &inodeHeader, inode, err //TODO: implement ALL cases default: return nil, nil, errors.New("Inode type is unrecognized: " + strconv.FormatInt(int64(inodeHeader.InodeType), 2)) diff --git a/internal/inode/util.go b/internal/inode/util.go new file mode 100644 index 0000000..03241b5 --- /dev/null +++ b/internal/inode/util.go @@ -0,0 +1,10 @@ +package inode + +//ProcessInodeRef processes an inode reference and returns two values +//The first value is the inode table offset. AKA, it's where the metadata block of the inode STARTS. +//The second value is the offset of the inode, INSIDE of the metadata. +func ProcessInodeRef(inodeRef uint64) (tableOffset uint32, metaOffset uint16) { + tableOffset = uint32(inodeRef >> 16) + metaOffset = uint16(inodeRef &^ 0xFFFFFFFF0000) + return +} diff --git a/reader.go b/reader.go index 0955825..c72ab59 100644 --- a/reader.go +++ b/reader.go @@ -9,9 +9,6 @@ import ( //Reader is a reader which implements Reader, ReaderAt, and Seeker, all with an accesible offset (for reasons) type Reader struct { - io.Reader - io.ReaderAt - io.Seeker rdr io.ReaderAt offset int64 } diff --git a/squashfs.go b/squashfs.go index a677a4d..065bc24 100644 --- a/squashfs.go +++ b/squashfs.go @@ -3,7 +3,10 @@ package squashfs import ( "encoding/binary" "errors" + "fmt" "io" + + "github.com/CalebQ42/GoSquashfs/internal/inode" ) var ( @@ -34,7 +37,7 @@ func NewSquashfs(reader io.ReaderAt) (*Squashfs, error) { flags := superblock.GetFlags() var compressionOptions CompressionOptions switch superblock.Compression { - case zlibCompression: + case gzipCompression: if flags.CompressorOptions { var gzipOpRaw gzipOptionsRaw err = binary.Read(&rdr, binary.LittleEndian, &gzipOpRaw) @@ -69,9 +72,39 @@ func NewSquashfs(reader io.ReaderAt) (*Squashfs, error) { }, nil } +func (s *Squashfs) readRootDirectoryTable() error { + offset, metaOffset := inode.ProcessInodeRef(s.super.RootInode) + meta, err := s.parseMetadataAt(int64(s.super.InodeTableOffset) + int64(offset)) + if err != nil { + fmt.Println("Error processing metadata") + return err + } + _, err = meta.Data.Read(make([]byte, metaOffset)) + if err != nil { + fmt.Println("error reading forward to offset") + return err + } + common, _, err := inode.ProcessInode(&meta.Data, s.super.BlockSize) + if err != nil { + fmt.Println("Error reading inode") + return err + } + if common.InodeType != inode.BasicDirectoryType { + return errors.New("Not a basic directory") + } + // dirTable, err := directory.NewDirectory(meta.Data) + // if err != nil { + // return err + // } + // for _, entry := range dirTable.Entries { + // fmt.Println(entry.Name) + // } + return nil +} + //GetFlags return the SuperblockFlags of the Squashfs func (s *Squashfs) GetFlags() SuperblockFlags { - return s.super.GetFlags() + return s.flags } //metadata is a parsed metadata block @@ -90,13 +123,14 @@ func (m *metadata) close() { func (s *Squashfs) parseNextMetadata() (*metadata, error) { var metaHeader uint16 - err := binary.Read(s.rdr, binary.LittleEndian, metaHeader) + err := binary.Read(s.rdr, binary.LittleEndian, &metaHeader) if err != nil { return nil, err } + reader := io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(metaHeader)) if metaHeader&0x8000 == 0x8000 { metaHeader = metaHeader &^ 0x8000 - compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, s.rdr.offset, int64(metaHeader))) + compressRead, err := s.compressionOptions.Reader(reader) return &metadata{ Compressed: true, Size: metaHeader, @@ -106,19 +140,19 @@ func (s *Squashfs) parseNextMetadata() (*metadata, error) { return &metadata{ Compressed: false, Size: metaHeader, - Data: io.NewSectionReader(s.rdr, s.rdr.offset, int64(metaHeader)), + Data: reader, }, nil } func (s *Squashfs) parseMetadataAt(offset int64) (*metadata, error) { var metaHeader uint16 - err := binary.Read(s.rdr, binary.LittleEndian, metaHeader) - if err != nil { - return nil, err - } + var headerBytes []byte + headerBytes = make([]byte, 2) + s.rdr.ReadAt(headerBytes, offset) + metaHeader = binary.LittleEndian.Uint16(headerBytes) if metaHeader&0x8000 == 0x8000 { metaHeader = metaHeader &^ 0x8000 - compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, offset, int64(metaHeader))) + compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize))) return &metadata{ Compressed: true, Size: metaHeader, @@ -128,6 +162,6 @@ func (s *Squashfs) parseMetadataAt(offset int64) (*metadata, error) { return &metadata{ Compressed: false, Size: metaHeader, - Data: io.NewSectionReader(s.rdr, offset, int64(metaHeader)), + Data: io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)), }, nil } diff --git a/squashfs_test.go b/squashfs_test.go index 2cf4c66..72d469a 100644 --- a/squashfs_test.go +++ b/squashfs_test.go @@ -1,7 +1,6 @@ package squashfs import ( - "fmt" "io" "net/http" "os" @@ -36,8 +35,8 @@ func TestAppImageSquash(t *testing.T) { if err != nil { t.Error(err) } - fmt.Println(squash.GetFlags()) - t.Fatal("Testing") + err = squash.readRootDirectoryTable() + t.Fatal(err) } func TestCreateSquashFromAppImage(t *testing.T) { @@ -78,6 +77,7 @@ func TestCreateSquashFromAppImage(t *testing.T) { } func downloadTestAppImage(t *testing.T, dir string) { + //seems to time out. Need to fix that at some point appImage, err := os.Create(dir + "/" + appImageName) if err != nil { t.Fatal(err)