diff --git a/README.md b/README.md index d9b2340..e213989 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ I am focusing purely on unsquashing before squashing. # Working * Reading the header -* (Maybe) reading gzip compressed data # Not Working (Yet). Roughly in order. @@ -22,4 +21,5 @@ I am focusing purely on unsquashing before squashing. # Where I'm at -* I can read the metadata, but can't read inodes just yet. \ No newline at end of file +* Redid a bunch. Implemented a custom reader that can read across blocks. + * As of yet, doesn't seem to be reading things quite right (seems to be issue with encryption reading) \ No newline at end of file diff --git a/blocks.go b/blocks.go new file mode 100644 index 0000000..1681481 --- /dev/null +++ b/blocks.go @@ -0,0 +1,117 @@ +package squashfs + +import ( + "encoding/binary" + "fmt" + "io" + + "github.com/CalebQ42/GoSquashfs/bytereadwrite" +) + +type MetadataHeader struct { + rawHeader uint16 + Compressed bool + Size uint16 +} + +type BlockReader struct { + initalOffset int64 + offset int64 + squash *Squashfs + headers []MetadataHeader + dataCache []byte + dataOffset int64 +} + +//NewBlockReader creates a new BlockReader from a squashfs.Reader. Reads the first header and caches the first set of data. +func (s *Squashfs) NewBlockReader(offset int64) (*BlockReader, error) { + var br BlockReader + br.squash = s + br.initalOffset = offset + br.offset = offset + br.headers = make([]MetadataHeader, 0) + br.dataCache = make([]byte, 0) + err := br.parseNewBlock() + if err != nil { + fmt.Println("Problem creating BlockReader") + return nil, err + } + return &br, nil +} + +func (br *BlockReader) parseNewBlock() error { + var header MetadataHeader + err := binary.Read(io.NewSectionReader(&br.squash.r, br.offset, 2), binary.LittleEndian, &header.rawHeader) + if err != nil { + fmt.Println("Error while reading the header ", len(br.headers), " in BlockReader") + return err + } + header.Compressed = (header.rawHeader&0x8000 == 0x8000) + header.Size = header.rawHeader &^ 0x8000 + br.headers = append(br.headers, header) + br.offset += 2 + sectionReader := io.NewSectionReader(&br.squash.r, br.offset, br.offset+int64(header.Size)) + dataWriter := bytereadwrite.NewByteReaderWriter() + _, err = io.Copy(dataWriter, sectionReader) + br.offset += int64(header.Size) + if header.Compressed { + data, err := br.squash.compression.Decompress(dataWriter) + if err != nil { + fmt.Println("Error while reading the encrypted data block in header", len(br.headers)) + return err + } + br.dataCache = append(br.dataCache, data...) + return nil + } + if err != nil { + fmt.Println("Error while reading uncompressed data in header", len(br.headers)) + return err + } + br.dataCache = append(br.dataCache, dataWriter.GetBytes()...) + return nil +} + +//Read reads data into p. If it reaches EOF, tries to read a new block and add it's data to the cache +func (br *BlockReader) Read(p []byte) (n int, err error) { + read := 0 + for { + byter := bytereadwrite.NewByteReaderWriterFromBytes(br.dataCache[br.dataOffset:]) + temp, err := byter.Read(p[read:]) + br.dataOffset += int64(temp) + read += temp + if err == nil { + return read, nil + } else if err != io.EOF { + return read, err + } + if err == io.EOF { + err = br.parseNewBlock() + if err != nil { + fmt.Println("Error while reading a new block") + return read, err + } + } + } +} + +//ReadAt reads data into p from the offset. If it reaches EOF, tries to read a new block and add it's data to the cache. +// +//Offset is reletive to the offset set on creation. +func (br *BlockReader) ReadAt(p []byte, offset int) (n int, err error) { + read := 0 + for err == io.EOF { + byter := bytereadwrite.NewByteReaderWriterFromBytes(br.dataCache[offset+read:]) + temp, inErr := byter.Read(p[read:]) + err = inErr + read += temp + br.dataOffset += int64(temp) + if err == io.EOF { + inErr = br.parseNewBlock() + if inErr != nil { + fmt.Println("Error while reading a new block") + return read, inErr + } + } + } + return +} diff --git a/bytereadwrite/bytereadwrite.go b/bytereadwrite/bytereadwrite.go new file mode 100644 index 0000000..b73a73b --- /dev/null +++ b/bytereadwrite/bytereadwrite.go @@ -0,0 +1,60 @@ +package bytereadwrite + +import "io" + +//ByteReaderWriter allows you to read to and from a byte slice. When writing, it expands the slice to accomodate any data. +type ByteReaderWriter struct { + byts []byte + offset int +} + +//NewByteReaderWriter creates a ByteReaderWriter with an internal byte slice of the given length. +func NewByteReaderWriter() *ByteReaderWriter { + return &ByteReaderWriter{ + byts: make([]byte, 0), + offset: 0, + } +} + +//NewByteReaderWriter creates a ByteReaderWriter with an internal byte slice of the given length. +func NewByteReaderWriterWithLength(length int) *ByteReaderWriter { + return &ByteReaderWriter{ + byts: make([]byte, length), + offset: 0, + } +} + +//NewByteReaderWriterFromBytes creates a new ByteReaderWriter initialized with the given bytes +func NewByteReaderWriterFromBytes(byts []byte) *ByteReaderWriter { + return &ByteReaderWriter{ + byts: byts, + offset: 0, + } +} + +//GetBytes return the underlyting byte slice of the readerwriter +func (bwr *ByteReaderWriter) GetBytes() []byte { + return bwr.byts +} + +//Read reads the bytes. +func (bwr *ByteReaderWriter) Read(byt []byte) (int, error) { + if len(bwr.byts) < bwr.offset+len(byt) { + bytesWritten := len(bwr.byts) - bwr.offset + for i := 0; i < bytesWritten; i++ { + byt[i] = bwr.byts[i+bwr.offset] + } + return bytesWritten, io.EOF + } + for i := 0; i < len(byt); i++ { + byt[i] = bwr.byts[bwr.offset+i] + } + bwr.offset += len(byt) + return len(byt), nil +} + +//Write writes to the end of the bytes. WILL expand to accept the incoming bytes. +func (bwr *ByteReaderWriter) Write(byts []byte) (int, error) { + bwr.byts = append(bwr.byts, byts...) + return len(byts), nil +} diff --git a/compressionoptions.go b/compressionoptions.go index 069ea6d..3e4189e 100644 --- a/compressionoptions.go +++ b/compressionoptions.go @@ -4,7 +4,7 @@ import ( "compress/gzip" "io" - "gopkg.in/src-d/go-git.v4/utils/ioutil" + "github.com/CalebQ42/GoSquashfs/bytereadwrite" ) const ( @@ -18,19 +18,18 @@ const ( //TODO: implement for each type of Options type CompressionOptions interface { - Decompress(*io.SectionReader, int) ([]byte, error) + Decompress(io.Reader) ([]byte, error) DecompressCopy(*io.Reader, *io.Writer) (int, error) - Compress(*io.SectionReader, int) ([]byte, error) + Compress(*io.Reader) ([]byte, error) CompressCopy(*io.Reader, *io.Writer) (int, error) - Reader(io.Reader) (*io.ReadCloser, error) } //TODO: Allow creation of options for compression. type gzipOptionsRaw struct { - compressionLevel int32 - windowSize int16 - strategies int16 + CompressionLevel int32 + WindowSize int16 + Strategies int16 } //GzipOptions is the options used for gzip compression. Backed by the raw format, with strategies parsed. @@ -44,51 +43,6 @@ type GzipOptions struct { FixedStretegy bool } -type byteWriterReader struct { - byts []byte - offset int -} - -func newByteReadWrite(length int) *byteWriterReader { - return &byteWriterReader{ - byts: make([]byte, length), - offset: 0, - } -} - -func newByteReadWriteFromBytes(byts []byte) *byteWriterReader { - return &byteWriterReader{ - byts: byts, - offset: 0, - } -} - -func (bwr *byteWriterReader) getBytes() []byte { - return bwr.byts -} - -//Read reads the bytes. -func (bwr *byteWriterReader) Read(byt []byte) (int, error) { - if len(bwr.byts) < bwr.offset+len(byt) { - bytesWritten := len(bwr.byts) - bwr.offset - for i := 0; i < bytesWritten; i++ { - byt[i] = bwr.byts[i+bwr.offset] - } - return bytesWritten, io.EOF - } - for i := 0; i < len(byt); i++ { - byt[i] = bwr.byts[bwr.offset+i] - } - bwr.offset += len(byt) - return len(byt), nil -} - -//Write writes to the bytes. WILL expand to accept the incoming bytes. -func (bwr *byteWriterReader) Write(byts []byte) (int, error) { - bwr.byts = append(bwr.byts, byts...) - return len(byts), nil -} - func NewGzipOptions(raw gzipOptionsRaw) *GzipOptions { //TODO: parse strategies return &GzipOptions{ @@ -96,18 +50,18 @@ func NewGzipOptions(raw gzipOptionsRaw) *GzipOptions { } } -func (gzipOp *GzipOptions) Decompress(rdr *io.SectionReader, blockSize int) ([]byte, error) { +func (gzipOp *GzipOptions) Decompress(rdr io.Reader) ([]byte, error) { gzipRdr, err := gzip.NewReader(rdr) - defer gzipRdr.Close() + // defer gzipRdr.Close() if err != nil { return nil, err } - bytrw := newByteReadWrite(0) + bytrw := bytereadwrite.NewByteReaderWriter() _, err = io.Copy(bytrw, gzipRdr) if err != nil { - return bytrw.byts, err + return nil, err } - return bytrw.byts, nil + return bytrw.GetBytes(), nil } func (gzipOp *GzipOptions) DecompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) { @@ -120,15 +74,15 @@ func (gzipOp *GzipOptions) DecompressCopy(rdr *io.Reader, wrt *io.Writer) (int, return int(n), err } -func (gzipOp *GzipOptions) Compress(rdr *io.SectionReader, blockSize int) ([]byte, error) { - bytWrt := newByteReadWrite(0) +func (gzipOp *GzipOptions) Compress(rdr *io.Reader) ([]byte, error) { + bytWrt := bytereadwrite.NewByteReaderWriter() gzipWrt := gzip.NewWriter(bytWrt) //TODO: allow setting level defer gzipWrt.Close() - _, err := io.Copy(gzipWrt, rdr) + _, err := io.Copy(gzipWrt, *rdr) if err != nil { - return bytWrt.byts, err + return bytWrt.GetBytes(), err } - return bytWrt.byts, nil + return bytWrt.GetBytes(), nil } func (gzipOp *GzipOptions) CompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) { @@ -138,15 +92,9 @@ func (gzipOp *GzipOptions) CompressCopy(rdr *io.Reader, wrt *io.Writer) (int, er return int(n), err } -func (gzipOp *GzipOptions) Reader(rdr io.Reader) (*io.ReadCloser, error) { - read, err := gzip.NewReader(rdr) - redClo := ioutil.NewReadCloser(read, read) - return &redClo, err -} - type xzOptionsRaw struct { - dictionarySize int32 - executableFilters int32 + DictionarySize int32 + ExecutableFilters int32 } type XzOptions struct { @@ -163,18 +111,18 @@ type XzOptions struct { func NewXzOption(raw xzOptionsRaw) XzOptions { return XzOptions{ raw: &raw, - Execx86: raw.executableFilters&0x1 == 0x1, - ExecPower: raw.executableFilters&0x2 == 0x2, - Execa64: raw.executableFilters&0x4 == 0x4, - ExecArm: raw.executableFilters&0x8 == 0x8, - ExecArmThumb: raw.executableFilters&0x10 == 0x10, - ExecSparc: raw.executableFilters&0x20 == 0x20, + Execx86: raw.ExecutableFilters&0x1 == 0x1, + ExecPower: raw.ExecutableFilters&0x2 == 0x2, + Execa64: raw.ExecutableFilters&0x4 == 0x4, + ExecArm: raw.ExecutableFilters&0x8 == 0x8, + ExecArmThumb: raw.ExecutableFilters&0x10 == 0x10, + ExecSparc: raw.ExecutableFilters&0x20 == 0x20, } } type lz4OptionsRaw struct { - version int32 - flags int32 + Version int32 + Flags int32 } //ZstdOptions is the options set for zstdOptions @@ -183,6 +131,6 @@ type ZstdOptions struct { } type lzoOptionsRaw struct { - algorithm int32 - compressionLevel int32 + Algorithm int32 + CompressionLevel int32 } diff --git a/internal/directory/directory.go b/internal/directory/directory.go index 4194d82..84ed25a 100644 --- a/internal/directory/directory.go +++ b/internal/directory/directory.go @@ -54,6 +54,7 @@ func NewDirectory(rdr io.Reader) (*Directory, error) { if err != nil { return nil, err } + fmt.Println(hdr) headers := hdr.Count / 256 if headers%256 > 0 { headers++ diff --git a/internal/inode/processInode.go b/internal/inode/processInode.go index 7ada4c9..1b89d26 100644 --- a/internal/inode/processInode.go +++ b/internal/inode/processInode.go @@ -10,7 +10,7 @@ import ( const ( //The inode type from inode.Common.InodeType - BasicDirectoryType = iota + 1 + BasicDirectoryType = iota BasicFileType BasicSymlinkType BasicBlockDeviceType @@ -27,65 +27,64 @@ const ( ) //ProcessInode processes the next inode in the given reader -func ProcessInode(rdr *io.Reader, blockSize uint32) (*Common, interface{}, error) { +func ProcessInode(rdr io.Reader, blockSize uint32) (*Common, interface{}, error) { var inodeHeader Common - err := binary.Read(*rdr, binary.LittleEndian, &inodeHeader) + err := binary.Read(rdr, binary.LittleEndian, &inodeHeader) if err != nil { return nil, nil, err } switch inodeHeader.InodeType { case BasicDirectoryType: var inode BasicDirectory - err = binary.Read(*rdr, binary.LittleEndian, &inode) + err = binary.Read(rdr, binary.LittleEndian, &inode) return &inodeHeader, &inode, err case BasicFileType: - inode, err := NewBasicFile(rdr, blockSize) + inode, err := NewBasicFile(&rdr, blockSize) return &inodeHeader, inode, err case BasicSymlinkType: - inode, err := NewBasicSymlink(rdr) + inode, err := NewBasicSymlink(&rdr) return &inodeHeader, inode, err case BasicBlockDeviceType: var inode BasicDevice - err = binary.Read(*rdr, binary.LittleEndian, &inode) + err = binary.Read(rdr, binary.LittleEndian, &inode) return &inodeHeader, inode, err case BasicCharDeviceType: var inode BasicDevice - err = binary.Read(*rdr, binary.LittleEndian, &inode) + err = binary.Read(rdr, binary.LittleEndian, &inode) return &inodeHeader, inode, err case BasicFifoType: var inode BasicIPC - err = binary.Read(*rdr, binary.LittleEndian, &inode) + err = binary.Read(rdr, binary.LittleEndian, &inode) return &inodeHeader, inode, err case BasicSocketType: var inode BasicIPC - err = binary.Read(*rdr, binary.LittleEndian, &inode) + err = binary.Read(rdr, binary.LittleEndian, &inode) return &inodeHeader, inode, err case ExtendedDirectoryType: - inode, err := NewExtendedDirectory(rdr) + inode, err := NewExtendedDirectory(&rdr) return &inodeHeader, inode, err case ExtendedFileType: - inode, err := NewExtendedFile(rdr, blockSize) + inode, err := NewExtendedFile(&rdr, blockSize) return &inodeHeader, inode, err case ExtendedSymlinkType: - inode, err := NewExtendedSymlink(rdr) + inode, err := NewExtendedSymlink(&rdr) return &inodeHeader, inode, err case ExtendedBlockDeviceType: var inode ExtendedDevice - err = binary.Read(*rdr, binary.LittleEndian, &inode) + err = binary.Read(rdr, binary.LittleEndian, &inode) return &inodeHeader, inode, err case ExtendedCharDeviceType: var inode ExtendedDevice - err = binary.Read(*rdr, binary.LittleEndian, &inode) + err = binary.Read(rdr, binary.LittleEndian, &inode) return &inodeHeader, inode, err case ExtendedFifoType: var inode ExtendedIPC - err = binary.Read(*rdr, binary.LittleEndian, &inode) + err = binary.Read(rdr, binary.LittleEndian, &inode) return &inodeHeader, inode, err case ExtendedSocketType: var inode ExtendedIPC - err = binary.Read(*rdr, binary.LittleEndian, &inode) + err = binary.Read(rdr, binary.LittleEndian, &inode) return &inodeHeader, inode, err - //TODO: implement ALL cases default: return nil, nil, errors.New("Inode type is unrecognized: " + strconv.FormatInt(int64(inodeHeader.InodeType), 2)) } diff --git a/internal/inode/util.go b/internal/inode/util.go index 03241b5..550e1bf 100644 --- a/internal/inode/util.go +++ b/internal/inode/util.go @@ -3,8 +3,8 @@ package inode //ProcessInodeRef processes an inode reference and returns two values //The first value is the inode table offset. AKA, it's where the metadata block of the inode STARTS. //The second value is the offset of the inode, INSIDE of the metadata. -func ProcessInodeRef(inodeRef uint64) (tableOffset uint32, metaOffset uint16) { - tableOffset = uint32(inodeRef >> 16) - metaOffset = uint16(inodeRef &^ 0xFFFFFFFF0000) +func ProcessInodeRef(inodeRef uint64) (tableOffset uint64, metaOffset uint64) { + tableOffset = inodeRef >> 16 + metaOffset = inodeRef &^ 0xFFFFFFFF0000 return } diff --git a/old/bytereadwrite.go b/old/bytereadwrite.go new file mode 100644 index 0000000..89d7f81 --- /dev/null +++ b/old/bytereadwrite.go @@ -0,0 +1,48 @@ +package squashfs + +import "io" + +type byteWriterReader struct { + byts []byte + offset int +} + +func newByteReadWrite(length int) *byteWriterReader { + return &byteWriterReader{ + byts: make([]byte, length), + offset: 0, + } +} + +func newByteReadWriteFromBytes(byts []byte) *byteWriterReader { + return &byteWriterReader{ + byts: byts, + offset: 0, + } +} + +func (bwr *byteWriterReader) getBytes() []byte { + return bwr.byts +} + +//Read reads the bytes. +func (bwr *byteWriterReader) Read(byt []byte) (int, error) { + if len(bwr.byts) < bwr.offset+len(byt) { + bytesWritten := len(bwr.byts) - bwr.offset + for i := 0; i < bytesWritten; i++ { + byt[i] = bwr.byts[i+bwr.offset] + } + return bytesWritten, io.EOF + } + for i := 0; i < len(byt); i++ { + byt[i] = bwr.byts[bwr.offset+i] + } + bwr.offset += len(byt) + return len(byt), nil +} + +//Write writes to the bytes. WILL expand to accept the incoming bytes. +func (bwr *byteWriterReader) Write(byts []byte) (int, error) { + bwr.byts = append(bwr.byts, byts...) + return len(byts), nil +} diff --git a/old/compressionoptions.go b/old/compressionoptions.go new file mode 100644 index 0000000..d2e99ab --- /dev/null +++ b/old/compressionoptions.go @@ -0,0 +1,143 @@ +package squashfs + +import ( + "compress/zlib" + "io" + + "gopkg.in/src-d/go-git.v4/utils/ioutil" +) + +const ( + zlibCompression = 1 + iota + lzmaCompression + lzoCompression + xzCompression + lz4Compression + zstdCompression +) + +//TODO: implement for each type of Options +type CompressionOptions interface { + Decompress(*io.SectionReader) ([]byte, error) + DecompressCopy(*io.Reader, *io.Writer) (int, error) + Compress(*io.SectionReader) ([]byte, error) + CompressCopy(*io.Reader, *io.Writer) (int, error) + Reader(io.Reader) (*io.ReadCloser, error) +} + +//TODO: Allow creation of options for compression. + +type zlibOptionsRaw struct { + compressionLevel int32 + windowSize int16 + strategies int16 +} + +//ZlibOptions is the options used for zlib compression. Backed by the raw format, with strategies parsed. +type ZlibOptions struct { + CompressionOptions + raw *zlibOptionsRaw + DefaultStrategy bool + FilteredStrategy bool + HuffmanOnlyStrategy bool + RunLengthEncodedStrategy bool + FixedStretegy bool +} + +func NewZlibOptions(raw zlibOptionsRaw) *ZlibOptions { + //TODO: parse strategies + return &ZlibOptions{ + raw: &raw, + } +} + +func (zlibOp *ZlibOptions) Decompress(rdr *io.SectionReader) ([]byte, error) { + zlibRdr, err := zlib.NewReader(rdr) + defer zlibRdr.Close() + if err != nil { + return nil, err + } + bytrw := newByteReadWrite(0) + _, err = io.Copy(bytrw, zlibRdr) + if err != nil { + return bytrw.byts, err + } + return bytrw.byts, nil +} + +func (zlibOp *ZlibOptions) DecompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) { + zlibRdr, err := zlib.NewReader(*rdr) + defer zlibRdr.Close() + if err != nil { + return 0, err + } + n, err := io.Copy(*wrt, zlibRdr) + return int(n), err +} + +func (zlibOp *ZlibOptions) Compress(rdr *io.SectionReader) ([]byte, error) { + bytWrt := newByteReadWrite(0) + zlibWrt := zlib.NewWriter(bytWrt) //TODO: allow setting level + defer zlibWrt.Close() + _, err := io.Copy(zlibWrt, rdr) + if err != nil { + return bytWrt.byts, err + } + return bytWrt.byts, nil +} + +func (zlibOp *ZlibOptions) CompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) { + zlibWrt := zlib.NewWriter(*wrt) //TODO: allow setting level + defer zlibWrt.Close() + n, err := io.Copy(zlibWrt, *rdr) + return int(n), err +} + +func (zlibOp *ZlibOptions) Reader(rdr io.Reader) (*io.ReadCloser, error) { + read, err := zlib.NewReader(rdr) + redClo := ioutil.NewReadCloser(NewByteBufferedReader(read), read) + return &redClo, err +} + +type xzOptionsRaw struct { + dictionarySize int32 + executableFilters int32 +} + +type XzOptions struct { + CompressionOptions //TODO: Remove + raw *xzOptionsRaw + Execx86 bool + ExecPower bool + Execa64 bool + ExecArm bool + ExecArmThumb bool + ExecSparc bool +} + +func NewXzOption(raw xzOptionsRaw) XzOptions { + return XzOptions{ + raw: &raw, + Execx86: raw.executableFilters&0x1 == 0x1, + ExecPower: raw.executableFilters&0x2 == 0x2, + Execa64: raw.executableFilters&0x4 == 0x4, + ExecArm: raw.executableFilters&0x8 == 0x8, + ExecArmThumb: raw.executableFilters&0x10 == 0x10, + ExecSparc: raw.executableFilters&0x20 == 0x20, + } +} + +type lz4OptionsRaw struct { + version int32 + flags int32 +} + +//ZstdOptions is the options set for zstdOptions +type ZstdOptions struct { + CompressionLevel int32 //CompressionLevel should be between 1 and 22 +} + +type lzoOptionsRaw struct { + algorithm int32 + compressionLevel int32 +} diff --git a/old/reader.go b/old/reader.go new file mode 100644 index 0000000..c72ab59 --- /dev/null +++ b/old/reader.go @@ -0,0 +1,59 @@ +package squashfs + +import ( + "errors" + "io" +) + +//TODO: possible custom reader because I'm havng some issuse... + +//Reader is a reader which implements Reader, ReaderAt, and Seeker, all with an accesible offset (for reasons) +type Reader struct { + rdr io.ReaderAt + offset int64 +} + +//NewReader creates a squashfs.Reader from a io.ReaderAt +func NewReader(baseReader io.ReaderAt) Reader { + return Reader{ + rdr: baseReader, + offset: 0, + } +} + +//Read reads len(byt) into byt. Advances the internal offset +func (r *Reader) Read(byt []byte) (int, error) { + n, err := r.rdr.ReadAt(byt, r.offset) + r.offset += int64(n) + return n, err +} + +//ReadAt wraps the internal io.ReadAt's function. DOES NOT advance the internal offset for Read function. +//Returns how many bytes were read. +func (r *Reader) ReadAt(byt []byte, offset int64) (int, error) { + return r.rdr.ReadAt(byt, offset) +} + +//ReadAtFromOffset is the same as ReadAt, but the given offset is offset by the internal offset. DOES NOT advance the internal offset. +//Returns how many bytes were read. +func (r *Reader) ReadAtFromOffset(byt []byte, offset int64) (int, error) { + offset += r.offset + return r.rdr.ReadAt(byt, offset) +} + +//Seek advances the internal offset. SeekEnd DOES NOT work +//Might not be necessary, but here just in case +func (r *Reader) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekCurrent: + n, err := r.Read(make([]byte, offset)) + return int64(n), err + case io.SeekStart: + r.offset = 0 + n, err := r.Read(make([]byte, offset)) + return int64(n), err + case io.SeekEnd: + return 0, errors.New("SeekEnd is NOT supported") + } + return 0, errors.New("incorrect whence") +} diff --git a/reader_old.go b/old/reader_old.go similarity index 100% rename from reader_old.go rename to old/reader_old.go diff --git a/reader_test.go b/old/reader_test.go similarity index 100% rename from reader_test.go rename to old/reader_test.go diff --git a/old/squashfs.go b/old/squashfs.go new file mode 100644 index 0000000..7b499df --- /dev/null +++ b/old/squashfs.go @@ -0,0 +1,181 @@ +package squashfs + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/CalebQ42/GoSquashfs/old/internal/inode" +) + +var ( + //ErrNotMagical happens when making a new Squashfs and it doesn't have the magic number + ErrNotMagical = errors.New("Not Magical") +) + +//Squashfs is a squashfs backed by a ReadSeeker. +type Squashfs struct { + rdr *Reader //underlying reader + offset int + super Superblock + flags SuperblockFlags + compressionOptions CompressionOptions +} + +//NewSquashfs creates a new Squashfs backed by the given reader +func NewSquashfs(reader io.ReaderAt) (*Squashfs, error) { + rdr := NewReader(reader) + var superblock Superblock + err := binary.Read(&rdr, binary.LittleEndian, &superblock) + if err != nil { + return nil, err + } + if superblock.Magic != squashfsMagic { + return nil, ErrNotMagical + } + flags := superblock.GetFlags() + var compressionOptions CompressionOptions + switch superblock.Compression { + case zlibCompression: + if flags.CompressorOptions { + var zlibOpRaw zlibOptionsRaw + err = binary.Read(&rdr, binary.LittleEndian, &zlibOpRaw) + if err != nil { + return nil, err + } + compressionOptions = NewZlibOptions(zlibOpRaw) + } else { + compressionOptions = NewZlibOptions(zlibOptionsRaw{}) + } + case xzCompression: + if flags.CompressorOptions { + var xzOpRaw xzOptionsRaw + err = binary.Read(&rdr, binary.LittleEndian, xzOpRaw) + if err != nil { + return nil, err + } + compressionOptions = NewXzOption(xzOpRaw) + } else { + compressionOptions = NewXzOption(xzOptionsRaw{}) + } + default: + //TODO: all the compression options + return nil, errors.New("This type of compression isn't supported yet") + } + //TODO: parse more info + return &Squashfs{ + rdr: &rdr, + super: superblock, + flags: flags, + compressionOptions: compressionOptions, + }, nil +} + +func (s *Squashfs) readRootDirectoryTable() error { + + offset, metaOffset := inode.ProcessInodeRef(s.super.RootInode) + meta, err := s.parseMetadataAt(int64(s.super.InodeTableOffset) + int64(offset)) + if err != nil { + fmt.Println("Error processing metadata") + return err + } + _, err = meta.Data.Read(make([]byte, metaOffset)) + if err != nil { + fmt.Println("error reading forward to offset") + return err + } + common, _, err := inode.ProcessInode(&meta.Data, s.super.BlockSize) + if err != nil { + fmt.Println("Error reading inode") + return err + } + if common.InodeType != inode.BasicDirectoryType { + return errors.New("Not a basic directory") + } + // rootDir := inodeInterface.(inode.BasicDirectory) + // dirTable, err := directory.NewDirectory(meta.Data) + // if err != nil { + // return err + // } + // for _, entry := range dirTable.Entries { + // fmt.Println(entry.Name) + // } + return nil +} + +//GetFlags return the SuperblockFlags of the Squashfs +func (s *Squashfs) GetFlags() SuperblockFlags { + return s.flags +} + +//metadata is a parsed metadata block +type metadata struct { + Compressed bool + Size uint16 + Data io.Reader +} + +func (m *metadata) close() { + switch m.Data.(type) { + case io.ReadCloser: + m.Data.(io.ReadCloser).Close() + } +} + +func (s *Squashfs) parseNextMetadata() (*metadata, error) { + var metaHeader uint16 + err := binary.Read(s.rdr, binary.LittleEndian, &metaHeader) + if err != nil { + return nil, err + } + reader := io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(metaHeader&^0x8000)) + if metaHeader&0x8000 == 0x8000 { + metaHeader = metaHeader &^ 0x8000 + compressRead, err := s.compressionOptions.Reader(reader) + return &metadata{ + Compressed: true, + Size: metaHeader, + Data: *compressRead, + }, err + } + return &metadata{ + Compressed: false, + Size: metaHeader, + Data: reader, + }, nil +} + +func (s *Squashfs) parseMetadataAt(offset int64) (*metadata, error) { + // tmp := s.rdr.offset + // meta, err := s.parseNextMetadata() + // s.rdr.offset = tmp + // return meta, err + var metaHeader uint16 + var headerBytes []byte + headerBytes = make([]byte, 2) + s.rdr.ReadAt(headerBytes, offset) + metaHeader = binary.LittleEndian.Uint16(headerBytes) + if metaHeader&0x8000 == 0x8000 { + fmt.Println("Compressed") + metaHeader = metaHeader &^ 0x8000 + compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize))) + return &metadata{ + Compressed: true, + Size: metaHeader, + Data: *compressRead, + }, err + } + return &metadata{ + Compressed: false, + Size: metaHeader, + Data: io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)), + }, nil + //TODO: figure out why things break when I use a straight zlib reader, but not when I use a byte reader writer + // byts, err := s.compressionOptions.Decompress(io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize))) + // return &metadata{ + // Compressed: false, + // Size: metaHeader, + // Data: newByteReadWriteFromBytes(byts), + // }, err +} diff --git a/old/squashfs_test.go b/old/squashfs_test.go new file mode 100644 index 0000000..72d469a --- /dev/null +++ b/old/squashfs_test.go @@ -0,0 +1,106 @@ +package squashfs + +import ( + "io" + "net/http" + "os" + "testing" + + appimage "github.com/CalebQ42/GoAppImage" +) + +const ( + downloadURL = "https://github.com/zilti/code-oss.AppImage/releases/download/continuous/Code_OSS-x86_64.AppImage" + appImageName = "Code_OSS.AppImage" + squashfsName = "Code_OSS.Squashfs" +) + +func TestAppImageSquash(t *testing.T) { + t.Parallel() + wd, err := os.Getwd() + if err != nil { + t.Error(err) + } + squashFil, err := os.Open(wd + "/testing/" + squashfsName) + if os.IsNotExist(err) { + TestCreateSquashFromAppImage(t) + squashFil, err = os.Open(wd + "/testing/" + squashfsName) + if err != nil { + t.Error(err) + } + } + defer squashFil.Close() + stat, _ := squashFil.Stat() + squash, err := NewSquashfs(io.NewSectionReader(squashFil, 0, stat.Size())) + if err != nil { + t.Error(err) + } + err = squash.readRootDirectoryTable() + t.Fatal(err) +} + +func TestCreateSquashFromAppImage(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + err = os.Mkdir(wd+"/testing", 0777) + if err != nil && !os.IsExist(err) { + t.Fatal(err) + } + _, err = os.Open(wd + "/testing/" + appImageName) + if os.IsNotExist(err) { + downloadTestAppImage(t, wd+"/testing") + _, err = os.Open(wd + "/testing/" + appImageName) + if err != nil { + t.Fatal(err) + } + } else if err != nil { + t.Fatal(err) + } + ai := appimage.NewAppImage(wd + "/testing/" + appImageName) + aiFil, err := os.Open(wd + "/testing/" + appImageName) + if err != nil { + t.Fatal(err) + } + defer aiFil.Close() + aiFil.Seek(ai.Offset, 0) + os.Remove(wd + "/testing/" + squashfsName) + aiSquash, err := os.Create(wd + "/testing/" + squashfsName) + if err != nil { + t.Fatal(err) + } + _, err = io.Copy(aiSquash, aiFil) + if err != nil { + t.Fatal(err) + } +} + +func downloadTestAppImage(t *testing.T, dir string) { + //seems to time out. Need to fix that at some point + appImage, err := os.Create(dir + "/" + appImageName) + if err != nil { + t.Fatal(err) + } + defer appImage.Close() + check := http.Client{ + CheckRedirect: func(r *http.Request, via []*http.Request) error { + r.URL.Opaque = r.URL.Path + return nil + }, + } + resp, err := check.Get(downloadURL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + _, err = io.Copy(appImage, resp.Body) + if err != nil { + t.Fatal(err) + } +} + +func TestLookInsideSquash(t *testing.T) { + t.Parallel() + //TODO +} diff --git a/writer_old.go b/old/writer_old.go similarity index 100% rename from writer_old.go rename to old/writer_old.go diff --git a/writer_test.go b/old/writer_test.go similarity index 100% rename from writer_test.go rename to old/writer_test.go diff --git a/squashfs.go b/squashfs.go index 065bc24..08b9715 100644 --- a/squashfs.go +++ b/squashfs.go @@ -2,166 +2,73 @@ package squashfs import ( "encoding/binary" - "errors" "fmt" "io" + "github.com/CalebQ42/GoSquashfs/internal/directory" "github.com/CalebQ42/GoSquashfs/internal/inode" ) -var ( - //ErrNotMagical happens when making a new Squashfs and it doesn't have the magic number - ErrNotMagical = errors.New("Not Magical") -) - -//Squashfs is a squashfs backed by a ReadSeeker. type Squashfs struct { - rdr *Reader //underlying reader - offset int - super Superblock - flags SuperblockFlags - compressionOptions CompressionOptions + r Reader + super Superblock + flags SuperblockFlags + compression CompressionOptions } -//NewSquashfs creates a new Squashfs backed by the given reader -func NewSquashfs(reader io.ReaderAt) (*Squashfs, error) { - rdr := NewReader(reader) - var superblock Superblock - err := binary.Read(&rdr, binary.LittleEndian, &superblock) +func NewSquashfs(rdr io.ReaderAt) (*Squashfs, error) { + var squash Squashfs + squash.r = NewReader(rdr) + err := binary.Read(&squash.r, binary.LittleEndian, &squash.super) if err != nil { return nil, err } - if superblock.Magic != squashfsMagic { - return nil, ErrNotMagical - } - flags := superblock.GetFlags() - var compressionOptions CompressionOptions - switch superblock.Compression { + squash.flags = squash.super.getFlags() + switch squash.super.Compression { case gzipCompression: - if flags.CompressorOptions { - var gzipOpRaw gzipOptionsRaw - err = binary.Read(&rdr, binary.LittleEndian, &gzipOpRaw) - if err != nil { - return nil, err - } - compressionOptions = NewGzipOptions(gzipOpRaw) - } else { - compressionOptions = NewGzipOptions(gzipOptionsRaw{}) - } - case xzCompression: - if flags.CompressorOptions { - var xzOpRaw xzOptionsRaw - err = binary.Read(&rdr, binary.LittleEndian, xzOpRaw) - if err != nil { - return nil, err - } - compressionOptions = NewXzOption(xzOpRaw) - } else { - compressionOptions = NewXzOption(xzOptionsRaw{}) + var raw gzipOptionsRaw + err := binary.Read(&squash.r, binary.LittleEndian, &raw) + if err != nil { + return nil, err } + squash.compression = NewGzipOptions(raw) default: - //TODO: all the compression options - return nil, errors.New("This type of compression isn't supported yet") + fmt.Println("Other compression options are not currently supported") + return nil, err } - //TODO: parse more info - return &Squashfs{ - rdr: &rdr, - super: superblock, - flags: flags, - compressionOptions: compressionOptions, - }, nil + return &squash, nil } -func (s *Squashfs) readRootDirectoryTable() error { +func (s *Squashfs) printDirTable() error { offset, metaOffset := inode.ProcessInodeRef(s.super.RootInode) - meta, err := s.parseMetadataAt(int64(s.super.InodeTableOffset) + int64(offset)) + br, err := s.NewBlockReader(int64(offset)) if err != nil { - fmt.Println("Error processing metadata") return err } - _, err = meta.Data.Read(make([]byte, metaOffset)) + fmt.Println(offset, metaOffset) + br.dataOffset = int64(metaOffset) + _, inodeType, err := inode.ProcessInode(br, s.super.BlockSize) if err != nil { - fmt.Println("error reading forward to offset") return err } - common, _, err := inode.ProcessInode(&meta.Data, s.super.BlockSize) + rootDir := inodeType.(*inode.BasicDirectory) + fmt.Println(*rootDir) + br, err = s.NewBlockReader(int64(s.super.DirectoryTableOffset) + int64(rootDir.DirectoryIndex)) if err != nil { - fmt.Println("Error reading inode") return err } - if common.InodeType != inode.BasicDirectoryType { - return errors.New("Not a basic directory") + br.dataOffset = int64(rootDir.DirectoryOffset) + dir, err := directory.NewDirectory(br) + if err != nil { + return err + } + for _, entry := range dir.Entries { + fmt.Println(entry.Name) } - // dirTable, err := directory.NewDirectory(meta.Data) - // if err != nil { - // return err - // } - // for _, entry := range dirTable.Entries { - // fmt.Println(entry.Name) - // } return nil } -//GetFlags return the SuperblockFlags of the Squashfs +//GetFlags returns the SuperblockFlags from the Superblock func (s *Squashfs) GetFlags() SuperblockFlags { return s.flags } - -//metadata is a parsed metadata block -type metadata struct { - Compressed bool - Size uint16 - Data io.Reader -} - -func (m *metadata) close() { - switch m.Data.(type) { - case io.ReadCloser: - m.Data.(io.ReadCloser).Close() - } -} - -func (s *Squashfs) parseNextMetadata() (*metadata, error) { - var metaHeader uint16 - err := binary.Read(s.rdr, binary.LittleEndian, &metaHeader) - if err != nil { - return nil, err - } - reader := io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(metaHeader)) - if metaHeader&0x8000 == 0x8000 { - metaHeader = metaHeader &^ 0x8000 - compressRead, err := s.compressionOptions.Reader(reader) - return &metadata{ - Compressed: true, - Size: metaHeader, - Data: *compressRead, - }, err - } - return &metadata{ - Compressed: false, - Size: metaHeader, - Data: reader, - }, nil -} - -func (s *Squashfs) parseMetadataAt(offset int64) (*metadata, error) { - var metaHeader uint16 - var headerBytes []byte - headerBytes = make([]byte, 2) - s.rdr.ReadAt(headerBytes, offset) - metaHeader = binary.LittleEndian.Uint16(headerBytes) - if metaHeader&0x8000 == 0x8000 { - metaHeader = metaHeader &^ 0x8000 - compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize))) - return &metadata{ - Compressed: true, - Size: metaHeader, - Data: *compressRead, - }, err - } - return &metadata{ - Compressed: false, - Size: metaHeader, - Data: io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)), - }, nil -} diff --git a/squashfs_test.go b/squashfs_test.go index 72d469a..048e1d0 100644 --- a/squashfs_test.go +++ b/squashfs_test.go @@ -35,7 +35,7 @@ func TestAppImageSquash(t *testing.T) { if err != nil { t.Error(err) } - err = squash.readRootDirectoryTable() + err = squash.printDirTable() t.Fatal(err) } diff --git a/superblock.go b/superblock.go index 59550ec..b33080b 100644 --- a/superblock.go +++ b/superblock.go @@ -42,8 +42,7 @@ type SuperblockFlags struct { UncompressedIDs bool } -//GetFlags returns the Flags parsed into a SuperblockFlags -func (s *Superblock) GetFlags() SuperblockFlags { +func (s *Superblock) getFlags() SuperblockFlags { return SuperblockFlags{ UncompressedInodes: s.Flags&0x1 == 0x1, UncompressedData: s.Flags&0x2 == 0x2, diff --git a/uncompress.go b/uncompress.go deleted file mode 100644 index d161d00..0000000 --- a/uncompress.go +++ /dev/null @@ -1,14 +0,0 @@ -package squashfs - -import "io" - -func uncompressData(data []byte, compressionType int) []byte { - //TODO: check compression type and uncompress the data - return make([]byte, 0) -} - -//same os uncompressData, but uses a reader instead. reader's seek will be -func uncompressReaderData(reader *io.Reader, compressionType int) []byte { - //TODO: check compression type and uncompress the data - return make([]byte, 0) -}