Restarted some stuff so I can do it better.

Made a reader that can reade across data blocks if necessary
Still can't get things to read right

README.md

@@ -10,7 +10,6 @@ I am focusing purely on unsquashing before squashing.
 # Working
 
 * Reading the header
-* (Maybe) reading gzip compressed data
 
 # Not Working (Yet). Roughly in order.
 
@@ -22,4 +21,5 @@ I am focusing purely on unsquashing before squashing.
 
 # Where I'm at
 
-* I can read the metadata, but can't read inodes just yet.
+* Redid a bunch. Implemented a custom reader that can read across blocks.
+    * As of yet, doesn't seem to be reading things quite right (seems to be issue with encryption reading)

blocks.go

@@ -0,0 +1,117 @@
+package squashfs
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	"github.com/CalebQ42/GoSquashfs/bytereadwrite"
+)
+
+type MetadataHeader struct {
+	rawHeader  uint16
+	Compressed bool
+	Size       uint16
+}
+
+type BlockReader struct {
+	initalOffset int64
+	offset       int64
+	squash       *Squashfs
+	headers      []MetadataHeader
+	dataCache    []byte
+	dataOffset   int64
+}
+
+//NewBlockReader creates a new BlockReader from a squashfs.Reader. Reads the first header and caches the first set of data.
+func (s *Squashfs) NewBlockReader(offset int64) (*BlockReader, error) {
+	var br BlockReader
+	br.squash = s
+	br.initalOffset = offset
+	br.offset = offset
+	br.headers = make([]MetadataHeader, 0)
+	br.dataCache = make([]byte, 0)
+	err := br.parseNewBlock()
+	if err != nil {
+		fmt.Println("Problem creating BlockReader")
+		return nil, err
+	}
+	return &br, nil
+}
+
+func (br *BlockReader) parseNewBlock() error {
+	var header MetadataHeader
+	err := binary.Read(io.NewSectionReader(&br.squash.r, br.offset, 2), binary.LittleEndian, &header.rawHeader)
+	if err != nil {
+		fmt.Println("Error while reading the header ", len(br.headers), " in BlockReader")
+		return err
+	}
+	header.Compressed = (header.rawHeader&0x8000 == 0x8000)
+	header.Size = header.rawHeader &^ 0x8000
+	br.headers = append(br.headers, header)
+	br.offset += 2
+	sectionReader := io.NewSectionReader(&br.squash.r, br.offset, br.offset+int64(header.Size))
+	dataWriter := bytereadwrite.NewByteReaderWriter()
+	_, err = io.Copy(dataWriter, sectionReader)
+	br.offset += int64(header.Size)
+	if header.Compressed {
+		data, err := br.squash.compression.Decompress(dataWriter)
+		if err != nil {
+			fmt.Println("Error while reading the encrypted data block in header", len(br.headers))
+			return err
+		}
+		br.dataCache = append(br.dataCache, data...)
+		return nil
+	}
+	if err != nil {
+		fmt.Println("Error while reading uncompressed data in header", len(br.headers))
+		return err
+	}
+	br.dataCache = append(br.dataCache, dataWriter.GetBytes()...)
+	return nil
+}
+
+//Read reads data into p. If it reaches EOF, tries to read a new block and add it's data to the cache
+func (br *BlockReader) Read(p []byte) (n int, err error) {
+	read := 0
+	for {
+		byter := bytereadwrite.NewByteReaderWriterFromBytes(br.dataCache[br.dataOffset:])
+		temp, err := byter.Read(p[read:])
+		br.dataOffset += int64(temp)
+		read += temp
+		if err == nil {
+			return read, nil
+		} else if err != io.EOF {
+			return read, err
+		}
+		if err == io.EOF {
+			err = br.parseNewBlock()
+			if err != nil {
+				fmt.Println("Error while reading a new block")
+				return read, err
+			}
+		}
+	}
+}
+
+//ReadAt reads data into p from the offset. If it reaches EOF, tries to read a new block and add it's data to the cache.
+//
+//Offset is reletive to the offset set on creation.
+func (br *BlockReader) ReadAt(p []byte, offset int) (n int, err error) {
+	read := 0
+	for err == io.EOF {
+		byter := bytereadwrite.NewByteReaderWriterFromBytes(br.dataCache[offset+read:])
+		temp, inErr := byter.Read(p[read:])
+		err = inErr
+		read += temp
+		br.dataOffset += int64(temp)
+		if err == io.EOF {
+			inErr = br.parseNewBlock()
+			if inErr != nil {
+				fmt.Println("Error while reading a new block")
+				return read, inErr
+			}
+		}
+	}
+	return
+}

bytereadwrite/bytereadwrite.go

@@ -0,0 +1,60 @@
+package bytereadwrite
+
+import "io"
+
+//ByteReaderWriter allows you to read to and from a byte slice. When writing, it expands the slice to accomodate any data.
+type ByteReaderWriter struct {
+	byts   []byte
+	offset int
+}
+
+//NewByteReaderWriter creates a ByteReaderWriter with an internal byte slice of the given length.
+func NewByteReaderWriter() *ByteReaderWriter {
+	return &ByteReaderWriter{
+		byts:   make([]byte, 0),
+		offset: 0,
+	}
+}
+
+//NewByteReaderWriter creates a ByteReaderWriter with an internal byte slice of the given length.
+func NewByteReaderWriterWithLength(length int) *ByteReaderWriter {
+	return &ByteReaderWriter{
+		byts:   make([]byte, length),
+		offset: 0,
+	}
+}
+
+//NewByteReaderWriterFromBytes creates a new ByteReaderWriter initialized with the given bytes
+func NewByteReaderWriterFromBytes(byts []byte) *ByteReaderWriter {
+	return &ByteReaderWriter{
+		byts:   byts,
+		offset: 0,
+	}
+}
+
+//GetBytes return the underlyting byte slice of the readerwriter
+func (bwr *ByteReaderWriter) GetBytes() []byte {
+	return bwr.byts
+}
+
+//Read reads the bytes.
+func (bwr *ByteReaderWriter) Read(byt []byte) (int, error) {
+	if len(bwr.byts) < bwr.offset+len(byt) {
+		bytesWritten := len(bwr.byts) - bwr.offset
+		for i := 0; i < bytesWritten; i++ {
+			byt[i] = bwr.byts[i+bwr.offset]
+		}
+		return bytesWritten, io.EOF
+	}
+	for i := 0; i < len(byt); i++ {
+		byt[i] = bwr.byts[bwr.offset+i]
+	}
+	bwr.offset += len(byt)
+	return len(byt), nil
+}
+
+//Write writes to the end of the bytes. WILL expand to accept the incoming bytes.
+func (bwr *ByteReaderWriter) Write(byts []byte) (int, error) {
+	bwr.byts = append(bwr.byts, byts...)
+	return len(byts), nil
+}

compressionoptions.go

@@ -4,7 +4,7 @@ import (
 	"compress/gzip"
 	"io"
 
-	"gopkg.in/src-d/go-git.v4/utils/ioutil"
+	"github.com/CalebQ42/GoSquashfs/bytereadwrite"
 )
 
 const (
@@ -18,19 +18,18 @@ const (
 
 //TODO: implement for each type of Options
 type CompressionOptions interface {
-	Decompress(*io.SectionReader, int) ([]byte, error)
+	Decompress(io.Reader) ([]byte, error)
 	DecompressCopy(*io.Reader, *io.Writer) (int, error)
-	Compress(*io.SectionReader, int) ([]byte, error)
+	Compress(*io.Reader) ([]byte, error)
 	CompressCopy(*io.Reader, *io.Writer) (int, error)
-	Reader(io.Reader) (*io.ReadCloser, error)
 }
 
 //TODO: Allow creation of options for compression.
 
 type gzipOptionsRaw struct {
-	compressionLevel int32
+	CompressionLevel int32
-	windowSize       int16
+	WindowSize       int16
-	strategies       int16
+	Strategies       int16
 }
 
 //GzipOptions is the options used for gzip compression. Backed by the raw format, with strategies parsed.
@@ -44,51 +43,6 @@ type GzipOptions struct {
 	FixedStretegy            bool
 }
 
-type byteWriterReader struct {
-	byts   []byte
-	offset int
-}
-
-func newByteReadWrite(length int) *byteWriterReader {
-	return &byteWriterReader{
-		byts:   make([]byte, length),
-		offset: 0,
-	}
-}
-
-func newByteReadWriteFromBytes(byts []byte) *byteWriterReader {
-	return &byteWriterReader{
-		byts:   byts,
-		offset: 0,
-	}
-}
-
-func (bwr *byteWriterReader) getBytes() []byte {
-	return bwr.byts
-}
-
-//Read reads the bytes.
-func (bwr *byteWriterReader) Read(byt []byte) (int, error) {
-	if len(bwr.byts) < bwr.offset+len(byt) {
-		bytesWritten := len(bwr.byts) - bwr.offset
-		for i := 0; i < bytesWritten; i++ {
-			byt[i] = bwr.byts[i+bwr.offset]
-		}
-		return bytesWritten, io.EOF
-	}
-	for i := 0; i < len(byt); i++ {
-		byt[i] = bwr.byts[bwr.offset+i]
-	}
-	bwr.offset += len(byt)
-	return len(byt), nil
-}
-
-//Write writes to the bytes. WILL expand to accept the incoming bytes.
-func (bwr *byteWriterReader) Write(byts []byte) (int, error) {
-	bwr.byts = append(bwr.byts, byts...)
-	return len(byts), nil
-}
-
 func NewGzipOptions(raw gzipOptionsRaw) *GzipOptions {
 	//TODO: parse strategies
 	return &GzipOptions{
@@ -96,18 +50,18 @@ func NewGzipOptions(raw gzipOptionsRaw) *GzipOptions {
 	}
 }
 
-func (gzipOp *GzipOptions) Decompress(rdr *io.SectionReader, blockSize int) ([]byte, error) {
+func (gzipOp *GzipOptions) Decompress(rdr io.Reader) ([]byte, error) {
 	gzipRdr, err := gzip.NewReader(rdr)
-	defer gzipRdr.Close()
+	// defer gzipRdr.Close()
 	if err != nil {
 		return nil, err
 	}
-	bytrw := newByteReadWrite(0)
+	bytrw := bytereadwrite.NewByteReaderWriter()
 	_, err = io.Copy(bytrw, gzipRdr)
 	if err != nil {
-		return bytrw.byts, err
+		return nil, err
 	}
-	return bytrw.byts, nil
+	return bytrw.GetBytes(), nil
 }
 
 func (gzipOp *GzipOptions) DecompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) {
@@ -120,15 +74,15 @@ func (gzipOp *GzipOptions) DecompressCopy(rdr *io.Reader, wrt *io.Writer) (int,
 	return int(n), err
 }
 
-func (gzipOp *GzipOptions) Compress(rdr *io.SectionReader, blockSize int) ([]byte, error) {
+func (gzipOp *GzipOptions) Compress(rdr *io.Reader) ([]byte, error) {
-	bytWrt := newByteReadWrite(0)
+	bytWrt := bytereadwrite.NewByteReaderWriter()
 	gzipWrt := gzip.NewWriter(bytWrt) //TODO: allow setting level
 	defer gzipWrt.Close()
-	_, err := io.Copy(gzipWrt, rdr)
+	_, err := io.Copy(gzipWrt, *rdr)
 	if err != nil {
-		return bytWrt.byts, err
+		return bytWrt.GetBytes(), err
 	}
-	return bytWrt.byts, nil
+	return bytWrt.GetBytes(), nil
 }
 
 func (gzipOp *GzipOptions) CompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) {
@@ -138,15 +92,9 @@ func (gzipOp *GzipOptions) CompressCopy(rdr *io.Reader, wrt *io.Writer) (int, er
 	return int(n), err
 }
 
-func (gzipOp *GzipOptions) Reader(rdr io.Reader) (*io.ReadCloser, error) {
-	read, err := gzip.NewReader(rdr)
-	redClo := ioutil.NewReadCloser(read, read)
-	return &redClo, err
-}
-
 type xzOptionsRaw struct {
-	dictionarySize    int32
+	DictionarySize    int32
-	executableFilters int32
+	ExecutableFilters int32
 }
 
 type XzOptions struct {
@@ -163,18 +111,18 @@ type XzOptions struct {
 func NewXzOption(raw xzOptionsRaw) XzOptions {
 	return XzOptions{
 		raw:          &raw,
-		Execx86:      raw.executableFilters&0x1 == 0x1,
+		Execx86:      raw.ExecutableFilters&0x1 == 0x1,
-		ExecPower:    raw.executableFilters&0x2 == 0x2,
+		ExecPower:    raw.ExecutableFilters&0x2 == 0x2,
-		Execa64:      raw.executableFilters&0x4 == 0x4,
+		Execa64:      raw.ExecutableFilters&0x4 == 0x4,
-		ExecArm:      raw.executableFilters&0x8 == 0x8,
+		ExecArm:      raw.ExecutableFilters&0x8 == 0x8,
-		ExecArmThumb: raw.executableFilters&0x10 == 0x10,
+		ExecArmThumb: raw.ExecutableFilters&0x10 == 0x10,
-		ExecSparc:    raw.executableFilters&0x20 == 0x20,
+		ExecSparc:    raw.ExecutableFilters&0x20 == 0x20,
 	}
 }
 
 type lz4OptionsRaw struct {
-	version int32
+	Version int32
-	flags   int32
+	Flags   int32
 }
 
 //ZstdOptions is the options set for zstdOptions
@@ -183,6 +131,6 @@ type ZstdOptions struct {
 }
 
 type lzoOptionsRaw struct {
-	algorithm        int32
+	Algorithm        int32
-	compressionLevel int32
+	CompressionLevel int32
 }

internal/directory/directory.go

@@ -54,6 +54,7 @@ func NewDirectory(rdr io.Reader) (*Directory, error) {
 	if err != nil {
 		return nil, err
 	}
+	fmt.Println(hdr)
 	headers := hdr.Count / 256
 	if headers%256 > 0 {
 		headers++

internal/inode/processInode.go

@@ -10,7 +10,7 @@ import (
 const (
 	//The inode type from inode.Common.InodeType
 
-	BasicDirectoryType = iota + 1
+	BasicDirectoryType = iota
 	BasicFileType
 	BasicSymlinkType
 	BasicBlockDeviceType
@@ -27,65 +27,64 @@ const (
 )
 
 //ProcessInode processes the next inode in the given reader
-func ProcessInode(rdr *io.Reader, blockSize uint32) (*Common, interface{}, error) {
+func ProcessInode(rdr io.Reader, blockSize uint32) (*Common, interface{}, error) {
 	var inodeHeader Common
-	err := binary.Read(*rdr, binary.LittleEndian, &inodeHeader)
+	err := binary.Read(rdr, binary.LittleEndian, &inodeHeader)
 	if err != nil {
 		return nil, nil, err
 	}
 	switch inodeHeader.InodeType {
 	case BasicDirectoryType:
 		var inode BasicDirectory
-		err = binary.Read(*rdr, binary.LittleEndian, &inode)
+		err = binary.Read(rdr, binary.LittleEndian, &inode)
 		return &inodeHeader, &inode, err
 	case BasicFileType:
-		inode, err := NewBasicFile(rdr, blockSize)
+		inode, err := NewBasicFile(&rdr, blockSize)
 		return &inodeHeader, inode, err
 	case BasicSymlinkType:
-		inode, err := NewBasicSymlink(rdr)
+		inode, err := NewBasicSymlink(&rdr)
 		return &inodeHeader, inode, err
 	case BasicBlockDeviceType:
 		var inode BasicDevice
-		err = binary.Read(*rdr, binary.LittleEndian, &inode)
+		err = binary.Read(rdr, binary.LittleEndian, &inode)
 		return &inodeHeader, inode, err
 	case BasicCharDeviceType:
 		var inode BasicDevice
-		err = binary.Read(*rdr, binary.LittleEndian, &inode)
+		err = binary.Read(rdr, binary.LittleEndian, &inode)
 		return &inodeHeader, inode, err
 	case BasicFifoType:
 		var inode BasicIPC
-		err = binary.Read(*rdr, binary.LittleEndian, &inode)
+		err = binary.Read(rdr, binary.LittleEndian, &inode)
 		return &inodeHeader, inode, err
 	case BasicSocketType:
 		var inode BasicIPC
-		err = binary.Read(*rdr, binary.LittleEndian, &inode)
+		err = binary.Read(rdr, binary.LittleEndian, &inode)
 		return &inodeHeader, inode, err
 	case ExtendedDirectoryType:
-		inode, err := NewExtendedDirectory(rdr)
+		inode, err := NewExtendedDirectory(&rdr)
 		return &inodeHeader, inode, err
 	case ExtendedFileType:
-		inode, err := NewExtendedFile(rdr, blockSize)
+		inode, err := NewExtendedFile(&rdr, blockSize)
 		return &inodeHeader, inode, err
 	case ExtendedSymlinkType:
-		inode, err := NewExtendedSymlink(rdr)
+		inode, err := NewExtendedSymlink(&rdr)
 		return &inodeHeader, inode, err
 	case ExtendedBlockDeviceType:
 		var inode ExtendedDevice
-		err = binary.Read(*rdr, binary.LittleEndian, &inode)
+		err = binary.Read(rdr, binary.LittleEndian, &inode)
 		return &inodeHeader, inode, err
 	case ExtendedCharDeviceType:
 		var inode ExtendedDevice
-		err = binary.Read(*rdr, binary.LittleEndian, &inode)
+		err = binary.Read(rdr, binary.LittleEndian, &inode)
 		return &inodeHeader, inode, err
 	case ExtendedFifoType:
 		var inode ExtendedIPC
-		err = binary.Read(*rdr, binary.LittleEndian, &inode)
+		err = binary.Read(rdr, binary.LittleEndian, &inode)
 		return &inodeHeader, inode, err
 	case ExtendedSocketType:
 		var inode ExtendedIPC
-		err = binary.Read(*rdr, binary.LittleEndian, &inode)
+		err = binary.Read(rdr, binary.LittleEndian, &inode)
 		return &inodeHeader, inode, err
-	//TODO: implement ALL cases
 	default:
 		return nil, nil, errors.New("Inode type is unrecognized: " + strconv.FormatInt(int64(inodeHeader.InodeType), 2))
 	}

internal/inode/util.go

@@ -3,8 +3,8 @@ package inode
 //ProcessInodeRef processes an inode reference and returns two values
 //The first value is the inode table offset. AKA, it's where the metadata block of the inode STARTS.
 //The second value is the offset of the inode, INSIDE of the metadata.
-func ProcessInodeRef(inodeRef uint64) (tableOffset uint32, metaOffset uint16) {
+func ProcessInodeRef(inodeRef uint64) (tableOffset uint64, metaOffset uint64) {
-	tableOffset = uint32(inodeRef >> 16)
+	tableOffset = inodeRef >> 16
-	metaOffset = uint16(inodeRef &^ 0xFFFFFFFF0000)
+	metaOffset = inodeRef &^ 0xFFFFFFFF0000
 	return
 }

old/bytereadwrite.go

@@ -0,0 +1,48 @@
+package squashfs
+
+import "io"
+
+type byteWriterReader struct {
+	byts   []byte
+	offset int
+}
+
+func newByteReadWrite(length int) *byteWriterReader {
+	return &byteWriterReader{
+		byts:   make([]byte, length),
+		offset: 0,
+	}
+}
+
+func newByteReadWriteFromBytes(byts []byte) *byteWriterReader {
+	return &byteWriterReader{
+		byts:   byts,
+		offset: 0,
+	}
+}
+
+func (bwr *byteWriterReader) getBytes() []byte {
+	return bwr.byts
+}
+
+//Read reads the bytes.
+func (bwr *byteWriterReader) Read(byt []byte) (int, error) {
+	if len(bwr.byts) < bwr.offset+len(byt) {
+		bytesWritten := len(bwr.byts) - bwr.offset
+		for i := 0; i < bytesWritten; i++ {
+			byt[i] = bwr.byts[i+bwr.offset]
+		}
+		return bytesWritten, io.EOF
+	}
+	for i := 0; i < len(byt); i++ {
+		byt[i] = bwr.byts[bwr.offset+i]
+	}
+	bwr.offset += len(byt)
+	return len(byt), nil
+}
+
+//Write writes to the bytes. WILL expand to accept the incoming bytes.
+func (bwr *byteWriterReader) Write(byts []byte) (int, error) {
+	bwr.byts = append(bwr.byts, byts...)
+	return len(byts), nil
+}

old/compressionoptions.go

@@ -0,0 +1,143 @@
+package squashfs
+
+import (
+	"compress/zlib"
+	"io"
+
+	"gopkg.in/src-d/go-git.v4/utils/ioutil"
+)
+
+const (
+	zlibCompression = 1 + iota
+	lzmaCompression
+	lzoCompression
+	xzCompression
+	lz4Compression
+	zstdCompression
+)
+
+//TODO: implement for each type of Options
+type CompressionOptions interface {
+	Decompress(*io.SectionReader) ([]byte, error)
+	DecompressCopy(*io.Reader, *io.Writer) (int, error)
+	Compress(*io.SectionReader) ([]byte, error)
+	CompressCopy(*io.Reader, *io.Writer) (int, error)
+	Reader(io.Reader) (*io.ReadCloser, error)
+}
+
+//TODO: Allow creation of options for compression.
+
+type zlibOptionsRaw struct {
+	compressionLevel int32
+	windowSize       int16
+	strategies       int16
+}
+
+//ZlibOptions is the options used for zlib compression. Backed by the raw format, with strategies parsed.
+type ZlibOptions struct {
+	CompressionOptions
+	raw                      *zlibOptionsRaw
+	DefaultStrategy          bool
+	FilteredStrategy         bool
+	HuffmanOnlyStrategy      bool
+	RunLengthEncodedStrategy bool
+	FixedStretegy            bool
+}
+
+func NewZlibOptions(raw zlibOptionsRaw) *ZlibOptions {
+	//TODO: parse strategies
+	return &ZlibOptions{
+		raw: &raw,
+	}
+}
+
+func (zlibOp *ZlibOptions) Decompress(rdr *io.SectionReader) ([]byte, error) {
+	zlibRdr, err := zlib.NewReader(rdr)
+	defer zlibRdr.Close()
+	if err != nil {
+		return nil, err
+	}
+	bytrw := newByteReadWrite(0)
+	_, err = io.Copy(bytrw, zlibRdr)
+	if err != nil {
+		return bytrw.byts, err
+	}
+	return bytrw.byts, nil
+}
+
+func (zlibOp *ZlibOptions) DecompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) {
+	zlibRdr, err := zlib.NewReader(*rdr)
+	defer zlibRdr.Close()
+	if err != nil {
+		return 0, err
+	}
+	n, err := io.Copy(*wrt, zlibRdr)
+	return int(n), err
+}
+
+func (zlibOp *ZlibOptions) Compress(rdr *io.SectionReader) ([]byte, error) {
+	bytWrt := newByteReadWrite(0)
+	zlibWrt := zlib.NewWriter(bytWrt) //TODO: allow setting level
+	defer zlibWrt.Close()
+	_, err := io.Copy(zlibWrt, rdr)
+	if err != nil {
+		return bytWrt.byts, err
+	}
+	return bytWrt.byts, nil
+}
+
+func (zlibOp *ZlibOptions) CompressCopy(rdr *io.Reader, wrt *io.Writer) (int, error) {
+	zlibWrt := zlib.NewWriter(*wrt) //TODO: allow setting level
+	defer zlibWrt.Close()
+	n, err := io.Copy(zlibWrt, *rdr)
+	return int(n), err
+}
+
+func (zlibOp *ZlibOptions) Reader(rdr io.Reader) (*io.ReadCloser, error) {
+	read, err := zlib.NewReader(rdr)
+	redClo := ioutil.NewReadCloser(NewByteBufferedReader(read), read)
+	return &redClo, err
+}
+
+type xzOptionsRaw struct {
+	dictionarySize    int32
+	executableFilters int32
+}
+
+type XzOptions struct {
+	CompressionOptions //TODO: Remove
+	raw                *xzOptionsRaw
+	Execx86            bool
+	ExecPower          bool
+	Execa64            bool
+	ExecArm            bool
+	ExecArmThumb       bool
+	ExecSparc          bool
+}
+
+func NewXzOption(raw xzOptionsRaw) XzOptions {
+	return XzOptions{
+		raw:          &raw,
+		Execx86:      raw.executableFilters&0x1 == 0x1,
+		ExecPower:    raw.executableFilters&0x2 == 0x2,
+		Execa64:      raw.executableFilters&0x4 == 0x4,
+		ExecArm:      raw.executableFilters&0x8 == 0x8,
+		ExecArmThumb: raw.executableFilters&0x10 == 0x10,
+		ExecSparc:    raw.executableFilters&0x20 == 0x20,
+	}
+}
+
+type lz4OptionsRaw struct {
+	version int32
+	flags   int32
+}
+
+//ZstdOptions is the options set for zstdOptions
+type ZstdOptions struct {
+	CompressionLevel int32 //CompressionLevel should be between 1 and 22
+}
+
+type lzoOptionsRaw struct {
+	algorithm        int32
+	compressionLevel int32
+}

old/reader.go

@@ -0,0 +1,59 @@
+package squashfs
+
+import (
+	"errors"
+	"io"
+)
+
+//TODO: possible custom reader because I'm havng some issuse...
+
+//Reader is a reader which implements Reader, ReaderAt, and Seeker, all with an accesible offset (for reasons)
+type Reader struct {
+	rdr    io.ReaderAt
+	offset int64
+}
+
+//NewReader creates a squashfs.Reader from a io.ReaderAt
+func NewReader(baseReader io.ReaderAt) Reader {
+	return Reader{
+		rdr:    baseReader,
+		offset: 0,
+	}
+}
+
+//Read reads len(byt) into byt. Advances the internal offset
+func (r *Reader) Read(byt []byte) (int, error) {
+	n, err := r.rdr.ReadAt(byt, r.offset)
+	r.offset += int64(n)
+	return n, err
+}
+
+//ReadAt wraps the internal io.ReadAt's function. DOES NOT advance the internal offset for Read function.
+//Returns how many bytes were read.
+func (r *Reader) ReadAt(byt []byte, offset int64) (int, error) {
+	return r.rdr.ReadAt(byt, offset)
+}
+
+//ReadAtFromOffset is the same as ReadAt, but the given offset is offset by the internal offset. DOES NOT advance the internal offset.
+//Returns how many bytes were read.
+func (r *Reader) ReadAtFromOffset(byt []byte, offset int64) (int, error) {
+	offset += r.offset
+	return r.rdr.ReadAt(byt, offset)
+}
+
+//Seek advances the internal offset. SeekEnd DOES NOT work
+//Might not be necessary, but here just in case
+func (r *Reader) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekCurrent:
+		n, err := r.Read(make([]byte, offset))
+		return int64(n), err
+	case io.SeekStart:
+		r.offset = 0
+		n, err := r.Read(make([]byte, offset))
+		return int64(n), err
+	case io.SeekEnd:
+		return 0, errors.New("SeekEnd is NOT supported")
+	}
+	return 0, errors.New("incorrect whence")
+}

old/squashfs.go

@@ -0,0 +1,181 @@
+package squashfs
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/CalebQ42/GoSquashfs/old/internal/inode"
+)
+
+var (
+	//ErrNotMagical happens when making a new Squashfs and it doesn't have the magic number
+	ErrNotMagical = errors.New("Not Magical")
+)
+
+//Squashfs is a squashfs backed by a ReadSeeker.
+type Squashfs struct {
+	rdr                *Reader //underlying reader
+	offset             int
+	super              Superblock
+	flags              SuperblockFlags
+	compressionOptions CompressionOptions
+}
+
+//NewSquashfs creates a new Squashfs backed by the given reader
+func NewSquashfs(reader io.ReaderAt) (*Squashfs, error) {
+	rdr := NewReader(reader)
+	var superblock Superblock
+	err := binary.Read(&rdr, binary.LittleEndian, &superblock)
+	if err != nil {
+		return nil, err
+	}
+	if superblock.Magic != squashfsMagic {
+		return nil, ErrNotMagical
+	}
+	flags := superblock.GetFlags()
+	var compressionOptions CompressionOptions
+	switch superblock.Compression {
+	case zlibCompression:
+		if flags.CompressorOptions {
+			var zlibOpRaw zlibOptionsRaw
+			err = binary.Read(&rdr, binary.LittleEndian, &zlibOpRaw)
+			if err != nil {
+				return nil, err
+			}
+			compressionOptions = NewZlibOptions(zlibOpRaw)
+		} else {
+			compressionOptions = NewZlibOptions(zlibOptionsRaw{})
+		}
+	case xzCompression:
+		if flags.CompressorOptions {
+			var xzOpRaw xzOptionsRaw
+			err = binary.Read(&rdr, binary.LittleEndian, xzOpRaw)
+			if err != nil {
+				return nil, err
+			}
+			compressionOptions = NewXzOption(xzOpRaw)
+		} else {
+			compressionOptions = NewXzOption(xzOptionsRaw{})
+		}
+	default:
+		//TODO: all the compression options
+		return nil, errors.New("This type of compression isn't supported yet")
+	}
+	//TODO: parse more info
+	return &Squashfs{
+		rdr:                &rdr,
+		super:              superblock,
+		flags:              flags,
+		compressionOptions: compressionOptions,
+	}, nil
+}
+
+func (s *Squashfs) readRootDirectoryTable() error {
+
+	offset, metaOffset := inode.ProcessInodeRef(s.super.RootInode)
+	meta, err := s.parseMetadataAt(int64(s.super.InodeTableOffset) + int64(offset))
+	if err != nil {
+		fmt.Println("Error processing metadata")
+		return err
+	}
+	_, err = meta.Data.Read(make([]byte, metaOffset))
+	if err != nil {
+		fmt.Println("error reading forward to offset")
+		return err
+	}
+	common, _, err := inode.ProcessInode(&meta.Data, s.super.BlockSize)
+	if err != nil {
+		fmt.Println("Error reading inode")
+		return err
+	}
+	if common.InodeType != inode.BasicDirectoryType {
+		return errors.New("Not a basic directory")
+	}
+	// rootDir := inodeInterface.(inode.BasicDirectory)
+	// dirTable, err := directory.NewDirectory(meta.Data)
+	// if err != nil {
+	// 	return err
+	// }
+	// for _, entry := range dirTable.Entries {
+	// 	fmt.Println(entry.Name)
+	// }
+	return nil
+}
+
+//GetFlags return the SuperblockFlags of the Squashfs
+func (s *Squashfs) GetFlags() SuperblockFlags {
+	return s.flags
+}
+
+//metadata is a parsed metadata block
+type metadata struct {
+	Compressed bool
+	Size       uint16
+	Data       io.Reader
+}
+
+func (m *metadata) close() {
+	switch m.Data.(type) {
+	case io.ReadCloser:
+		m.Data.(io.ReadCloser).Close()
+	}
+}
+
+func (s *Squashfs) parseNextMetadata() (*metadata, error) {
+	var metaHeader uint16
+	err := binary.Read(s.rdr, binary.LittleEndian, &metaHeader)
+	if err != nil {
+		return nil, err
+	}
+	reader := io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(metaHeader&^0x8000))
+	if metaHeader&0x8000 == 0x8000 {
+		metaHeader = metaHeader &^ 0x8000
+		compressRead, err := s.compressionOptions.Reader(reader)
+		return &metadata{
+			Compressed: true,
+			Size:       metaHeader,
+			Data:       *compressRead,
+		}, err
+	}
+	return &metadata{
+		Compressed: false,
+		Size:       metaHeader,
+		Data:       reader,
+	}, nil
+}
+
+func (s *Squashfs) parseMetadataAt(offset int64) (*metadata, error) {
+	// tmp := s.rdr.offset
+	// meta, err := s.parseNextMetadata()
+	// s.rdr.offset = tmp
+	// return meta, err
+	var metaHeader uint16
+	var headerBytes []byte
+	headerBytes = make([]byte, 2)
+	s.rdr.ReadAt(headerBytes, offset)
+	metaHeader = binary.LittleEndian.Uint16(headerBytes)
+	if metaHeader&0x8000 == 0x8000 {
+		fmt.Println("Compressed")
+		metaHeader = metaHeader &^ 0x8000
+		compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)))
+		return &metadata{
+			Compressed: true,
+			Size:       metaHeader,
+			Data:       *compressRead,
+		}, err
+	}
+	return &metadata{
+		Compressed: false,
+		Size:       metaHeader,
+		Data:       io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)),
+	}, nil
+	//TODO: figure out why things break when I use a straight zlib reader, but not when I use a byte reader writer
+	// byts, err := s.compressionOptions.Decompress(io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)))
+	// return &metadata{
+	// 	Compressed: false,
+	// 	Size:       metaHeader,
+	// 	Data:       newByteReadWriteFromBytes(byts),
+	// }, err
+}

old/squashfs_test.go

@@ -0,0 +1,106 @@
+package squashfs
+
+import (
+	"io"
+	"net/http"
+	"os"
+	"testing"
+
+	appimage "github.com/CalebQ42/GoAppImage"
+)
+
+const (
+	downloadURL  = "https://github.com/zilti/code-oss.AppImage/releases/download/continuous/Code_OSS-x86_64.AppImage"
+	appImageName = "Code_OSS.AppImage"
+	squashfsName = "Code_OSS.Squashfs"
+)
+
+func TestAppImageSquash(t *testing.T) {
+	t.Parallel()
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Error(err)
+	}
+	squashFil, err := os.Open(wd + "/testing/" + squashfsName)
+	if os.IsNotExist(err) {
+		TestCreateSquashFromAppImage(t)
+		squashFil, err = os.Open(wd + "/testing/" + squashfsName)
+		if err != nil {
+			t.Error(err)
+		}
+	}
+	defer squashFil.Close()
+	stat, _ := squashFil.Stat()
+	squash, err := NewSquashfs(io.NewSectionReader(squashFil, 0, stat.Size()))
+	if err != nil {
+		t.Error(err)
+	}
+	err = squash.readRootDirectoryTable()
+	t.Fatal(err)
+}
+
+func TestCreateSquashFromAppImage(t *testing.T) {
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.Mkdir(wd+"/testing", 0777)
+	if err != nil && !os.IsExist(err) {
+		t.Fatal(err)
+	}
+	_, err = os.Open(wd + "/testing/" + appImageName)
+	if os.IsNotExist(err) {
+		downloadTestAppImage(t, wd+"/testing")
+		_, err = os.Open(wd + "/testing/" + appImageName)
+		if err != nil {
+			t.Fatal(err)
+		}
+	} else if err != nil {
+		t.Fatal(err)
+	}
+	ai := appimage.NewAppImage(wd + "/testing/" + appImageName)
+	aiFil, err := os.Open(wd + "/testing/" + appImageName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer aiFil.Close()
+	aiFil.Seek(ai.Offset, 0)
+	os.Remove(wd + "/testing/" + squashfsName)
+	aiSquash, err := os.Create(wd + "/testing/" + squashfsName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = io.Copy(aiSquash, aiFil)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func downloadTestAppImage(t *testing.T, dir string) {
+	//seems to time out. Need to fix that at some point
+	appImage, err := os.Create(dir + "/" + appImageName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer appImage.Close()
+	check := http.Client{
+		CheckRedirect: func(r *http.Request, via []*http.Request) error {
+			r.URL.Opaque = r.URL.Path
+			return nil
+		},
+	}
+	resp, err := check.Get(downloadURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(appImage, resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestLookInsideSquash(t *testing.T) {
+	t.Parallel()
+	//TODO
+}

squashfs.go

@@ -2,166 +2,73 @@ package squashfs
 
 import (
 	"encoding/binary"
-	"errors"
 	"fmt"
 	"io"
 
+	"github.com/CalebQ42/GoSquashfs/internal/directory"
 	"github.com/CalebQ42/GoSquashfs/internal/inode"
 )
 
-var (
-	//ErrNotMagical happens when making a new Squashfs and it doesn't have the magic number
-	ErrNotMagical = errors.New("Not Magical")
-)
-
-//Squashfs is a squashfs backed by a ReadSeeker.
 type Squashfs struct {
-	rdr                *Reader //underlying reader
+	r           Reader
-	offset             int
+	super       Superblock
-	super              Superblock
+	flags       SuperblockFlags
-	flags              SuperblockFlags
+	compression CompressionOptions
-	compressionOptions CompressionOptions
 }
 
-//NewSquashfs creates a new Squashfs backed by the given reader
+func NewSquashfs(rdr io.ReaderAt) (*Squashfs, error) {
-func NewSquashfs(reader io.ReaderAt) (*Squashfs, error) {
+	var squash Squashfs
-	rdr := NewReader(reader)
+	squash.r = NewReader(rdr)
-	var superblock Superblock
+	err := binary.Read(&squash.r, binary.LittleEndian, &squash.super)
-	err := binary.Read(&rdr, binary.LittleEndian, &superblock)
 	if err != nil {
 		return nil, err
 	}
-	if superblock.Magic != squashfsMagic {
+	squash.flags = squash.super.getFlags()
-		return nil, ErrNotMagical
+	switch squash.super.Compression {
-	}
-	flags := superblock.GetFlags()
-	var compressionOptions CompressionOptions
-	switch superblock.Compression {
 	case gzipCompression:
-		if flags.CompressorOptions {
+		var raw gzipOptionsRaw
-			var gzipOpRaw gzipOptionsRaw
+		err := binary.Read(&squash.r, binary.LittleEndian, &raw)
-			err = binary.Read(&rdr, binary.LittleEndian, &gzipOpRaw)
+		if err != nil {
-			if err != nil {
+			return nil, err
-				return nil, err
-			}
-			compressionOptions = NewGzipOptions(gzipOpRaw)
-		} else {
-			compressionOptions = NewGzipOptions(gzipOptionsRaw{})
-		}
-	case xzCompression:
-		if flags.CompressorOptions {
-			var xzOpRaw xzOptionsRaw
-			err = binary.Read(&rdr, binary.LittleEndian, xzOpRaw)
-			if err != nil {
-				return nil, err
-			}
-			compressionOptions = NewXzOption(xzOpRaw)
-		} else {
-			compressionOptions = NewXzOption(xzOptionsRaw{})
 		}
+		squash.compression = NewGzipOptions(raw)
 	default:
-		//TODO: all the compression options
+		fmt.Println("Other compression options are not currently supported")
-		return nil, errors.New("This type of compression isn't supported yet")
+		return nil, err
 	}
-	//TODO: parse more info
+	return &squash, nil
-	return &Squashfs{
-		rdr:                &rdr,
-		super:              superblock,
-		flags:              flags,
-		compressionOptions: compressionOptions,
-	}, nil
 }
 
-func (s *Squashfs) readRootDirectoryTable() error {
+func (s *Squashfs) printDirTable() error {
 	offset, metaOffset := inode.ProcessInodeRef(s.super.RootInode)
-	meta, err := s.parseMetadataAt(int64(s.super.InodeTableOffset) + int64(offset))
+	br, err := s.NewBlockReader(int64(offset))
 	if err != nil {
-		fmt.Println("Error processing metadata")
 		return err
 	}
-	_, err = meta.Data.Read(make([]byte, metaOffset))
+	fmt.Println(offset, metaOffset)
+	br.dataOffset = int64(metaOffset)
+	_, inodeType, err := inode.ProcessInode(br, s.super.BlockSize)
 	if err != nil {
-		fmt.Println("error reading forward to offset")
 		return err
 	}
-	common, _, err := inode.ProcessInode(&meta.Data, s.super.BlockSize)
+	rootDir := inodeType.(*inode.BasicDirectory)
+	fmt.Println(*rootDir)
+	br, err = s.NewBlockReader(int64(s.super.DirectoryTableOffset) + int64(rootDir.DirectoryIndex))
 	if err != nil {
-		fmt.Println("Error reading inode")
 		return err
 	}
-	if common.InodeType != inode.BasicDirectoryType {
+	br.dataOffset = int64(rootDir.DirectoryOffset)
-		return errors.New("Not a basic directory")
+	dir, err := directory.NewDirectory(br)
+	if err != nil {
+		return err
+	}
+	for _, entry := range dir.Entries {
+		fmt.Println(entry.Name)
 	}
-	// dirTable, err := directory.NewDirectory(meta.Data)
-	// if err != nil {
-	// 	return err
-	// }
-	// for _, entry := range dirTable.Entries {
-	// 	fmt.Println(entry.Name)
-	// }
 	return nil
 }
 
-//GetFlags return the SuperblockFlags of the Squashfs
+//GetFlags returns the SuperblockFlags from the Superblock
 func (s *Squashfs) GetFlags() SuperblockFlags {
 	return s.flags
 }
-
-//metadata is a parsed metadata block
-type metadata struct {
-	Compressed bool
-	Size       uint16
-	Data       io.Reader
-}
-
-func (m *metadata) close() {
-	switch m.Data.(type) {
-	case io.ReadCloser:
-		m.Data.(io.ReadCloser).Close()
-	}
-}
-
-func (s *Squashfs) parseNextMetadata() (*metadata, error) {
-	var metaHeader uint16
-	err := binary.Read(s.rdr, binary.LittleEndian, &metaHeader)
-	if err != nil {
-		return nil, err
-	}
-	reader := io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(metaHeader))
-	if metaHeader&0x8000 == 0x8000 {
-		metaHeader = metaHeader &^ 0x8000
-		compressRead, err := s.compressionOptions.Reader(reader)
-		return &metadata{
-			Compressed: true,
-			Size:       metaHeader,
-			Data:       *compressRead,
-		}, err
-	}
-	return &metadata{
-		Compressed: false,
-		Size:       metaHeader,
-		Data:       reader,
-	}, nil
-}
-
-func (s *Squashfs) parseMetadataAt(offset int64) (*metadata, error) {
-	var metaHeader uint16
-	var headerBytes []byte
-	headerBytes = make([]byte, 2)
-	s.rdr.ReadAt(headerBytes, offset)
-	metaHeader = binary.LittleEndian.Uint16(headerBytes)
-	if metaHeader&0x8000 == 0x8000 {
-		metaHeader = metaHeader &^ 0x8000
-		compressRead, err := s.compressionOptions.Reader(io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)))
-		return &metadata{
-			Compressed: true,
-			Size:       metaHeader,
-			Data:       *compressRead,
-		}, err
-	}
-	return &metadata{
-		Compressed: false,
-		Size:       metaHeader,
-		Data:       io.NewSectionReader(s.rdr, s.rdr.offset, s.rdr.offset+int64(s.super.BlockSize)),
-	}, nil
-}

squashfs_test.go

@@ -35,7 +35,7 @@ func TestAppImageSquash(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	err = squash.readRootDirectoryTable()
+	err = squash.printDirTable()
 	t.Fatal(err)
 }
 

superblock.go

@@ -42,8 +42,7 @@ type SuperblockFlags struct {
 	UncompressedIDs       bool
 }
 
-//GetFlags returns the Flags parsed into a SuperblockFlags
+func (s *Superblock) getFlags() SuperblockFlags {
-func (s *Superblock) GetFlags() SuperblockFlags {
 	return SuperblockFlags{
 		UncompressedInodes:    s.Flags&0x1 == 0x1,
 		UncompressedData:      s.Flags&0x2 == 0x2,

uncompress.go

@@ -1,14 +0,0 @@
-package squashfs
-
-import "io"
-
-func uncompressData(data []byte, compressionType int) []byte {
-	//TODO: check compression type and uncompress the data
-	return make([]byte, 0)
-}
-
-//same os uncompressData, but uses a reader instead. reader's seek will be
-func uncompressReaderData(reader *io.Reader, compressionType int) []byte {
-	//TODO: check compression type and uncompress the data
-	return make([]byte, 0)
-}