Tweaks to make FromReader work

Now allows creation of a squashfs.Reader from an io.Reader

internal/rawreader/rawreader.go

@@ -0,0 +1,135 @@
+package rawreader
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+func ConvertReader(r io.Reader) (RawReader, error) {
+	if rr, ok := r.(RawReader); ok {
+		return rr, nil
+	}
+	if rs, is := r.(io.ReadSeeker); is {
+		return &fromReadSeeker{
+			ReadSeeker: rs,
+		}, nil
+	}
+	buf := new(bytes.Buffer)
+	_, err := io.Copy(buf, r)
+	if err != nil {
+		return nil, err
+	}
+	return &fromReader{
+		data: buf.Bytes(),
+	}, nil
+}
+
+func ConvertReaderAt(r io.ReaderAt) RawReader {
+	if rr, ok := r.(RawReader); ok {
+		return rr
+	}
+	return &fromReaderAt{
+		ReaderAt: r,
+	}
+}
+
+//TODO: Add way to discard data from fromReader
+//RawReader implements the needed interfaces for reading a squashfs archive.
+type RawReader interface {
+	io.ReadSeeker
+	io.ReaderAt
+}
+
+type fromReader struct {
+	data []byte
+	off  int
+}
+
+func (r *fromReader) ReadAt(p []byte, off int64) (n int, err error) {
+	n = len(p)
+	if int(off)+len(p) > len(r.data) {
+		n = len(r.data) - int(off)
+		err = io.EOF
+	}
+	if n < 0 {
+		n = 0
+	}
+	for i := 0; i < n; i++ {
+		p[i] = r.data[int(off)+i]
+	}
+	return
+}
+
+func (r *fromReader) Seek(off int64, whence int) (n int64, err error) {
+	switch whence {
+	case io.SeekEnd:
+		r.off = len(r.data) - int(off)
+		if r.off < 0 {
+			r.off = 0
+			err = io.EOF
+		}
+	case io.SeekCurrent:
+		r.off += int(off)
+	case io.SeekStart:
+		r.off = int(off)
+	}
+	if r.off > len(r.data) {
+		r.off = len(r.data)
+		return int64(r.off), io.EOF
+	}
+	return int64(r.off), err
+}
+
+func (r *fromReader) Read(p []byte) (n int, err error) {
+	n = len(p)
+	if r.off+len(p) > len(r.data) {
+		n = len(r.data) - r.off
+		err = io.EOF
+	}
+	if n < 0 {
+		n = 0
+	}
+	for i := 0; i < n; i++ {
+		p[i] = r.data[r.off+i]
+	}
+	return
+}
+
+type fromReadSeeker struct {
+	io.ReadSeeker
+}
+
+func (r *fromReadSeeker) ReadAt(p []byte, off int64) (n int, err error) {
+	tmp, _ := r.Seek(0, io.SeekCurrent)
+	defer r.Seek(tmp, io.SeekStart)
+	_, err = r.Seek(off, io.SeekStart)
+	if err != nil {
+		return
+	}
+	return r.Read(p)
+}
+
+type fromReaderAt struct {
+	io.ReaderAt
+
+	off int
+}
+
+func (r *fromReaderAt) Read(p []byte) (n int, err error) {
+	n, err = r.ReadAt(p, int64(r.off))
+	r.off += n
+	return
+}
+
+func (r *fromReaderAt) Seek(off int64, whence int) (n int64, err error) {
+	switch whence {
+	case io.SeekEnd:
+		return 0, errors.New("cannot SeekEnd RawReader")
+	case io.SeekCurrent:
+		r.off += int(off)
+	case io.SeekStart:
+		r.off = int(off)
+	}
+	return int64(r.off), nil
+}

reader.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/CalebQ42/squashfs/internal/compression"
 	"github.com/CalebQ42/squashfs/internal/inode"
+	"github.com/CalebQ42/squashfs/internal/rawreader"
 )
 
 const (
@@ -20,16 +21,12 @@ var (
 	errNoMagic = errors.New("magic number doesn't match. Either isn't a squashfs or corrupted")
 	//ErrIncompatibleCompression is returned if the compression type in the superblock doesn't work.
 	errIncompatibleCompression = errors.New("compression type unsupported")
-	//ErrOptions is returned when compression options that I haven't tested is set. When this is returned, the Reader is also returned.
-	ErrOptions = errors.New("possibly incompatible compressor options")
 )
 
-//TODO: implement fs.FS, possibly more FS types for compatibility. Most of this work will mostly be handed off to root anyway so this shouldn't be too difficult.
-
 //Reader processes and reads a squashfs archive.
 type Reader struct {
 	FS
-	r            *io.SectionReader
+	r            rawreader.RawReader
 	decompressor compression.Decompressor
 	fragOffsets  []uint64
 	idTable      []uint32
@@ -40,140 +37,163 @@ type Reader struct {
 //NewSquashfsReader returns a new squashfs.Reader from an io.ReaderAt
 func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 	var rdr Reader
-	err := binary.Read(io.NewSectionReader(r, 0, int64(binary.Size(rdr.super))), binary.LittleEndian, &rdr.super)
+	rdr.r = rawreader.ConvertReaderAt(r)
+	err := rdr.Init()
 	if err != nil {
 		return nil, err
 	}
-	rdr.r = io.NewSectionReader(r, 0, int64(rdr.super.BytesUsed))
+	return &rdr, nil
-	if rdr.super.Magic != magic {
+}
-		return nil, errNoMagic
+
+//NewSquashfsReaderFromReader returns a new squashfs.Reader from an io.Reader.
+//If the io.Reader implements io.Seeker, the seek functions are used.
+//It is NOT recommended to use a pure io.Reader as due to how squashfs
+//archives are formatted, the ENTIRETY of the io.Reader's data is loaded into
+//memory first before it can be used.
+func NewSquashfsReaderFromReader(r io.Reader) (*Reader, error) {
+	var rdr Reader
+	var err error
+	rdr.r, err = rawreader.ConvertReader(r)
+	if err != nil {
+		return nil, err
 	}
-	if rdr.super.BlockLog != uint16(math.Log2(float64(rdr.super.BlockSize))) {
+	err = rdr.Init()
-		return nil, errors.New("BlockSize and BlockLog doesn't match. The archive is probably corrupt")
+	if err != nil {
+		return nil, err
 	}
-	rdr.r.Seek(96, io.SeekStart)
+	return &rdr, nil
-	hasUnsupportedOptions := false
+}
-	rdr.flags = rdr.super.GetFlags()
+
-	if rdr.flags.compressorOptions {
+func (r *Reader) Init() error {
-		switch rdr.super.CompressionType {
+	err := binary.Read(r.r, binary.LittleEndian, &r.super)
+	if err != nil {
+		return err
+	}
+	if r.super.Magic != magic {
+		return errNoMagic
+	}
+	if r.super.BlockLog != uint16(math.Log2(float64(r.super.BlockSize))) {
+		return errors.New("BlockSize and BlockLog doesn't match. The archive is probably corrupt")
+	}
+	r.r.Seek(96, io.SeekStart)
+	r.flags = r.super.GetFlags()
+	if r.flags.compressorOptions {
+		switch r.super.CompressionType {
 		case GzipCompression:
 			var gzip *compression.Gzip
-			gzip, err = compression.NewGzipCompressorWithOptions(rdr.r)
+			gzip, err = compression.NewGzipCompressorWithOptions(r.r)
 			if err != nil {
-				return nil, err
+				return err
 			}
-			if gzip.HasCustomWindow || gzip.HasStrategies {
+			r.decompressor = gzip
-				hasUnsupportedOptions = true
-			}
-			rdr.decompressor = gzip
 		case XzCompression:
 			var xz *compression.Xz
-			xz, err = compression.NewXzCompressorWithOptions(rdr.r)
+			xz, err = compression.NewXzCompressorWithOptions(r.r)
 			if err != nil {
-				return nil, err
+				return err
 			}
-			rdr.decompressor = xz
+			r.decompressor = xz
 		case LzoCompression:
 			var lz *compression.Lzo
-			lz, err = compression.NewLzoCompressorWithOptions(rdr.r)
+			lz, err = compression.NewLzoCompressorWithOptions(r.r)
 			if err != nil {
-				return nil, err
+				return err
 			}
-			rdr.decompressor = lz
+			r.decompressor = lz
 		case Lz4Compression:
 			var lz4 *compression.Lz4
-			lz4, err = compression.NewLz4CompressorWithOptions(rdr.r)
+			lz4, err = compression.NewLz4CompressorWithOptions(r.r)
 			if err != nil {
-				return nil, err
+				return err
 			}
-			rdr.decompressor = lz4
+			r.decompressor = lz4
 		case ZstdCompression:
 			var zstd *compression.Zstd
-			zstd, err = compression.NewZstdCompressorWithOptions(rdr.r)
+			zstd, err = compression.NewZstdCompressorWithOptions(r.r)
 			if err != nil {
-				return nil, err
+				return err
 			}
-			rdr.decompressor = zstd
+			r.decompressor = zstd
 		default:
-			return nil, errIncompatibleCompression
+			return errIncompatibleCompression
 		}
 	} else {
-		switch rdr.super.CompressionType {
+		switch r.super.CompressionType {
 		case GzipCompression:
-			rdr.decompressor = &compression.Gzip{}
+			r.decompressor = &compression.Gzip{}
 		case LzmaCompression:
-			rdr.decompressor = &compression.Lzma{}
+			r.decompressor = &compression.Lzma{}
 		case LzoCompression:
-			rdr.decompressor = &compression.Lzo{}
+			r.decompressor = &compression.Lzo{}
 		case XzCompression:
-			rdr.decompressor = &compression.Xz{}
+			r.decompressor = &compression.Xz{}
 		case Lz4Compression:
-			rdr.decompressor = &compression.Lz4{}
+			r.decompressor = &compression.Lz4{}
 		case ZstdCompression:
-			rdr.decompressor = &compression.Zstd{}
+			r.decompressor = &compression.Zstd{}
 		default:
 			//TODO: all compression types.
-			return nil, errIncompatibleCompression
+			return errIncompatibleCompression
 		}
 	}
-	fragBlocks := int(math.Ceil(float64(rdr.super.FragCount) / 512))
+	fragBlocks := int(math.Ceil(float64(r.super.FragCount) / 512))
 	if fragBlocks > 0 {
-		offset := int64(rdr.super.FragTableStart)
+		offset := int64(r.super.FragTableStart)
 		for i := 0; i < fragBlocks; i++ {
 			tmp := make([]byte, 8)
-			_, err = r.ReadAt(tmp, offset)
+			_, err = r.r.ReadAt(tmp, offset)
 			if err != nil {
-				return nil, err
+				return err
 			}
-			rdr.fragOffsets = append(rdr.fragOffsets, binary.LittleEndian.Uint64(tmp))
+			r.fragOffsets = append(r.fragOffsets, binary.LittleEndian.Uint64(tmp))
 			offset += 8
 		}
 	}
-	unread := rdr.super.IDCount
+	unread := r.super.IDCount
-	blockOffsets := make([]uint64, int(math.Ceil(float64(rdr.super.IDCount)/2048)))
+	blockOffsets := make([]uint64, int(math.Ceil(float64(r.super.IDCount)/2048)))
-	rdr.r.Seek(int64(rdr.super.IDTableStart), io.SeekStart)
+	_, err = r.r.Seek(int64(r.super.IDTableStart), io.SeekStart)
+	if err != nil {
+		return err
+	}
 	for i := range blockOffsets {
-		err = binary.Read(rdr.r, binary.LittleEndian, &blockOffsets[i])
+		err = binary.Read(r.r, binary.LittleEndian, &blockOffsets[i])
 		if err != nil {
-			return nil, err
+			return err
 		}
 		var idRdr *metadataReader
-		idRdr, err = rdr.newMetadataReader(int64(blockOffsets[i]))
+		idRdr, err = r.newMetadataReader(int64(blockOffsets[i]))
 		if err != nil {
-			return nil, err
+			return err
 		}
 		read := uint16(math.Min(float64(unread), 2048))
 		for i := uint16(0); i < read; i++ {
 			var tmp uint32
 			err = binary.Read(idRdr, binary.LittleEndian, &tmp)
 			if err != nil {
-				return nil, err
+				return err
 			}
-			rdr.idTable = append(rdr.idTable, tmp)
+			r.idTable = append(r.idTable, tmp)
 		}
 		unread -= read
 	}
-	metaRdr, err := rdr.newMetadataReaderFromInodeRef(rdr.super.RootInodeRef)
+	metaRdr, err := r.newMetadataReaderFromInodeRef(r.super.RootInodeRef)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	i, err := inode.ProcessInode(metaRdr, rdr.super.BlockSize)
+	i, err := inode.ProcessInode(metaRdr, r.super.BlockSize)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	entries, err := rdr.readDirFromInode(i)
+	entries, err := r.readDirFromInode(i)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	rdr.FS = FS{
+	r.FS = FS{
-		r:       &rdr,
+		r:       r,
 		name:    "/",
 		entries: entries,
 	}
-	if hasUnsupportedOptions {
+	return nil
-		return &rdr, ErrOptions
-	}
-	return &rdr, nil
 }
 
 //ModTime is the last time the file was modified/created.

reader_test.go

@@ -14,9 +14,10 @@ import (
 )
 
 const (
-	downloadURL  = "https://github.com/srevinsaju/Firefox-Appimage/releases/download/firefox-v84.0.r20201221152838/firefox-84.0.r20201221152838-x86_64.AppImage"
+	appImageURL  = "https://github.com/srevinsaju/Firefox-Appimage/releases/download/firefox-v84.0.r20201221152838/firefox-84.0.r20201221152838-x86_64.AppImage"
 	appImageName = "firefox-84.0.r20201221152838-x86_64.AppImage"
-	squashfsName = "balenaEtcher-1.5.113-x64.AppImage.sfs"
+	squashfsURL  = "https://darkstorm.tech/LinuxPATest.sfs"
+	squashfsName = "LinuxPATest.sfs"
 )
 
 func TestSquashfs(t *testing.T) {
@@ -25,6 +26,13 @@ func TestSquashfs(t *testing.T) {
 		t.Fatal(err)
 	}
 	squashFil, err := os.Open(wd + "/testing/" + squashfsName)
+	if os.IsNotExist(err) {
+		err = downloadTestSquash(wd + "/testing")
+		if err != nil {
+			t.Fatal(err)
+		}
+		squashFil, err = os.Open(wd + "/testing/" + squashfsName)
+	}
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -32,19 +40,33 @@ func TestSquashfs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	fmt.Println("stuff", rdr.super.CompressionType)
+	os.RemoveAll(wd + "/testing/" + squashfsName + ".d")
-	// fil := rdr.GetFileAtPath("*.desktop")
+	op := DefaultOptions()
-	// if fil == nil {
+	op.Verbose = true
-	// 	t.Fatal("Can't find desktop fil")
+	err = rdr.ExtractWithOptions(wd+"/testing/"+squashfsName+".d", op)
-	// }
+	if err != nil {
-	// errs := fil.ExtractTo(wd + "/testing")
+		t.Fatal(err)
-	// if len(errs) > 0 {
+	}
-	// 	t.Fatal(errs)
+	t.Fatal("No Problems")
-	// }
+}
-	// errs = rdr.ExtractTo(wd + "/testing/" + squashfsName + ".d")
+
-	// if len(errs) > 0 {
+func TestSquashfsFromReader(t *testing.T) {
-	// 	t.Fatal(errs)
+	resp, err := http.DefaultClient.Get(squashfsURL)
-	// }
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	rdr, err := NewSquashfsReaderFromReader(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.RemoveAll("testing/" + squashfsName + ".d")
+	op := DefaultOptions()
+	op.Verbose = true
+	err = rdr.ExtractWithOptions("testing/"+squashfsName+".d", op)
+	if err != nil {
+		t.Fatal(err)
+	}
 	t.Fatal("No Problems")
 }
 
@@ -171,7 +193,7 @@ func downloadTestAppImage(dir string) error {
 			return nil
 		},
 	}
-	resp, err := check.Get(downloadURL)
+	resp, err := check.Get(appImageURL)
 	if err != nil {
 		return err
 	}
@@ -183,6 +205,32 @@ func downloadTestAppImage(dir string) error {
 	return nil
 }
 
+func downloadTestSquash(dir string) error {
+	//seems to time out on slow connections. Might fix that at some point... or not. It's just a test...
+	os.Mkdir(dir, os.ModePerm)
+	sfs, err := os.Create(dir + "/" + squashfsName)
+	if err != nil {
+		return err
+	}
+	defer sfs.Close()
+	check := http.Client{
+		CheckRedirect: func(r *http.Request, _ []*http.Request) error {
+			r.URL.Opaque = r.URL.Path
+			return nil
+		},
+	}
+	resp, err := check.Get(squashfsURL)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(sfs, resp.Body)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func TestCreateSquashFromAppImage(t *testing.T) {
 	wd, err := os.Getwd()
 	if err != nil {