Removed close function from squashfs.File

This seems to cause issues in ver specific circumstances and ultimately, isn't needed.

README.md

@@ -8,7 +8,7 @@ Currently has support for reading squashfs files and extracting files and folder
 
 The only major thing missing from squashfs reading is Xattr parsing.
 
-Special thanks to https://dr-emann.github.io/squashfs/ for some VERY important information in an easy to understand format.
+Special thanks to <https://dr-emann.github.io/squashfs/> for some VERY important information in an easy to understand format.
 Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree/master/internal/squashfs) as I referenced it to figure some things out (and double check others).
 
-# [TODO](https://github.com/CalebQ42/squashfs/projects/1?fullscreen=true)
+## [TODO](https://github.com/CalebQ42/squashfs/projects/1?fullscreen=true)

datareader.go

@@ -18,11 +18,11 @@ var (
 //DataReader reads data from data blocks.
 type dataReader struct {
 	r             *Reader
-	offset        int64 //offset relative to the beginning of the squash file
 	blocks        []dataBlock
-	curBlock      int //Which block in sizes is currently cached
 	curData       []byte
-	curReadOffset int //offset relative to the currently cached data
+	offset        int64 //offset relative to the beginning of the squash file
+	curBlock      int   //Which block in sizes is currently cached
+	curReadOffset int   //offset relative to the currently cached data
 }
 
 //DataBlock holds info about a given data block from it's size
@@ -145,12 +145,6 @@ func (d *dataReader) readCurBlock() error {
 	return err
 }
 
-//Close frees up the curData from memory
-func (d *dataReader) Close() error {
-	d.curData = nil
-	return nil
-}
-
 func (d *dataReader) Read(p []byte) (int, error) {
 	if d.curData == nil {
 		err := d.readCurBlock()
@@ -175,12 +169,12 @@ func (d *dataReader) Read(p []byte) (int, error) {
 			d.curReadOffset = 0
 		}
 		for ; read < len(p); read++ {
-			d.curReadOffset++
 			if d.curReadOffset < len(d.curData) {
 				p[read] = d.curData[d.curReadOffset]
 			} else {
 				break
 			}
+			d.curReadOffset++
 		}
 	}
 	if read != len(p) {

file.go

@@ -30,13 +30,14 @@ var (
 //
 //Implements os.FileInfo and io.ReadCloser
 type File struct {
-	Parent  *File        //The parent directory. Should ALWAYS be a folder. If it's the root directory, will be nil
-	Reader  io.Reader    //Underlying reader. When writing, will probably be an os.File. When reading this is kept nil UNTIL reading to save memory.
-	name    string       //The name of the file or folder. Root folder will not have a name ("")
-	path    string       //The path to the folder the File is located in.
-	r       *Reader      //The squashfs.Reader where this file is contained.
-	in      *inode.Inode //Underlyting inode when reading.
-	filType int          //The file's type, using inode types.
+	Reader  io.Reader
+	Parent  *File
+	r       *Reader //Underlying reader. When writing, will probably be an os.File. When reading this is kept nil UNTIL reading to save memory.
+	in      *inode.Inode
+	name    string
+	path    string
+	filType int //The file's type, using inode types.
+
 }
 
 //get a File from a directory.entry
@@ -355,7 +356,6 @@ func (f *File) ExtractWithOptions(path string, dereferenceSymlink, unbreakSymlin
 			return
 		}
 		finishChan := make(chan []error)
-		defer close(finishChan)
 		for _, child := range children {
 			go func(child *File) {
 				if f.name == "" {
@@ -407,7 +407,6 @@ func (f *File) ExtractWithOptions(path string, dereferenceSymlink, unbreakSymlin
 			errs = append(errs, err)
 			return
 		}
-		f.Close()
 		fil.Chown(int(f.r.idTable[f.in.Header.UID]), int(f.r.idTable[f.in.Header.GID]))
 		//don't mention anything when it fails. Because it fails often. Probably has something to do about uid & gid 0
 		// if err != nil {
@@ -479,21 +478,6 @@ func (f *File) ExtractWithOptions(path string, dereferenceSymlink, unbreakSymlin
 	return
 }
 
-//Close frees up the memory held up by the underlying reader. Should NOT be called when writing.
-//When reading, Close is safe to use, but any subsequent Read calls resets to the beginning of the file.
-func (f *File) Close() error {
-	if f.IsDir() {
-		return errNotFile
-	}
-	if f.Reader != nil {
-		if closer, is := f.Reader.(io.Closer); is {
-			closer.Close()
-		}
-		f.Reader = nil
-	}
-	return nil
-}
-
 //Read from the file. Doesn't do anything fancy, just pases it to the underlying io.Reader. If a directory, return io.EOF.
 func (f *File) Read(p []byte) (int, error) {
 	if f.IsDir() {

filereader.go

@@ -57,15 +57,6 @@ func (r *Reader) newFileReader(in *inode.Inode) (*fileReader, error) {
 	return &rdr, nil
 }
 
-//Close runs Close on the data reader and frees the fragmentdata
-func (f *fileReader) Close() error {
-	if f.data != nil {
-		f.data.Close()
-	}
-	f.fragmentData = nil
-	return nil
-}
-
 func (f *fileReader) Read(p []byte) (int, error) {
 	if f.fragOnly {
 		n, err := bytes.NewBuffer(f.fragmentData[f.read:]).Read(p)

go.mod

@@ -7,11 +7,11 @@ require (
 	github.com/adrg/xdg v0.2.3 // indirect
 	github.com/google/go-cmp v0.5.4 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
-	github.com/klauspost/compress v1.11.3
+	github.com/klauspost/compress v1.11.4
 	github.com/kr/text v0.2.0 // indirect
-	github.com/pierrec/lz4/v4 v4.1.1
+	github.com/pierrec/lz4/v4 v4.1.2
 	github.com/smartystreets/assertions v1.2.0 // indirect
-	github.com/ulikunitz/xz v0.5.8
+	github.com/ulikunitz/xz v0.5.9
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect

go.sum

@@ -18,8 +18,8 @@ github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3
 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc=
-github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.4 h1:kz40R/YWls3iqT9zX9AHN3WoVsrAWVyui5sxuLqiXqU=
+github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -29,8 +29,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/pierrec/lz4/v4 v4.1.1 h1:cS6aGkNLJr4u+UwaA21yp+gbWN3WJWtKo1axmPDObMA=
-github.com/pierrec/lz4/v4 v4.1.1/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM=
+github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
@@ -42,8 +42,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ=
-github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I=
+github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 go.lsp.dev/uri v0.3.0 h1:KcZJmh6nFIBeJzTugn5JTU6OOyG0lDOo3R9KwTxTYbo=
 go.lsp.dev/uri v0.3.0/go.mod h1:P5sbO1IQR+qySTWOCnhnK7phBx+W3zbLqSMDJNTw88I=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=

metadata.go

@@ -16,9 +16,9 @@ type metadata struct {
 //MetadataReader is a block reader for metadata. It will automatically read the next block, when it reaches the end of a block.
 type metadataReader struct {
 	s          *Reader
-	offset     int64
 	headers    []*metadata
 	data       []byte
+	offset     int64
 	readOffset int
 }
 

reader.go

@@ -29,11 +29,11 @@ var (
 //Reader processes and reads a squashfs archive.
 type Reader struct {
 	r            io.ReaderAt
-	super        superblock
-	flags        superblockFlags
 	decompressor compression.Decompressor
 	fragOffsets  []uint64
 	idTable      []uint32
+	super        superblock
+	flags        superblockFlags
 }
 
 //NewSquashfsReader returns a new squashfs.Reader from an io.ReaderAt
@@ -54,7 +54,7 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 	rdr.flags = rdr.super.GetFlags()
 	if rdr.flags.CompressorOptions {
 		switch rdr.super.CompressionType {
-		case gzipCompression:
+		case GzipCompression:
 			gzip, err := compression.NewGzipCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 8))
 			if err != nil {
 				return nil, err
@@ -63,7 +63,7 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 				hasUnsupportedOptions = true
 			}
 			rdr.decompressor = gzip
-		case xzCompression:
+		case XzCompression:
 			xz, err := compression.NewXzCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 8))
 			if err != nil {
 				return nil, err
@@ -72,7 +72,7 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 				return nil, errors.New("XZ compression options has filters. These are not yet supported")
 			}
 			rdr.decompressor = xz
-		case lz4Compression:
+		case Lz4Compression:
 			lz4, err := compression.NewLz4CompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 8))
 			if err != nil {
 				return nil, err
@@ -81,26 +81,26 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 				hasUnsupportedOptions = true
 			}
 			rdr.decompressor = lz4
-		case zstdCompression:
+		case ZstdCompression:
 			zstd, err := compression.NewZstdCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 4))
 			if err != nil {
 				return nil, err
 			}
 			rdr.decompressor = zstd
 		default:
-			return nil, errCompressorOptions
+			return nil, errIncompatibleCompression
 		}
 	} else {
 		switch rdr.super.CompressionType {
-		case gzipCompression:
+		case GzipCompression:
 			rdr.decompressor = &compression.Gzip{}
-		case lzmaCompression:
+		case LzmaCompression:
 			rdr.decompressor = &compression.Lzma{}
-		case xzCompression:
+		case XzCompression:
 			rdr.decompressor = &compression.Xz{}
-		case lz4Compression:
+		case Lz4Compression:
 			rdr.decompressor = &compression.Lz4{}
-		case zstdCompression:
+		case ZstdCompression:
 			rdr.decompressor = &compression.Zstd{}
 		default:
 			//TODO: all compression types.

squash_test.go

@@ -1,6 +1,7 @@
 package squashfs
 
 import (
+	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -10,12 +11,11 @@ import (
 )
 
 const (
-	downloadURL  = "https://github.com/Swordfish90/cool-retro-term/releases/download/1.1.1/Cool-Retro-Term-1.1.1-x86_64.AppImage"
-	appImageName = "Cool-Retro-Term.AppImage"
-	squashfsName = "airootfs.sfs" //testing with a ArchLinux root fs from the live img
+	downloadURL  = "https://github.com/srevinsaju/Firefox-Appimage/releases/download/firefox-v84.0.r20201221152838/firefox-84.0.r20201221152838-x86_64.AppImage"
+	appImageName = "firefox-84.0.r20201221152838-x86_64.AppImage"
+	squashfsName = "balenaEtcher-1.5.113-x64.AppImage.sfs" //testing with a ArchLinux root fs from the live img
 )
 
-//Right now, don't use. Arch linux sfs uses XZ compression with filters, which isn't supported
 func TestSquashfs(t *testing.T) {
 	wd, err := os.Getwd()
 	if err != nil {
@@ -29,10 +29,20 @@ func TestSquashfs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	os.RemoveAll(wd + "/testing/" + squashfsName + ".d")
-	root, _ := rdr.GetRootFolder()
-	errs := root.ExtractWithOptions(wd+"/testing/"+squashfsName+".d", false, false, os.ModePerm, true)
-	t.Fatal(errs)
+	fmt.Println("stuff", rdr.super.CompressionType)
+	fil := rdr.GetFileAtPath("*.desktop")
+	if fil == nil {
+		t.Fatal("Can't find desktop fil")
+	}
+	errs := fil.ExtractTo(wd + "/testing")
+	if len(errs) > 0 {
+		t.Fatal(errs)
+	}
+	errs = rdr.ExtractTo(wd + "/testing/" + squashfsName + ".d")
+	if len(errs) > 0 {
+		t.Fatal(errs)
+	}
+	t.Fatal("No Problems")
 }
 
 func TestAppImage(t *testing.T) {
@@ -58,7 +68,8 @@ func TestAppImage(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	errs := rdr.ExtractTo(wd + "/testing/cool-retro")
+	os.RemoveAll(wd + "testing/firefox")
+	errs := rdr.ExtractTo(wd + "/testing/firefox")
 	if len(errs) > 0 {
 		t.Fatal(errs)
 	}

superblock.go

@@ -1,12 +1,13 @@
 package squashfs
 
+//The types of compression supported by squashfs.
 const (
-	gzipCompression = 1 + iota
-	lzmaCompression
-	lzoCompression
-	xzCompression
-	lz4Compression
-	zstdCompression
+	GzipCompression = 1 + iota
+	LzmaCompression
+	LzoCompression
+	XzCompression
+	Lz4Compression
+	ZstdCompression
 )
 
 //Superblock contains important information about a squashfs file. Located at the very front of the archive.

writer.go

@@ -0,0 +1,181 @@
+package squashfs
+
+import (
+	"errors"
+	"log"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/CalebQ42/squashfs/internal/inode"
+)
+
+//Writer is an interface to write a squashfs. Doesn't write until you call Write (TODO: maybe not do Write...).
+//If AllowErrors is true, when errors are encountered, it just prints to the log instead of failing.
+type Writer struct {
+	files           map[string][]*File
+	symlinkTable    map[string]string //[oldpath]newpath
+	symTableTemp    map[string]string
+	directories     []string
+	compression     int
+	ResolveSymlinks bool
+	AllowErrors     bool
+}
+
+//NewWriter creates a new squashfs.Writer with the default settings (gzip compression, autoresolving symlinks, and allowErrors)
+func NewWriter() (*Writer, error) {
+	return NewWriterWithOptions(true, true, GzipCompression)
+}
+
+//NewWriterWithOptions creates a new squashfs.Writer with the given options.
+//ResolveSymlinks tries to make sure symlinks aren't broken. It will either try to make the link's location work
+func NewWriterWithOptions(resolveSymlinks, allowErrors bool, compressionType int) (*Writer, error) {
+	if compressionType < 0 || compressionType > 6 {
+		return nil, errors.New("Incorrect compression type")
+	}
+	if compressionType == 3 {
+		return nil, errors.New("Lzo compression is not currently supported")
+	}
+	out := Writer{
+		files: map[string][]*File{
+			"/": make([]*File, 0),
+		},
+		ResolveSymlinks: resolveSymlinks,
+		AllowErrors:     allowErrors,
+		compression:     compressionType,
+	}
+	if resolveSymlinks {
+		out.symlinkTable = make(map[string]string)
+	}
+	return &out, nil
+}
+
+type fileError struct {
+	err   error
+	files []*File
+}
+
+//convertFile converts the given os.File to a squashfs.File. Returns the errors and converted file to the channels.
+func (w *Writer) convertFile(squashfsPath string, file *os.File, subDir bool, fileErrChan chan fileError) {
+	var out fileError
+	var fil File
+	fil.Reader = file
+	fil.path = squashfsPath
+	fil.name = path.Base(file.Name())
+	mode := fil.Mode()
+
+	if mode.IsRegular() {
+		fil.filType = inode.BasicFileType
+		goto successExit
+	} else if mode.IsDir() {
+		fil.filType = inode.BasicSymlinkType
+		subDirs, err := file.Readdirnames(-1)
+		if err != nil {
+			if w.AllowErrors && !subDir {
+				log.Println("Can't get sub-directories for", file.Name())
+				log.Println(err)
+			} else {
+				out.err = err
+			}
+			goto failExit
+		}
+		subDirChan := make(chan fileError)
+		for _, filName := range subDirs {
+			go func(filename string, returnChan chan fileError) {
+				subFil, err := os.Open(filename)
+				if err != nil {
+					out.err = err
+					returnChan <- fileError{
+						err: err,
+					}
+					return
+				}
+				w.convertFile(fil.Path(), subFil, true, subDirChan)
+			}(file.Name()+filName, subDirChan)
+		}
+		for range subDirs {
+			filErr := <-subDirChan
+			if filErr.err != nil {
+				if w.AllowErrors && !subDir {
+					log.Println("Error while adding subdirectory of", file.Name())
+					log.Println(filErr.err)
+				} else if subDir {
+					if out.err == nil {
+						out.err = filErr.err
+					}
+				} else {
+					out.err = err
+					goto failExit
+				}
+				continue
+			}
+			out.files = append(out.files, filErr.files...)
+		}
+		goto successExit
+	} else if mode&os.ModeSymlink == os.ModeSymlink {
+		fil.filType = inode.BasicSymlinkType
+		symLocation, err := os.Readlink(file.Name())
+		if err != nil {
+			if w.AllowErrors && !subDir {
+				log.Println("Error while getting symlink's information for", file.Name())
+				log.Println(err)
+			} else {
+				out.err = err
+			}
+			goto failExit
+		}
+		if w.ResolveSymlinks {
+			if val, ok := w.symlinkTable[symLocation]; ok {
+				symLocation = val
+			} else if val, ok := w.symTableTemp[symLocation]; ok {
+				symLocation = val
+			} else {
+				//TODO: either add the file, or place the file in this location. Maybe defer this until after all the other files are added?
+			}
+		}
+		//TODO: store the symLocation inside the File somehow....
+	}
+	if w.AllowErrors && !subDir {
+		log.Println("Unsupported file type for", file.Name())
+	} else {
+		out.err = errors.New("Unsupported file type")
+	}
+failExit: //before this is used, make sure to log or set the error.
+	fileErrChan <- out
+	return
+successExit:
+	out.files = []*File{&fil}
+	fileErrChan <- out
+	return
+}
+
+//AddFilesToPath adds the give os.Files to the given path within the squashfs archive.
+//If AllowErrors is true, this will ALWAYS return nil
+func (w *Writer) AddFilesToPath(squashfsPath string, files ...*os.File) error {
+	squashfsPath = path.Clean(squashfsPath)
+	if strings.HasPrefix(squashfsPath, "/") {
+		squashfsPath = strings.TrimPrefix(squashfsPath, "/")
+	}
+	if squashfsPath == "." {
+		squashfsPath = "/"
+	}
+	fileErrChan := make(chan fileError)
+	for _, fil := range files {
+		go w.convertFile(squashfsPath, fil, false, fileErrChan)
+	}
+	return errors.New("Not yet ready")
+}
+
+//AddFiles adds all files given to the root directory
+//If AllowErrors is true, this will ALWAYS return nil
+func (w *Writer) AddFiles(files ...*os.File) error {
+	return w.AddFilesToPath("/", files...)
+}
+
+//RemoveFileAt removes the file at filepath from the Writer.
+//If multiple files match the given filepath (such as if there are wildcards), all matching files are removed.
+//If one or more files are removed, returns true.
+func (w *Writer) RemoveFileAt(filepath string) bool {
+	//TODO
+	return false
+}