Merge branch 'main' into go1.16_fs3

README.md

@@ -13,6 +13,6 @@ Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree
 
 ## Performane
 
-This library, decompressing the firefox AppImage and using go tests, takes about twice as long as `unsquashfs` on my quad core laptop. (~1 second with the libarary and about half a second with `unsquashfs`)
+This library, decompressing the Firefox AppImage and using go tests, takes about twice as long as `unsquashfs` on my quad core laptop. (~1 second with the library and about half a second with `unsquashfs`)
 
 ## [TODO](https://github.com/CalebQ42/squashfs/projects/1?fullscreen=true)

superblock.go

@@ -35,30 +35,36 @@ type superblock struct {
 
 //SuperblockFlags is the series of flags describing how a squashfs archive is packed.
 type SuperblockFlags struct {
-	//If true, inodes are stored uncompressed
+	//If true, inodes are stored uncompressed.
 	UncompressedInodes bool
-	//If true, data is stored uncompressed
+	//If true, data is stored uncompressed.
 	UncompressedData bool
 	check            bool
-	//If true, fragments are stored uncompressed
+	//If true, fragments are stored uncompressed.
 	UncompressedFragments bool
-	//If true, ALL data is stored in sequential data blocks instead of utilizing fragments
+	//If true, ALL data is stored in sequential data blocks instead of utilizing fragments.
 	NoFragments bool
 	//If true, the last block of data will always be stored as a fragment if it's less then the block size.
 	AlwaysFragments bool
-	//If true, duplicate files are only stored once.
+	//If true, duplicate files are only stored once. (Currently unsupported)
-	Duplicates bool
+	RemoveDuplicates bool
-	//If true, the export table is populated
+	//If true, the export table is populated. (Currently unsupported)
 	Exportable bool
-	//If true, the xattr table is uncompressed
+	//If true, the xattr table is uncompressed. (Currently unsupported)
 	UncompressedXattr bool
-	//If true, the xattr table is not populated
+	//If true, the xattr table is not populated. (Currently unsupported)
 	NoXattr           bool
 	compressorOptions bool
-	//If true, the UID/GID table is stored uncompressed
+	//If true, the UID/GID table is stored uncompressed.
 	UncompressedIDs bool
 }
 
+//DefaultFlags are the default SuperblockFlags that are used.
+var DefaultFlags = SuperblockFlags{
+	RemoveDuplicates: true,
+	Exportable:       true,
+}
+
 //GetFlags returns a SuperblockFlags for a given superblock.
 func (s *superblock) GetFlags() SuperblockFlags {
 	return SuperblockFlags{
@@ -68,7 +74,7 @@ func (s *superblock) GetFlags() SuperblockFlags {
 		UncompressedFragments: s.Flags&0x8 == 0x8,
 		NoFragments:           s.Flags&0x10 == 0x10,
 		AlwaysFragments:       s.Flags&0x20 == 0x20,
-		Duplicates:            s.Flags&0x40 == 0x40,
+		RemoveDuplicates:      s.Flags&0x40 == 0x40,
 		Exportable:            s.Flags&0x80 == 0x80,
 		UncompressedXattr:     s.Flags&0x100 == 0x100,
 		NoXattr:               s.Flags&0x200 == 0x200,
@@ -98,7 +104,7 @@ func (s *SuperblockFlags) ToUint() uint16 {
 	if s.AlwaysFragments {
 		out = out | 0x20
 	}
-	if s.Duplicates {
+	if s.RemoveDuplicates {
 		out = out | 0x40
 	}
 	if s.Exportable {

writer.go

@@ -3,6 +3,7 @@ package squashfs
 import (
 	"errors"
 	"io"
+	"io/fs"
 	"log"
 	"os"
 	"path"
@@ -30,6 +31,10 @@ type Writer struct {
 	//Currently Duplicates, Exportable, UncompressedXattr, NoXattr values are ignored
 	Flags       SuperblockFlags
 	allowErrors bool
+
+	//variables used when actually writing.
+	superblock superblock
+	frags      []fragment
 }
 
 //NewWriter creates a new with the default options (Gzip compression and allow errors)
@@ -58,6 +63,7 @@ func NewWriterWithOptions(compressionType int, allowErrors bool) (*Writer, error
 		compressionType: compressionType,
 		allowErrors:     allowErrors,
 		BlockSize:       uint32(1048576),
+		Flags:           DefaultFlags,
 	}, nil
 }
 
@@ -67,10 +73,11 @@ type fileHolder struct {
 	path        string
 	name        string
 	symLocation string
-	UID         int
+	blockSizes  []uint32
 	GUID        int
 	perm        int
-	size        uint32
+	size        uint64
+	UID         int
 	folder      bool
 	symlink     bool
 }
@@ -99,7 +106,9 @@ func (w *Writer) AddFileTo(filepath string, file *os.File) error {
 		return errors.New("File already exists at " + filepath)
 	}
 	var holder fileHolder
-	holder.path, holder.name = path.Dir(filepath), path.Base(filepath)
+	holder.path = path.Dir(filepath)
+	holder.name = path.Base(filepath)
+	holder.reader = file
 	stat, err := file.Stat()
 	if err != nil {
 		return err
@@ -168,7 +177,7 @@ func (w *Writer) AddFileTo(filepath string, file *os.File) error {
 //AddReaderTo adds the data from the given reader to the archive as a file located at the given filepath.
 //Data from the reader is not read until the squashfs archive is writen.
 //If the given reader implements io.Closer, it will be closed after it is fully read.
-func (w *Writer) AddReaderTo(filepath string, reader io.Reader) error {
+func (w *Writer) AddReaderTo(filepath string, reader io.Reader, size uint64) error {
 	filepath = path.Clean(filepath)
 	if !strings.HasPrefix(filepath, "/") {
 		filepath = "/" + filepath
@@ -177,7 +186,9 @@ func (w *Writer) AddReaderTo(filepath string, reader io.Reader) error {
 		return errors.New("File already exists at " + filepath)
 	}
 	var holder fileHolder
-	holder.path, holder.name = path.Dir(filepath), path.Base(filepath)
+	holder.path = path.Dir(filepath)
+	holder.name = path.Base(filepath)
+	holder.size = size
 	holder.reader = reader
 	if _, ok := w.structure[holder.path]; ok {
 		w.folders = append(w.folders, holder.path)
@@ -187,6 +198,28 @@ func (w *Writer) AddReaderTo(filepath string, reader io.Reader) error {
 	return nil
 }
 
+//AddFolderTo adds a folder at the given path. IF the folder is already present, it sets the folder's permissions.
+//If the path points to a non-folder (such as a file or symlink), an error is returned
+func (w *Writer) AddFolderTo(folderpath string, permission fs.FileMode) error {
+	folderpath = path.Clean(folderpath)
+	tmp := w.holderAt(folderpath)
+	if tmp != nil {
+		if !tmp.folder {
+			return errors.New("Path is not a folder: " + folderpath)
+		}
+		tmp.perm = int(permission.Perm())
+		return nil
+	}
+	file := fileHolder{
+		path:   path.Dir(folderpath),
+		name:   path.Base(folderpath),
+		perm:   int(permission | fs.ModePerm),
+		folder: true,
+	}
+	w.structure[file.path] = append(w.structure[file.path], &file)
+	return nil
+}
+
 //Remove tries to remove the file(s) at the given filepath. If wildcards are used, it will remove all files that match.
 //Returns true if one or more files are removed.
 func (w *Writer) Remove(filepath string) bool {

writer_fragment.go

@@ -0,0 +1,21 @@
+package squashfs
+
+type fragment struct {
+	w     *Writer
+	files []*fileHolder
+	sizes []uint32
+}
+
+func (f *fragment) SizeLeft() uint32 {
+	totalSize := uint32(0)
+	for _, siz := range f.sizes {
+		totalSize += siz
+	}
+	return f.w.BlockSize - uint32(totalSize)
+}
+
+func (f *fragment) AddFragment(fil *fileHolder) {
+	//SizeLeft should already be checked
+	f.files = append(f.files, fil)
+	f.sizes = append(f.sizes, fil.blockSizes[len(fil.blockSizes)-1])
+}

writer_inodes.go

@@ -0,0 +1,23 @@
+package squashfs
+
+func (w *Writer) countInodes() (out uint32) {
+	for _, files := range w.structure {
+		out++
+		out += uint32(len(files))
+	}
+	return
+}
+
+//intilialize the block sizes. These values will be overwritten with their compressed sizes later.
+func (w *Writer) calculateBlockSizes(fil *fileHolder) {
+	tmp := fil.size
+	for {
+		if tmp < uint64(w.BlockSize) {
+			fil.blockSizes = append(fil.blockSizes, uint32(tmp))
+			break
+		}
+		tmp -= uint64(w.BlockSize)
+		fil.blockSizes = append(fil.blockSizes, w.BlockSize)
+	}
+	return
+}

writer_write.go

@@ -3,31 +3,41 @@ package squashfs
 import (
 	"errors"
 	"io"
+	"io/fs"
 	"math"
 	"time"
 )
 
-func (w Writer) countInodes() (out uint32) {
+func (w *Writer) fixFolders() error {
-	out++ // for the root indode
+	for folder := range w.structure {
-	for _, fold := range w.folders {
+		if folder == "/" || w.Contains(folder) {
-		out += uint32(len(w.structure[fold]))
+			continue
 		}
-	return
+		err := w.AddFolderTo(folder, fs.ModePerm)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 //WriteTo attempts to write the archive to the given io.Writer.
-func (w *Writer) WriteTo(write io.WriteSeeker) (int64, error) {
+//Folder that aren't present (such as if you add a file at /folder/file, but not the folder /folder)
+//are added with full permission (777).
+func (w *Writer) WriteTo(write io.Writer) (int64, error) {
+	err := w.fixFolders()
+	if err != nil {
+		return 0, err
+	}
 	if w.BlockSize > 1048576 {
 		w.BlockSize = 1048576
 	} else if w.BlockSize < 4096 {
 		w.BlockSize = 4096
 	}
-	w.Flags.Duplicates = false
+	w.Flags.RemoveDuplicates = false
 	w.Flags.Exportable = false
 	w.Flags.NoXattr = true
-	//TODO: set forced Flag values
+	w.superblock = superblock{
-	//TODO: make sure we aren't missing folders
-	super := superblock{
 		Magic:           magic,
 		InodeCount:      w.countInodes(),
 		CreationTime:    uint32(time.Now().Unix()),