diff --git a/README.md b/README.md index ffb180c..c2e8a25 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,6 @@ Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree ## Performane -This library, decompressing the firefox AppImage and using go tests, takes about twice as long as `unsquashfs` on my quad core laptop. (~1 second with the libarary and about half a second with `unsquashfs`) +This library, decompressing the Firefox AppImage and using go tests, takes about twice as long as `unsquashfs` on my quad core laptop. (~1 second with the library and about half a second with `unsquashfs`) ## [TODO](https://github.com/CalebQ42/squashfs/projects/1?fullscreen=true) diff --git a/superblock.go b/superblock.go index 8134304..0f846b8 100644 --- a/superblock.go +++ b/superblock.go @@ -35,30 +35,36 @@ type superblock struct { //SuperblockFlags is the series of flags describing how a squashfs archive is packed. type SuperblockFlags struct { - //If true, inodes are stored uncompressed + //If true, inodes are stored uncompressed. UncompressedInodes bool - //If true, data is stored uncompressed + //If true, data is stored uncompressed. UncompressedData bool check bool - //If true, fragments are stored uncompressed + //If true, fragments are stored uncompressed. UncompressedFragments bool - //If true, ALL data is stored in sequential data blocks instead of utilizing fragments + //If true, ALL data is stored in sequential data blocks instead of utilizing fragments. NoFragments bool //If true, the last block of data will always be stored as a fragment if it's less then the block size. AlwaysFragments bool - //If true, duplicate files are only stored once. - Duplicates bool - //If true, the export table is populated + //If true, duplicate files are only stored once. (Currently unsupported) + RemoveDuplicates bool + //If true, the export table is populated. (Currently unsupported) Exportable bool - //If true, the xattr table is uncompressed + //If true, the xattr table is uncompressed. (Currently unsupported) UncompressedXattr bool - //If true, the xattr table is not populated + //If true, the xattr table is not populated. (Currently unsupported) NoXattr bool compressorOptions bool - //If true, the UID/GID table is stored uncompressed + //If true, the UID/GID table is stored uncompressed. UncompressedIDs bool } +//DefaultFlags are the default SuperblockFlags that are used. +var DefaultFlags = SuperblockFlags{ + RemoveDuplicates: true, + Exportable: true, +} + //GetFlags returns a SuperblockFlags for a given superblock. func (s *superblock) GetFlags() SuperblockFlags { return SuperblockFlags{ @@ -68,7 +74,7 @@ func (s *superblock) GetFlags() SuperblockFlags { UncompressedFragments: s.Flags&0x8 == 0x8, NoFragments: s.Flags&0x10 == 0x10, AlwaysFragments: s.Flags&0x20 == 0x20, - Duplicates: s.Flags&0x40 == 0x40, + RemoveDuplicates: s.Flags&0x40 == 0x40, Exportable: s.Flags&0x80 == 0x80, UncompressedXattr: s.Flags&0x100 == 0x100, NoXattr: s.Flags&0x200 == 0x200, @@ -98,7 +104,7 @@ func (s *SuperblockFlags) ToUint() uint16 { if s.AlwaysFragments { out = out | 0x20 } - if s.Duplicates { + if s.RemoveDuplicates { out = out | 0x40 } if s.Exportable { diff --git a/writer.go b/writer.go index ec70b0c..7a7bbb9 100644 --- a/writer.go +++ b/writer.go @@ -3,6 +3,7 @@ package squashfs import ( "errors" "io" + "io/fs" "log" "os" "path" @@ -30,6 +31,10 @@ type Writer struct { //Currently Duplicates, Exportable, UncompressedXattr, NoXattr values are ignored Flags SuperblockFlags allowErrors bool + + //variables used when actually writing. + superblock superblock + frags []fragment } //NewWriter creates a new with the default options (Gzip compression and allow errors) @@ -58,6 +63,7 @@ func NewWriterWithOptions(compressionType int, allowErrors bool) (*Writer, error compressionType: compressionType, allowErrors: allowErrors, BlockSize: uint32(1048576), + Flags: DefaultFlags, }, nil } @@ -67,10 +73,11 @@ type fileHolder struct { path string name string symLocation string - UID int + blockSizes []uint32 GUID int perm int - size uint32 + size uint64 + UID int folder bool symlink bool } @@ -99,7 +106,9 @@ func (w *Writer) AddFileTo(filepath string, file *os.File) error { return errors.New("File already exists at " + filepath) } var holder fileHolder - holder.path, holder.name = path.Dir(filepath), path.Base(filepath) + holder.path = path.Dir(filepath) + holder.name = path.Base(filepath) + holder.reader = file stat, err := file.Stat() if err != nil { return err @@ -168,7 +177,7 @@ func (w *Writer) AddFileTo(filepath string, file *os.File) error { //AddReaderTo adds the data from the given reader to the archive as a file located at the given filepath. //Data from the reader is not read until the squashfs archive is writen. //If the given reader implements io.Closer, it will be closed after it is fully read. -func (w *Writer) AddReaderTo(filepath string, reader io.Reader) error { +func (w *Writer) AddReaderTo(filepath string, reader io.Reader, size uint64) error { filepath = path.Clean(filepath) if !strings.HasPrefix(filepath, "/") { filepath = "/" + filepath @@ -177,7 +186,9 @@ func (w *Writer) AddReaderTo(filepath string, reader io.Reader) error { return errors.New("File already exists at " + filepath) } var holder fileHolder - holder.path, holder.name = path.Dir(filepath), path.Base(filepath) + holder.path = path.Dir(filepath) + holder.name = path.Base(filepath) + holder.size = size holder.reader = reader if _, ok := w.structure[holder.path]; ok { w.folders = append(w.folders, holder.path) @@ -187,6 +198,28 @@ func (w *Writer) AddReaderTo(filepath string, reader io.Reader) error { return nil } +//AddFolderTo adds a folder at the given path. IF the folder is already present, it sets the folder's permissions. +//If the path points to a non-folder (such as a file or symlink), an error is returned +func (w *Writer) AddFolderTo(folderpath string, permission fs.FileMode) error { + folderpath = path.Clean(folderpath) + tmp := w.holderAt(folderpath) + if tmp != nil { + if !tmp.folder { + return errors.New("Path is not a folder: " + folderpath) + } + tmp.perm = int(permission.Perm()) + return nil + } + file := fileHolder{ + path: path.Dir(folderpath), + name: path.Base(folderpath), + perm: int(permission | fs.ModePerm), + folder: true, + } + w.structure[file.path] = append(w.structure[file.path], &file) + return nil +} + //Remove tries to remove the file(s) at the given filepath. If wildcards are used, it will remove all files that match. //Returns true if one or more files are removed. func (w *Writer) Remove(filepath string) bool { diff --git a/compress.go b/writer_compress.go similarity index 100% rename from compress.go rename to writer_compress.go diff --git a/writer_fragment.go b/writer_fragment.go new file mode 100644 index 0000000..165be64 --- /dev/null +++ b/writer_fragment.go @@ -0,0 +1,21 @@ +package squashfs + +type fragment struct { + w *Writer + files []*fileHolder + sizes []uint32 +} + +func (f *fragment) SizeLeft() uint32 { + totalSize := uint32(0) + for _, siz := range f.sizes { + totalSize += siz + } + return f.w.BlockSize - uint32(totalSize) +} + +func (f *fragment) AddFragment(fil *fileHolder) { + //SizeLeft should already be checked + f.files = append(f.files, fil) + f.sizes = append(f.sizes, fil.blockSizes[len(fil.blockSizes)-1]) +} diff --git a/writer_inodes.go b/writer_inodes.go new file mode 100644 index 0000000..685efbe --- /dev/null +++ b/writer_inodes.go @@ -0,0 +1,23 @@ +package squashfs + +func (w *Writer) countInodes() (out uint32) { + for _, files := range w.structure { + out++ + out += uint32(len(files)) + } + return +} + +//intilialize the block sizes. These values will be overwritten with their compressed sizes later. +func (w *Writer) calculateBlockSizes(fil *fileHolder) { + tmp := fil.size + for { + if tmp < uint64(w.BlockSize) { + fil.blockSizes = append(fil.blockSizes, uint32(tmp)) + break + } + tmp -= uint64(w.BlockSize) + fil.blockSizes = append(fil.blockSizes, w.BlockSize) + } + return +} diff --git a/writer_write.go b/writer_write.go index 9cbe922..17b56ab 100644 --- a/writer_write.go +++ b/writer_write.go @@ -3,31 +3,41 @@ package squashfs import ( "errors" "io" + "io/fs" "math" "time" ) -func (w Writer) countInodes() (out uint32) { - out++ // for the root indode - for _, fold := range w.folders { - out += uint32(len(w.structure[fold])) +func (w *Writer) fixFolders() error { + for folder := range w.structure { + if folder == "/" || w.Contains(folder) { + continue + } + err := w.AddFolderTo(folder, fs.ModePerm) + if err != nil { + return err + } } - return + return nil } //WriteTo attempts to write the archive to the given io.Writer. -func (w *Writer) WriteTo(write io.WriteSeeker) (int64, error) { +//Folder that aren't present (such as if you add a file at /folder/file, but not the folder /folder) +//are added with full permission (777). +func (w *Writer) WriteTo(write io.Writer) (int64, error) { + err := w.fixFolders() + if err != nil { + return 0, err + } if w.BlockSize > 1048576 { w.BlockSize = 1048576 } else if w.BlockSize < 4096 { w.BlockSize = 4096 } - w.Flags.Duplicates = false + w.Flags.RemoveDuplicates = false w.Flags.Exportable = false w.Flags.NoXattr = true - //TODO: set forced Flag values - //TODO: make sure we aren't missing folders - super := superblock{ + w.superblock = superblock{ Magic: magic, InodeCount: w.countInodes(), CreationTime: uint32(time.Now().Unix()),