From 8ab566521d18b4e675251bdac64d0934b5ba86c9 Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Wed, 27 Jan 2021 12:54:29 +0300 Subject: [PATCH 1/3] fix: typo in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ffb180c..c2e8a25 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,6 @@ Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree ## Performane -This library, decompressing the firefox AppImage and using go tests, takes about twice as long as `unsquashfs` on my quad core laptop. (~1 second with the libarary and about half a second with `unsquashfs`) +This library, decompressing the Firefox AppImage and using go tests, takes about twice as long as `unsquashfs` on my quad core laptop. (~1 second with the library and about half a second with `unsquashfs`) ## [TODO](https://github.com/CalebQ42/squashfs/projects/1?fullscreen=true) From 8a91e1a1c1e7310273bf136ed29f3bc26ecc98f3 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Tue, 23 Feb 2021 22:26:15 -0600 Subject: [PATCH 2/3] Getting back into it... --- superblock.go | 30 ++++++++++++++++++------------ writer.go | 33 +++++++++++++++++++++++++++++++-- writer_write.go | 16 ++++++++++++++-- 3 files changed, 63 insertions(+), 16 deletions(-) diff --git a/superblock.go b/superblock.go index 8134304..0f846b8 100644 --- a/superblock.go +++ b/superblock.go @@ -35,30 +35,36 @@ type superblock struct { //SuperblockFlags is the series of flags describing how a squashfs archive is packed. type SuperblockFlags struct { - //If true, inodes are stored uncompressed + //If true, inodes are stored uncompressed. UncompressedInodes bool - //If true, data is stored uncompressed + //If true, data is stored uncompressed. UncompressedData bool check bool - //If true, fragments are stored uncompressed + //If true, fragments are stored uncompressed. UncompressedFragments bool - //If true, ALL data is stored in sequential data blocks instead of utilizing fragments + //If true, ALL data is stored in sequential data blocks instead of utilizing fragments. NoFragments bool //If true, the last block of data will always be stored as a fragment if it's less then the block size. AlwaysFragments bool - //If true, duplicate files are only stored once. - Duplicates bool - //If true, the export table is populated + //If true, duplicate files are only stored once. (Currently unsupported) + RemoveDuplicates bool + //If true, the export table is populated. (Currently unsupported) Exportable bool - //If true, the xattr table is uncompressed + //If true, the xattr table is uncompressed. (Currently unsupported) UncompressedXattr bool - //If true, the xattr table is not populated + //If true, the xattr table is not populated. (Currently unsupported) NoXattr bool compressorOptions bool - //If true, the UID/GID table is stored uncompressed + //If true, the UID/GID table is stored uncompressed. UncompressedIDs bool } +//DefaultFlags are the default SuperblockFlags that are used. +var DefaultFlags = SuperblockFlags{ + RemoveDuplicates: true, + Exportable: true, +} + //GetFlags returns a SuperblockFlags for a given superblock. func (s *superblock) GetFlags() SuperblockFlags { return SuperblockFlags{ @@ -68,7 +74,7 @@ func (s *superblock) GetFlags() SuperblockFlags { UncompressedFragments: s.Flags&0x8 == 0x8, NoFragments: s.Flags&0x10 == 0x10, AlwaysFragments: s.Flags&0x20 == 0x20, - Duplicates: s.Flags&0x40 == 0x40, + RemoveDuplicates: s.Flags&0x40 == 0x40, Exportable: s.Flags&0x80 == 0x80, UncompressedXattr: s.Flags&0x100 == 0x100, NoXattr: s.Flags&0x200 == 0x200, @@ -98,7 +104,7 @@ func (s *SuperblockFlags) ToUint() uint16 { if s.AlwaysFragments { out = out | 0x20 } - if s.Duplicates { + if s.RemoveDuplicates { out = out | 0x40 } if s.Exportable { diff --git a/writer.go b/writer.go index 6521210..5939109 100644 --- a/writer.go +++ b/writer.go @@ -3,6 +3,7 @@ package squashfs import ( "errors" "io" + "io/fs" "log" "os" "path" @@ -29,6 +30,9 @@ type Writer struct { //Currently Duplicates, Exportable, UncompressedXattr, NoXattr values are ignored Flags SuperblockFlags allowErrors bool + + //variables used when actually writing. + superblock superblock } //NewWriter creates a new with the default options (Gzip compression and allow errors) @@ -54,6 +58,7 @@ func NewWriterWithOptions(compressionType int, allowErrors bool) (*Writer, error compressionType: compressionType, allowErrors: allowErrors, BlockSize: uint32(1048576), + Flags: DefaultFlags, }, nil } @@ -95,7 +100,8 @@ func (w *Writer) AddFileTo(filepath string, file *os.File) error { return errors.New("File already exists at " + filepath) } var holder fileHolder - holder.path, holder.name = path.Split(filepath) + holder.path = path.Dir(filepath) + holder.name = path.Base(filepath) holder.reader = file stat, err := file.Stat() if err != nil { @@ -167,12 +173,35 @@ func (w *Writer) AddReaderTo(filepath string, reader io.Reader) error { return errors.New("File already exists at " + filepath) } var holder fileHolder - holder.path, holder.name = path.Split(filepath) + holder.path = path.Dir(filepath) + holder.name = path.Base(filepath) holder.reader = reader w.structure[holder.path] = append(w.structure[holder.path], &holder) return nil } +//AddFolderTo adds a folder at the given path. IF the folder is already present, it sets the folder's permissions. +//If the path points to a non-folder (such as a file or symlink), an error is returned +func (w *Writer) AddFolderTo(folderpath string, permission fs.FileMode) error { + folderpath = path.Clean(folderpath) + tmp := w.holderAt(folderpath) + if tmp != nil { + if !tmp.folder { + return errors.New("Path is not a folder: " + folderpath) + } + tmp.perm = int(permission.Perm()) + return nil + } + file := fileHolder{ + path: path.Dir(folderpath), + name: path.Base(folderpath), + perm: int(permission | fs.ModePerm), + folder: true, + } + w.structure[file.path] = append(w.structure[file.path], &file) + return nil +} + //Remove tries to remove the file(s) at the given filepath. If wildcards are used, it will remove all files that match. //Returns true if one or more files are removed. func (w *Writer) Remove(filepath string) bool { diff --git a/writer_write.go b/writer_write.go index 0ee8bab..5f3de19 100644 --- a/writer_write.go +++ b/writer_write.go @@ -4,8 +4,18 @@ import ( "errors" "io" "math" + "path" ) +func (w *Writer) fixFolders() error { + for folder := range w.structure { + if folder == "/" { + continue + } + dir, name := path.Dir(folder), path.Base(folder) + } +} + //WriteTo attempts to write the archive to the given io.Writer. func (w *Writer) WriteTo(write io.Writer) (int64, error) { if w.BlockSize > 1048576 { @@ -13,8 +23,10 @@ func (w *Writer) WriteTo(write io.Writer) (int64, error) { } else if w.BlockSize < 4096 { w.BlockSize = 4096 } - //TODO: set forced Flag values - _ = superblock{ + w.Flags.RemoveDuplicates = false + w.Flags.Exportable = false + w.Flags.NoXattr = true + w.superblock = superblock{ Magic: magic, BlockSize: w.BlockSize, BlockLog: uint16(math.Log2(float64(w.BlockSize))), From a1239352502e7b5f0108f7b2d1f39fa71db754f5 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Wed, 24 Feb 2021 05:58:18 -0600 Subject: [PATCH 3/3] Added more necessary parts to compression. --- writer.go | 9 ++++++--- compress.go => writer_compress.go | 0 writer_fragment.go | 21 +++++++++++++++++++++ writer_inodes.go | 23 +++++++++++++++++++++++ writer_write.go | 19 ++++++++++++++++--- 5 files changed, 66 insertions(+), 6 deletions(-) rename compress.go => writer_compress.go (100%) create mode 100644 writer_fragment.go create mode 100644 writer_inodes.go diff --git a/writer.go b/writer.go index 5939109..5fd70ce 100644 --- a/writer.go +++ b/writer.go @@ -33,6 +33,7 @@ type Writer struct { //variables used when actually writing. superblock superblock + frags []fragment } //NewWriter creates a new with the default options (Gzip compression and allow errors) @@ -68,10 +69,11 @@ type fileHolder struct { path string name string symLocation string - UID int + blockSizes []uint32 GUID int perm int - size uint32 + size uint64 + UID int folder bool symlink bool } @@ -164,7 +166,7 @@ func (w *Writer) AddFileTo(filepath string, file *os.File) error { //AddReaderTo adds the data from the given reader to the archive as a file located at the given filepath. //Data from the reader is not read until the squashfs archive is writen. //If the given reader implements io.Closer, it will be closed after it is fully read. -func (w *Writer) AddReaderTo(filepath string, reader io.Reader) error { +func (w *Writer) AddReaderTo(filepath string, reader io.Reader, size uint64) error { filepath = path.Clean(filepath) if !strings.HasPrefix(filepath, "/") { filepath = "/" + filepath @@ -175,6 +177,7 @@ func (w *Writer) AddReaderTo(filepath string, reader io.Reader) error { var holder fileHolder holder.path = path.Dir(filepath) holder.name = path.Base(filepath) + holder.size = size holder.reader = reader w.structure[holder.path] = append(w.structure[holder.path], &holder) return nil diff --git a/compress.go b/writer_compress.go similarity index 100% rename from compress.go rename to writer_compress.go diff --git a/writer_fragment.go b/writer_fragment.go new file mode 100644 index 0000000..165be64 --- /dev/null +++ b/writer_fragment.go @@ -0,0 +1,21 @@ +package squashfs + +type fragment struct { + w *Writer + files []*fileHolder + sizes []uint32 +} + +func (f *fragment) SizeLeft() uint32 { + totalSize := uint32(0) + for _, siz := range f.sizes { + totalSize += siz + } + return f.w.BlockSize - uint32(totalSize) +} + +func (f *fragment) AddFragment(fil *fileHolder) { + //SizeLeft should already be checked + f.files = append(f.files, fil) + f.sizes = append(f.sizes, fil.blockSizes[len(fil.blockSizes)-1]) +} diff --git a/writer_inodes.go b/writer_inodes.go new file mode 100644 index 0000000..685efbe --- /dev/null +++ b/writer_inodes.go @@ -0,0 +1,23 @@ +package squashfs + +func (w *Writer) countInodes() (out uint32) { + for _, files := range w.structure { + out++ + out += uint32(len(files)) + } + return +} + +//intilialize the block sizes. These values will be overwritten with their compressed sizes later. +func (w *Writer) calculateBlockSizes(fil *fileHolder) { + tmp := fil.size + for { + if tmp < uint64(w.BlockSize) { + fil.blockSizes = append(fil.blockSizes, uint32(tmp)) + break + } + tmp -= uint64(w.BlockSize) + fil.blockSizes = append(fil.blockSizes, w.BlockSize) + } + return +} diff --git a/writer_write.go b/writer_write.go index 5f3de19..db1a4e9 100644 --- a/writer_write.go +++ b/writer_write.go @@ -3,21 +3,32 @@ package squashfs import ( "errors" "io" + "io/fs" "math" - "path" + "time" ) func (w *Writer) fixFolders() error { for folder := range w.structure { - if folder == "/" { + if folder == "/" || w.Contains(folder) { continue } - dir, name := path.Dir(folder), path.Base(folder) + err := w.AddFolderTo(folder, fs.ModePerm) + if err != nil { + return err + } } + return nil } //WriteTo attempts to write the archive to the given io.Writer. +//Folder that aren't present (such as if you add a file at /folder/file, but not the folder /folder) +//are added with full permission (777). func (w *Writer) WriteTo(write io.Writer) (int64, error) { + err := w.fixFolders() + if err != nil { + return 0, err + } if w.BlockSize > 1048576 { w.BlockSize = 1048576 } else if w.BlockSize < 4096 { @@ -28,6 +39,8 @@ func (w *Writer) WriteTo(write io.Writer) (int64, error) { w.Flags.NoXattr = true w.superblock = superblock{ Magic: magic, + InodeCount: w.countInodes(), + CreationTime: uint32(time.Now().Unix()), BlockSize: w.BlockSize, BlockLog: uint16(math.Log2(float64(w.BlockSize))), CompressionType: uint16(w.compressionType),