diff --git a/superblock.go b/superblock.go index 0f846b8..8a46007 100644 --- a/superblock.go +++ b/superblock.go @@ -45,7 +45,7 @@ type SuperblockFlags struct { //If true, ALL data is stored in sequential data blocks instead of utilizing fragments. NoFragments bool //If true, the last block of data will always be stored as a fragment if it's less then the block size. - AlwaysFragments bool + AlwaysFragment bool //If true, duplicate files are only stored once. (Currently unsupported) RemoveDuplicates bool //If true, the export table is populated. (Currently unsupported) @@ -73,7 +73,7 @@ func (s *superblock) GetFlags() SuperblockFlags { check: s.Flags&0x4 == 0x4, UncompressedFragments: s.Flags&0x8 == 0x8, NoFragments: s.Flags&0x10 == 0x10, - AlwaysFragments: s.Flags&0x20 == 0x20, + AlwaysFragment: s.Flags&0x20 == 0x20, RemoveDuplicates: s.Flags&0x40 == 0x40, Exportable: s.Flags&0x80 == 0x80, UncompressedXattr: s.Flags&0x100 == 0x100, @@ -101,7 +101,7 @@ func (s *SuperblockFlags) ToUint() uint16 { if s.NoFragments { out = out | 0x10 } - if s.AlwaysFragments { + if s.AlwaysFragment { out = out | 0x20 } if s.RemoveDuplicates { diff --git a/writer.go b/writer.go index 7a7bbb9..ba0fe2d 100644 --- a/writer.go +++ b/writer.go @@ -19,9 +19,11 @@ import ( type Writer struct { compressor compression.Compressor structure map[string][]*fileHolder - symlinkTable map[string]string //[oldpath]newpath + symlinkTable map[string]string folders []string uidGUIDTable []int + frags []fragment + superblock superblock compressionType int //BlockSize is how large the data blocks are. Can be between 4096 (4KB) and 1048576 (1 MB). //If BlockSize is not inside that range, it will be set to within the range before writing. @@ -31,10 +33,6 @@ type Writer struct { //Currently Duplicates, Exportable, UncompressedXattr, NoXattr values are ignored Flags SuperblockFlags allowErrors bool - - //variables used when actually writing. - superblock superblock - frags []fragment } //NewWriter creates a new with the default options (Gzip compression and allow errors) @@ -52,7 +50,7 @@ func NewWriterWithOptions(compressionType int, allowErrors bool) (*Writer, error if compressionType == 3 { return nil, errors.New("Lzo compression is not (currently) supported") } - return &Writer{ + writer := &Writer{ structure: map[string][]*fileHolder{ "/": make([]*fileHolder, 0), }, @@ -64,7 +62,20 @@ func NewWriterWithOptions(compressionType int, allowErrors bool) (*Writer, error allowErrors: allowErrors, BlockSize: uint32(1048576), Flags: DefaultFlags, - }, nil + } + switch compressionType { + case 1: + writer.compressor = &compression.Gzip{} + case 2: + writer.compressor = &compression.Lzma{} + case 4: + writer.compressor = &compression.Xz{} + case 5: + writer.compressor = &compression.Lz4{} + case 6: + writer.compressor = &compression.Zstd{} + } + return writer, nil } //fileHolder holds the necessary information about a given file inside of a squashfs @@ -80,6 +91,9 @@ type fileHolder struct { UID int folder bool symlink bool + + fragIndex int + fragOffset int } //AddFile attempts to add an os.File to the archive's root directory. diff --git a/writer_fragment.go b/writer_fragment.go index 165be64..b0ea9eb 100644 --- a/writer_fragment.go +++ b/writer_fragment.go @@ -16,6 +16,54 @@ func (f *fragment) SizeLeft() uint32 { func (f *fragment) AddFragment(fil *fileHolder) { //SizeLeft should already be checked + fil.fragOffset = len(f.files) f.files = append(f.files, fil) f.sizes = append(f.sizes, fil.blockSizes[len(fil.blockSizes)-1]) } + +func (w *Writer) addToFragments(fil *fileHolder) { + fragSize := fil.blockSizes[len(fil.blockSizes)-1] + //only fragment if the final block is less then 80% of a full block or AlwaysFragment + if w.Flags.AlwaysFragment || fragSize < uint32(float32(w.BlockSize)*0.8) { + var possibleFrags []int + for i := range w.frags { + left := w.frags[i].SizeLeft() + if left == fragSize { + fil.fragIndex = i + w.frags[i].AddFragment(fil) + return + } else if left > fragSize { + possibleFrags = append(possibleFrags, i) + } + } + if len(possibleFrags) > 0 { + fil.fragIndex = possibleFrags[0] + } else { + fil.fragIndex = len(w.frags) + w.frags = append(w.frags, fragment{ + w: w, + files: []*fileHolder{fil}, + sizes: []uint32{fragSize}, + }) + } + } +} + +func (w *Writer) calculateFragsAndBlockSizes() { + for _, files := range w.structure { + for i := range files { + files[i].fragIndex = -1 + files[i].blockSizes = make([]uint32, files[i].size/uint64(w.BlockSize)) + for j := range files[i].blockSizes { + files[i].blockSizes[j] = w.BlockSize + } + fragSize := uint32(files[i].size % uint64(w.BlockSize)) + if fragSize > 0 { + files[i].blockSizes = append(files[i].blockSizes, fragSize) + if !w.Flags.NoFragments { + w.addToFragments(files[i]) + } + } + } + } +} diff --git a/writer_inodes.go b/writer_inodes.go index 685efbe..e9a857d 100644 --- a/writer_inodes.go +++ b/writer_inodes.go @@ -7,17 +7,3 @@ func (w *Writer) countInodes() (out uint32) { } return } - -//intilialize the block sizes. These values will be overwritten with their compressed sizes later. -func (w *Writer) calculateBlockSizes(fil *fileHolder) { - tmp := fil.size - for { - if tmp < uint64(w.BlockSize) { - fil.blockSizes = append(fil.blockSizes, uint32(tmp)) - break - } - tmp -= uint64(w.BlockSize) - fil.blockSizes = append(fil.blockSizes, w.BlockSize) - } - return -} diff --git a/writer_write.go b/writer_write.go index 17b56ab..03e8609 100644 --- a/writer_write.go +++ b/writer_write.go @@ -24,6 +24,8 @@ func (w *Writer) fixFolders() error { //WriteTo attempts to write the archive to the given io.Writer. //Folder that aren't present (such as if you add a file at /folder/file, but not the folder /folder) //are added with full permission (777). +// +//Not working. Yet. func (w *Writer) WriteTo(write io.Writer) (int64, error) { err := w.fixFolders() if err != nil { @@ -37,6 +39,7 @@ func (w *Writer) WriteTo(write io.Writer) (int64, error) { w.Flags.RemoveDuplicates = false w.Flags.Exportable = false w.Flags.NoXattr = true + w.calculateFragsAndBlockSizes() w.superblock = superblock{ Magic: magic, InodeCount: w.countInodes(), @@ -49,6 +52,5 @@ func (w *Writer) WriteTo(write io.Writer) (int64, error) { MajorVersion: 4, MinorVersion: 0, } - _ = super return 0, errors.New("I SAID DON'T") }