From 305f261d1051764c845a418b9ad4524cb5df2952 Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Sun, 12 Sep 2021 05:26:47 -0500 Subject: [PATCH] Add Lzo decompressor and Xz decompressor with filters --- go.mod | 11 +- go.sum | 4 + internal/compression/lzo.go | 30 +++ internal/compression/xz.go | 29 +-- reader.go | 12 +- writer.go | 412 ---------------------------------- writer_compress.go | 124 ---------- writer_compression_options.go | 15 -- writer_fragment.go | 92 -------- writer_inodes.go | 28 --- writer_test.go | 16 -- writer_write.go | 71 ------ 12 files changed, 60 insertions(+), 784 deletions(-) create mode 100644 internal/compression/lzo.go delete mode 100644 writer.go delete mode 100644 writer_compress.go delete mode 100644 writer_compression_options.go delete mode 100644 writer_fragment.go delete mode 100644 writer_inodes.go delete mode 100644 writer_test.go delete mode 100644 writer_write.go diff --git a/go.mod b/go.mod index 3abffad..49a8c66 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/CalebQ42/squashfs -go 1.16 +go 1.17 require ( github.com/CalebQ42/GoAppImage v0.5.0 @@ -12,3 +12,12 @@ require ( github.com/ulikunitz/xz v0.5.10 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) + +require ( + github.com/golang/snappy v0.0.3 // indirect + github.com/google/go-cmp v0.5.5 // indirect + github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e + github.com/therootcompany/xz v1.0.1 + go.lsp.dev/uri v0.3.0 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect +) diff --git a/go.sum b/go.sum index 0593691..308dfe6 100644 --- a/go.sum +++ b/go.sum @@ -156,6 +156,8 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e h1:dCWirM5F3wMY+cmRda/B1BiPsFtmzXqV9b0hLWtVBMs= +github.com/rasky/go-lzo v0.0.0-20200203143853-96a758eda86e/go.mod h1:9leZcVcItj6m9/CfHY5Em/iBrCz7js8LcRQGTKEEv2M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -187,6 +189,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= +github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= diff --git a/internal/compression/lzo.go b/internal/compression/lzo.go new file mode 100644 index 0000000..740646b --- /dev/null +++ b/internal/compression/lzo.go @@ -0,0 +1,30 @@ +package compression + +import ( + "encoding/binary" + "io" + + lzo "github.com/rasky/go-lzo" +) + +type Lzo struct { + Algorithm int32 + Level int32 +} + +func NewLzoCompressorWithOptions(rdr io.Reader) (*Lzo, error) { + var lz Lzo + err := binary.Read(rdr, binary.LittleEndian, &lz) + if err != nil { + return nil, err + } + return &lz, nil +} + +func (l Lzo) Decompress(rdr io.Reader) ([]byte, error) { + byt, err := lzo.Decompress1X(rdr, 0, 0) + if err != nil { + return nil, err + } + return byt, nil +} diff --git a/internal/compression/xz.go b/internal/compression/xz.go index 8f8fc58..272d11c 100644 --- a/internal/compression/xz.go +++ b/internal/compression/xz.go @@ -5,44 +5,29 @@ import ( "encoding/binary" "io" - "github.com/ulikunitz/xz" + "github.com/therootcompany/xz" + + wrtXz "github.com/ulikunitz/xz" ) -type xzInit struct { - DictionarySize int32 - Filters int32 -} - -//Xz is a Xz decompressor. type Xz struct { DictionarySize int32 - HasFilters bool + Filters int32 } //NewXzCompressorWithOptions creates a new Xz compressor/decompressor that reads the compressor options from the given reader. func NewXzCompressorWithOptions(rdr io.Reader) (*Xz, error) { var x Xz - var init xzInit - err := binary.Read(rdr, binary.LittleEndian, &init) + err := binary.Read(rdr, binary.LittleEndian, &x) if err != nil { return nil, err } - x.DictionarySize = init.DictionarySize - //TODO: When I can do filters, parse the filters - if init.Filters != 0 { - x.HasFilters = true - } return &x, nil } //Decompress decompresses all the data from the rdr and returns the uncompressed bytes. func (x *Xz) Decompress(rdr io.Reader) ([]byte, error) { - r, err := xz.NewReader(rdr) - if err != nil { - return nil, err - } - r.DictCap = int(x.DictionarySize) - err = r.Verify() + r, err := xz.NewReader(rdr, 0) if err != nil { return nil, err } @@ -57,7 +42,7 @@ func (x *Xz) Decompress(rdr io.Reader) ([]byte, error) { //Compress implements compression.Compress func (x *Xz) Compress(data []byte) ([]byte, error) { var buf bytes.Buffer - w, err := xz.NewWriter(&buf) + w, err := wrtXz.NewWriter(&buf) if err != nil { return nil, err } diff --git a/reader.go b/reader.go index 41de824..0ed5f53 100644 --- a/reader.go +++ b/reader.go @@ -72,10 +72,14 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) { if err != nil { return nil, err } - if xz.HasFilters { - return nil, errors.New("XZ compression options has filters. These are not yet supported") - } rdr.decompressor = xz + case LzoCompression: + var lz *compression.Lzo + lz, err = compression.NewLzoCompressorWithOptions(rdr.r) + if err != nil { + return nil, err + } + rdr.decompressor = lz case Lz4Compression: var lz4 *compression.Lz4 lz4, err = compression.NewLz4CompressorWithOptions(rdr.r) @@ -99,6 +103,8 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) { rdr.decompressor = &compression.Gzip{} case LzmaCompression: rdr.decompressor = &compression.Lzma{} + case LzoCompression: + rdr.decompressor = &compression.Lzo{} case XzCompression: rdr.decompressor = &compression.Xz{} case Lz4Compression: diff --git a/writer.go b/writer.go deleted file mode 100644 index ae7fbc4..0000000 --- a/writer.go +++ /dev/null @@ -1,412 +0,0 @@ -package squashfs - -import ( - "errors" - "io" - "io/fs" - "log" - "os" - "path" - "sort" - "strings" - "syscall" - - "github.com/CalebQ42/squashfs/internal/compression" - "github.com/CalebQ42/squashfs/internal/inode" -) - -//Writer is used to creaste squashfs archives. Currently unusable. -//TODO: Make usable -type Writer struct { - compressor compression.Compressor - structure map[string][]*fileHolder - symlinkTable map[string]string - folders []string - uidGUIDTable []int - frags []fragment - superblock superblock - rootInode inode.Inode - //since we need some information from the actually compressed and writen data for tables - //we need to write the data FIRST, but make sure there's enough space for the tables. - dataOffset int - compressionType int - //BlockSize is how large the data blocks are. Can be between 4096 (4KB) and 1048576 (1 MB). - //If BlockSize is not inside that range, it will be set to within the range before writing. - //Default is 1048576. - BlockSize uint32 - //Flags are the SuperblockFlags used when writing the archive. - //Currently Duplicates, Exportable, UncompressedXattr, NoXattr values are ignored - Flags SuperblockFlags - allowErrors bool -} - -//NewWriter creates a new with the default options (Gzip compression and allow errors) -func NewWriter() (*Writer, error) { - return NewWriterWithOptions(GzipCompression, true) -} - -//NewWriterWithOptions creates a new squashfs.Writer with the given options. -//compressionType can be of any types, except LZO (which this library doesn't have support for yet) -//allowErrors determines, when adding folders, if it allows errors encountered with it's sub-directories -//and instead just logs the errors. -func NewWriterWithOptions(compressionType int, allowErrors bool) (*Writer, error) { - if compressionType < 0 || compressionType > 6 { - return nil, errors.New("incorrect compression type") - } - if compressionType == 3 { - return nil, errors.New("LZO compression is not (currently) supported") - } - writer := &Writer{ - structure: map[string][]*fileHolder{ - "/": make([]*fileHolder, 0), - }, - folders: []string{ - "/", - }, - symlinkTable: make(map[string]string), - compressionType: compressionType, - allowErrors: allowErrors, - BlockSize: uint32(1048576), - Flags: DefaultFlags, - } - switch compressionType { - case 1: - writer.compressor = &compression.Gzip{} - case 2: - writer.compressor = &compression.Lzma{} - case 4: - writer.compressor = &compression.Xz{} - case 5: - writer.compressor = &compression.Lz4{} - case 6: - writer.compressor = &compression.Zstd{} - } - return writer, nil -} - -//fileHolder holds the necessary information about a given file inside of a squashfs -type fileHolder struct { - reader io.Reader - path string - name string - symLocation string - blockSizes []uint32 - GUID int - perm int - size uint64 //when folder, size is of the directory entry(s) in the Directory Table. - UID int - folder bool - symlink bool - - fragIndex int - fragOffset int - inode inode.Inode -} - -//AddFile attempts to add an os.File to the archive's root directory. -func (w *Writer) AddFile(file *os.File) error { - return w.AddFileToFolder("/", file) -} - -//AddFileToFolder adds the given file to the squashfs archive, placing it inside the given folder. -func (w *Writer) AddFileToFolder(folder string, file *os.File) error { - name := path.Base(file.Name()) - if !strings.HasSuffix(folder, "/") { - folder += "/" - } - return w.AddFileTo(folder+name, file) -} - -//AddFileTo adds the given file to the squashfs archive at the given filepath. -func (w *Writer) AddFileTo(filepath string, file *os.File) error { - filepath = path.Clean(filepath) - if !strings.HasPrefix(filepath, "/") { - filepath = "/" + filepath - } - if w.Contains(filepath) { - return errors.New("File already exists at " + filepath) - } - var holder fileHolder - holder.path = path.Dir(filepath) - holder.name = path.Base(filepath) - holder.reader = file - stat, err := file.Stat() - if err != nil { - return err - } - holder.folder = stat.IsDir() - holder.symlink = (stat.Mode()&os.ModeSymlink == os.ModeSymlink) - holder.perm = int(stat.Mode().Perm()) - //Thanks to https://stackoverflow.com/questions/58179647/getting-uid-and-gid-of-a-file for uid and guid getting - if stat, ok := stat.Sys().(*syscall.Stat_t); ok { - holder.UID = int(stat.Uid) - holder.GUID = int(stat.Gid) - } - if sort.SearchInts(w.uidGUIDTable, holder.UID) == len(w.uidGUIDTable) { - w.uidGUIDTable = append(w.uidGUIDTable, holder.UID) - sort.Ints(w.uidGUIDTable) - } - if sort.SearchInts(w.uidGUIDTable, holder.GUID) == len(w.uidGUIDTable) { - w.uidGUIDTable = append(w.uidGUIDTable, holder.GUID) - sort.Ints(w.uidGUIDTable) - } - if holder.symlink { - holder.reader = file - var target string - target, err = os.Readlink(file.Name()) - if err != nil { - return err - } - holder.symLocation = target - } else if holder.folder { - var subDirNames []string - subDirNames, err = file.Readdirnames(-1) - if err != nil { - return err - } - dirsAdded := make([]string, 0) - for _, subDir := range subDirNames { - var fil *os.File - fil, err = os.Open(file.Name() + subDir) - if err != nil { - return err - } - err = w.AddFileToFolder(holder.path+"/"+holder.name, fil) - if err != nil && !w.allowErrors { - for _, dir := range dirsAdded { - w.Remove(dir) - } - return err - } else if err != nil { - log.Println("Error while adding", fil.Name()) - log.Println(err) - } - if !w.allowErrors { - dirsAdded = append(dirsAdded, holder.path+"/"+holder.name) - } - } - } else if stat.Mode().IsRegular() { - holder.reader = file - } else { - return errors.New("Unsupported file type " + file.Name()) - } - if _, ok := w.structure[holder.path]; ok { - if !w.Contains(holder.path) { - err = w.AddFolderTo(holder.path, fs.ModePerm) - if err != nil { - return err - } - } - w.folders = append(w.folders, holder.path) - sort.Strings(w.folders) - } - w.structure[holder.path] = append(w.structure[holder.path], &holder) - return nil -} - -//AddReaderTo adds the data from the given reader to the archive as a file located at the given filepath. -//Data from the reader is not read until the squashfs archive is writen. -//If the given reader implements io.Closer, it will be closed after it is fully read. -func (w *Writer) AddReaderTo(filepath string, reader io.Reader, size uint64) error { - filepath = path.Clean(filepath) - if !strings.HasPrefix(filepath, "/") { - filepath = "/" + filepath - } - if w.Contains(filepath) { - return errors.New("File already exists at " + filepath) - } - var holder fileHolder - holder.path = path.Dir(filepath) - holder.name = path.Base(filepath) - holder.size = size - holder.reader = reader - if _, ok := w.structure[holder.path]; ok { - if !w.Contains(holder.path) { - err := w.AddFolderTo(holder.path, fs.ModePerm) - if err != nil { - return err - } - } - w.folders = append(w.folders, holder.path) - sort.Strings(w.folders) - } - w.structure[holder.path] = append(w.structure[holder.path], &holder) - return nil -} - -//AddFolderTo adds a folder at the given path. IF the folder is already present, it sets the folder's permissions. -//If the path points to a non-folder (such as a file or symlink), an error is returned -func (w *Writer) AddFolderTo(folderpath string, permission fs.FileMode) error { - folderpath = path.Clean(folderpath) - tmp := w.holderAt(folderpath) - if tmp != nil { - if !tmp.folder { - return errors.New("Path is not a folder: " + folderpath) - } - tmp.perm = int(permission.Perm()) - return nil - } - file := fileHolder{ - path: path.Dir(folderpath), - name: path.Base(folderpath), - perm: int(permission.Perm()), - folder: true, - } - if _, ok := w.structure[file.path]; ok { - if !w.Contains(file.path) { - err := w.AddFolderTo(file.path, fs.ModePerm) - if err != nil { - return err - } - } - w.folders = append(w.folders, file.path) - sort.Strings(w.folders) - } - w.structure[file.path] = append(w.structure[file.path], &file) - return nil -} - -//Remove tries to remove the file(s) at the given filepath. If wildcards are used, it will remove all files that match. -//Returns true if one or more files are removed. -func (w *Writer) Remove(filepath string) bool { - var matchFound bool - filepath = path.Clean(filepath) - if !strings.HasPrefix(filepath, "/") { - filepath = "/" + filepath - } - dir, name := path.Split(filepath) - for structDir, files := range w.structure { - if match, _ := path.Match(dir, structDir); match { - for i, fil := range files { - if match, _ = path.Match(name, fil.name); match { - matchFound = true - if len(w.structure[structDir]) > 1 { - w.structure[structDir][i] = w.structure[structDir][len(w.structure[structDir])-1] - w.structure[structDir] = w.structure[structDir][:len(w.structure[structDir])-1] - } else { - w.structure[structDir] = nil - } - } - } - } - } - return matchFound -} - -//FixSymlinks will scan through the squashfs archive and try to find broken symlinks and fix them. -//This done by replacing the symlink with the target file and then pointing other symlinks to that file. -//If all symlinks can be resolved, the error slice will be nil, and the bool false, otherwise all errors occured will be in the slice. -func (w *Writer) FixSymlinks() (errs []error, problems bool) { - for dir, holderSlice := range w.structure { - for i, holder := range holderSlice { - if !holder.symlink { - continue - } - sym := holder.symLocation - if !path.IsAbs(holder.symLocation) { - sym = path.Join(dir, holder.symLocation) - } - if path, ok := w.symlinkTable[sym]; ok { - w.structure[dir][i].symLocation = path - continue - } - if path.IsAbs(sym) || strings.HasPrefix(sym, "../") { - var symFil *os.File - var err error - if strings.HasPrefix(sym, "../") { - holderFil, ok := holder.reader.(*os.File) - if !ok { - problems = true - errs = append(errs, errors.New("Cannot resolve symlink at "+dir+holder.name)) - continue - } - symFilPath := path.Dir(holderFil.Name()) - symFilPath = path.Join(symFilPath, holder.symLocation) - symFil, err = os.Open(symFilPath) - } else { - symFil, err = os.Open(sym) - } - if err != nil { - problems = true - errs = append(errs, err) - continue - } - suc := w.Remove(dir + holder.name) - if !suc { - problems = true - errs = append(errs, errors.New("Cannot resolve symlink at "+dir+holder.name)) - continue - } - err = w.AddFileTo(dir+holder.name, symFil) - if err != nil { - w.structure[dir] = append(w.structure[dir], holder) - problems = true - errs = append(errs, err) - continue - } - w.symlinkTable[sym] = dir + holder.name - } else { - symHolder := w.holderAt(sym) - if symHolder != nil { - w.symlinkTable[sym] = sym - continue - } - holderFil, ok := holder.reader.(*os.File) - if !ok { - problems = true - errs = append(errs, errors.New("Cannot resolve symlink at "+dir+holder.name)) - continue - } - symFilPath := path.Dir(holderFil.Name()) - symFilPath = path.Join(symFilPath, holder.symLocation) - symFil, err := os.Open(symFilPath) - if err != nil { - problems = true - errs = append(errs, err) - continue - } - err = w.AddFileTo(sym, symFil) - if err != nil { - problems = true - errs = append(errs, err) - continue - } - w.symlinkTable[sym] = sym - } - } - } - return -} - -func (w *Writer) holderAt(filepath string) *fileHolder { - filepath = path.Clean(filepath) - if !strings.HasPrefix(filepath, "/") { - filepath = "/" + filepath - } - dir, name := path.Split(filepath) - if holderSlice, ok := w.structure[dir]; ok { - for _, holder := range holderSlice { - if holder.name == name { - return holder - } - } - } - return nil -} - -//Contains returns whether a file is present at the given filepath -func (w *Writer) Contains(filepath string) bool { - filepath = path.Clean(filepath) - if !strings.HasPrefix(filepath, "/") { - filepath = "/" + filepath - } - dir, name := path.Split(filepath) - if holderSlice, ok := w.structure[dir]; ok { - for _, holder := range holderSlice { - if holder.name == name { - return true - } - } - } - return false -} diff --git a/writer_compress.go b/writer_compress.go deleted file mode 100644 index af88746..0000000 --- a/writer_compress.go +++ /dev/null @@ -1,124 +0,0 @@ -package squashfs - -import ( - "io" - "reflect" - "sync" -) - -func (w *Writer) compressData(data []byte) ([]byte, error) { - if reflect.DeepEqual(data, make([]byte, len(data))) { - return nil, nil - } - if w.Flags.UncompressedData || w.compressor == nil { - return data, nil - } - compressedData, err := w.compressor.Compress(data) - if err != nil { - return nil, err - } - if len(data) <= len(compressedData) { - return data, nil - } - return compressedData, nil -} - -//Writes the given fileHolder to the WriterAt at the given offset. -//If fil.Reader implements io.ReaderAt, the process is threaded. -func (w *Writer) writeFile(fil *fileHolder, write io.WriterAt, startOffset int64) (endOffset int64, err error) { - endOffset = startOffset - var sizes []uint32 - if fil.fragIndex != -1 { - sizes = fil.blockSizes[:len(fil.blockSizes)-1] - } else { - sizes = fil.blockSizes - } - if rdrAt, ok := fil.reader.(io.ReaderAt); ok { - type writeReturn struct { - err error - byts []byte - i int - } - out := make(chan *writeReturn) - var filOffset int64 - var sync sync.WaitGroup - sync.Add(len(sizes)) - for i, size := range sizes { - go func(offset int64, size uint32, i int) { - var ret writeReturn - ret.i = i - defer func() { - out <- &ret - }() - ret.byts = make([]byte, size) - _, ret.err = rdrAt.ReadAt(ret.byts, offset) - if ret.err != nil { - return - } - ret.byts, ret.err = w.compressData(ret.byts) - sync.Done() - }(filOffset, size, i) - filOffset += int64(size) - } - var curInd int - var holdingArea []*writeReturn - for curInd < len(sizes) { - var tmp *writeReturn - for _, ret := range holdingArea { - if ret.i == curInd { - tmp = ret - break - } - } - if tmp == nil { - tmp = <-out - if tmp.err != nil { - sync.Wait() - return endOffset, tmp.err - } - if tmp.i != curInd { - holdingArea = append(holdingArea, tmp) - continue - } - } - fil.blockSizes[curInd] = uint32(len(tmp.byts)) - if len(tmp.byts) == int(w.BlockSize) { - //set uncompressed bit if not compressed - fil.blockSizes[curInd] |= (1 << 24) - } - var n int - n, err = write.WriteAt(tmp.byts, endOffset) - endOffset += int64(n) - if err != nil { - sync.Wait() - return - } - curInd++ - } - return - } - var byts []byte - for i, size := range sizes { - byts = make([]byte, size) - _, err = fil.reader.Read(byts) - if err != nil { - return - } - byts, err = w.compressData(byts) - if err != nil { - return - } - fil.blockSizes[i] = uint32(len(byts)) - if len(byts) == int(w.BlockSize) { - //set uncompressed bit if not compressed - fil.blockSizes[i] |= (1 << 24) - } - var n int - n, err = write.WriteAt(byts, endOffset) - endOffset += int64(n) - if err != nil { - return - } - } - return -} diff --git a/writer_compression_options.go b/writer_compression_options.go deleted file mode 100644 index f02eda6..0000000 --- a/writer_compression_options.go +++ /dev/null @@ -1,15 +0,0 @@ -package squashfs - -//TODO: Allow settings the options - -// func (w *Writer) SetGzipOptions() error {} - -// func (w *Writer) SetLzmaOptions() error {} - -// func (w *Writer) SetLzoOptions() error {} - -// func (w *Writer) SetXzOptions() error {} - -// func (w *Writer) SetLz4Options() error {} - -// func (w *Writer) SetZstdOptions() error {} diff --git a/writer_fragment.go b/writer_fragment.go deleted file mode 100644 index 399ecd0..0000000 --- a/writer_fragment.go +++ /dev/null @@ -1,92 +0,0 @@ -package squashfs - -import ( - "bytes" - "io" -) - -type fragment struct { - w *Writer - files []*fileHolder - sizes []uint32 -} - -func (f *fragment) sizeLeft() uint32 { - totalSize := uint32(0) - for _, siz := range f.sizes { - totalSize += siz - } - return f.w.BlockSize - uint32(totalSize) -} - -func (f *fragment) addFragment(fil *fileHolder) { - //SizeLeft should already be checked - fil.fragOffset = len(f.files) - f.files = append(f.files, fil) - f.sizes = append(f.sizes, fil.blockSizes[len(fil.blockSizes)-1]) -} - -//TODO: give info about the frags for the frag table. -func (w *Writer) writeFragments(write io.WriterAt, off int64) (newOff int64, err error) { - newOff = off - var buf bytes.Buffer - var byts []byte - var n int - for _, frag := range w.frags { - blockOffset := 0 - for i, fil := range frag.files { - _, err = io.CopyN(&buf, fil.reader, int64(frag.sizes[i])) - if err != nil { - return - } - fil.fragOffset = blockOffset - blockOffset += int(frag.sizes[i]) - } - if !w.Flags.UncompressedFragments && w.compressor != nil { - byts, err = w.compressor.Compress(buf.Bytes()) - if err != nil { - return - } - } else { - byts = buf.Bytes() - } - n, err = write.WriteAt(byts, newOff) - newOff += int64(n) - if err != nil { - return - } - buf.Reset() - } - return -} - -func (w *Writer) addToFragments(fil *fileHolder) { - fragSize := fil.blockSizes[len(fil.blockSizes)-1] - //only fragment if the final block is less then 80% of a full block or AlwaysFragment. - //TODO: Make this check after looking at all fragment blocks. Below option is better, but this is easier to implement... - if w.Flags.AlwaysFragment || fragSize < uint32(float32(w.BlockSize)*0.8) { - //Try to slot the fragment into a fragment that has the perfect size left. If not, just pick the first one. - //TODO: possibly make this more efficient, possibly by calculating fragments all at once and seeing which combos match BlockSize perfectly. - var possibleFrags []int - for i := range w.frags { - left := w.frags[i].sizeLeft() - if left == fragSize { - fil.fragIndex = i - w.frags[i].addFragment(fil) - return - } else if left > fragSize { - possibleFrags = append(possibleFrags, i) - } - } - if len(possibleFrags) > 0 { - fil.fragIndex = possibleFrags[0] - } else { - fil.fragIndex = len(w.frags) - w.frags = append(w.frags, fragment{ - w: w, - files: []*fileHolder{fil}, - sizes: []uint32{fragSize}, - }) - } - } -} diff --git a/writer_inodes.go b/writer_inodes.go deleted file mode 100644 index 109a586..0000000 --- a/writer_inodes.go +++ /dev/null @@ -1,28 +0,0 @@ -package squashfs - -import ( - "encoding/binary" - "io" - - "github.com/CalebQ42/squashfs/internal/inode" -) - -func (w *Writer) countInodes() (out uint32) { - out++ //for the root directory - for _, files := range w.structure { - out += uint32(len(files)) - } - return -} - -func (w *Writer) setupInodes() (size int, err error) { - w.rootInode.Type = inode.DirType - //setup - size += binary.Size(w.rootInode) - return -} - -func (w *Writer) writeInodeTable(wrt io.WriterAt, off int64) (newOff int64, err error) { - newOff = off - return -} diff --git a/writer_test.go b/writer_test.go deleted file mode 100644 index 4656645..0000000 --- a/writer_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package squashfs - -import ( - "os" - "testing" -) - -func TestWrite(t *testing.T) { - os.Remove("testing/test.sfs") - os.Mkdir("testing", os.ModePerm) - test, err := os.Create("testing/test.sfs") - if err != nil { - t.Fatal(err) - } - _ = test -} diff --git a/writer_write.go b/writer_write.go deleted file mode 100644 index ecfea0a..0000000 --- a/writer_write.go +++ /dev/null @@ -1,71 +0,0 @@ -package squashfs - -import ( - "errors" - "io" - "math" - "os" - "time" -) - -//WriteToFilename creates the squashfs archive with the given filepath. -func (w *Writer) WriteToFilename(filepath string) error { - newFil, err := os.Create(filepath) - if err != nil { - return err - } - _, err = w.WriteTo(newFil) - return err -} - -//WriteTo attempts to write the archive to the given io.WriterAt. -// -//Not working. Yet. -func (w *Writer) WriteTo(write io.WriterAt) (int64, error) { - if w.BlockSize > 1048576 { - w.BlockSize = 1048576 - } else if w.BlockSize < 4096 { - w.BlockSize = 4096 - } - w.Flags.RemoveDuplicates = false - w.Flags.Exportable = false - w.Flags.NoXattr = true - w.calculateFragsAndBlockSizes() - w.superblock = superblock{ - Magic: magic, - InodeCount: w.countInodes(), - CreationTime: uint32(time.Now().Unix()), - BlockSize: w.BlockSize, - CompressionType: uint16(w.compressionType), - BlockLog: uint16(math.Log2(float64(w.BlockSize))), - Flags: w.Flags.ToUint(), - IDCount: uint16(len(w.uidGUIDTable)), - MajorVersion: 4, - MinorVersion: 0, - } - w.dataOffset = 96 //superblock size - //write compression options - //write/calculate compressed data sizes - - return 0, errors.New("i said don't") -} - -//splits up the size of files into -func (w *Writer) calculateFragsAndBlockSizes() { - for _, files := range w.structure { - for i := range files { - files[i].fragIndex = -1 - files[i].blockSizes = make([]uint32, files[i].size/uint64(w.BlockSize)) - for j := range files[i].blockSizes { - files[i].blockSizes[j] = w.BlockSize - } - fragSize := uint32(files[i].size % uint64(w.BlockSize)) - if fragSize > 0 { - files[i].blockSizes = append(files[i].blockSizes, fragSize) - if !w.Flags.NoFragments { - w.addToFragments(files[i]) - } - } - } - } -}