From 5c3bf8d528444b4f21c785812f9118e0be03550a Mon Sep 17 00:00:00 2001 From: Caleb Gardner Date: Wed, 9 Dec 2020 01:40:30 -0600 Subject: [PATCH] Implemented the rest of the compression types Haven't implemented LZO due to limited libraries --- README.md | 4 +++- go.mod | 2 ++ go.sum | 4 ++++ internal/compression/gzip.go | 28 ++++++++++++++++++++++++++- internal/compression/lz4.go | 37 ++++++++++++++++++++++++++++++++++++ internal/compression/zstd.go | 36 +++++++++++++++++++++++++++++++++++ reader.go | 36 ++++++++++++++++++++++++++++++++++- 7 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 internal/compression/lz4.go create mode 100644 internal/compression/zstd.go diff --git a/README.md b/README.md index e7a812f..4f993ed 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# squashfs [![PkgGoDev](https://pkg.go.dev/badge/github.com/CalebQ42/squashfs)](https://pkg.go.dev/github.com/CalebQ42/squashfs) +# squashfs + +[![PkgGoDev](https://pkg.go.dev/badge/github.com/CalebQ42/squashfs)](https://pkg.go.dev/github.com/CalebQ42/squashfs) [![Go Report Card](https://goreportcard.com/badge/github.com/CalebQ42/squashfs)](https://goreportcard.com/report/github.com/CalebQ42/squashfs) A PURE Go library to read and write squashfs. diff --git a/go.mod b/go.mod index f7f2384..3765da2 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,9 @@ require ( github.com/adrg/xdg v0.2.3 // indirect github.com/google/go-cmp v0.5.4 // indirect github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect + github.com/klauspost/compress v1.11.3 github.com/kr/text v0.2.0 // indirect + github.com/pierrec/lz4/v4 v4.1.1 github.com/smartystreets/assertions v1.2.0 // indirect github.com/ulikunitz/xz v0.5.8 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/go.sum b/go.sum index 26b6a80..00c259a 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -27,6 +29,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pierrec/lz4/v4 v4.1.1 h1:cS6aGkNLJr4u+UwaA21yp+gbWN3WJWtKo1axmPDObMA= +github.com/pierrec/lz4/v4 v4.1.1/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= diff --git a/internal/compression/gzip.go b/internal/compression/gzip.go index 19ddafd..98f3cd4 100644 --- a/internal/compression/gzip.go +++ b/internal/compression/gzip.go @@ -3,11 +3,37 @@ package compression import ( "bytes" "compress/zlib" + "encoding/binary" "io" ) +type gzipInit struct { + CompressionLevel int32 + WindowSize int16 + Strategies int16 +} + //Gzip is a decompressor for gzip type compression. Uses zlib for compression and decompression -type Gzip struct{} +type Gzip struct { + CompressionLevel int32 + HasCustomWindow bool + HasStrategies bool +} + +//NewGzipCompressorWithOptions creates a new gzip compressor/decompressor with options read from the given reader. +func NewGzipCompressorWithOptions(r io.Reader) (*Gzip, error) { + var gzip Gzip + var init gzipInit + err := binary.Read(r, binary.LittleEndian, &init) + if err != nil { + return nil, err + } + gzip.CompressionLevel = init.CompressionLevel + //TODO: proper support for window size and strategies + gzip.HasCustomWindow = init.WindowSize != 15 + gzip.HasStrategies = init.Strategies != 0 && init.Strategies != 1 + return &gzip, nil +} //Decompress reads the entirety of the given reader and returns it uncompressed as a byte slice. func (g *Gzip) Decompress(r io.Reader) ([]byte, error) { diff --git a/internal/compression/lz4.go b/internal/compression/lz4.go new file mode 100644 index 0000000..2cbfd89 --- /dev/null +++ b/internal/compression/lz4.go @@ -0,0 +1,37 @@ +package compression + +import ( + "bytes" + "encoding/binary" + "io" + + "github.com/pierrec/lz4/v4" +) + +//Lz4 is a Lz4 Compressor/Decompressor +type Lz4 struct { + HC bool +} + +//NewLz4CompressorWithOptions creates a new lz4 compressor/decompressor with options read from the given reader. +func NewLz4CompressorWithOptions(r io.Reader) (*Lz4, error) { + var lz4 Lz4 + var init struct { + Version int32 + Flags int32 + } + err := binary.Read(r, binary.LittleEndian, &init) + if err != nil { + return nil, err + } + lz4.HC = init.Flags == 1 + return &lz4, nil +} + +//Decompress decompresses all data from r and returns the uncompressed bytes +func (l *Lz4) Decompress(r io.Reader) ([]byte, error) { + rdr := lz4.NewReader(r) + var buf bytes.Buffer + _, err := io.Copy(&buf, rdr) + return buf.Bytes(), err +} diff --git a/internal/compression/zstd.go b/internal/compression/zstd.go new file mode 100644 index 0000000..38b88b1 --- /dev/null +++ b/internal/compression/zstd.go @@ -0,0 +1,36 @@ +package compression + +import ( + "bytes" + "encoding/binary" + "io" + + "github.com/klauspost/compress/zstd" +) + +//Zstd is a zstd compressor/decompressor +type Zstd struct { + CompressionLevel int32 +} + +//NewZstdCompressorWithOptions creates a new Zstd with options read from the given reader +func NewZstdCompressorWithOptions(r io.Reader) (*Zstd, error) { + var zstd Zstd + err := binary.Read(r, binary.LittleEndian, &zstd) + if err != nil { + return nil, err + } + return &zstd, nil +} + +//Decompress decompresses all data from the reader and returns the uncompressed data +func (z *Zstd) Decompress(r io.Reader) ([]byte, error) { + rdr, err := zstd.NewReader(r) + if err != nil { + return nil, err + } + defer rdr.Close() + var buf bytes.Buffer + _, err = io.Copy(&buf, rdr) + return buf.Bytes(), err +} diff --git a/reader.go b/reader.go index 97dd69d..671b058 100644 --- a/reader.go +++ b/reader.go @@ -21,6 +21,8 @@ var ( errIncompatibleCompression = errors.New("Compression type unsupported") //ErrCompressorOptions is returned if compressor options is present. It's not currently supported. errCompressorOptions = errors.New("Compressor options is not currently supported") + //ErrOptions is returned when compression options that I haven't tested is set. When this is returned, the Reader is also returned. + ErrOptions = errors.New("Possibly incompatible compressor options") ) //Reader processes and reads a squashfs archive. @@ -35,6 +37,7 @@ type Reader struct { //NewSquashfsReader returns a new squashfs.Reader from an io.ReaderAt func NewSquashfsReader(r io.ReaderAt) (*Reader, error) { + hasUnsupportedOptions := false var rdr Reader rdr.r = r err := binary.Read(io.NewSectionReader(rdr.r, 0, int64(binary.Size(rdr.super))), binary.LittleEndian, &rdr.super) @@ -50,8 +53,17 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) { rdr.flags = rdr.super.GetFlags() if rdr.flags.CompressorOptions { switch rdr.super.CompressionType { + case gzipCompression: + gzip, err := compression.NewGzipCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 8)) + if err != nil { + return nil, err + } + if gzip.HasCustomWindow || gzip.HasStrategies { + hasUnsupportedOptions = true + } + rdr.decompressor = gzip case xzCompression: - xz, err := compression.NewXzCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 1000)) //1000 is technically too much, but it's just an easy way to do it. + xz, err := compression.NewXzCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 8)) if err != nil { return nil, err } @@ -59,6 +71,21 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) { return nil, errors.New("XZ compression options has filters. These are not yet supported") } rdr.decompressor = xz + case lz4Compression: + lz4, err := compression.NewLz4CompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 8)) + if err != nil { + return nil, err + } + if lz4.HC { + hasUnsupportedOptions = true + } + rdr.decompressor = lz4 + case zstdCompression: + zstd, err := compression.NewZstdCompressorWithOptions(io.NewSectionReader(rdr.r, int64(binary.Size(rdr.super)), 4)) + if err != nil { + return nil, err + } + rdr.decompressor = zstd default: return nil, errCompressorOptions } @@ -70,6 +97,10 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) { rdr.decompressor = &compression.Lzma{} case xzCompression: rdr.decompressor = &compression.Xz{} + case lz4Compression: + rdr.decompressor = &compression.Lz4{} + case zstdCompression: + rdr.decompressor = &compression.Zstd{} default: //TODO: all compression types. return nil, errIncompatibleCompression @@ -111,6 +142,9 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) { } unread -= read } + if hasUnsupportedOptions { + return &rdr, ErrOptions + } return &rdr, nil }