diff --git a/README.md b/README.md index df4b04a..061c43d 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,4 @@ Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree ## Performance -This library, decompressing the Firefox AppImage and using go tests, takes about twice as long as `unsquashfs` on my quad core laptop. (~1 second with the library and about half a second with `unsquashfs`). - -**My recents tests have shown the Firefox AppImage might be an outlier and this library might be considerably slower (4x ~ 6x time slower then `unsquashfs`)** +Testing on a zstd compressed file, my library is anywhere from 5x ~ 7x slower then `unsquashfs` diff --git a/internal/data/fullreader.go b/internal/data/fullreader.go index 19bf3ea..1ada283 100644 --- a/internal/data/fullreader.go +++ b/internal/data/fullreader.go @@ -49,27 +49,26 @@ func (r FullReader) process(index int, offset int64, od *outDat, out chan *outDa od.data = make([]byte, r.blockSize) return } - // rdr := io.LimitReader(toreader.NewReader(r.r, offset), int64(size)) if size == r.sizes[index] { - //Special workaround for zstd for increased performancce. - if zstd, ok := r.d.(*decompress.Zstd); ok { - od.data = make([]byte, size) - _, od.err = r.r.ReadAt(od.data, offset) - if od.err == nil { - od.data, od.err = zstd.Decode(od.data) - } - } else { - var rdr io.ReadCloser - rdr, od.err = r.d.Reader(io.LimitReader(toreader.NewReader(r.r, offset), int64(size))) + if dec, ok := r.d.(decompress.Decoder); ok { + dat := make([]byte, size) + _, od.err = r.r.ReadAt(dat, offset) if od.err != nil { return } - od.data = make([]byte, r.blockSize) - var read int - read, od.err = rdr.Read(od.data) - od.data = od.data[:read] - rdr.Close() + od.data, od.err = dec.Decode(dat, int(r.blockSize)) + return } + var rdr io.ReadCloser + rdr, od.err = r.d.Reader(io.LimitReader(toreader.NewReader(r.r, offset), int64(size))) + if od.err != nil { + return + } + od.data = make([]byte, r.blockSize) + var read int + read, od.err = rdr.Read(od.data) + od.data = od.data[:read] + rdr.Close() } else { od.data = make([]byte, size) _, od.err = r.r.ReadAt(od.data, offset) @@ -206,40 +205,59 @@ func (r FullReader) WriteTo(w io.Writer) (n int64, err error) { go r.process(i, int64(offset), od, out) offset += uint64(realSize(r.sizes[i])) } - var cur int - cache := make(map[int]outDat) - var tmpN int - for dat := range out { - if dat.err != nil { - err = dat.err - pol.Put(dat) - return - } - if dat.i != cur { - cache[dat.i] = *dat - pol.Put(dat) - continue - } - tmpN, err = w.Write(dat.data) - n += int64(tmpN) - pol.Put(dat) - if err != nil { - return - } - cur++ - var ok bool - var curDat outDat - for { - curDat, ok = cache[cur] - if !ok { - break + wt, ok := w.(io.WriterAt) + if !ok { + var cur int + cache := make(map[int]outDat) + var tmpN int + var dat *outDat + for cur < len(r.sizes) { + dat = <-out + defer pol.Put(dat) + if dat.err != nil { + err = dat.err + return } - tmpN, err = w.Write(curDat.data) + if dat.i != cur { + cache[dat.i] = *dat + continue + } + tmpN, err = w.Write(dat.data) n += int64(tmpN) if err != nil { return } cur++ + var ok bool + var curDat outDat + for { + curDat, ok = cache[cur] + if !ok { + break + } + tmpN, err = w.Write(curDat.data) + n += int64(tmpN) + if err != nil { + return + } + cur++ + } + } + } else { + var done int + var dat *outDat + for done < len(r.sizes) { + dat = <-out + defer pol.Put(dat) + if dat.err != nil { + err = dat.err + return + } + _, err = wt.WriteAt(dat.data, int64(dat.i*int(r.blockSize))) + if err != nil { + return + } + done++ } } return diff --git a/internal/data/reader.go b/internal/data/reader.go index d4c51c2..e5c2c96 100644 --- a/internal/data/reader.go +++ b/internal/data/reader.go @@ -53,14 +53,14 @@ func (r *Reader) advance() (err error) { } else { r.cur = io.LimitReader(r.master, int64(size)) if size == r.blockSizes[0] { - if r.d.Resetable() { + if rs, ok := r.d.(decompress.Resetable); ok { if r.comRdr == nil { r.cur, err = r.d.Reader(r.cur) if err != nil { return } } else { - err = r.d.Reset(r.comRdr, r.cur) + err = rs.Reset(r.comRdr, r.cur) r.cur = r.comRdr } } else { diff --git a/internal/decompress/gzip.go b/internal/decompress/gzip.go index 879686f..26feccd 100644 --- a/internal/decompress/gzip.go +++ b/internal/decompress/gzip.go @@ -12,8 +12,6 @@ func (g GZip) Reader(src io.Reader) (io.ReadCloser, error) { return zlib.NewReader(src) } -func (g GZip) Resetable() bool { return true } - func (g GZip) Reset(old, src io.Reader) error { return old.(zlib.Resetter).Reset(src, nil) } diff --git a/internal/decompress/interface.go b/internal/decompress/interface.go index af522f1..0850b84 100644 --- a/internal/decompress/interface.go +++ b/internal/decompress/interface.go @@ -1,19 +1,22 @@ package decompress import ( - "errors" "io" ) -var ErrNotResetable = errors.New("decompressor not resetable") - type Decompressor interface { //Creates a new decompressor reading from src. Reader(src io.Reader) (io.ReadCloser, error) - //Reports whether Reset will work or not. - Resetable() bool +} + +type Resetable interface { //Reset attempts to re-use an old decompressor with new data. //Will return ErrNotResetable if not Resetable(). //Must ALWAYS be provided with a reader created with Reader. Reset(old, src io.Reader) error } + +type Decoder interface { + //Decodes a chunk of data all at once. + Decode(in []byte, outSize int) ([]byte, error) +} diff --git a/internal/decompress/lz4.go b/internal/decompress/lz4.go index c2bab28..ebad497 100644 --- a/internal/decompress/lz4.go +++ b/internal/decompress/lz4.go @@ -12,9 +12,16 @@ func (l Lz4) Reader(r io.Reader) (io.ReadCloser, error) { return io.NopCloser(lz4.NewReader(r)), nil } -func (l Lz4) Resetable() bool { return true } - func (l Lz4) Reset(old, src io.Reader) error { old.(*lz4.Reader).Reset(src) return nil } + +func (l Lz4) Decode(in []byte, outSize int) (out []byte, err error) { + out = make([]byte, outSize) + outLen, err := lz4.UncompressBlock(in, out) + if outLen < outSize { + out = out[:outLen] + } + return +} diff --git a/internal/decompress/lzma.go b/internal/decompress/lzma.go index f93b29d..9add4ae 100644 --- a/internal/decompress/lzma.go +++ b/internal/decompress/lzma.go @@ -12,7 +12,3 @@ func (l Lzma) Reader(r io.Reader) (io.ReadCloser, error) { rdr, err := lzma.NewReader(r) return io.NopCloser(rdr), err } - -func (l Lzma) Resetable() bool { return false } - -func (l Lzma) Reset(old, src io.Reader) error { return ErrNotResetable } diff --git a/internal/decompress/lzo.go b/internal/decompress/lzo.go index 15d0f2e..76333e7 100644 --- a/internal/decompress/lzo.go +++ b/internal/decompress/lzo.go @@ -16,7 +16,3 @@ func (l Lzo) Reader(r io.Reader) (io.ReadCloser, error) { } return io.NopCloser(bytes.NewReader(cache)), nil } - -func (l Lzo) Resetable() bool { return false } - -func (l Lzo) Reset(old, src io.Reader) error { return ErrNotResetable } diff --git a/internal/decompress/xz.go b/internal/decompress/xz.go index ad260bd..8bc2c80 100644 --- a/internal/decompress/xz.go +++ b/internal/decompress/xz.go @@ -13,8 +13,6 @@ func (x Xz) Reader(r io.Reader) (io.ReadCloser, error) { return io.NopCloser(rdr), err } -func (x Xz) Resetable() bool { return true } - func (x Xz) Reset(old, src io.Reader) error { return old.(*xz.Reader).Reset(src) } diff --git a/internal/decompress/zstd.go b/internal/decompress/zstd.go index 544500d..c4ba07b 100644 --- a/internal/decompress/zstd.go +++ b/internal/decompress/zstd.go @@ -15,15 +15,13 @@ func (z Zstd) Reader(src io.Reader) (io.ReadCloser, error) { return r.IOReadCloser(), err } -func (z Zstd) Resetable() bool { return true } - func (z Zstd) Reset(old, src io.Reader) error { return old.(*zstd.Decoder).Reset(src) } -func (z *Zstd) Decode(in []byte) (out []byte, err error) { +func (z Zstd) Decode(in []byte, outSize int) ([]byte, error) { if z.writeToReader == nil { z.writeToReader, _ = zstd.NewReader(nil) } - return z.writeToReader.DecodeAll(in, nil) + return z.writeToReader.DecodeAll(in, make([]byte, outSize)) } diff --git a/internal/metadata/reader.go b/internal/metadata/reader.go index 0768084..3fd0499 100644 --- a/internal/metadata/reader.go +++ b/internal/metadata/reader.go @@ -26,7 +26,7 @@ func realSize(siz uint16) uint16 { } func (r *Reader) advance() (err error) { - if !r.d.Resetable() { + if _, ok := r.d.(decompress.Resetable); !ok { if clr, ok := r.cur.(io.Closer); ok { clr.Close() } @@ -39,14 +39,14 @@ func (r *Reader) advance() (err error) { size := realSize(raw) r.cur = io.LimitReader(r.master, int64(size)) if size == raw { - if r.d.Resetable() { + if rs, ok := r.d.(decompress.Resetable); ok { if r.comRdr == nil { r.cur, err = r.d.Reader(r.cur) if err != nil { return } } else { - err = r.d.Reset(r.comRdr, r.cur) + err = rs.Reset(r.comRdr, r.cur) r.cur = r.comRdr } } else {