Potential workaround for poor zstd performance

Performance is still not great, but better
This commit is contained in:
Caleb Gardner
2022-06-21 01:09:33 -05:00
parent 1b934de04d
commit 83dfa77b7d
12 changed files with 110 additions and 30 deletions
+18 -5
View File
@@ -40,6 +40,7 @@ type outDat struct {
func (r FullReader) process(index int, offset int64, out chan outDat) {
var err error
var dat []byte
var rdr io.ReadCloser
size := realSize(r.sizes[index])
if size == 0 {
out <- outDat{
@@ -49,12 +50,24 @@ func (r FullReader) process(index int, offset int64, out chan outDat) {
}
return
}
rdr := io.LimitReader(toreader.NewReader(r.r, offset), int64(size))
// rdr := io.LimitReader(toreader.NewReader(r.r, offset), int64(size))
if size == r.sizes[index] {
rdr, err = r.d.Reader(rdr)
}
if err == nil {
dat, err = io.ReadAll(rdr)
//Special workaround for zstd for increased performancce.
if zstd, ok := r.d.(*decompress.Zstd); ok {
dat = make([]byte, size)
_, err = r.r.ReadAt(dat, offset)
if err == nil {
dat, err = zstd.Decode(dat)
}
} else {
rdr, err = r.d.Reader(io.LimitReader(toreader.NewReader(r.r, offset), int64(size)))
if err == nil {
dat, err = io.ReadAll(rdr)
}
}
} else {
dat = make([]byte, size)
_, err = r.r.ReadAt(dat, offset)
}
out <- outDat{
i: index,