Reset to zero

This commit is contained in:
Caleb Gardner
2023-12-19 03:23:24 -06:00
parent fcd8c4c85b
commit d4d1b2c2b2
35 changed files with 0 additions and 3014 deletions
-235
View File
@@ -1,235 +0,0 @@
package data
import (
"io"
"github.com/CalebQ42/squashfs/internal/decompress"
"github.com/CalebQ42/squashfs/internal/toreader"
)
type FullReader struct {
r io.ReaderAt
d decompress.Decompressor
fragRdr func() (io.Reader, error)
sizes []uint32
blockSize uint32
start uint64
fileSize uint64
}
func NewFullReader(r io.ReaderAt, start uint64, d decompress.Decompressor, blockSizes []uint32, blockSize uint32, fileSize uint64) *FullReader {
return &FullReader{
r: r,
start: start,
blockSize: blockSize,
sizes: blockSizes,
d: d,
fileSize: fileSize,
}
}
func (r *FullReader) AddFragment(rdr func() (io.Reader, error)) {
r.fragRdr = rdr
r.sizes = append(r.sizes, 0)
}
type outDat struct {
err error
data []byte
i int
}
func (r FullReader) process(index int, offset int64, out chan outDat) {
var err error
var dat []byte
var rdr io.ReadCloser
size := realSize(r.sizes[index])
if size == 0 {
outSize := r.blockSize
if r.fileSize < uint64(r.blockSize) {
outSize = uint32(r.fileSize)
}
out <- outDat{
i: index,
err: nil,
data: make([]byte, outSize),
}
return
}
// rdr := io.LimitReader(toreader.NewReader(r.r, offset), int64(size))
if size == r.sizes[index] {
if dec, ok := r.d.(decompress.Decoder); ok {
dat = make([]byte, size)
_, err = r.r.ReadAt(dat, offset)
if err == nil {
dat, err = dec.Decode(dat)
}
} else {
rdr, err = r.d.Reader(io.LimitReader(toreader.NewReader(r.r, offset), int64(size)))
if err == nil {
dat, err = io.ReadAll(rdr)
}
}
} else {
dat = make([]byte, size)
_, err = r.r.ReadAt(dat, offset)
}
out <- outDat{
i: index,
err: err,
data: dat,
}
if clr, ok := rdr.(io.Closer); ok {
clr.Close()
}
}
func (r FullReader) ReadAt(p []byte, off int64) (n int, err error) {
out := make(chan outDat, len(r.sizes))
offset := r.start
num := len(r.sizes)
start := off / int64(r.blockSize)
end := len(p) / int(r.blockSize)
if end%int(r.blockSize) > 0 {
end++
}
if end > len(r.sizes) {
if r.fragRdr != nil {
end = len(r.sizes)
} else {
end = len(r.sizes) + 1
}
}
for i := 0; i < num; i++ {
if i < int(start) || i > end {
offset += uint64(realSize(r.sizes[i]))
continue
}
if i == num-1 && r.fragRdr != nil {
go func() {
rdr, e := r.fragRdr()
if e != nil {
out <- outDat{
i: num - 1,
err: e,
}
return
}
dat, e := io.ReadAll(rdr)
out <- outDat{
i: num - 1,
err: e,
data: dat,
}
if clr, ok := rdr.(io.Closer); ok {
clr.Close()
}
}()
continue
}
go r.process(i, int64(offset), out)
offset += uint64(realSize(r.sizes[i]))
}
cache := make(map[int]outDat)
for cur := start; cur < int64(end); {
dat := <-out
if dat.err != nil {
err = dat.err
return
}
if dat.i != int(cur) {
cache[dat.i] = dat
continue
}
if cur == start {
dat.data = dat.data[off%int64(r.blockSize):]
}
for i := range dat.data {
p[n+i] = dat.data[i]
}
n += len(dat.data)
cur++
var ok bool
for {
dat, ok = cache[int(cur)]
if !ok {
break
}
for i := range dat.data {
p[n+i] = dat.data[i]
}
n += len(dat.data)
cur++
delete(cache, int(cur))
}
}
if n < len(p) {
err = io.EOF
}
return
}
func (r FullReader) WriteTo(w io.Writer) (n int64, err error) {
out := make(chan outDat, len(r.sizes))
offset := r.start
num := len(r.sizes)
for i := 0; i < num; i++ {
if i == num-1 && r.fragRdr != nil {
go func() {
rdr, e := r.fragRdr()
if err != nil {
out <- outDat{
i: num - 1,
err: e,
}
return
}
dat, e := io.ReadAll(rdr)
out <- outDat{
i: num - 1,
err: e,
data: dat,
}
if clr, ok := rdr.(io.Closer); ok {
clr.Close()
}
}()
continue
}
go r.process(i, int64(offset), out)
offset += uint64(realSize(r.sizes[i]))
}
cache := make(map[int]outDat)
var tmpN int
for cur := 0; cur < num; {
dat := <-out
if dat.err != nil {
err = dat.err
return
}
if dat.i != cur {
cache[dat.i] = dat
continue
}
tmpN, err = w.Write(dat.data)
n += int64(tmpN)
if err != nil {
return
}
cur++
var ok bool
for {
dat, ok = cache[cur]
if !ok {
break
}
tmpN, err = w.Write(dat.data)
n += int64(tmpN)
if err != nil {
return
}
cur++
}
}
return
}
-104
View File
@@ -1,104 +0,0 @@
package data
import (
"bytes"
"io"
"github.com/CalebQ42/squashfs/internal/decompress"
)
type Reader struct {
master io.Reader
cur io.Reader
fragRdr io.Reader
d decompress.Decompressor
comRdr io.Reader
blockSizes []uint32
blockSize uint32
resetable bool
fileSize uint64
}
func NewReader(r io.Reader, d decompress.Decompressor, blockSizes []uint32, blockSize uint32, fileSize uint64) *Reader {
return &Reader{
d: d,
master: r,
blockSizes: blockSizes,
blockSize: blockSize,
resetable: true,
fileSize: fileSize,
}
}
func (r *Reader) AddFragment(rdr io.Reader) {
r.fragRdr = rdr
r.blockSizes = append(r.blockSizes, 0)
}
func realSize(siz uint32) uint32 {
return siz &^ (1 << 24)
}
func (r *Reader) advance() (err error) {
if clr, ok := r.cur.(io.Closer); ok {
clr.Close()
}
if len(r.blockSizes) == 0 {
return io.EOF
}
if len(r.blockSizes) == 1 && r.fragRdr != nil {
r.cur = r.fragRdr
} else {
size := realSize(r.blockSizes[0])
if size == 0 {
outSize := r.blockSize
if r.fileSize < uint64(r.blockSize) {
outSize = uint32(r.fileSize)
}
r.cur = bytes.NewReader(make([]byte, outSize))
} else {
r.cur = io.LimitReader(r.master, int64(size))
if size == r.blockSizes[0] {
if rs, ok := r.d.(decompress.Resetable); ok {
if r.comRdr == nil {
r.cur, err = r.d.Reader(r.cur)
if err != nil {
return
}
} else {
err = rs.Reset(r.comRdr, r.cur)
r.cur = r.comRdr
}
} else {
r.cur, err = r.d.Reader(r.cur)
}
}
}
}
r.blockSizes = r.blockSizes[1:]
return
}
func (r *Reader) Read(p []byte) (n int, err error) {
if r.cur == nil {
err = r.advance()
if err != nil {
return
}
}
n, err = r.cur.Read(p)
if err == io.EOF {
err = r.advance()
if err != nil {
return
}
var tmpN int
tmp := make([]byte, len(p)-n)
tmpN, err = r.Read(tmp)
for i := range tmp {
p[n+i] = tmp[i]
}
n += tmpN
}
return
}
-17
View File
@@ -1,17 +0,0 @@
package decompress
import (
"io"
"github.com/klauspost/compress/zlib"
)
type GZip struct{}
func (g GZip) Reader(src io.Reader) (io.ReadCloser, error) {
return zlib.NewReader(src)
}
func (g GZip) Reset(old, src io.Reader) error {
return old.(zlib.Resetter).Reset(src, nil)
}
-22
View File
@@ -1,22 +0,0 @@
package decompress
import (
"io"
)
type Decompressor interface {
//Creates a new decompressor reading from src.
Reader(src io.Reader) (io.ReadCloser, error)
}
type Resetable interface {
//Reset attempts to re-use an old decompressor with new data.
//Will return ErrNotResetable if not Resetable().
//Must ALWAYS be provided with a reader created with Reader.
Reset(old, src io.Reader) error
}
type Decoder interface {
//Decodes a chunk of data all at once.
Decode(in []byte) ([]byte, error)
}
-18
View File
@@ -1,18 +0,0 @@
package decompress
import (
"io"
"github.com/pierrec/lz4/v4"
)
type Lz4 struct{}
func (l Lz4) Reader(r io.Reader) (io.ReadCloser, error) {
return io.NopCloser(lz4.NewReader(r)), nil
}
func (l Lz4) Reset(old, src io.Reader) error {
old.(*lz4.Reader).Reset(src)
return nil
}
-14
View File
@@ -1,14 +0,0 @@
package decompress
import (
"io"
"github.com/ulikunitz/xz/lzma"
)
type Lzma struct{}
func (l Lzma) Reader(r io.Reader) (io.ReadCloser, error) {
rdr, err := lzma.NewReader(r)
return io.NopCloser(rdr), err
}
-18
View File
@@ -1,18 +0,0 @@
package decompress
import (
"bytes"
"io"
"github.com/rasky/go-lzo"
)
type Lzo struct{}
func (l Lzo) Reader(r io.Reader) (io.ReadCloser, error) {
cache, err := lzo.Decompress1X(r, 0, 0)
if err != nil {
return nil, err
}
return io.NopCloser(bytes.NewReader(cache)), nil
}
-18
View File
@@ -1,18 +0,0 @@
package decompress
import (
"io"
"github.com/therootcompany/xz"
)
type Xz struct{}
func (x Xz) Reader(r io.Reader) (io.ReadCloser, error) {
rdr, err := xz.NewReader(r, 0)
return io.NopCloser(rdr), err
}
func (x Xz) Reset(old, src io.Reader) error {
return old.(*xz.Reader).Reset(src)
}
-27
View File
@@ -1,27 +0,0 @@
package decompress
import (
"io"
"github.com/klauspost/compress/zstd"
)
type Zstd struct {
writeToReader *zstd.Decoder
}
func (z Zstd) Reader(src io.Reader) (io.ReadCloser, error) {
r, err := zstd.NewReader(src)
return r.IOReadCloser(), err
}
func (z Zstd) Reset(old, src io.Reader) error {
return old.(*zstd.Decoder).Reset(src)
}
func (z Zstd) Decode(in []byte) ([]byte, error) {
if z.writeToReader == nil {
z.writeToReader, _ = zstd.NewReader(nil)
}
return z.writeToReader.DecodeAll(in, nil)
}
-80
View File
@@ -1,80 +0,0 @@
package directory
import (
"bytes"
"encoding/binary"
"io"
)
type header struct {
Entries uint32
InodeStart uint32
Num uint32
}
type entryInit struct {
Offset uint16
NumOffset int16
Type uint16
NameSize uint16
}
type entry struct {
entryInit
Name []byte
}
type Entry struct {
Name string
BlockStart uint32
Type uint16
Offset uint16
Num uint32
}
func readEntry(r io.Reader) (e entry, err error) {
err = binary.Read(r, binary.LittleEndian, &e.entryInit)
if err != nil {
return
}
e.Name = make([]byte, e.NameSize+1)
err = binary.Read(r, binary.LittleEndian, &e.Name)
return
}
func ReadEntries(rdr io.Reader, size uint32) (e []Entry, err error) {
dat := make([]byte, size-3)
rdr.Read(dat)
r := bytes.NewReader(dat)
var h header
var en entry
for {
err = binary.Read(r, binary.LittleEndian, &h)
if err == io.EOF {
err = nil
return
} else if err != nil {
return
}
h.Entries++
for i := 0; i < int(h.Entries); i++ {
if i != 0 && i%256 == 0 {
err = binary.Read(r, binary.LittleEndian, &h)
if err != nil {
return
}
}
en, err = readEntry(r)
if err != nil {
return
}
e = append(e, Entry{
Name: string(en.Name),
BlockStart: h.InodeStart,
Type: en.Type,
Offset: en.Offset,
Num: h.Num + uint32(en.NumOffset),
})
}
}
}
-65
View File
@@ -1,65 +0,0 @@
package inode
import (
"encoding/binary"
"io"
)
type Directory struct {
BlockStart uint32
LinkCount uint32
Size uint16
Offset uint16
ParentNum uint32
}
type eDirectoryInit struct {
LinkCount uint32
Size uint32
BlockStart uint32
ParentNum uint32
IndCount uint16
Offset uint16
XattrInd uint32
}
type EDirectory struct {
eDirectoryInit
Indexes []DirectoryIndex
}
type directoryIndexInit struct {
Ind uint32
Start uint32
NameSize uint32
}
type DirectoryIndex struct {
directoryIndexInit
Name []byte
}
func ReadDir(r io.Reader) (d Directory, err error) {
err = binary.Read(r, binary.LittleEndian, &d)
return
}
func ReadEDir(r io.Reader) (d EDirectory, err error) {
err = binary.Read(r, binary.LittleEndian, &d.eDirectoryInit)
if err != nil {
return
}
d.Indexes = make([]DirectoryIndex, d.IndCount)
for i := range d.Indexes {
err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].directoryIndexInit)
if err != nil {
return
}
d.Indexes[i].Name = make([]byte, d.Indexes[i].NameSize+1)
err = binary.Read(r, binary.LittleEndian, &d.Indexes[i].Name)
if err != nil {
return
}
}
return
}
-62
View File
@@ -1,62 +0,0 @@
package inode
import (
"encoding/binary"
"io"
"math"
)
type fileInit struct {
BlockStart uint32
FragInd uint32
FragOffset uint32
Size uint32
}
type File struct {
fileInit
BlockSizes []uint32
}
type eFileInit struct {
BlockStart uint64
Size uint64
Sparse uint64
LinkCount uint32
FragInd uint32
FragOffset uint32
XattrInd uint32
}
type EFile struct {
eFileInit
BlockSizes []uint32
}
func ReadFile(r io.Reader, blockSize uint32) (f File, err error) {
err = binary.Read(r, binary.LittleEndian, &f.fileInit)
if err != nil {
return
}
toRead := int(math.Floor(float64(f.Size) / float64(blockSize)))
if f.FragInd == 0xFFFFFFFF && f.Size%blockSize > 0 {
toRead++
}
f.BlockSizes = make([]uint32, toRead)
err = binary.Read(r, binary.LittleEndian, &f.BlockSizes)
return
}
func ReadEFile(r io.Reader, blockSize uint32) (f EFile, err error) {
err = binary.Read(r, binary.LittleEndian, &f.eFileInit)
if err != nil {
return
}
toRead := int(math.Floor(float64(f.Size) / float64(blockSize)))
if f.FragInd == 0xFFFFFFFF && f.Size%uint64(blockSize) > 0 {
toRead++
}
f.BlockSizes = make([]uint32, toRead)
err = binary.Read(r, binary.LittleEndian, &f.BlockSizes)
return
}
-143
View File
@@ -1,143 +0,0 @@
package inode
import (
"encoding/binary"
"errors"
"io"
"io/fs"
"strconv"
)
const (
Dir = uint16(iota + 1)
Fil
Sym
Block
Char
Fifo
Sock
EDir
EFil
ESym
EBlock
EChar
EFifo
ESock
)
type Header struct {
Type uint16
Perm uint16
UidInd uint16
GidInd uint16
ModTime uint32
Num uint32
}
type Inode struct {
Header
Data any
}
func Read(r io.Reader, blockSize uint32) (i Inode, err error) {
err = binary.Read(r, binary.LittleEndian, &i.Header)
if err != nil {
return
}
switch i.Type {
case Dir:
i.Data, err = ReadDir(r)
case Fil:
i.Data, err = ReadFile(r, blockSize)
case Sym:
i.Data, err = ReadSym(r)
case Block:
fallthrough
case Char:
i.Data, err = ReadDevice(r)
case Fifo:
fallthrough
case Sock:
i.Data, err = ReadIPC(r)
case EDir:
i.Data, err = ReadEDir(r)
case EFil:
i.Data, err = ReadEFile(r, blockSize)
case ESym:
i.Data, err = ReadESym(r)
case EBlock:
fallthrough
case EChar:
i.Data, err = ReadEDevice(r)
case EFifo:
fallthrough
case ESock:
i.Data, err = ReadEIPC(r)
default:
return i, errors.New("invalid inode type " + strconv.Itoa(int(i.Type)))
}
return
}
func (i Inode) Mode() (out fs.FileMode) {
out = fs.FileMode(i.Perm)
switch i.Data.(type) {
case Directory:
out |= fs.ModeDir
case EDirectory:
out |= fs.ModeDir
case Symlink:
out |= fs.ModeSymlink
case ESymlink:
out |= fs.ModeSymlink
case Device:
out |= fs.ModeDevice
case EDevice:
out |= fs.ModeDevice
case IPC:
out |= fs.ModeNamedPipe
case EIPC:
out |= fs.ModeNamedPipe
}
return
}
func (i Inode) LinkCount() uint32 {
switch i.Data.(type) {
case EFile:
return i.Data.(EFile).LinkCount
case Directory:
return i.Data.(Directory).LinkCount
case EDirectory:
return i.Data.(EDirectory).LinkCount
case Device:
return i.Data.(Device).LinkCount
case EDevice:
return i.Data.(EDevice).LinkCount
case IPC:
return i.Data.(IPC).LinkCount
case EIPC:
return i.Data.(EIPC).LinkCount
case Symlink:
return i.Data.(Symlink).LinkCount
case ESymlink:
return i.Data.(ESymlink).LinkCount
default:
return 0
}
}
func (i Inode) Size() uint64 {
switch i.Data.(type) {
case File:
return uint64(i.Data.(File).Size)
case EFile:
return i.Data.(EFile).Size
// case Directory:
// return uint64(i.Data.(Directory).Size)
// case EDirectory:
// return uint64(i.Data.(EDirectory).Size)
default:
return 0
}
}
-45
View File
@@ -1,45 +0,0 @@
package inode
import (
"encoding/binary"
"io"
)
type Device struct {
LinkCount uint32
Dev uint32
}
type EDevice struct {
Device
XattrInd uint32
}
func ReadDevice(r io.Reader) (d Device, err error) {
err = binary.Read(r, binary.LittleEndian, &d)
return
}
func ReadEDevice(r io.Reader) (d EDevice, err error) {
err = binary.Read(r, binary.LittleEndian, &d)
return
}
type IPC struct {
LinkCount uint32
}
type EIPC struct {
IPC
XattrInd uint32
}
func ReadIPC(r io.Reader) (i IPC, err error) {
err = binary.Read(r, binary.LittleEndian, &i)
return
}
func ReadEIPC(r io.Reader) (i EIPC, err error) {
err = binary.Read(r, binary.LittleEndian, &i)
return
}
-46
View File
@@ -1,46 +0,0 @@
package inode
import (
"encoding/binary"
"io"
)
type symlinkInit struct {
LinkCount uint32
TargetSize uint32
}
type Symlink struct {
symlinkInit
Target []byte
}
type ESymlink struct {
symlinkInit
Target []byte
XattrInd uint32
}
func ReadSym(r io.Reader) (s Symlink, err error) {
err = binary.Read(r, binary.LittleEndian, &s.symlinkInit)
if err != nil {
return
}
s.Target = make([]byte, s.TargetSize)
err = binary.Read(r, binary.LittleEndian, &s.Target)
return
}
func ReadESym(r io.Reader) (s ESymlink, err error) {
err = binary.Read(r, binary.LittleEndian, &s.symlinkInit)
if err != nil {
return
}
s.Target = make([]byte, s.TargetSize)
err = binary.Read(r, binary.LittleEndian, &s.Target)
if err != nil {
return
}
err = binary.Read(r, binary.LittleEndian, &s.XattrInd)
return
}
-81
View File
@@ -1,81 +0,0 @@
package metadata
import (
"encoding/binary"
"io"
"github.com/CalebQ42/squashfs/internal/decompress"
)
type Reader struct {
master io.Reader
cur io.Reader
d decompress.Decompressor
comRdr io.Reader
}
func NewReader(master io.Reader, d decompress.Decompressor) *Reader {
return &Reader{
master: master,
d: d,
}
}
func realSize(siz uint16) uint16 {
return siz &^ 0x8000
}
func (r *Reader) advance() (err error) {
if _, ok := r.d.(decompress.Resetable); !ok {
if clr, ok := r.cur.(io.Closer); ok {
clr.Close()
}
}
var raw uint16
err = binary.Read(r.master, binary.LittleEndian, &raw)
if err != nil {
return
}
size := realSize(raw)
r.cur = io.LimitReader(r.master, int64(size))
if size == raw {
if rs, ok := r.d.(decompress.Resetable); ok {
if r.comRdr == nil {
r.cur, err = r.d.Reader(r.cur)
if err != nil {
return
}
} else {
err = rs.Reset(r.comRdr, r.cur)
r.cur = r.comRdr
}
} else {
r.cur, err = r.d.Reader(r.cur)
}
}
return
}
func (r *Reader) Read(p []byte) (n int, err error) {
if r.cur == nil {
err = r.advance()
if err != nil {
return
}
}
n, err = r.cur.Read(p)
if err == io.EOF {
err = r.advance()
if err != nil {
return
}
var tmpN int
tmp := make([]byte, len(p)-n)
tmpN, err = r.Read(tmp)
for i := 0; i < tmpN; i++ {
p[n+i] = tmp[i]
}
n += tmpN
}
return
}
-23
View File
@@ -1,23 +0,0 @@
package threadmanager
type Manager struct {
c chan int
}
func NewManager(maxRoutines int) *Manager {
m := &Manager{
c: make(chan int, maxRoutines),
}
for i := 0; i < maxRoutines; i++ {
m.c <- i
}
return m
}
func (m *Manager) Lock() int {
return <-m.c
}
func (m *Manager) Unlock(n int) {
m.c <- n
}
-19
View File
@@ -1,19 +0,0 @@
package toreader
import "io"
type OffsetReader struct {
r io.ReaderAt
off int64
}
func NewOffsetReader(r io.ReaderAt, off int64) *OffsetReader {
return &OffsetReader{
r: r,
off: off,
}
}
func (r OffsetReader) ReadAt(p []byte, off int64) (n int, e error) {
return r.r.ReadAt(p, off+r.off)
}
-25
View File
@@ -1,25 +0,0 @@
package toreader
import "io"
type Reader struct {
r io.ReaderAt
off int64
}
func NewReader(r io.ReaderAt, start int64) *Reader {
return &Reader{
r: r,
off: start,
}
}
func (r *Reader) Read(p []byte) (n int, err error) {
n, err = r.r.ReadAt(p, r.off)
r.off += int64(n)
return
}
func (r Reader) Offset() int64 {
return r.off
}
-24
View File
@@ -1,24 +0,0 @@
package toreader
import "io"
type ReaderAt struct {
d []byte
}
func NewReaderAt(r io.Reader) (ra *ReaderAt, err error) {
ra = new(ReaderAt)
ra.d, err = io.ReadAll(r)
return
}
func (r ReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
if int(off) >= len(r.d) {
return 0, io.EOF
}
n = copy(p, r.d[off:])
if n != len(p) {
err = io.EOF
}
return
}