First version of File interface.

This will allow you to easily find and extract files.
Extraction of whole folders coming next. (Maybe)

README.md

@@ -21,9 +21,9 @@ Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree
 
 # Not Working (Yet). Not necessarily in order.
 
-* Rename repo so it's easier to import
 * Provide an easy interface to find and list files and their properties
     * Maybe squashfs.File
+* Make device, socket, symlink, and all extended types of inode work properly. (I need to find an archive that uses it first.)
 * Extracting files
     * from inodes.
     * from file info.
@@ -41,4 +41,4 @@ Thanks also to [distri's squashfs library](https://github.com/distr1/distri/tree
 
 # Where I'm at
 
-* v0.1 is the first working version!
+* Working on the File interface that should make it easier to deal with squashfs files. I'm also trying to make them capable for when I get squashing working.

compression.go

@@ -11,6 +11,10 @@ type decompressor interface {
 	Decompress(io.Reader) ([]byte, error)
 }
 
+type compressor interface {
+	Compress(io.Reader) ([]byte, error)
+}
+
 //ZlibDecompressor is a decompressor for gzip type compression
 type zlibDecompressor struct{}
 

datareader.go

@@ -10,9 +10,9 @@ import (
 
 var (
 	//ErrInodeNotFile is given when giving an inode, but the function requires a file inode.
-	ErrInodeNotFile = errors.New("Given inode is NOT a file type")
+	errInodeNotFile = errors.New("Given inode is NOT a file type")
 	//ErrInodeOnlyFragment is given when trying to make a DataReader from an inode, but the inode only had data in a fragment
-	ErrInodeOnlyFragment = errors.New("Given inode ONLY has fragment data")
+	errInodeOnlyFragment = errors.New("Given inode ONLY has fragment data")
 )
 
 //DataReader reads data from data blocks.
@@ -66,7 +66,7 @@ func (r *Reader) newDataReaderFromInode(i *inode.Inode) (*dataReader, error) {
 	case inode.BasicFileType:
 		fil := i.Info.(inode.BasicFile)
 		if fil.Init.BlockStart == 0 {
-			return nil, ErrInodeOnlyFragment
+			return nil, errInodeOnlyFragment
 		}
 		rdr.offset = int64(fil.Init.BlockStart)
 		for _, sizes := range fil.BlockSizes {
@@ -78,7 +78,7 @@ func (r *Reader) newDataReaderFromInode(i *inode.Inode) (*dataReader, error) {
 	case inode.ExtFileType:
 		fil := i.Info.(inode.ExtendedFile)
 		if fil.Init.BlockStart == 0 {
-			return nil, ErrInodeOnlyFragment
+			return nil, errInodeOnlyFragment
 		}
 		rdr.offset = int64(fil.Init.BlockStart)
 		for _, sizes := range fil.BlockSizes {
@@ -88,7 +88,7 @@ func (r *Reader) newDataReaderFromInode(i *inode.Inode) (*dataReader, error) {
 			rdr.blocks = rdr.blocks[:len(rdr.blocks)-1]
 		}
 	default:
-		return nil, ErrInodeNotFile
+		return nil, errInodeNotFile
 	}
 	err := rdr.readCurBlock()
 	if err != nil {
@@ -145,7 +145,16 @@ func (d *dataReader) readCurBlock() error {
 	return err
 }
 
+//Close frees up the curData from memory
+func (d *dataReader) Close() error {
+	d.curData = nil
+	return nil
+}
+
 func (d *dataReader) Read(p []byte) (int, error) {
+	if d.curData == nil {
+		d.readCurBlock()
+	}
 	if d.curReadOffset+len(p) < len(d.curData) {
 		for i := 0; i < len(p); i++ {
 			p[i] = d.curData[d.curReadOffset+i]

file.go

@@ -0,0 +1,136 @@
+package squashfs
+
+import (
+	"errors"
+	"io"
+
+	"github.com/CalebQ42/squashfs/internal/directory"
+	"github.com/CalebQ42/squashfs/internal/inode"
+)
+
+var (
+	//ErrNotDirectory is returned when you're trying to do directory things with a non-directory
+	ErrNotDirectory = errors.New("File is not a directory")
+	//ErrNotFile is returned when you're trying to do file things with a directory
+	ErrNotFile = errors.New("File is not a file")
+	//ErrNotReading is returned when running functions that are only meant to be used when reading a squashfs
+	ErrNotReading = errors.New("Function only supported when reading a squashfs")
+)
+
+//File is the main way to interact with files within squashfs, or when putting files into a squashfs.
+//File can be either a file or folder. When reading from a squashfs, it reads from the datablocks.
+//When writing, this holds the information on WHERE the file will be placed inside the archive.
+type File struct {
+	Name    string       //The name of the file or folder. Root folder will not have a name ("")
+	Parent  *File        //The parent directory. If it's the root directory, will be nil
+	Reader  io.Reader    //Underlying reader. When writing, will probably be an os.File. When reading this is kept nil UNTIL reading to save memory.
+	Path    string       //The path to the folder the File is located in.
+	r       *Reader      //The squashfs.Reader where this file is contained.
+	in      *inode.Inode //Underlyting inode when reading.
+	filType int          //The file's type, using inode types.
+}
+
+//get a File from a directory.entry
+func (r *Reader) newFileFromDirEntry(entry *directory.Entry) (fil *File, err error) {
+	fil = new(File)
+	fil.in, err = r.getInodeFromEntry(entry)
+	if err != nil {
+		return nil, err
+	}
+	fil.Name = entry.Name
+	fil.r = r
+	fil.filType = fil.in.Type
+	return
+}
+
+//GetChildren returns a *squashfs.File slice of every direct child of the directory. If the File is not a directory, will return ErrNotDirectory
+func (f *File) GetChildren() (children []*File, err error) {
+	children = make([]*File, 0)
+	if f.r == nil {
+		return nil, ErrNotReading
+	}
+	if !f.IsDir() {
+		return nil, ErrNotDirectory
+	}
+	dir, err := f.r.readDirFromInode(f.in)
+	if err != nil {
+		return
+	}
+	var fil *File
+	for _, entry := range dir.Entries {
+		fil, err = f.r.newFileFromDirEntry(&entry)
+		if err != nil {
+			return
+		}
+		fil.Parent = f
+		if f.Name != "" {
+			fil.Path = f.Path + "/" + f.Name
+		}
+		children = append(children, fil)
+	}
+	return
+}
+
+//GetChildrenRecursively returns ALL children. Goes down ALL folder paths.
+func (f *File) GetChildrenRecursively() (children []*File, err error) {
+	children = make([]*File, 0)
+	if f.r == nil {
+		return nil, ErrNotReading
+	}
+	if !f.IsDir() {
+		return nil, ErrNotDirectory
+	}
+	chil, err := f.GetChildren()
+	if err != nil {
+		return
+	}
+	var childFolders []*File
+	for _, child := range chil {
+		children = append(children, child)
+		if child.IsDir() {
+			childFolders = append(childFolders, child)
+		}
+	}
+	for _, folds := range childFolders {
+		var childs []*File
+		childs, err = folds.GetChildrenRecursively()
+		if err != nil {
+			return
+		}
+		children = append(children, childs...)
+	}
+	return
+}
+
+//IsDir returns if the file is a directory.
+func (f *File) IsDir() bool {
+	return f.filType == inode.BasicDirectoryType || f.filType == inode.ExtDirType
+}
+
+//Close frees up the memory held up by the underlying reader. Should NOT be called when writing.
+//When reading, Close is safe to use, but any subsequent Read calls resets to the beginning of the file.
+func (f *File) Close() error {
+	if f.IsDir() {
+		return ErrNotFile
+	}
+	if closer, is := f.Reader.(io.Closer); is {
+		closer.Close()
+	}
+	f.Reader = nil
+	return nil
+}
+
+//Read from the file. Doesn't do anything fancy, just pases it to the underlying io.Reader. If a directory, return io.EOF.
+func (f *File) Read(p []byte) (int, error) {
+	if f.IsDir() {
+		return 0, io.EOF
+	}
+	var err error
+	if f.Reader == nil && f.r != nil {
+		f.Reader, err = f.r.newFileReader(f.in)
+		if err != nil {
+			return 0, err
+		}
+	}
+	return f.Reader.Read(p)
+}

filereader.go

@@ -9,9 +9,10 @@ import (
 )
 
 //FileReader provides a io.Reader interface for files within a squashfs archive
-type FileReader struct {
+type fileReader struct {
 	r            *Reader
 	data         *dataReader
+	in           *inode.Inode
 	fragmentData []byte
 	fragged      bool
 	fragOnly     bool
@@ -25,13 +26,9 @@ var (
 )
 
 //ReadFile provides a squashfs.FileReader for the file at the given location.
-func (r *Reader) ReadFile(location string) (*FileReader, error) {
-	var rdr FileReader
-	rdr.r = r
-	in, err := r.getInodeFromPath(location)
-	if err != nil {
-		return nil, err
-	}
+func (r *Reader) newFileReader(in *inode.Inode) (*fileReader, error) {
+	var rdr fileReader
+	rdr.in = in
 	if in.Type != inode.BasicFileType && in.Type != inode.ExtFileType {
 		return nil, ErrPathIsNotFile
 	}
@@ -47,6 +44,7 @@ func (r *Reader) ReadFile(location string) (*FileReader, error) {
 		rdr.fragOnly = fil.Init.BlockStart == 0
 		rdr.FileSize = int(fil.Init.Size)
 	}
+	var err error
 	if rdr.fragged {
 		rdr.fragmentData, err = r.getFragmentDataFromInode(in)
 		if err != nil {
@@ -59,7 +57,16 @@ func (r *Reader) ReadFile(location string) (*FileReader, error) {
 	return &rdr, nil
 }
 
-func (f *FileReader) Read(p []byte) (int, error) {
+//Close runs Close on the data reader and frees the fragmentdata
+func (f *fileReader) Close() error {
+	if f.data != nil {
+		f.data.Close()
+	}
+	f.fragmentData = nil
+	return nil
+}
+
+func (f *fileReader) Read(p []byte) (int, error) {
 	if f.fragOnly {
 		n, err := bytes.NewBuffer(f.fragmentData[f.read:]).Read(p)
 		f.read += n
@@ -72,6 +79,9 @@ func (f *FileReader) Read(p []byte) (int, error) {
 	n, err := f.data.Read(p)
 	read += n
 	if f.fragged && err == io.EOF {
+		if f.fragmentData == nil {
+			f.fragmentData, err = f.r.getFragmentDataFromInode(f.in)
+		}
 		n, err = bytes.NewBuffer(f.fragmentData).Read(p[read:])
 		read += n
 		if err != nil {

reader.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 	"math"
+	"strings"
 
 	"github.com/CalebQ42/squashfs/internal/inode"
 )
@@ -22,7 +23,7 @@ var (
 	ErrCompressorOptions = errors.New("Compressor options is not currently supported")
 	//ErrFragmentTableIssues is returned if there's trouble reading the fragment table when creating a reader.
 	//When this is returned, the reader is still returned.
-	ErrFragmentTableIssues = errors.New("Trouble while reading the fragment table")
+	errFragmentTableIssues = errors.New("Trouble while reading the fragment table")
 )
 
 //Reader processes and reads a squashfs archive.
@@ -73,50 +74,120 @@ func NewSquashfsReader(r io.ReaderAt) (*Reader, error) {
 	return &rdr, nil
 }
 
-//GetFilesList returns a list of ALL files in the squashfs, going down every folder.
-//Folders end in /
-func (r *Reader) GetFilesList() ([]string, error) {
-	inoderdr, err := r.newMetadataReaderFromInodeRef(r.super.RootInodeRef)
+//GetRootFolder returns a squashfs.File that references the root directory of the squashfs archive.
+func (r *Reader) GetRootFolder() (root *File, err error) {
+	root = new(File)
+	mr, err := r.newMetadataReaderFromInodeRef(r.super.RootInodeRef)
 	if err != nil {
 		return nil, err
 	}
-	i, err := inode.ProcessInode(inoderdr, r.super.BlockSize)
+	root.in, err = inode.ProcessInode(mr, r.super.BlockSize)
 	if err != nil {
 		return nil, err
 	}
-	paths, err := r.readDir(i)
-	if err != nil {
-		return nil, err
-	}
-	return paths, nil
+	root.Path = "/"
+	root.filType = root.in.Type
+	root.r = r
+	return root, nil
 }
 
-//readDir returns a list of all decendents of a given inode. Inode given MUST be a directory type.
-func (r *Reader) readDir(i *inode.Inode) (paths []string, err error) {
-	dir, err := r.readDirFromInode(i)
+//GetAllFiles returns a slice of ALL files and folders contained in the squashfs.
+func (r *Reader) GetAllFiles() (fils []*File, err error) {
+	root, err := r.GetRootFolder()
 	if err != nil {
-		return
+		return nil, err
 	}
-	for _, entry := range dir.Entries {
-		if entry.Init.Type == inode.BasicDirectoryType {
-			paths = append(paths)
-			i, err = r.getInodeFromEntry(&entry)
+	return root.GetChildrenRecursively()
+}
+
+//FindFile returns the first file (in the same order as Reader.GetAllFiles) that the given function returns true for. Returns nil if nothing is found.
+func (r *Reader) FindFile(query func(*File) bool) *File {
+	root, err := r.GetRootFolder()
+	if err != nil {
+		return nil
+	}
+	fils, err := root.GetChildren()
+	if err != nil {
+		return nil
+	}
+	var childrenDirs []*File
+	for _, fil := range fils {
+		if query(fil) {
+			return fil
+		}
+		if fil.IsDir() {
+			childrenDirs = append(childrenDirs, fil)
+		}
+	}
+	for len(childrenDirs) != 0 {
+		var tmp []*File
+		for _, dirs := range childrenDirs {
+			chil, err := dirs.GetChildren()
 			if err != nil {
-				return
+				return nil
 			}
-			var subPaths []string
-			subPaths, err = r.readDir(i)
-			if err != nil {
-				return
+			for _, child := range chil {
+				if query(child) {
+					return child
+				}
+				if child.IsDir() {
+					tmp = append(tmp, child)
+				}
 			}
-			for pathI := range subPaths {
-				subPaths[pathI] = entry.Name + "/" + subPaths[pathI]
-			}
-			paths = append(paths, entry.Name+"/")
-			paths = append(paths, subPaths...)
-		} else {
-			paths = append(paths, entry.Name)
+		}
+		childrenDirs = tmp
+	}
+	return nil
+}
+
+//FindAll returns all files where the given function returns true.
+func (r *Reader) FindAll(query func(*File) bool) (all []*File) {
+	root, err := r.GetRootFolder()
+	if err != nil {
+		return nil
+	}
+	fils, err := root.GetChildrenRecursively()
+	if err != nil {
+		return nil
+	}
+	for _, fil := range fils {
+		if query(fil) {
+			all = append(all, fil)
 		}
 	}
 	return
 }
+
+//GetFileAtPath will return the file at the given path. If the file cannot be found, will return nil.
+func (r *Reader) GetFileAtPath(path string) *File {
+	path = strings.TrimSuffix(strings.TrimPrefix(path, "/"), "/")
+	pathDirs := strings.Split(path, "/")
+	dir, err := r.GetRootFolder()
+	if err != nil {
+		return nil
+	}
+	children, err := dir.GetChildren()
+	if err != nil {
+		return nil
+	}
+	for _, folder := range pathDirs {
+		for _, child := range children {
+			if child.Name == folder {
+				dir = child
+				if dir.IsDir() {
+					children, err = dir.GetChildren()
+					if err != nil {
+						return nil
+					}
+				} else {
+					children = make([]*File, 0)
+				}
+				break
+			}
+		}
+	}
+	if dir.Path+"/"+dir.Name == "/"+path {
+		return dir
+	}
+	return nil
+}

squash_test.go

@@ -1,9 +1,11 @@
 package squashfs
 
 import (
+	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"strings"
 	"testing"
 
 	goappimage "github.com/CalebQ42/GoAppImage"
@@ -12,7 +14,6 @@ import (
 const (
 	downloadURL  = "https://github.com/zilti/code-oss.AppImage/releases/download/continuous/Code_OSS-x86_64.AppImage"
 	appImageName = "Code_OSS.AppImage"
-	squashfsName = "Code_OSS.Squashfs"
 )
 
 func TestMain(t *testing.T) {
@@ -21,78 +22,54 @@ func TestMain(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	squashFil, err := os.Open(wd + "/testing/" + squashfsName)
-	if os.IsNotExist(err) {
-		TestCreateSquashFromAppImage(t)
-		squashFil, err = os.Open(wd + "/testing/" + squashfsName)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-	defer squashFil.Close()
-	stat, _ := squashFil.Stat()
-	rdr, err := NewSquashfsReader(io.NewSectionReader(squashFil, 0, stat.Size()))
-	if err != nil {
-		t.Fatal(err)
-	}
-	//testing code to print out the directory structure
-	// rdr.GetFileStructure()
-	extractionFil := "code-oss.desktop"
-	os.Remove(wd + "/testing/" + extractionFil)
-	desk, err := os.Create(wd + "/testing/" + extractionFil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ext, err := rdr.ReadFile(extractionFil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, err = io.Copy(desk, ext)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Fatal("No problems here!")
-}
-
-func TestCreateSquashFromAppImage(t *testing.T) {
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = os.Mkdir(wd+"/testing", 0777)
-	if err != nil && !os.IsExist(err) {
-		t.Fatal(err)
-	}
-	_, err = os.Open(wd + "/testing/" + appImageName)
+	aiFil, err := os.Open(wd + "/testing/" + appImageName)
 	if os.IsNotExist(err) {
 		downloadTestAppImage(t, wd+"/testing")
-		_, err = os.Open(wd + "/testing/" + appImageName)
+		aiFil, err = os.Open(wd + "/testing/" + appImageName)
 		if err != nil {
 			t.Fatal(err)
 		}
 	} else if err != nil {
 		t.Fatal(err)
 	}
-	ai := goappimage.NewAppImage(wd + "/testing/" + appImageName)
-	aiFil, err := os.Open(wd + "/testing/" + appImageName)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer aiFil.Close()
-	aiFil.Seek(ai.Offset, 0)
-	os.Remove(wd + "/testing/" + squashfsName)
-	aiSquash, err := os.Create(wd + "/testing/" + squashfsName)
+	stat, _ := aiFil.Stat()
+	ai := goappimage.NewAppImage(wd + "/testing/" + appImageName)
+	rdr, err := NewSquashfsReader(io.NewSectionReader(aiFil, ai.Offset, stat.Size()-ai.Offset))
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = io.Copy(aiSquash, aiFil)
+	rdr.FindAll(func(fil *File) bool {
+		return strings.HasSuffix(fil.Name, ".desktop")
+	})
+	fils, err := rdr.GetAllFiles()
 	if err != nil {
 		t.Fatal(err)
 	}
+	for _, fil := range fils {
+		fmt.Println(fil.Path + "/" + fil.Name)
+	}
+	// extractionFil := "code-oss.desktop"
+	// os.Remove(wd + "/testing/" + extractionFil)
+	// desk, err := os.Create(wd + "/testing/" + extractionFil)
+	// if err != nil {
+	// 	t.Fatal(err)
+	// }
+	// ext := rdr.GetFileAtPath(extractionFil)
+	// if ext == nil {
+	// 	t.Fatal("Cannot find file")
+	// }
+	// defer ext.Close()
+	// _, err = io.Copy(desk, ext)
+	// if err != nil {
+	// 	t.Fatal(err)
+	// }
+	t.Fatal("No problems here!")
 }
 
 func downloadTestAppImage(t *testing.T, dir string) {
-	//seems to time out. Need to fix that at some point
+	//seems to time out on slow connections. Might fix that at some point... or not
+	os.Mkdir(dir, 0777)
 	appImage, err := os.Create(dir + "/" + appImageName)
 	if err != nil {
 		t.Fatal(err)