Testing better large file support

reader_file.go

@@ -219,99 +219,107 @@ func (f File) GetSymlinkFile() *File {
 
 // ExtractionOptions are available options on how to extract.
 type ExtractionOptions struct {
-	LogOutput          io.Writer   //Where error log should write. If nil, uses os.Stdout. Has no effect if verbose is false.
+	LogOutput          io.Writer   //Where error log should write.
 	DereferenceSymlink bool        //Replace symlinks with the target file.
 	UnbreakSymlink     bool        //Try to make sure symlinks remain unbroken when extracted, without changing the symlink.
 	Verbose            bool        //Prints extra info to log on an error.
-	IgnorePerm         bool        //ignore the file's permission and instead use FolderPerm.
+	IgnorePerm         bool        //Ignore file's permissions and instead use FolderPerm.
-	FolderPerm         fs.FileMode //The permissions used when creating the extraction folder. Defaults to 0755.
+	FolderPerm         fs.FileMode //Permission to use when making the root folder if it doesn't exist.
-}
+	notFirst           bool
-
-// DefaultOptions is the default ExtractionOptions.
-func DefaultOptions() ExtractionOptions {
-	return ExtractionOptions{
-		FolderPerm: 0755,
-	}
 }
 
 // ExtractTo extracts the File to the given folder with the default options.
 // If the File is a directory, it instead extracts the directory's contents to the folder.
 func (f File) ExtractTo(folder string) error {
-	return f.ExtractWithOptions(folder, DefaultOptions())
+	return f.realExtract(folder, &ExtractionOptions{})
 }
 
 // ExtractSymlink extracts the File to the folder with the DereferenceSymlink option.
 // If the File is a directory, it instead extracts the directory's contents to the folder.
 func (f File) ExtractSymlink(folder string) error {
-	op := DefaultOptions()
+	return f.realExtract(folder, &ExtractionOptions{})
-	op.DereferenceSymlink = true
-	return f.ExtractWithOptions(folder, op)
 }
 
 // ExtractWithOptions extracts the File to the given folder with the given ExtrationOptions.
 // If the File is a directory, it instead extracts the directory's contents to the folder.
-func (f File) ExtractWithOptions(folder string, op ExtractionOptions) error {
+func (f File) ExtractWithOptions(folder string, op *ExtractionOptions) error {
-	if op.Verbose {
+	if op.Verbose && op.LogOutput != nil {
-		if op.LogOutput == nil {
-			op.LogOutput = os.Stdout
-		}
 		log.SetOutput(op.LogOutput)
 	}
 	return f.realExtract(folder, op)
 }
 
-func (f File) realExtract(folder string, op ExtractionOptions) error {
+func (f File) realExtract(folder string, op *ExtractionOptions) (err error) {
-	err := os.MkdirAll(folder, op.FolderPerm)
+	extDir := folder + "/" + f.e.Name
-	folder = filepath.Clean(folder)
+	if !op.notFirst {
-	if err != nil && !os.IsExist(err) {
+		op.notFirst = true
-		if op.Verbose {
+		if f.IsDir() {
-			log.Println("Error while creating extraction folder")
+			extDir = folder
+			_, err = os.Open(folder)
+			if err != nil && os.IsNotExist(err) {
+				if op.IgnorePerm {
+					err = os.Mkdir(extDir, op.FolderPerm|(f.Mode()&fs.ModeType))
+				} else {
+					err = os.Mkdir(extDir, f.Mode())
+				}
+			}
+			if err != nil {
+				if op.Verbose {
+					log.Println("Error while making", folder)
+				}
+				return
+			}
 		}
-		return err
 	}
 	switch {
 	case f.IsDir():
-		filFS, _ := f.FS()
+		if folder != extDir && f.e.Name != "" {
-		var ents []directory.Entry
+			if op.IgnorePerm {
-		ents, err = f.r.readDirectory(f.i)
+				err = os.Mkdir(extDir, op.FolderPerm|(f.Mode()&fs.ModeType))
+			} else {
+				err = os.Mkdir(extDir, f.Mode())
+			}
+			if err != nil {
+				if op.Verbose {
+					log.Println("Error while making directory", extDir)
+				}
+				return
+			}
+		}
+		var filFS *FS
+		filFS, err = f.FS()
 		if err != nil {
 			if op.Verbose {
-				log.Println("Error while reading children of", f.path())
+				log.Println("Error while converting", f.path(), "to FS")
 			}
 			return err
 		}
-		errChan := make(chan error)
+		errChan := make(chan error, len(filFS.e))
-		for i := 0; i < len(ents); i++ {
+		files := make([]directory.Entry, 0)
-			go func(ent directory.Entry) {
+		//Focus on making the folder tree first...
-				fil, goErr := f.r.newFile(ent, filFS)
+		var i int
-				if goErr != nil {
+		for i = 0; i < len(filFS.e); i++ {
-					if op.Verbose {
+			if filFS.e[i].Type == inode.Fil {
-						log.Println("Error while reading info for", filepath.Join(f.path(), ent.Name))
+				files = append(files, filFS.e[i])
-					}
+			} else {
-					errChan <- goErr
+				go func() {
-					return
+					errChan <- f.ExtractWithOptions(extDir, op)
-				}
+				}()
-				if fil.IsDir() {
+			}
-					perm := f.Mode()
-					if op.IgnorePerm {
-						perm = (op.FolderPerm & fs.ModePerm) | (perm & fs.ModeType)
-					}
-					err = os.Mkdir(filepath.Join(folder, fil.e.Name), perm)
-					if err != nil {
-						if op.Verbose {
-							log.Println("Error while creating", filepath.Join(folder, fil.e.Name))
-						}
-						errChan <- err
-						return
-					}
-					errChan <- fil.realExtract(filepath.Join(folder, fil.e.Name), op)
-				} else {
-					errChan <- fil.realExtract(folder, op)
-				}
-				fil.Close()
-			}(ents[i])
 		}
-		for i := 0; i < len(ents); i++ {
+		for i = 0; i < len(filFS.e)-len(files); i++ {
+			err = <-errChan
+			if err != nil {
+				return err
+			}
+		}
+		//Then we extract the files.
+		for i = 0; i < len(files); i++ {
+			go func() {
+				errChan <- f.ExtractWithOptions(extDir, op)
+			}()
+		}
+		for i = 0; i < len(files); i++ {
 			err = <-errChan
 			if err != nil {
 				return err
@@ -319,19 +327,19 @@ func (f File) realExtract(folder string, op ExtractionOptions) error {
 		}
 	case f.IsRegular():
 		var fil *os.File
-		fil, err = os.Create(folder + "/" + f.e.Name)
+		fil, err = os.Create(extDir)
 		if os.IsExist(err) {
-			os.Remove(folder + "/" + f.e.Name)
+			os.Remove(extDir)
-			fil, err = os.Create(folder + "/" + f.e.Name)
+			fil, err = os.Create(extDir)
 			if err != nil {
 				if op.Verbose {
-					log.Println("Error while creating", folder+"/"+f.e.Name)
+					log.Println("Error while creating", extDir)
 				}
 				return err
 			}
 		} else if err != nil {
 			if op.Verbose {
-				log.Println("Error while creating", folder+"/"+f.e.Name)
+				log.Println("Error while creating", extDir)
 			}
 			return err
 		}
@@ -339,12 +347,12 @@ func (f File) realExtract(folder string, op ExtractionOptions) error {
 		_, err = io.Copy(fil, f)
 		if err != nil {
 			if op.Verbose {
-				log.Println("Error while copying data to", folder+"/"+f.e.Name)
+				log.Println("Error while copying data to", extDir)
 			}
 			return err
 		}
 		if op.IgnorePerm {
-			os.Chmod(fil.Name(), op.FolderPerm)
+			os.Chmod(fil.Name(), op.FolderPerm|(f.Mode()&fs.ModeType))
 		} else {
 			os.Chmod(fil.Name(), f.Mode())
 		}
@@ -354,7 +362,7 @@ func (f File) realExtract(folder string, op ExtractionOptions) error {
 			fil := f.GetSymlinkFile()
 			if fil == nil {
 				if op.Verbose {
-					log.Println("Symlink path(", symPath, ") is unobtainable:", folder+"/"+f.e.Name)
+					log.Println("Symlink path(", symPath, ") is unobtainable:", extDir)
 				}
 				return errors.New("cannot get symlink target")
 			}
@@ -362,7 +370,7 @@ func (f File) realExtract(folder string, op ExtractionOptions) error {
 			err = fil.realExtract(folder, op)
 			if err != nil {
 				if op.Verbose {
-					log.Println("Error while extracting the symlink's file:", folder+"/"+f.e.Name)
+					log.Println("Error while extracting the symlink's file:", extDir)
 				}
 				return err
 			}
@@ -371,39 +379,39 @@ func (f File) realExtract(folder string, op ExtractionOptions) error {
 			fil := f.GetSymlinkFile()
 			if fil == nil {
 				if op.Verbose {
-					log.Println("Symlink path(", symPath, ") is unobtainable:", folder+"/"+f.e.Name)
+					log.Println("Symlink path(", symPath, ") is unobtainable:", extDir)
 				}
 				return errors.New("cannot get symlink target")
 			}
-			extractLoc := filepath.Clean(folder + "/" + filepath.Dir(symPath))
+			extractLoc := filepath.Join(folder, filepath.Dir(symPath))
 			err = fil.realExtract(extractLoc, op)
 			if err != nil {
 				if op.Verbose {
-					log.Println("Error while extracting ", folder+"/"+f.e.Name)
+					log.Println("Error while extracting ", extDir)
 				}
 				return err
 			}
 		}
-		err = os.Symlink(f.SymlinkPath(), folder+"/"+f.e.Name)
+		err = os.Symlink(f.SymlinkPath(), extDir)
 		if os.IsExist(err) {
-			os.Remove(folder + "/" + f.e.Name)
+			os.Remove(extDir)
-			err = os.Symlink(f.SymlinkPath(), folder+"/"+f.e.Name)
+			err = os.Symlink(f.SymlinkPath(), extDir)
 		}
 		if err != nil {
 			if op.Verbose {
-				log.Println("Error while making symlink:", folder+"/"+f.e.Name)
+				log.Println("Error while making symlink:", extDir)
 			}
 			return err
 		}
 		if op.IgnorePerm {
-			os.Chmod(folder+"/"+f.e.Name, op.FolderPerm)
+			os.Chmod(extDir, op.FolderPerm|(f.Mode()&fs.ModeType))
 		} else {
-			os.Chmod(folder+"/"+f.e.Name, f.Mode())
+			os.Chmod(extDir, f.Mode())
 		}
 	case f.isDeviceOrFifo():
 		if runtime.GOOS == "windows" {
 			if op.Verbose {
-				log.Println(folder+"/"+f.e.Name, "ignored since it's a device link and can't be created on Windows.")
+				log.Println(extDir, "ignored since it's a device link and can't be created on Windows.")
 			}
 			return nil
 		}
@@ -422,13 +430,13 @@ func (f File) realExtract(folder string, op ExtractionOptions) error {
 		} else { //Fifo IPC
 			if runtime.GOOS == "darwin" {
 				if op.Verbose {
-					log.Println(folder+"/"+f.e.Name, "ignored since it's a Fifo file and can't be created on Darwin.")
+					log.Println(extDir, "ignored since it's a Fifo file and can't be created on Darwin.")
 				}
 				return nil
 			}
 			typ = "p"
 		}
-		cmd := exec.Command("mknod", folder+"/"+f.e.Name, typ)
+		cmd := exec.Command("mknod", extDir, typ)
 		if typ != "p" {
 			maj, min := f.deviceDevices()
 			cmd.Args = append(cmd.Args, strconv.Itoa(int(maj)), strconv.Itoa(int(min)))
@@ -440,18 +448,18 @@ func (f File) realExtract(folder string, op ExtractionOptions) error {
 		err = cmd.Run()
 		if err != nil {
 			if op.Verbose {
-				log.Println("Error while running mknod for", folder+"/"+f.e.Name)
+				log.Println("Error while running mknod for", extDir)
 			}
 			return err
 		}
 		if op.IgnorePerm {
-			os.Chmod(folder+"/"+f.e.Name, op.FolderPerm)
+			os.Chmod(extDir, op.FolderPerm|(f.Mode()&fs.ModeType))
 		} else {
-			os.Chmod(folder+"/"+f.e.Name, f.Mode())
+			os.Chmod(extDir, f.Mode())
 		}
 	case f.e.Type == inode.Sock:
 		if op.Verbose {
-			log.Println(folder+"/"+f.e.Name, "ignored since it's a socket file.")
+			log.Println(extDir, "ignored since it's a socket file.")
 		}
 	default:
 		return errors.New("Unsupported file type. Inode type: " + strconv.Itoa(int(f.i.Type)))

squashfs_test.go

@@ -106,7 +106,6 @@ func BenchmarkRace(b *testing.B) {
 }
 
 func TestExtractQuick(t *testing.T) {
-
 	//First, setup everything and extract the archive using the library and unsquashfs
 
 	// tmpDir := b.TempDir()
@@ -123,9 +122,7 @@ func TestExtractQuick(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	op := squashfs.DefaultOptions()
+	err = rdr.ExtractWithOptions(libPath, &squashfs.ExtractionOptions{Verbose: true})
-	op.Verbose = true
-	err = rdr.ExtractWithOptions(libPath, op)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -176,9 +173,7 @@ func TestSingleFile(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	op := squashfs.DefaultOptions()
+	err = f.(*squashfs.File).ExtractWithOptions("testing", &squashfs.ExtractionOptions{Verbose: true})
-	op.Verbose = true
-	err = f.(*squashfs.File).ExtractWithOptions("testing", op)
 	if err != nil {
 		t.Fatal(err)
 	}