re-factoring volume file scanning code

add export function
fix bugs on deleted byte counting

5 files changed, 149 insertions, 50 deletions

diff --git a/weed-fs/src/cmd/weed/export.go b/weed-fs/src/cmd/weed/export.go
new file mode 100644
index 000000000..cd972f24c
--- /dev/null
+++ b/weed-fs/src/cmd/weed/export.go
@@ -0,0 +1,120 @@
+package main
+
+import (
+	"archive/tar"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"pkg/directory"
+	"pkg/storage"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func init() {
+	cmdExport.Run = runExport // break init cycle
+	cmdExport.IsDebug = cmdExport.Flag.Bool("debug", false, "enable debug mode")
+}
+
+var cmdExport = &Command{
+	UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar",
+	Short:     "export files out of one volume",
+	Long: `export all files in a volume
+
+  `,
+}
+
+var (
+	exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "data directory to store files")
+	exportVolumeId   = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+	dest             = cmdExport.Flag.String("o", "", "output tar file name")
+	tarFh            *tar.Writer
+	tarHeader        tar.Header
+	counter          int
+)
+
+func runExport(cmd *Command, args []string) bool {
+
+	if *exportVolumeId == -1 {
+		return false
+	}
+
+	var err error
+	if strings.HasSuffix(*dest, ".tar") {
+		var fh *os.File
+		if *dest == "" {
+			fh = os.Stdout
+		} else {
+			if fh, err = os.Create(*dest); err != nil {
+				log.Fatalf("cannot open output tar %s: %s", *dest, err)
+			}
+		}
+		defer fh.Close()
+		tarFh = tar.NewWriter(fh)
+		defer tarFh.Close()
+		t := time.Now()
+		tarHeader = tar.Header{Mode: 0644,
+			ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
+			Typeflag:   tar.TypeReg,
+			AccessTime: t, ChangeTime: t}
+	}
+
+	fileName := strconv.Itoa(*exportVolumeId)
+	vid := storage.VolumeId(*exportVolumeId)
+	indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
+	if err != nil {
+		log.Fatalf("Create Volume Index [ERROR] %s\n", err)
+	}
+	defer indexFile.Close()
+
+	nm := storage.LoadNeedleMap(indexFile)
+
+	err = storage.ScanVolumeFile(*exportVolumePath, vid, func(superBlock storage.SuperBlock) error {
+		return nil
+	}, func(n *storage.Needle, offset uint32) error {
+		debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped())
+		nv, ok := nm.Get(n.Id)
+		if ok && nv.Size > 0 {
+			return walker(vid, n)
+		} else {
+			if !ok {
+				debug("This seems deleted", n.Id)
+			} else {
+				debug("Id", n.Id, "size", n.Size)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		log.Fatalf("Export Volume File [ERROR] %s\n", err)
+	}
+	return true
+}
+
+func walker(vid storage.VolumeId, n *storage.Needle) (err error) {
+	nm := fmt.Sprintf("%s/%d#%s", n.Mime, n.Id, n.Name)
+	if n.IsGzipped() && path.Ext(nm) != ".gz" {
+		nm = nm + ".gz"
+	}
+	if tarFh != nil {
+		tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
+		if err = tarFh.WriteHeader(&tarHeader); err != nil {
+			return err
+		}
+		_, err = tarFh.Write(n.Data)
+	} else {
+		fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
+			directory.NewFileId(vid, n.Id, n.Cookie).String(),
+			n.Name,
+			n.DataSize,
+      n.IsGzipped(),
+			n.Mime,
+			)
+	}
+	if err == nil {
+		counter++
+	}
+	return
+}
diff --git a/weed-fs/src/cmd/weed/fix.go b/weed-fs/src/cmd/weed/fix.go
index deee9a81f..95fb0a7e2 100644
--- a/weed-fs/src/cmd/weed/fix.go
+++ b/weed-fs/src/cmd/weed/fix.go
@@ -33,46 +33,32 @@ func runFix(cmd *Command, args []string) bool {
 	}
 
 	fileName := strconv.Itoa(*fixVolumeId)
-	dataFile, e := os.OpenFile(path.Join(*fixVolumePath, fileName+".dat"), os.O_RDONLY, 0644)
-	if e != nil {
-		log.Fatalf("Read Volume [ERROR] %s\n", e)
-	}
-	defer dataFile.Close()
-	indexFile, ie := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_WRONLY|os.O_CREATE, 0644)
-	if ie != nil {
-		log.Fatalf("Create Volume Index [ERROR] %s\n", ie)
+	indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		log.Fatalf("Create Volume Index [ERROR] %s\n", err)
 	}
 	defer indexFile.Close()
 
-	dataFile.Seek(0, 0)
-	header := make([]byte, storage.SuperBlockSize)
-	if _, e := dataFile.Read(header); e != nil {
-		log.Fatalf("cannot read superblock: %s", e)
-	}
-
-	ver, _, e := storage.ParseSuperBlock(header)
-	if e != nil {
-		log.Fatalf("error parsing superblock: %s", e)
-	}
-
-	n, rest, e := storage.ReadNeedleHeader(dataFile, ver)
-	if e != nil {
-		log.Fatalf("error reading needle header: %s", e)
-	}
-	dataFile.Seek(int64(rest), 1)
 	nm := storage.NewNeedleMap(indexFile)
-	offset := uint32(storage.SuperBlockSize)
-	for n != nil {
-		debug("key", n.Id, "volume offset", offset, "data_size", n.Size, "rest", rest)
+	defer nm.Close()
+
+	vid := storage.VolumeId(*fixVolumeId)
+	err = storage.ScanVolumeFile(*fixVolumePath, vid, func(superBlock storage.SuperBlock) error {
+		return nil
+	}, func(n *storage.Needle, offset uint32) error {
+		debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped())
 		if n.Size > 0 {
 			count, pe := nm.Put(n.Id, offset/storage.NeedlePaddingSize, n.Size)
 			debug("saved", count, "with error", pe)
+		}else{
+      debug("skipping deleted file ...")
+      nm.Delete(n.Id)
 		}
-		offset += rest + 16
-		if n, rest, e = storage.ReadNeedleHeader(dataFile, ver); e != nil {
-			log.Fatalf("error reading needle header: %s", e)
-		}
-		dataFile.Seek(int64(rest), 1)
+		return nil
+	})
+	if err != nil {
+		log.Fatalf("Export Volume File [ERROR] %s\n", err)
 	}
+
 	return true
 }
diff --git a/weed-fs/src/cmd/weed/master.go b/weed-fs/src/cmd/weed/master.go
index 07abcf0dc..ccd827172 100644
--- a/weed-fs/src/cmd/weed/master.go
+++ b/weed-fs/src/cmd/weed/master.go
@@ -128,24 +128,13 @@ func dirStatusHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func volumeVacuumHandler(w http.ResponseWriter, r *http.Request) {
-	count := 0
-	rt, err := storage.NewReplicationTypeFromString(r.FormValue("replication"))
-	if err == nil {
-		if count, err = strconv.Atoi(r.FormValue("count")); err == nil {
-			if topo.FreeSpace() < count*rt.GetCopyCount() {
-				err = errors.New("Only " + strconv.Itoa(topo.FreeSpace()) + " volumes left! Not enough for " + strconv.Itoa(count*rt.GetCopyCount()))
-			} else {
-				count, err = vg.GrowByCountAndType(count, rt, topo)
-			}
-		}
-	}
-	if err != nil {
-		w.WriteHeader(http.StatusNotAcceptable)
-		writeJson(w, r, map[string]string{"error": err.Error()})
-	} else {
-		w.WriteHeader(http.StatusNotAcceptable)
-		writeJson(w, r, map[string]interface{}{"count": count})
+	gcThreshold := r.FormValue("garbageThreshold")
+	if gcThreshold == "" {
+		gcThreshold = *garbageThreshold
 	}
+	debug("garbageThreshold =", gcThreshold)
+	topo.Vacuum(gcThreshold)
+	dirStatusHandler(w, r)
 }
 
 func volumeGrowHandler(w http.ResponseWriter, r *http.Request) {
@@ -208,6 +197,8 @@ func runMaster(cmd *Command, args []string) bool {
 	http.HandleFunc("/dir/status", dirStatusHandler)
 	http.HandleFunc("/vol/grow", volumeGrowHandler)
 	http.HandleFunc("/vol/status", volumeStatusHandler)
+	http.HandleFunc("/vol/vacuum", volumeVacuumHandler)
+
 	http.HandleFunc("/", redirectHandler)
 
 	topo.StartRefreshWritableVolumes(*garbageThreshold)
diff --git a/weed-fs/src/cmd/weed/upload.go b/weed-fs/src/cmd/weed/upload.go
index 8ce0674f4..5df29f6a3 100644
--- a/weed-fs/src/cmd/weed/upload.go
+++ b/weed-fs/src/cmd/weed/upload.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"path"
 	"pkg/operation"
 	"pkg/util"
 	"strconv"
@@ -65,7 +66,7 @@ func upload(filename string, server string, fid string) (int, error) {
 		debug("Failed to open file:", filename)
 		return 0, err
 	}
-	ret, e := operation.Upload("http://"+server+"/"+fid, filename, fh)
+	ret, e := operation.Upload("http://"+server+"/"+fid, path.Base(filename), fh)
 	if e != nil {
 		return 0, e
 	}
diff --git a/weed-fs/src/cmd/weed/weed.go b/weed-fs/src/cmd/weed/weed.go
index bc05a9c2c..c03cb68ac 100644
--- a/weed-fs/src/cmd/weed/weed.go
+++ b/weed-fs/src/cmd/weed/weed.go
@@ -26,6 +26,7 @@ var commands = []*Command{
 	cmdShell,
 	cmdVersion,
 	cmdVolume,
+	cmdExport,
 }
 
 var exitStatus = 0