support compacting a volume

4 files changed, 52 insertions, 9 deletions

diff --git a/weed-fs/src/cmd/weed/fix.go b/weed-fs/src/cmd/weed/fix.go
index acf2d6643..e75bfa2f6 100644
--- a/weed-fs/src/cmd/weed/fix.go
+++ b/weed-fs/src/cmd/weed/fix.go
@@ -47,17 +47,19 @@ func runFix(cmd *Command, args []string) bool {
   //skip the volume super block
   dataFile.Seek(storage.SuperBlockSize, 0)
 
-  n, length := storage.ReadNeedle(dataFile)
+  n, rest := storage.ReadNeedle(dataFile)
+  dataFile.Seek(int64(rest), 1)
   nm := storage.NewNeedleMap(indexFile)
   offset := uint32(storage.SuperBlockSize)
   for n != nil {
-    debug("key", n.Id, "volume offset", offset, "data_size", n.Size, "length", length)
+    debug("key", n.Id, "volume offset", offset, "data_size", n.Size, "rest", rest)
     if n.Size > 0 {
       count, pe := nm.Put(n.Id, offset/8, n.Size)
       debug("saved", count, "with error", pe)
     }
-    offset += length
-    n, length = storage.ReadNeedle(dataFile)
+    offset += rest+16
+    n, rest = storage.ReadNeedle(dataFile)
+    dataFile.Seek(int64(rest), 1)
   }
   return true
 }
diff --git a/weed-fs/src/cmd/weed/master.go b/weed-fs/src/cmd/weed/master.go
index 759e5fdc9..b2b114a35 100644
--- a/weed-fs/src/cmd/weed/master.go
+++ b/weed-fs/src/cmd/weed/master.go
@@ -122,6 +122,27 @@ func dirStatusHandler(w http.ResponseWriter, r *http.Request) {
 	writeJson(w, r, m)
 }
 
+func volumeVacuumHandler(w http.ResponseWriter, r *http.Request) {
+  count := 0
+  rt, err := storage.NewReplicationTypeFromString(r.FormValue("replication"))
+  if err == nil {
+    if count, err = strconv.Atoi(r.FormValue("count")); err == nil {
+      if topo.FreeSpace() < count*rt.GetCopyCount() {
+        err = errors.New("Only " + strconv.Itoa(topo.FreeSpace()) + " volumes left! Not enough for " + strconv.Itoa(count*rt.GetCopyCount()))
+      } else {
+        count, err = vg.GrowByCountAndType(count, rt, topo)
+      }
+    }
+  }
+  if err != nil {
+    w.WriteHeader(http.StatusNotAcceptable)
+    writeJson(w, r, map[string]string{"error": err.Error()})
+  } else {
+    w.WriteHeader(http.StatusNotAcceptable)
+    writeJson(w, r, map[string]interface{}{"count": count})
+  }
+}
+
 func volumeGrowHandler(w http.ResponseWriter, r *http.Request) {
 	count := 0
 	rt, err := storage.NewReplicationTypeFromString(r.FormValue("replication"))
diff --git a/weed-fs/src/cmd/weed/version.go b/weed-fs/src/cmd/weed/version.go
index aa2396b11..f63b5bd70 100644
--- a/weed-fs/src/cmd/weed/version.go
+++ b/weed-fs/src/cmd/weed/version.go
@@ -6,7 +6,7 @@ import (
 )
 
 const (
-	VERSION = "0.23"
+	VERSION = "0.24"
 )
 
 var cmdVersion = &Command{
diff --git a/weed-fs/src/cmd/weed/volume.go b/weed-fs/src/cmd/weed/volume.go
index 259dbfded..9e6f3e9ac 100644
--- a/weed-fs/src/cmd/weed/volume.go
+++ b/weed-fs/src/cmd/weed/volume.go
@@ -55,7 +55,25 @@ func assignVolumeHandler(w http.ResponseWriter, r *http.Request) {
 	} else {
 		writeJson(w, r, map[string]string{"error": err.Error()})
 	}
-	debug("volume =", r.FormValue("volume"), ", replicationType =", r.FormValue("replicationType"), ", error =", err)
+	debug("assign volume =", r.FormValue("volume"), ", replicationType =", r.FormValue("replicationType"), ", error =", err)
+}
+func vacuumVolumeCompactHandler(w http.ResponseWriter, r *http.Request) {
+	err := store.CompactVolume(r.FormValue("volume"))
+	if err == nil {
+		writeJson(w, r, map[string]string{"error": ""})
+	} else {
+		writeJson(w, r, map[string]string{"error": err.Error()})
+	}
+	debug("compacted volume =", r.FormValue("volume"), ", error =", err)
+}
+func vacuumVolumeCommitHandler(w http.ResponseWriter, r *http.Request) {
+  count, err := store.CommitCompactVolume(r.FormValue("volume"))
+  if err == nil {
+    writeJson(w, r, map[string]interface{}{"error": "", "size":count})
+  } else {
+    writeJson(w, r, map[string]string{"error": err.Error()})
+  }
+  debug("commit compact volume =", r.FormValue("volume"), ", error =", err)
 }
 func storeHandler(w http.ResponseWriter, r *http.Request) {
 	switch r.Method {
@@ -250,9 +268,9 @@ func distributedOperation(volumeId storage.VolumeId, op func(location operation.
 }
 
 func runVolume(cmd *Command, args []string) bool {
-  if *vMaxCpu < 1 {
-    *vMaxCpu = runtime.NumCPU()
-  }
+	if *vMaxCpu < 1 {
+		*vMaxCpu = runtime.NumCPU()
+	}
 	runtime.GOMAXPROCS(*vMaxCpu)
 	fileInfo, err := os.Stat(*volumeFolder)
 	if err != nil {
@@ -273,6 +291,8 @@ func runVolume(cmd *Command, args []string) bool {
 	http.HandleFunc("/", storeHandler)
 	http.HandleFunc("/status", statusHandler)
 	http.HandleFunc("/admin/assign_volume", assignVolumeHandler)
+	http.HandleFunc("/admin/vacuum_volume_compact", vacuumVolumeCompactHandler)
+	http.HandleFunc("/admin/vacuum_volume_commit", vacuumVolumeCommitHandler)
 
 	go func() {
 		for {