add cmd/dump - a dumper

Walk needed to be added to NeedleMap and CompactMap, to be able to add WalkKeys and WalkValues to volume. This is needed for iterating through all the stored needles in a volume - this was dump's purpose.

5 files changed, 164 insertions, 70 deletions

diff --git a/weed-fs/src/cmd/dump/main.go b/weed-fs/src/cmd/dump/main.go
new file mode 100644
index 000000000..e3e151eb7
--- /dev/null
+++ b/weed-fs/src/cmd/dump/main.go
@@ -0,0 +1,96 @@
+// Copyright Tamás Gulácsi 2013 All rights reserved
+// Use of this source is governed by the same rules as the weed-fs library.
+// If this would be ambigous, than Apache License 2.0 has to be used.
+//
+// dump dumps the files of a volume to tar or unique files.
+// Each file will have id#mimetype#original_name file format
+
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"flag"
+	"fmt"
+	// "io"
+	"log"
+	"os"
+	"pkg/storage"
+	"strings"
+	"time"
+)
+
+var (
+	volumePath = flag.String("dir", "/tmp", "volume directory")
+	volumeId   = flag.Int("id", 0, "volume Id")
+	dest       = flag.String("out", "-", "output path. Produces tar if path ends with .tar; creates files otherwise.")
+	tarFh      *tar.Writer
+	tarHeader  tar.Header
+	counter    int
+)
+
+func main() {
+	var err error
+
+	flag.Parse()
+
+	if *dest == "-" {
+		*dest = ""
+	}
+	if *dest == "" || strings.HasSuffix(*dest, ".tar") {
+		var fh *os.File
+		if *dest == "" {
+			fh = os.Stdout
+		} else {
+			if fh, err = os.Create(*dest); err != nil {
+				log.Printf("cannot open output tar %s: %s", *dest, err)
+				return
+			}
+		}
+		defer fh.Close()
+		tarFh = tar.NewWriter(fh)
+		defer tarFh.Close()
+		t := time.Now()
+		tarHeader = tar.Header{Mode: 0644,
+			ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
+			Typeflag:   tar.TypeReg,
+			AccessTime: t, ChangeTime: t}
+	}
+
+	v, err := storage.NewVolume(*volumePath, storage.VolumeId(*volumeId), storage.CopyNil)
+	if v == nil || v.Version() == 0 || err != nil {
+		log.Printf("cannot load volume %d from %s (%s): %s", *volumeId, *volumePath, v, err)
+		return
+	}
+	log.Printf("volume: %s (ver. %d)", v, v.Version())
+	if err := v.WalkValues(walker); err != nil {
+		log.Printf("error while walking: %s", err)
+		return
+	}
+
+	log.Printf("%d files written.", counter)
+}
+
+func walker(n *storage.Needle) (err error) {
+	// log.Printf("Id=%d Size=%d Name=%s mime=%s", n.Id, n.Size, n.Name, n.Mime)
+	nm := fmt.Sprintf("%d#%s#%s", n.Id, bytes.Replace(n.Mime, []byte{'/'}, []byte{'_'}, -1), n.Name)
+	// log.Print(nm)
+	if tarFh != nil {
+		tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
+		if err = tarFh.WriteHeader(&tarHeader); err != nil {
+			return err
+		}
+		_, err = tarFh.Write(n.Data)
+	} else {
+		if fh, e := os.Create(*dest + "/" + nm); e != nil {
+			return e
+		} else {
+			defer fh.Close()
+			_, err = fh.Write(n.Data)
+		}
+	}
+	if err == nil {
+		counter++
+	}
+	return
+}
diff --git a/weed-fs/src/cmd/weed/command.go b/weed-fs/src/cmd/weed/command.go
index 8c725cafb..4d68ff151 100644
--- a/weed-fs/src/cmd/weed/command.go
+++ b/weed-fs/src/cmd/weed/command.go
@@ -1,53 +1,52 @@
 package main
 
 import (
-  "flag"
-  "fmt"
-  "os"
-  "strings"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
 )
 
 type Command struct {
-  // Run runs the command.
-  // The args are the arguments after the command name.
-  Run func(cmd *Command, args []string) bool
+	// Run runs the command.
+	// The args are the arguments after the command name.
+	Run func(cmd *Command, args []string) bool
 
-  // UsageLine is the one-line usage message.
-  // The first word in the line is taken to be the command name.
-  UsageLine string
+	// UsageLine is the one-line usage message.
+	// The first word in the line is taken to be the command name.
+	UsageLine string
 
-  // Short is the short description shown in the 'go help' output.
-  Short string
+	// Short is the short description shown in the 'go help' output.
+	Short string
 
-  // Long is the long message shown in the 'go help <this-command>' output.
-  Long string
-
-  // Flag is a set of flags specific to this command.
-  Flag flag.FlagSet
+	// Long is the long message shown in the 'go help <this-command>' output.
+	Long string
 
+	// Flag is a set of flags specific to this command.
+	Flag flag.FlagSet
 }
 
 // Name returns the command's name: the first word in the usage line.
 func (c *Command) Name() string {
-  name := c.UsageLine
-  i := strings.Index(name, " ")
-  if i >= 0 {
-    name = name[:i]
-  }
-  return name
+	name := c.UsageLine
+	i := strings.Index(name, " ")
+	if i >= 0 {
+		name = name[:i]
+	}
+	return name
 }
 
 func (c *Command) Usage() {
-  fmt.Fprintf(os.Stderr, "Example: weed %s\n", c.UsageLine)
-  fmt.Fprintf(os.Stderr, "Default Usage:\n")
-  c.Flag.PrintDefaults()
-  fmt.Fprintf(os.Stderr, "Description:\n")
-  fmt.Fprintf(os.Stderr, "  %s\n", strings.TrimSpace(c.Long))
-  os.Exit(2)
+	fmt.Fprintf(os.Stderr, "Example: weed %s\n", c.UsageLine)
+	fmt.Fprintf(os.Stderr, "Default Usage:\n")
+	c.Flag.PrintDefaults()
+	fmt.Fprintf(os.Stderr, "Description:\n")
+	fmt.Fprintf(os.Stderr, "  %s\n", strings.TrimSpace(c.Long))
+	os.Exit(2)
 }
 
 // Runnable reports whether the command can be run; otherwise
 // it is a documentation pseudo-command such as importpath.
 func (c *Command) Runnable() bool {
-  return c.Run != nil
+	return c.Run != nil
 }
diff --git a/weed-fs/src/cmd/weed/shell.go b/weed-fs/src/cmd/weed/shell.go
index 78a4b9eb1..daf0b7e1f 100644
--- a/weed-fs/src/cmd/weed/shell.go
+++ b/weed-fs/src/cmd/weed/shell.go
@@ -1,54 +1,53 @@
 package main
 
 import (
-  "bufio"
-  "os"
-  "fmt"
+	"bufio"
+	"fmt"
+	"os"
 )
 
 func init() {
-  cmdShell.Run = runShell // break init cycle
+	cmdShell.Run = runShell // break init cycle
 }
 
 var cmdShell = &Command{
-  UsageLine: "shell",
-  Short:     "run interactive commands, now just echo",
-  Long: `run interactive commands.
+	UsageLine: "shell",
+	Short:     "run interactive commands, now just echo",
+	Long: `run interactive commands.
 
   `,
 }
 
-var (
-)
+var ()
 
 func runShell(command *Command, args []string) bool {
-  r := bufio.NewReader(os.Stdin)
-  o := bufio.NewWriter(os.Stdout)
-  e := bufio.NewWriter(os.Stderr)
-  prompt := func () { 
-    o.WriteString("> ")
-    o.Flush()
-  };
-  readLine := func () string { 
-    ret, err := r.ReadString('\n')
-    if err != nil { 
-       fmt.Fprint(e,err); 
-       os.Exit(1)
-    }
-    return ret
-  }
-  execCmd := func (cmd string) int { 
-    if cmd != "" { 
-      o.WriteString(cmd)
-    }
-    return 0
-  }
+	r := bufio.NewReader(os.Stdin)
+	o := bufio.NewWriter(os.Stdout)
+	e := bufio.NewWriter(os.Stderr)
+	prompt := func() {
+		o.WriteString("> ")
+		o.Flush()
+	}
+	readLine := func() string {
+		ret, err := r.ReadString('\n')
+		if err != nil {
+			fmt.Fprint(e, err)
+			os.Exit(1)
+		}
+		return ret
+	}
+	execCmd := func(cmd string) int {
+		if cmd != "" {
+			o.WriteString(cmd)
+		}
+		return 0
+	}
 
-  cmd := ""
-  for { 
-    prompt()
-    cmd = readLine()
-    execCmd(cmd)
-  } 
-  return true
+	cmd := ""
+	for {
+		prompt()
+		cmd = readLine()
+		execCmd(cmd)
+	}
+	return true
 }
diff --git a/weed-fs/src/cmd/weed/upload.go b/weed-fs/src/cmd/weed/upload.go
index e25930b5d..5707fda56 100644
--- a/weed-fs/src/cmd/weed/upload.go
+++ b/weed-fs/src/cmd/weed/upload.go
@@ -67,7 +67,7 @@ func upload(filename string, server string, fid string) (int, error) {
 	}
 	ret, e := operation.Upload("http://"+server+"/"+fid, filename, fh)
 	if e != nil {
-	  return 0, e
+		return 0, e
 	}
 	return ret.Size, e
 }
diff --git a/weed-fs/src/cmd/weed/weed.go b/weed-fs/src/cmd/weed/weed.go
index 232520e75..e2eb41ced 100644
--- a/weed-fs/src/cmd/weed/weed.go
+++ b/weed-fs/src/cmd/weed/weed.go
@@ -175,9 +175,9 @@ func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) {
 	w.Header().Set("Content-Type", "application/javascript")
 	var bytes []byte
 	if r.FormValue("pretty") != "" {
-    bytes, _ = json.MarshalIndent(obj, "", "  ")
+		bytes, _ = json.MarshalIndent(obj, "", "  ")
 	} else {
-    bytes, _ = json.Marshal(obj)
+		bytes, _ = json.Marshal(obj)
 	}
 	callback := r.FormValue("callback")
 	if callback == "" {