diff options
| -rw-r--r-- | weed/command/command.go | 1 | ||||
| -rw-r--r-- | weed/command/filer_export.go | 107 | ||||
| -rw-r--r-- | weed/server/filer_server.go | 4 |
3 files changed, 110 insertions, 2 deletions
diff --git a/weed/command/command.go b/weed/command/command.go index 0d72ee0db..61f13c5e2 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -13,6 +13,7 @@ var Commands = []*Command{ cmdCompact, cmdCopy, cmdFix, + cmdFilerExport, cmdServer, cmdMaster, cmdFiler, diff --git a/weed/command/filer_export.go b/weed/command/filer_export.go new file mode 100644 index 000000000..ab8962630 --- /dev/null +++ b/weed/command/filer_export.go @@ -0,0 +1,107 @@ +package command + +import ( + "github.com/chrislusf/seaweedfs/weed/server" + "github.com/spf13/viper" + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/glog" +) + +func init() { + cmdFilerExport.Run = runFilerExport // break init cycle +} + +var cmdFilerExport = &Command{ + UsageLine: "filer.export -sourceStore=mysql -targetStroe=cassandra", + Short: "export meta data in filer store", + Long: `Iterate the file tree and export all metadata out + + Both source and target store: + * should be a store name already specified in filer.toml + * do not need to be enabled state + + If target store is empty, only the directory tree will be listed. + + `, +} + +var ( + // filerExportOutputFile = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree") + filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml") +) + +type statistics struct { + directoryCount int + fileCount int +} + +func runFilerExport(cmd *Command, args []string) bool { + + weed_server.LoadConfiguration("filer", true) + config := viper.GetViper() + + var sourceStore filer2.FilerStore + + for _, store := range filer2.Stores { + if store.GetName() == *filerExportSourceStore { + viperSub := config.Sub(store.GetName()) + if err := store.Initialize(viperSub); err != nil { + glog.Fatalf("Failed to initialize store for %s: %+v", + store.GetName(), err) + } else { + sourceStore = store + } + break + } + } + + if sourceStore == nil { + glog.Errorf("Failed to find source store %s", *filerExportSourceStore) + println("existing data sources are:") + for _, store := range filer2.Stores { + println(" " + store.GetName()) + } + return false + } + + stat := statistics{} + doTraverse(&stat, sourceStore, filer2.FullPath("/"), 0, printout) + + glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount) + + return true +} + +func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) { + + limit := 1000 + lastEntryName := "" + for { + entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit) + if err != nil { + break + } + for _, entry := range entries { + if fnErr := fn(level, entry); fnErr != nil { + glog.Errorf("failed to process entry: %s", entry.FullPath) + } + if entry.IsDirectory() { + stat.directoryCount++ + doTraverse(stat, filerStore, entry.FullPath, level+1, fn) + } else { + stat.fileCount++ + } + } + if len(entries) < limit { + break + } + } +} + +func printout(level int, entry *filer2.Entry) error { + for i := 0; i < level; i++ { + print(" ") + } + println(entry.FullPath.Name()) + return nil +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 7a4d1be7f..0150caf5e 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -49,7 +49,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) go fs.filer.KeepConnectedToMaster() - loadConfiguration("filer", true) + LoadConfiguration("filer", true) v := viper.GetViper() fs.filer.LoadConfiguration(v) @@ -69,7 +69,7 @@ func (fs *FilerServer) jwt(fileId string) security.EncodedJwt { return security.GenJwt(fs.secret, fileId) } -func loadConfiguration(configFileName string, required bool) { +func LoadConfiguration(configFileName string, required bool) { // find a filer store viper.SetConfigName(configFileName) // name of config file (without extension) |