diff options
| author | Chris Lu <chris.lu@gmail.com> | 2016-06-02 18:09:14 -0700 |
|---|---|---|
| committer | Chris Lu <chris.lu@gmail.com> | 2016-06-02 18:09:14 -0700 |
| commit | 5ce6bbf07672bf3f3c8d26cd2ce0e3e853a47c44 (patch) | |
| tree | 2e4dd2ad0a618ab2b7cdebcdb9c503526c31e2e8 /go/weed/export.go | |
| parent | caeffa3998adc060fa66c4cd77af971ff2d26c57 (diff) | |
| download | seaweedfs-5ce6bbf07672bf3f3c8d26cd2ce0e3e853a47c44.tar.xz seaweedfs-5ce6bbf07672bf3f3c8d26cd2ce0e3e853a47c44.zip | |
directory structure change to work with glide
glide has its own requirements. My previous workaround caused me some
code checkin errors. Need to fix this.
Diffstat (limited to 'go/weed/export.go')
| -rw-r--r-- | go/weed/export.go | 213 |
1 files changed, 0 insertions, 213 deletions
diff --git a/go/weed/export.go b/go/weed/export.go deleted file mode 100644 index 4417c65f4..000000000 --- a/go/weed/export.go +++ /dev/null @@ -1,213 +0,0 @@ -package main - -import ( - "archive/tar" - "bytes" - "fmt" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "text/template" - "time" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/storage" -) - -const ( - defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}` - timeFormat = "2006-01-02T15:04:05" -) - -var ( - export ExportOptions -) - -type ExportOptions struct { - dir *string - collection *string - volumeId *int -} - -var cmdExport = &Command{ - UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}} -newer='" + timeFormat + "'", - Short: "list or export files from one volume data file", - Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified. - - The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}. - - `, -} - -func init() { - cmdExport.Run = runExport // break init cycle - export.dir = cmdExport.Flag.String("dir", ".", "input data directory to store volume data files") - export.collection = cmdExport.Flag.String("collection", "", "the volume collection name") - export.volumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.") -} - -var ( - output = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout") - format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename formatted with {{.Mime}} {{.Id}} {{.Name}} {{.Ext}}") - newer = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone") - - tarOutputFile *tar.Writer - tarHeader tar.Header - fileNameTemplate *template.Template - fileNameTemplateBuffer = bytes.NewBuffer(nil) - newerThan time.Time - newerThanUnix int64 = -1 - localLocation, _ = time.LoadLocation("Local") -) - -func runExport(cmd *Command, args []string) bool { - - var err error - - if *newer != "" { - if newerThan, err = time.ParseInLocation(timeFormat, *newer, localLocation); err != nil { - fmt.Println("cannot parse 'newer' argument: " + err.Error()) - return false - } - newerThanUnix = newerThan.Unix() - } - - if *export.volumeId == -1 { - return false - } - - if *output != "" { - if *output != "-" && !strings.HasSuffix(*output, ".tar") { - fmt.Println("the output file", *output, "should be '-' or end with .tar") - return false - } - - if fileNameTemplate, err = template.New("name").Parse(*format); err != nil { - fmt.Println("cannot parse format " + *format + ": " + err.Error()) - return false - } - - var outputFile *os.File - if *output == "-" { - outputFile = os.Stdout - } else { - if outputFile, err = os.Create(*output); err != nil { - glog.Fatalf("cannot open output tar %s: %s", *output, err) - } - } - defer outputFile.Close() - tarOutputFile = tar.NewWriter(outputFile) - defer tarOutputFile.Close() - t := time.Now() - tarHeader = tar.Header{Mode: 0644, - ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(), - Typeflag: tar.TypeReg, - AccessTime: t, ChangeTime: t} - } - - fileName := strconv.Itoa(*export.volumeId) - if *export.collection != "" { - fileName = *export.collection + "_" + fileName - } - vid := storage.VolumeId(*export.volumeId) - indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644) - if err != nil { - glog.Fatalf("Create Volume Index [ERROR] %s\n", err) - } - defer indexFile.Close() - - needleMap, err := storage.LoadNeedleMap(indexFile) - if err != nil { - glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err) - } - - var version storage.Version - - err = storage.ScanVolumeFile(*export.dir, *export.collection, vid, - storage.NeedleMapInMemory, - func(superBlock storage.SuperBlock) error { - version = superBlock.Version() - return nil - }, true, func(n *storage.Needle, offset int64) error { - nv, ok := needleMap.Get(n.Id) - glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v", - n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv) - if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset { - if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) { - glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d", - n.LastModified, newerThanUnix) - return nil - } - return walker(vid, n, version) - } - if !ok { - glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size) - } else { - glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size) - } - return nil - }) - if err != nil { - glog.Fatalf("Export Volume File [ERROR] %s\n", err) - } - return true -} - -type nameParams struct { - Name string - Id uint64 - Mime string - Key string - Ext string -} - -func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) { - key := storage.NewFileIdFromNeedle(vid, n).String() - if tarOutputFile != nil { - fileNameTemplateBuffer.Reset() - if err = fileNameTemplate.Execute(fileNameTemplateBuffer, - nameParams{ - Name: string(n.Name), - Id: n.Id, - Mime: string(n.Mime), - Key: key, - Ext: filepath.Ext(string(n.Name)), - }, - ); err != nil { - return err - } - - fileName := fileNameTemplateBuffer.String() - - if n.IsGzipped() && path.Ext(fileName) != ".gz" { - fileName = fileName + ".gz" - } - - tarHeader.Name, tarHeader.Size = fileName, int64(len(n.Data)) - if n.HasLastModifiedDate() { - tarHeader.ModTime = time.Unix(int64(n.LastModified), 0) - } else { - tarHeader.ModTime = time.Unix(0, 0) - } - tarHeader.ChangeTime = tarHeader.ModTime - if err = tarOutputFile.WriteHeader(&tarHeader); err != nil { - return err - } - _, err = tarOutputFile.Write(n.Data) - } else { - size := n.DataSize - if version == storage.Version1 { - size = n.Size - } - fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n", - key, - n.Name, - size, - n.IsGzipped(), - n.Mime, - ) - } - return -} |
