aboutsummaryrefslogtreecommitdiff
path: root/weed/command/export.go
diff options
context:
space:
mode:
Diffstat (limited to 'weed/command/export.go')
-rw-r--r--weed/command/export.go135
1 files changed, 76 insertions, 59 deletions
diff --git a/weed/command/export.go b/weed/command/export.go
index 1202d687c..8c32b3f4d 100644
--- a/weed/command/export.go
+++ b/weed/command/export.go
@@ -4,6 +4,7 @@ import (
"archive/tar"
"bytes"
"fmt"
+ "io"
"os"
"path"
"path/filepath"
@@ -14,8 +15,10 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
+ "github.com/chrislusf/seaweedfs/weed/storage/needle"
+ "github.com/chrislusf/seaweedfs/weed/storage/needle_map"
+ "github.com/chrislusf/seaweedfs/weed/storage/super_block"
"github.com/chrislusf/seaweedfs/weed/storage/types"
- "io"
)
const (
@@ -66,10 +69,10 @@ var (
localLocation, _ = time.LoadLocation("Local")
)
-func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Version, deleted bool) {
- key := storage.NewFileIdFromNeedle(vid, n).String()
+func printNeedle(vid needle.VolumeId, n *needle.Needle, version needle.Version, deleted bool) {
+ key := needle.NewFileIdFromNeedle(vid, n).String()
size := n.DataSize
- if version == storage.Version1 {
+ if version == needle.Version1 {
size = n.Size
}
fmt.Printf("%s\t%s\t%d\t%t\t%s\t%s\t%s\t%t\n",
@@ -84,6 +87,62 @@ func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Versio
)
}
+type VolumeFileScanner4Export struct {
+ version needle.Version
+ counter int
+ needleMap *needle_map.MemDb
+ vid needle.VolumeId
+}
+
+func (scanner *VolumeFileScanner4Export) VisitSuperBlock(superBlock super_block.SuperBlock) error {
+ scanner.version = superBlock.Version
+ return nil
+
+}
+func (scanner *VolumeFileScanner4Export) ReadNeedleBody() bool {
+ return true
+}
+
+func (scanner *VolumeFileScanner4Export) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
+ needleMap := scanner.needleMap
+ vid := scanner.vid
+
+ nv, ok := needleMap.Get(n.Id)
+ glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
+ n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
+ if ok && nv.Size > 0 && nv.Size != types.TombstoneFileSize && nv.Offset.ToAcutalOffset() == offset {
+ if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
+ glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
+ n.LastModified, newerThanUnix)
+ return nil
+ }
+ scanner.counter++
+ if *limit > 0 && scanner.counter > *limit {
+ return io.EOF
+ }
+ if tarOutputFile != nil {
+ return writeFile(vid, n)
+ } else {
+ printNeedle(vid, n, scanner.version, false)
+ return nil
+ }
+ }
+ if !ok {
+ if *showDeleted && tarOutputFile == nil {
+ if n.DataSize > 0 {
+ printNeedle(vid, n, scanner.version, true)
+ } else {
+ n.Name = []byte("*tombstone")
+ printNeedle(vid, n, scanner.version, true)
+ }
+ }
+ glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
+ } else {
+ glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
+ }
+ return nil
+}
+
func runExport(cmd *Command, args []string) bool {
var err error
@@ -133,67 +192,25 @@ func runExport(cmd *Command, args []string) bool {
if *export.collection != "" {
fileName = *export.collection + "_" + fileName
}
- vid := storage.VolumeId(*export.volumeId)
- indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644)
- if err != nil {
- glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
- }
- defer indexFile.Close()
+ vid := needle.VolumeId(*export.volumeId)
- needleMap, err := storage.LoadBtreeNeedleMap(indexFile)
- if err != nil {
- glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
+ needleMap := needle_map.NewMemDb()
+ defer needleMap.Close()
+
+ if err := needleMap.LoadFromIdx(path.Join(*export.dir, fileName+".idx")); err != nil {
+ glog.Fatalf("cannot load needle map from %s.idx: %s", fileName, err)
}
- var version storage.Version
+ volumeFileScanner := &VolumeFileScanner4Export{
+ needleMap: needleMap,
+ vid: vid,
+ }
if tarOutputFile == nil {
fmt.Printf("key\tname\tsize\tgzip\tmime\tmodified\tttl\tdeleted\n")
}
- var counter = 0
-
- err = storage.ScanVolumeFile(*export.dir, *export.collection, vid,
- storage.NeedleMapInMemory,
- func(superBlock storage.SuperBlock) error {
- version = superBlock.Version()
- return nil
- }, true, func(n *storage.Needle, offset int64) error {
- nv, ok := needleMap.Get(n.Id)
- glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
- n.Id, offset, n.Size, n.DiskSize(version), n.IsGzipped(), ok, nv)
- if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
- if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
- glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
- n.LastModified, newerThanUnix)
- return nil
- }
- counter++
- if *limit > 0 && counter > *limit {
- return io.EOF
- }
- if tarOutputFile != nil {
- return writeFile(vid, n)
- } else {
- printNeedle(vid, n, version, false)
- return nil
- }
- }
- if !ok {
- if *showDeleted && tarOutputFile == nil {
- if n.DataSize > 0 {
- printNeedle(vid, n, version, true)
- } else {
- n.Name = []byte("*tombstone")
- printNeedle(vid, n, version, true)
- }
- }
- glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
- } else {
- glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
- }
- return nil
- })
+ err = storage.ScanVolumeFile(*export.dir, *export.collection, vid, storage.NeedleMapInMemory, volumeFileScanner)
if err != nil && err != io.EOF {
glog.Fatalf("Export Volume File [ERROR] %s\n", err)
}
@@ -208,8 +225,8 @@ type nameParams struct {
Ext string
}
-func writeFile(vid storage.VolumeId, n *storage.Needle) (err error) {
- key := storage.NewFileIdFromNeedle(vid, n).String()
+func writeFile(vid needle.VolumeId, n *needle.Needle) (err error) {
+ key := needle.NewFileIdFromNeedle(vid, n).String()
fileNameTemplateBuffer.Reset()
if err = fileNameTemplate.Execute(fileNameTemplateBuffer,
nameParams{
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 06:54:48 +0000