aboutsummaryrefslogtreecommitdiff
path: root/weed/command
diff options
context:
space:
mode:
Diffstat (limited to 'weed/command')
-rw-r--r--weed/command/export.go105
-rw-r--r--weed/command/fix.go48
2 files changed, 92 insertions, 61 deletions
diff --git a/weed/command/export.go b/weed/command/export.go
index 1202d687c..5c7e064ce 100644
--- a/weed/command/export.go
+++ b/weed/command/export.go
@@ -84,6 +84,62 @@ func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Versio
)
}
+type VolumeFileScanner4Export struct {
+ version storage.Version
+ counter int
+ needleMap *storage.NeedleMap
+ vid storage.VolumeId
+}
+
+func (scanner *VolumeFileScanner4Export) VisitSuperBlock(superBlock storage.SuperBlock) error {
+ scanner.version = superBlock.Version()
+ return nil
+
+}
+func (scanner *VolumeFileScanner4Export) ReadNeedleBody() bool {
+ return true
+}
+
+func (scanner *VolumeFileScanner4Export) VisitNeedle(n *storage.Needle, offset int64) error {
+ needleMap := scanner.needleMap
+ vid := scanner.vid
+
+ nv, ok := needleMap.Get(n.Id)
+ glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
+ n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
+ if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
+ if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
+ glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
+ n.LastModified, newerThanUnix)
+ return nil
+ }
+ scanner.counter++
+ if *limit > 0 && scanner.counter > *limit {
+ return io.EOF
+ }
+ if tarOutputFile != nil {
+ return writeFile(vid, n)
+ } else {
+ printNeedle(vid, n, scanner.version, false)
+ return nil
+ }
+ }
+ if !ok {
+ if *showDeleted && tarOutputFile == nil {
+ if n.DataSize > 0 {
+ printNeedle(vid, n, scanner.version, true)
+ } else {
+ n.Name = []byte("*tombstone")
+ printNeedle(vid, n, scanner.version, true)
+ }
+ }
+ glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
+ } else {
+ glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
+ }
+ return nil
+}
+
func runExport(cmd *Command, args []string) bool {
var err error
@@ -145,55 +201,16 @@ func runExport(cmd *Command, args []string) bool {
glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
}
- var version storage.Version
+ volumeFileScanner := &VolumeFileScanner4Export{
+ needleMap: needleMap,
+ vid: vid,
+ }
if tarOutputFile == nil {
fmt.Printf("key\tname\tsize\tgzip\tmime\tmodified\tttl\tdeleted\n")
}
- var counter = 0
-
- err = storage.ScanVolumeFile(*export.dir, *export.collection, vid,
- storage.NeedleMapInMemory,
- func(superBlock storage.SuperBlock) error {
- version = superBlock.Version()
- return nil
- }, true, func(n *storage.Needle, offset int64) error {
- nv, ok := needleMap.Get(n.Id)
- glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
- n.Id, offset, n.Size, n.DiskSize(version), n.IsGzipped(), ok, nv)
- if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
- if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
- glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
- n.LastModified, newerThanUnix)
- return nil
- }
- counter++
- if *limit > 0 && counter > *limit {
- return io.EOF
- }
- if tarOutputFile != nil {
- return writeFile(vid, n)
- } else {
- printNeedle(vid, n, version, false)
- return nil
- }
- }
- if !ok {
- if *showDeleted && tarOutputFile == nil {
- if n.DataSize > 0 {
- printNeedle(vid, n, version, true)
- } else {
- n.Name = []byte("*tombstone")
- printNeedle(vid, n, version, true)
- }
- }
- glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
- } else {
- glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
- }
- return nil
- })
+ err = storage.ScanVolumeFile(*export.dir, *export.collection, vid, storage.NeedleMapInMemory, volumeFileScanner)
if err != nil && err != io.EOF {
glog.Fatalf("Export Volume File [ERROR] %s\n", err)
}
diff --git a/weed/command/fix.go b/weed/command/fix.go
index 3643c9d58..a800978c6 100644
--- a/weed/command/fix.go
+++ b/weed/command/fix.go
@@ -28,6 +28,32 @@ var (
fixVolumeId = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
)
+type VolumeFileScanner4Fix struct {
+ version storage.Version
+ nm *storage.NeedleMap
+}
+
+func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock storage.SuperBlock) error {
+ scanner.version = superBlock.Version()
+ return nil
+
+}
+func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
+ return false
+}
+
+func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *storage.Needle, offset int64) error {
+ glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
+ if n.Size > 0 {
+ pe := scanner.nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
+ glog.V(2).Infof("saved %d with error %v", n.Size, pe)
+ } else {
+ glog.V(2).Infof("skipping deleted file ...")
+ return scanner.nm.Delete(n.Id, types.Offset(offset/types.NeedlePaddingSize))
+ }
+ return nil
+}
+
func runFix(cmd *Command, args []string) bool {
if *fixVolumeId == -1 {
@@ -48,24 +74,12 @@ func runFix(cmd *Command, args []string) bool {
nm := storage.NewBtreeNeedleMap(indexFile)
defer nm.Close()
- var version storage.Version
vid := storage.VolumeId(*fixVolumeId)
- err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid,
- storage.NeedleMapInMemory,
- func(superBlock storage.SuperBlock) error {
- version = superBlock.Version()
- return nil
- }, false, func(n *storage.Needle, offset int64) error {
- glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(version), n.IsGzipped())
- if n.Size > 0 {
- pe := nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
- glog.V(2).Infof("saved %d with error %v", n.Size, pe)
- } else {
- glog.V(2).Infof("skipping deleted file ...")
- return nm.Delete(n.Id, types.Offset(offset/types.NeedlePaddingSize))
- }
- return nil
- })
+ scanner := &VolumeFileScanner4Fix{
+ nm: nm,
+ }
+
+ err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, storage.NeedleMapInMemory, scanner)
if err != nil {
glog.Fatalf("Export Volume File [ERROR] %s\n", err)
os.Remove(indexFileName)
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 13:34:11 +0000