aboutsummaryrefslogtreecommitdiff
path: root/unmaintained/remove_duplicate_fids
diff options
context:
space:
mode:
Diffstat (limited to 'unmaintained/remove_duplicate_fids')
-rw-r--r--unmaintained/remove_duplicate_fids/remove_duplicate_fids.go109
1 files changed, 54 insertions, 55 deletions
diff --git a/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go b/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go
index 5716ffa90..4b37a64fb 100644
--- a/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go
+++ b/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go
@@ -1,92 +1,91 @@
package main
import (
- "flag"
-
- "github.com/chrislusf/seaweedfs/weed/glog"
- "github.com/chrislusf/seaweedfs/weed/storage"
- "github.com/chrislusf/seaweedfs/weed/storage/needle"
- "os"
- "path/filepath"
-
- "fmt"
+ "flag"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "github.com/chrislusf/seaweedfs/weed/glog"
+ "github.com/chrislusf/seaweedfs/weed/storage"
+ "github.com/chrislusf/seaweedfs/weed/storage/needle"
)
var (
- volumePath = flag.String("dir", "/tmp", "data directory to store files")
- volumeCollection = flag.String("collection", "", "the volume collection name")
- volumeId = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+ volumePath = flag.String("dir", "/tmp", "data directory to store files")
+ volumeCollection = flag.String("collection", "", "the volume collection name")
+ volumeId = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
)
-func Checksum(n* needle.Needle) string {
- return fmt.Sprintf("%s%x", n.Id, n.Cookie)
+func Checksum(n *needle.Needle) string {
+ return fmt.Sprintf("%s%x", n.Id, n.Cookie)
}
type VolumeFileScanner4SeeDat struct {
- version needle.Version
- block storage.SuperBlock
+ version needle.Version
+ block storage.SuperBlock
- dir string
- hashes map[string]bool
- dat * os.File
+ dir string
+ hashes map[string]bool
+ dat *os.File
}
func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.SuperBlock) error {
- scanner.version = superBlock.Version()
- scanner.block = superBlock
- return nil
+ scanner.version = superBlock.Version()
+ scanner.block = superBlock
+ return nil
}
func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
- return true
+ return true
}
func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64) error {
- if scanner.dat == nil {
- newDatFile, err := os.Create(filepath.Join(*volumePath, "dat_fixed"))
- if err != nil {
- glog.Fatalf("Write New Volume Data %v", err)
+ if scanner.dat == nil {
+ newDatFile, err := os.Create(filepath.Join(*volumePath, "dat_fixed"))
+ if err != nil {
+ glog.Fatalf("Write New Volume Data %v", err)
+ }
+ scanner.dat = newDatFile
+ scanner.dat.Write(scanner.block.Bytes())
}
- scanner.dat = newDatFile
- scanner.dat.Write(scanner.block.Bytes())
- }
- checksum := Checksum(n)
+ checksum := Checksum(n)
- if scanner.hashes[checksum] {
- glog.V(0).Infof("duplicate checksum:%s fid:%d,%s%x @ offset:%d", checksum, *volumeId, n.Id, n.Cookie, offset)
- return nil
- }
- scanner.hashes[checksum] = true
+ if scanner.hashes[checksum] {
+ glog.V(0).Infof("duplicate checksum:%s fid:%d,%s%x @ offset:%d", checksum, *volumeId, n.Id, n.Cookie, offset)
+ return nil
+ }
+ scanner.hashes[checksum] = true
- _, s, _, e := n.Append(scanner.dat, scanner.version)
- fmt.Printf("size %d error %v\n", s, e)
+ _, s, _, e := n.Append(scanner.dat, scanner.version)
+ fmt.Printf("size %d error %v\n", s, e)
- return nil
+ return nil
}
func main() {
- flag.Parse()
+ flag.Parse()
- vid := needle.VolumeId(*volumeId)
+ vid := needle.VolumeId(*volumeId)
- outpath, _ := filepath.Abs(filepath.Dir(os.Args[0]))
+ outpath, _ := filepath.Abs(filepath.Dir(os.Args[0]))
- scanner := &VolumeFileScanner4SeeDat{
- dir: filepath.Join(outpath, "out"),
- hashes: map[string]bool{},
- }
+ scanner := &VolumeFileScanner4SeeDat{
+ dir: filepath.Join(outpath, "out"),
+ hashes: map[string]bool{},
+ }
- if _, err := os.Stat(scanner.dir); err != nil {
- if err := os.MkdirAll(scanner.dir, os.ModePerm); err != nil {
- glog.Fatalf("could not create output dir : %s", err)
- }
- }
+ if _, err := os.Stat(scanner.dir); err != nil {
+ if err := os.MkdirAll(scanner.dir, os.ModePerm); err != nil {
+ glog.Fatalf("could not create output dir : %s", err)
+ }
+ }
- err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
- if err != nil {
- glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
- }
+ err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
+ if err != nil {
+ glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
+ }
}