diff options
Diffstat (limited to 'weed')
| -rw-r--r-- | weed/command/compact.go | 11 | ||||
| -rw-r--r-- | weed/storage/volume_vacuum.go | 69 |
2 files changed, 78 insertions, 2 deletions
diff --git a/weed/command/compact.go b/weed/command/compact.go index ba2fbf867..db11880ec 100644 --- a/weed/command/compact.go +++ b/weed/command/compact.go @@ -23,6 +23,7 @@ var ( compactVolumePath = cmdCompact.Flag.String("dir", ".", "data directory to store files") compactVolumeCollection = cmdCompact.Flag.String("collection", "", "volume collection name") compactVolumeId = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.") + compactMethod = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 or 1.") ) func runCompact(cmd *Command, args []string) bool { @@ -37,8 +38,14 @@ func runCompact(cmd *Command, args []string) bool { if err != nil { glog.Fatalf("Load Volume [ERROR] %s\n", err) } - if err = v.Compact(); err != nil { - glog.Fatalf("Compact Volume [ERROR] %s\n", err) + if *compactMethod == 0 { + if err = v.Compact(); err != nil { + glog.Fatalf("Compact Volume [ERROR] %s\n", err) + } + } else { + if err = v.Compact2(); err != nil { + glog.Fatalf("Compact Volume [ERROR] %s\n", err) + } } return true diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 9b9a27816..785e2ff88 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -23,6 +23,14 @@ func (v *Volume) Compact() error { glog.V(3).Infof("creating copies for volume %d ...", v.Id) return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx") } + +func (v *Volume) Compact2() error { + glog.V(3).Infof("Compact2 ...") + filePath := v.FileName() + glog.V(3).Infof("creating copies for volume %d ...", v.Id) + return v.copyDataBasedOnIndexFile(filePath+".cpd", filePath+".cpx") +} + func (v *Volume) commitCompact() error { glog.V(3).Infof("Committing vacuuming...") v.dataFileAccessLock.Lock() @@ -91,3 +99,64 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err erro return } + +func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) { + var ( + dst, idx, oldIndexFile *os.File + ) + if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil { + return + } + defer dst.Close() + + if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil { + return + } + defer idx.Close() + + if oldIndexFile, err = os.OpenFile(v.FileName()+".idx", os.O_RDONLY, 0644); err != nil { + return + } + defer oldIndexFile.Close() + + nm := NewNeedleMap(idx) + now := uint64(time.Now().Unix()) + + v.SuperBlock.CompactRevision++ + dst.Write(v.SuperBlock.Bytes()) + new_offset := int64(SuperBlockSize) + + WalkIndexFile(oldIndexFile, func(key uint64, offset, size uint32) error { + if size <= 0 { + return nil + } + + nv, ok := v.nm.Get(key) + if !ok { + return nil + } + + n := new(Needle) + n.ReadData(v.dataFile, int64(offset)*NeedlePaddingSize, size, v.Version()) + defer n.ReleaseMemory() + + if n.HasTtl() && now >= n.LastModified+uint64(v.Ttl.Minutes()*60) { + return nil + } + + glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv) + if nv.Offset == offset && nv.Size > 0 { + if err = nm.Put(n.Id, uint32(new_offset/NeedlePaddingSize), n.Size); err != nil { + return fmt.Errorf("cannot put needle: %s", err) + } + if _, err = n.Append(dst, v.Version()); err != nil { + return fmt.Errorf("cannot append needle: %s", err) + } + new_offset += n.DiskSize() + glog.V(3).Infoln("saving key", n.Id, "volume offset", offset, "=>", new_offset, "data_size", n.Size) + } + return nil + }) + + return +} |