generate ec01~ec14, generate ecx file with sorted needle values

5 files changed, 47 insertions, 21 deletions

diff --git a/weed/storage/needle_map/btree_map.go b/weed/storage/needle_map/btree_map.go
index 74b4952b7..a26c5e068 100644
--- a/weed/storage/needle_map/btree_map.go
+++ b/weed/storage/needle_map/btree_map.go
@@ -43,7 +43,7 @@ func (cm *BtreeMap) Get(key NeedleId) (*NeedleValue, bool) {
 }
 
 // Visit visits all entries or stop if any error when visiting
-func (cm *BtreeMap) Visit(visit func(NeedleValue) error) (ret error) {
+func (cm *BtreeMap) AscendingVisit(visit func(NeedleValue) error) (ret error) {
 	cm.tree.Ascend(func(item btree.Item) bool {
 		needle := item.(NeedleValue)
 		ret = visit(needle)
diff --git a/weed/storage/needle_map/compact_map.go b/weed/storage/needle_map/compact_map.go
index 193d17e47..8aee51ec9 100644
--- a/weed/storage/needle_map/compact_map.go
+++ b/weed/storage/needle_map/compact_map.go
@@ -244,24 +244,37 @@ func (cm *CompactMap) binarySearchCompactSection(key NeedleId) int {
 }
 
 // Visit visits all entries or stop if any error when visiting
-func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
+func (cm *CompactMap) AscendingVisit(visit func(NeedleValue) error) error {
 	for _, cs := range cm.list {
 		cs.RLock()
-		for i, v := range cs.overflow {
-			if err := visit(toNeedleValue(cs.overflowExtra[i], v, cs)); err != nil {
+		var i, j int
+		for i, j = 0, 0; i < len(cs.overflow) && j < len(cs.values) && j<cs.counter; {
+			if cs.overflow[i].Key < cs.values[j].Key {
+				if err := visit(toNeedleValue(cs.overflowExtra[i], cs.overflow[i], cs)); err != nil {
+					cs.RUnlock()
+					return err
+				}
+				i++
+			}else if cs.overflow[i].Key == cs.values[j].Key {
+				j++
+			}else{
+				if err := visit(toNeedleValue(cs.valuesExtra[j], cs.values[j], cs)); err != nil {
+					cs.RUnlock()
+					return err
+				}
+				j++
+			}
+		}
+		for ;i < len(cs.overflow);i++{
+			if err := visit(toNeedleValue(cs.overflowExtra[i], cs.overflow[i], cs)); err != nil {
 				cs.RUnlock()
 				return err
 			}
 		}
-		for i, v := range cs.values {
-			if i >= cs.counter {
-				break
-			}
-			if _, _, found := cs.findOverflowEntry(v.Key); !found {
-				if err := visit(toNeedleValue(cs.valuesExtra[i], v, cs)); err != nil {
-					cs.RUnlock()
-					return err
-				}
+		for ; j < len(cs.values)&& j<cs.counter;j++{
+			if err := visit(toNeedleValue(cs.valuesExtra[j], cs.values[j], cs)); err != nil {
+				cs.RUnlock()
+				return err
 			}
 		}
 		cs.RUnlock()
@@ -279,10 +292,10 @@ func toNeedleValue(snve SectionalNeedleValueExtra, snv SectionalNeedleValue, cs
 
 func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) (SectionalNeedleValue, SectionalNeedleValueExtra) {
 	return SectionalNeedleValue{
-			SectionalNeedleId(nv.Key - cs.start),
-			nv.Offset.OffsetLower,
-			nv.Size,
-		}, SectionalNeedleValueExtra{
-			nv.Offset.OffsetHigher,
-		}
+		SectionalNeedleId(nv.Key - cs.start),
+		nv.Offset.OffsetLower,
+		nv.Size,
+	}, SectionalNeedleValueExtra{
+		nv.Offset.OffsetHigher,
+	}
 }
diff --git a/weed/storage/needle_map/compact_map_test.go b/weed/storage/needle_map/compact_map_test.go
index 4894e59ad..3bad85727 100644
--- a/weed/storage/needle_map/compact_map_test.go
+++ b/weed/storage/needle_map/compact_map_test.go
@@ -19,7 +19,7 @@ func TestOverflow2(t *testing.T) {
 	m.Set(NeedleId(150158), ToOffset(8), 3000073)
 	m.Set(NeedleId(150162), ToOffset(8), 3000073)
 
-	m.Visit(func(value NeedleValue) error {
+	m.AscendingVisit(func(value NeedleValue) error {
 		println("needle key:", value.Key)
 		return nil
 	})
diff --git a/weed/storage/needle_map/needle_value.go b/weed/storage/needle_map/needle_value.go
index d0c0b9006..ef540b55e 100644
--- a/weed/storage/needle_map/needle_value.go
+++ b/weed/storage/needle_map/needle_value.go
@@ -2,6 +2,7 @@ package needle_map
 
 import (
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/google/btree"
 )
 
@@ -15,3 +16,15 @@ func (this NeedleValue) Less(than btree.Item) bool {
 	that := than.(NeedleValue)
 	return this.Key < that.Key
 }
+
+func (nv NeedleValue) ToBytes() []byte {
+	return ToBytes(nv.Key, nv.Offset, nv.Size)
+}
+
+func ToBytes(key NeedleId, offset Offset, size uint32) []byte {
+	bytes := make([]byte, NeedleIdSize+OffsetSize+SizeSize)
+	NeedleIdToBytes(bytes[0:NeedleIdSize], key)
+	OffsetToBytes(bytes[NeedleIdSize:NeedleIdSize+OffsetSize], offset)
+	util.Uint32toBytes(bytes[NeedleIdSize+OffsetSize:NeedleIdSize+OffsetSize+SizeSize], size)
+	return bytes
+}
diff --git a/weed/storage/needle_map/needle_value_map.go b/weed/storage/needle_map/needle_value_map.go
index acf38a4bf..0a5a00ef7 100644
--- a/weed/storage/needle_map/needle_value_map.go
+++ b/weed/storage/needle_map/needle_value_map.go
@@ -8,5 +8,5 @@ type NeedleValueMap interface {
 	Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32)
 	Delete(key NeedleId) uint32
 	Get(key NeedleId) (*NeedleValue, bool)
-	Visit(visit func(NeedleValue) error) error
+	AscendingVisit(visit func(NeedleValue) error) error
 }