aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--weed/storage/erasure_coding/ec_encoder.go50
-rw-r--r--weed/storage/erasure_coding/ec_test.go89
-rw-r--r--weed/storage/needle_map.go6
-rw-r--r--weed/storage/needle_map/btree_map.go2
-rw-r--r--weed/storage/needle_map/compact_map.go49
-rw-r--r--weed/storage/needle_map/compact_map_test.go2
-rw-r--r--weed/storage/needle_map/needle_value.go13
-rw-r--r--weed/storage/needle_map/needle_value_map.go2
8 files changed, 177 insertions, 36 deletions
diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go
index a873b7ce5..f200297cf 100644
--- a/weed/storage/erasure_coding/ec_encoder.go
+++ b/weed/storage/erasure_coding/ec_encoder.go
@@ -1,9 +1,11 @@
package erasure_coding
import (
+ "fmt"
"io"
"os"
+ "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/klauspost/reedsolomon"
)
@@ -14,7 +16,46 @@ const (
ErasureCodingSmallBlockSize = 1024 * 1024 // 1MB
)
-func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte) error {
+func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte, outputs []*os.File) error {
+
+ bufferSize := int64(len(buffers[0]))
+ batchCount := blockSize/bufferSize
+ if blockSize%bufferSize!=0 {
+ glog.Fatalf("unexpected block size %d buffer size %d", blockSize, bufferSize)
+ }
+
+ for b := int64(0); b < batchCount; b++ {
+ err := encodeDataOneBatch(file, enc, startOffset+b*bufferSize, blockSize, buffers, outputs)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func openEcFiles(baseFileName string) (files []*os.File, err error){
+ for i := 0; i< DataShardsCount+ParityShardsCount; i++{
+ fname := fmt.Sprintf("%s.ec%02d", baseFileName, i+1)
+ f, err := os.OpenFile(fname, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return files, fmt.Errorf("failed to open file %s: %v", fname, err)
+ }
+ files = append(files, f)
+ }
+ return
+}
+
+func closeEcFiles(files []*os.File){
+ for _, f := range files{
+ if f != nil {
+ f.Close()
+ }
+ }
+}
+
+
+func encodeDataOneBatch(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte, outputs []*os.File) error {
// read data into buffers
for i := 0; i < DataShardsCount; i++ {
@@ -36,5 +77,12 @@ func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize i
return err
}
+ for i := 0; i < DataShardsCount+ParityShardsCount; i++ {
+ _, err := outputs[i].Write(buffers[i])
+ if err != nil {
+ return err
+ }
+ }
+
return nil
}
diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go
index b7c3ae0dd..8942bfbe5 100644
--- a/weed/storage/erasure_coding/ec_test.go
+++ b/weed/storage/erasure_coding/ec_test.go
@@ -1,9 +1,13 @@
package erasure_coding
import (
+ "fmt"
"os"
"testing"
+ "github.com/chrislusf/seaweedfs/weed/storage"
+ "github.com/chrislusf/seaweedfs/weed/storage/needle_map"
+ "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/klauspost/reedsolomon"
)
@@ -11,8 +15,9 @@ func TestEncodingDecoding(t *testing.T) {
largeBlockSize := int64(10000)
smallBlockSize := int64(100)
bufferSize := 50
+ baseFileName := "1"
- file, err := os.OpenFile("1.dat", os.O_RDONLY, 0)
+ file, err := os.OpenFile(baseFileName+".dat", os.O_RDONLY, 0)
if err != nil {
t.Logf("failed to open dat file: %v", err)
}
@@ -22,31 +27,95 @@ func TestEncodingDecoding(t *testing.T) {
t.Logf("failed to stat dat file: %v", err)
}
- remainingSize := fi.Size()
- var processedSize int64
+ err = encodeDatFile(fi.Size(), err, baseFileName, bufferSize, largeBlockSize, file, smallBlockSize)
+ if err != nil {
+ t.Logf("failed to stat dat file: %v", err)
+ }
+ file.Close()
- enc, err := reedsolomon.New(DataShardsCount, ParityShardsCount)
+ err = writeSortedEcxFiles(baseFileName)
if err != nil {
- t.Logf("failed to create encoder: %v", err)
+ t.Logf("writeSortedEcxFiles: %v", err)
}
- buffers := make([][]byte, DataShardsCount+ParityShardsCount)
+ err = validateFiles(baseFileName)
+ if err != nil {
+ t.Logf("writeSortedEcxFiles: %v", err)
+ }
+}
+
+func encodeDatFile(remainingSize int64, err error, baseFileName string, bufferSize int, largeBlockSize int64, file *os.File, smallBlockSize int64) error {
+ var processedSize int64
+ enc, err := reedsolomon.New(DataShardsCount, ParityShardsCount)
+ if err != nil {
+ return fmt.Errorf("failed to create encoder: %v", err)
+ }
+ buffers := make([][]byte, DataShardsCount+ParityShardsCount)
+ outputs, err := openEcFiles(baseFileName)
+ defer closeEcFiles(outputs)
+ if err != nil {
+ return fmt.Errorf("failed to open dat file: %v", err)
+ }
for i, _ := range buffers {
buffers[i] = make([]byte, bufferSize)
}
-
for remainingSize > largeBlockSize*DataShardsCount {
- encodeData(file, enc, processedSize, largeBlockSize, buffers)
+ err = encodeData(file, enc, processedSize, largeBlockSize, buffers, outputs)
+ if err != nil {
+ return fmt.Errorf("failed to encode large chunk data: %v", err)
+ }
remainingSize -= largeBlockSize * DataShardsCount
processedSize += largeBlockSize * DataShardsCount
}
-
for remainingSize > 0 {
- encodeData(file, enc, processedSize, smallBlockSize, buffers)
+ encodeData(file, enc, processedSize, smallBlockSize, buffers, outputs)
+ if err != nil {
+ return fmt.Errorf("failed to encode small chunk data: %v", err)
+ }
remainingSize -= smallBlockSize * DataShardsCount
processedSize += smallBlockSize * DataShardsCount
}
+ return nil
+}
+
+func writeSortedEcxFiles(baseFileName string) (e error) {
+
+ var indexFile *os.File
+ if indexFile, e = os.OpenFile(baseFileName+".idx", os.O_RDONLY, 0644); e != nil {
+ return fmt.Errorf("cannot read Volume Index %s.idx: %v", baseFileName, e)
+ }
+
+ cm := needle_map.NewCompactMap()
+ storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
+ if !offset.IsZero() && size != types.TombstoneFileSize {
+ cm.Set(key, offset, size)
+ } else {
+ cm.Delete(key)
+ }
+ return nil
+ })
+
+ ecxFile, err := os.OpenFile(baseFileName+".ecx", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return fmt.Errorf("failed to open dat file: %v", err)
+ }
+ defer ecxFile.Close()
+
+ err = cm.AscendingVisit(func(value needle_map.NeedleValue) error {
+ bytes := value.ToBytes()
+ _, writeErr := ecxFile.Write(bytes)
+ return writeErr
+ })
+
+ if err != nil {
+ return fmt.Errorf("failed to open dat file: %v", err)
+ }
+ return nil
}
+func validateFiles(baseFileName string) error {
+ return nil
+
+}
diff --git a/weed/storage/needle_map.go b/weed/storage/needle_map.go
index 634685175..f2c88093f 100644
--- a/weed/storage/needle_map.go
+++ b/weed/storage/needle_map.go
@@ -62,10 +62,7 @@ func IdxFileEntry(bytes []byte) (key NeedleId, offset Offset, size uint32) {
return
}
func (nm *baseNeedleMapper) appendToIndexFile(key NeedleId, offset Offset, size uint32) error {
- bytes := make([]byte, NeedleIdSize+OffsetSize+SizeSize)
- NeedleIdToBytes(bytes[0:NeedleIdSize], key)
- OffsetToBytes(bytes[NeedleIdSize:NeedleIdSize+OffsetSize], offset)
- util.Uint32toBytes(bytes[NeedleIdSize+OffsetSize:NeedleIdSize+OffsetSize+SizeSize], size)
+ bytes := needle_map.ToBytes(key, offset, size)
nm.indexFileAccessLock.Lock()
defer nm.indexFileAccessLock.Unlock()
@@ -76,6 +73,7 @@ func (nm *baseNeedleMapper) appendToIndexFile(key NeedleId, offset Offset, size
_, err := nm.indexFile.Write(bytes)
return err
}
+
func (nm *baseNeedleMapper) IndexFileContent() ([]byte, error) {
nm.indexFileAccessLock.Lock()
defer nm.indexFileAccessLock.Unlock()
diff --git a/weed/storage/needle_map/btree_map.go b/weed/storage/needle_map/btree_map.go
index 74b4952b7..a26c5e068 100644
--- a/weed/storage/needle_map/btree_map.go
+++ b/weed/storage/needle_map/btree_map.go
@@ -43,7 +43,7 @@ func (cm *BtreeMap) Get(key NeedleId) (*NeedleValue, bool) {
}
// Visit visits all entries or stop if any error when visiting
-func (cm *BtreeMap) Visit(visit func(NeedleValue) error) (ret error) {
+func (cm *BtreeMap) AscendingVisit(visit func(NeedleValue) error) (ret error) {
cm.tree.Ascend(func(item btree.Item) bool {
needle := item.(NeedleValue)
ret = visit(needle)
diff --git a/weed/storage/needle_map/compact_map.go b/weed/storage/needle_map/compact_map.go
index 193d17e47..8aee51ec9 100644
--- a/weed/storage/needle_map/compact_map.go
+++ b/weed/storage/needle_map/compact_map.go
@@ -244,24 +244,37 @@ func (cm *CompactMap) binarySearchCompactSection(key NeedleId) int {
}
// Visit visits all entries or stop if any error when visiting
-func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
+func (cm *CompactMap) AscendingVisit(visit func(NeedleValue) error) error {
for _, cs := range cm.list {
cs.RLock()
- for i, v := range cs.overflow {
- if err := visit(toNeedleValue(cs.overflowExtra[i], v, cs)); err != nil {
+ var i, j int
+ for i, j = 0, 0; i < len(cs.overflow) && j < len(cs.values) && j<cs.counter; {
+ if cs.overflow[i].Key < cs.values[j].Key {
+ if err := visit(toNeedleValue(cs.overflowExtra[i], cs.overflow[i], cs)); err != nil {
+ cs.RUnlock()
+ return err
+ }
+ i++
+ }else if cs.overflow[i].Key == cs.values[j].Key {
+ j++
+ }else{
+ if err := visit(toNeedleValue(cs.valuesExtra[j], cs.values[j], cs)); err != nil {
+ cs.RUnlock()
+ return err
+ }
+ j++
+ }
+ }
+ for ;i < len(cs.overflow);i++{
+ if err := visit(toNeedleValue(cs.overflowExtra[i], cs.overflow[i], cs)); err != nil {
cs.RUnlock()
return err
}
}
- for i, v := range cs.values {
- if i >= cs.counter {
- break
- }
- if _, _, found := cs.findOverflowEntry(v.Key); !found {
- if err := visit(toNeedleValue(cs.valuesExtra[i], v, cs)); err != nil {
- cs.RUnlock()
- return err
- }
+ for ; j < len(cs.values)&& j<cs.counter;j++{
+ if err := visit(toNeedleValue(cs.valuesExtra[j], cs.values[j], cs)); err != nil {
+ cs.RUnlock()
+ return err
}
}
cs.RUnlock()
@@ -279,10 +292,10 @@ func toNeedleValue(snve SectionalNeedleValueExtra, snv SectionalNeedleValue, cs
func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) (SectionalNeedleValue, SectionalNeedleValueExtra) {
return SectionalNeedleValue{
- SectionalNeedleId(nv.Key - cs.start),
- nv.Offset.OffsetLower,
- nv.Size,
- }, SectionalNeedleValueExtra{
- nv.Offset.OffsetHigher,
- }
+ SectionalNeedleId(nv.Key - cs.start),
+ nv.Offset.OffsetLower,
+ nv.Size,
+ }, SectionalNeedleValueExtra{
+ nv.Offset.OffsetHigher,
+ }
}
diff --git a/weed/storage/needle_map/compact_map_test.go b/weed/storage/needle_map/compact_map_test.go
index 4894e59ad..3bad85727 100644
--- a/weed/storage/needle_map/compact_map_test.go
+++ b/weed/storage/needle_map/compact_map_test.go
@@ -19,7 +19,7 @@ func TestOverflow2(t *testing.T) {
m.Set(NeedleId(150158), ToOffset(8), 3000073)
m.Set(NeedleId(150162), ToOffset(8), 3000073)
- m.Visit(func(value NeedleValue) error {
+ m.AscendingVisit(func(value NeedleValue) error {
println("needle key:", value.Key)
return nil
})
diff --git a/weed/storage/needle_map/needle_value.go b/weed/storage/needle_map/needle_value.go
index d0c0b9006..ef540b55e 100644
--- a/weed/storage/needle_map/needle_value.go
+++ b/weed/storage/needle_map/needle_value.go
@@ -2,6 +2,7 @@ package needle_map
import (
. "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "github.com/chrislusf/seaweedfs/weed/util"
"github.com/google/btree"
)
@@ -15,3 +16,15 @@ func (this NeedleValue) Less(than btree.Item) bool {
that := than.(NeedleValue)
return this.Key < that.Key
}
+
+func (nv NeedleValue) ToBytes() []byte {
+ return ToBytes(nv.Key, nv.Offset, nv.Size)
+}
+
+func ToBytes(key NeedleId, offset Offset, size uint32) []byte {
+ bytes := make([]byte, NeedleIdSize+OffsetSize+SizeSize)
+ NeedleIdToBytes(bytes[0:NeedleIdSize], key)
+ OffsetToBytes(bytes[NeedleIdSize:NeedleIdSize+OffsetSize], offset)
+ util.Uint32toBytes(bytes[NeedleIdSize+OffsetSize:NeedleIdSize+OffsetSize+SizeSize], size)
+ return bytes
+}
diff --git a/weed/storage/needle_map/needle_value_map.go b/weed/storage/needle_map/needle_value_map.go
index acf38a4bf..0a5a00ef7 100644
--- a/weed/storage/needle_map/needle_value_map.go
+++ b/weed/storage/needle_map/needle_value_map.go
@@ -8,5 +8,5 @@ type NeedleValueMap interface {
Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32)
Delete(key NeedleId) uint32
Get(key NeedleId) (*NeedleValue, bool)
- Visit(visit func(NeedleValue) error) error
+ AscendingVisit(visit func(NeedleValue) error) error
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 17:26:29 +0000