aboutsummaryrefslogtreecommitdiff
path: root/weed/storage/erasure_coding
diff options
context:
space:
mode:
Diffstat (limited to 'weed/storage/erasure_coding')
-rw-r--r--weed/storage/erasure_coding/ec_locate.go12
-rw-r--r--weed/storage/erasure_coding/ec_shard.go6
-rw-r--r--weed/storage/erasure_coding/ec_test.go19
-rw-r--r--weed/storage/erasure_coding/ec_volume.go13
4 files changed, 29 insertions, 21 deletions
diff --git a/weed/storage/erasure_coding/ec_locate.go b/weed/storage/erasure_coding/ec_locate.go
index ee8af3382..562966f8f 100644
--- a/weed/storage/erasure_coding/ec_locate.go
+++ b/weed/storage/erasure_coding/ec_locate.go
@@ -69,3 +69,15 @@ func locateOffsetWithinBlocks(blockLength int64, offset int64) (blockIndex int,
innerBlockOffset = offset % blockLength
return
}
+
+func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64) {
+ ecFileOffset := interval.InnerBlockOffset
+ rowIndex := interval.BlockIndex / DataShardsCount
+ if interval.IsLargeBlock {
+ ecFileOffset += int64(rowIndex) * largeBlockSize
+ } else {
+ ecFileOffset += int64(interval.LargeBlockRowsCount)*largeBlockSize + int64(rowIndex)*smallBlockSize
+ }
+ ecFileIndex := interval.BlockIndex % DataShardsCount
+ return ShardId(ecFileIndex), ecFileOffset
+}
diff --git a/weed/storage/erasure_coding/ec_shard.go b/weed/storage/erasure_coding/ec_shard.go
index 084d3f03b..fa4bfcecd 100644
--- a/weed/storage/erasure_coding/ec_shard.go
+++ b/weed/storage/erasure_coding/ec_shard.go
@@ -105,3 +105,9 @@ func (shard *EcVolumeShard) findNeedleFromEcx(needleId types.NeedleId) (offset t
err = fmt.Errorf("needle id %d not found", needleId)
return
}
+
+func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error) {
+
+ return shard.ecdFile.ReadAt(buf, offset)
+
+}
diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go
index 83b0bc23a..e2e872dbe 100644
--- a/weed/storage/erasure_coding/ec_test.go
+++ b/weed/storage/erasure_coding/ec_test.go
@@ -105,10 +105,8 @@ func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size uin
intervals := LocateData(largeBlockSize, smallBlockSize, datSize, offset.ToAcutalOffset(), size)
- nLargeBlockRows := int(datSize / (largeBlockSize * DataShardsCount))
-
for i, interval := range intervals {
- if d, e := readOneInterval(interval, ecFiles, nLargeBlockRows); e != nil {
+ if d, e := readOneInterval(interval, ecFiles); e != nil {
return nil, e
} else {
if i == 0 {
@@ -122,21 +120,14 @@ func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size uin
return data, nil
}
-func readOneInterval(interval Interval, ecFiles []*os.File, nLargeBlockRows int) (data []byte, err error) {
- ecFileOffset := interval.InnerBlockOffset
- rowIndex := interval.BlockIndex / DataShardsCount
- if interval.IsLargeBlock {
- ecFileOffset += int64(rowIndex) * largeBlockSize
- } else {
- ecFileOffset += int64(nLargeBlockRows)*largeBlockSize + int64(rowIndex)*smallBlockSize
- }
+func readOneInterval(interval Interval, ecFiles []*os.File) (data []byte, err error) {
- ecFileIndex := interval.BlockIndex % DataShardsCount
+ ecFileOffset, ecFileIndex := interval.ToShardIdAndOffset(largeBlockSize, smallBlockSize)
data = make([]byte, interval.Size)
err = readFromFile(ecFiles[ecFileIndex], data, ecFileOffset)
{ // do some ec testing
- ecData, err := readFromOtherEcFiles(ecFiles, ecFileIndex, ecFileOffset, interval.Size)
+ ecData, err := readFromOtherEcFiles(ecFiles, int(ecFileIndex), ecFileOffset, interval.Size)
if err != nil {
return nil, fmt.Errorf("ec reconstruct error: %v", err)
}
@@ -198,7 +189,7 @@ func TestLocateData(t *testing.T) {
if len(intervals) != 1 {
t.Errorf("unexpected interval size %d", len(intervals))
}
- if !intervals[0].sameAs(Interval{0, 0, 1, false}) {
+ if !intervals[0].sameAs(Interval{0, 0, 1, false, 1}) {
t.Errorf("unexpected interval %+v", intervals[0])
}
diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go
index d57a28449..881e88d5f 100644
--- a/weed/storage/erasure_coding/ec_volume.go
+++ b/weed/storage/erasure_coding/ec_volume.go
@@ -6,6 +6,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
+ "github.com/chrislusf/seaweedfs/weed/storage/types"
)
type EcVolumeShards []*EcVolumeShard
@@ -71,19 +72,17 @@ func (shards *EcVolumeShards) ToVolumeEcShardInformationMessage() (messages []*m
return
}
-func (shards *EcVolumeShards) ReadEcShardNeedle(n *needle.Needle) (int, error) {
+func (shards *EcVolumeShards) LocateEcShardNeedle(n *needle.Needle) (offset types.Offset, size uint32, intervals []Interval, err error) {
shard := (*shards)[0]
// find the needle from ecx file
- offset, size, err := shard.findNeedleFromEcx(n.Id)
+ offset, size, err = shard.findNeedleFromEcx(n.Id)
if err != nil {
- return 0, err
+ return types.Offset{}, 0, nil, err
}
// calculate the locations in the ec shards
- intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shard.ecxFileSize, offset.ToAcutalOffset(), size)
+ intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shard.ecxFileSize, offset.ToAcutalOffset(), size)
- // TODO read the intervals
-
- return len(intervals), nil
+ return
}