aboutsummaryrefslogtreecommitdiff
path: root/weed/storage/erasure_coding
diff options
context:
space:
mode:
Diffstat (limited to 'weed/storage/erasure_coding')
-rw-r--r--weed/storage/erasure_coding/ec_locate.go17
-rw-r--r--weed/storage/erasure_coding/ec_test.go12
2 files changed, 19 insertions, 10 deletions
diff --git a/weed/storage/erasure_coding/ec_locate.go b/weed/storage/erasure_coding/ec_locate.go
index be7e085e6..4b092695c 100644
--- a/weed/storage/erasure_coding/ec_locate.go
+++ b/weed/storage/erasure_coding/ec_locate.go
@@ -5,25 +5,22 @@ import (
)
type Interval struct {
- BlockIndex int
+ BlockIndex int // the index of the block in either the large blocks or the small blocks
InnerBlockOffset int64
Size types.Size
- IsLargeBlock bool
+ IsLargeBlock bool // whether the block is a large block or a small block
LargeBlockRowsCount int
}
func LocateData(largeBlockLength, smallBlockLength int64, datSize int64, offset int64, size types.Size) (intervals []Interval) {
- blockIndex, isLargeBlock, innerBlockOffset := locateOffset(largeBlockLength, smallBlockLength, datSize, offset)
-
- // adding DataShardsCount*smallBlockLength to ensure we can derive the number of large block size from a shard size
- nLargeBlockRows := int((datSize + DataShardsCount*smallBlockLength) / (largeBlockLength * DataShardsCount))
+ blockIndex, isLargeBlock, nLargeBlockRows, innerBlockOffset := locateOffset(largeBlockLength, smallBlockLength, datSize, offset)
for size > 0 {
interval := Interval{
BlockIndex: blockIndex,
InnerBlockOffset: innerBlockOffset,
IsLargeBlock: isLargeBlock,
- LargeBlockRowsCount: nLargeBlockRows,
+ LargeBlockRowsCount: int(nLargeBlockRows),
}
blockRemaining := largeBlockLength - innerBlockOffset
@@ -41,7 +38,7 @@ func LocateData(largeBlockLength, smallBlockLength int64, datSize int64, offset
size -= interval.Size
blockIndex += 1
- if isLargeBlock && blockIndex == nLargeBlockRows*DataShardsCount {
+ if isLargeBlock && blockIndex == interval.LargeBlockRowsCount*DataShardsCount {
isLargeBlock = false
blockIndex = 0
}
@@ -51,9 +48,9 @@ func LocateData(largeBlockLength, smallBlockLength int64, datSize int64, offset
return
}
-func locateOffset(largeBlockLength, smallBlockLength int64, datSize int64, offset int64) (blockIndex int, isLargeBlock bool, innerBlockOffset int64) {
+func locateOffset(largeBlockLength, smallBlockLength int64, datSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) {
largeRowSize := largeBlockLength * DataShardsCount
- nLargeBlockRows := datSize / (largeBlockLength * DataShardsCount)
+ nLargeBlockRows = datSize / (largeBlockLength * DataShardsCount)
// if offset is within the large block area
if offset < nLargeBlockRows*largeRowSize {
diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go
index aa46cf6d4..fc0adbe9f 100644
--- a/weed/storage/erasure_coding/ec_test.go
+++ b/weed/storage/erasure_coding/ec_test.go
@@ -3,6 +3,7 @@ package erasure_coding
import (
"bytes"
"fmt"
+ "github.com/stretchr/testify/assert"
"math/rand"
"os"
"testing"
@@ -208,3 +209,14 @@ func (this Interval) sameAs(that Interval) bool {
this.BlockIndex == that.BlockIndex &&
this.Size == that.Size
}
+
+func TestLocateData2(t *testing.T) {
+ intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, 32205678320, 21479557912, 4194339)
+ assert.Equal(t, intervals, []Interval{
+ {BlockIndex: 4, InnerBlockOffset: 527128, Size: 521448, IsLargeBlock: false, LargeBlockRowsCount: 2},
+ {BlockIndex: 5, InnerBlockOffset: 0, Size: 1048576, IsLargeBlock: false, LargeBlockRowsCount: 2},
+ {BlockIndex: 6, InnerBlockOffset: 0, Size: 1048576, IsLargeBlock: false, LargeBlockRowsCount: 2},
+ {BlockIndex: 7, InnerBlockOffset: 0, Size: 1048576, IsLargeBlock: false, LargeBlockRowsCount: 2},
+ {BlockIndex: 8, InnerBlockOffset: 0, Size: 527163, IsLargeBlock: false, LargeBlockRowsCount: 2},
+ })
+}