1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
|
package erasure_coding
import (
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
type Interval struct {
BlockIndex int
InnerBlockOffset int64
Size types.Size
IsLargeBlock bool
LargeBlockRowsCount int
}
func LocateData(largeBlockLength, smallBlockLength int64, datSize int64, offset int64, size types.Size) (intervals []Interval) {
blockIndex, isLargeBlock, innerBlockOffset := locateOffset(largeBlockLength, smallBlockLength, datSize, offset)
// adding DataShardsCount*smallBlockLength to ensure we can derive the number of large block size from a shard size
nLargeBlockRows := int((datSize + DataShardsCount*smallBlockLength) / (largeBlockLength * DataShardsCount))
for size > 0 {
interval := Interval{
BlockIndex: blockIndex,
InnerBlockOffset: innerBlockOffset,
IsLargeBlock: isLargeBlock,
LargeBlockRowsCount: nLargeBlockRows,
}
blockRemaining := largeBlockLength - innerBlockOffset
if !isLargeBlock {
blockRemaining = smallBlockLength - innerBlockOffset
}
if int64(size) <= blockRemaining {
interval.Size = size
intervals = append(intervals, interval)
return
}
interval.Size = types.Size(blockRemaining)
intervals = append(intervals, interval)
size -= interval.Size
blockIndex += 1
if isLargeBlock && blockIndex == nLargeBlockRows*DataShardsCount {
isLargeBlock = false
blockIndex = 0
}
innerBlockOffset = 0
}
return
}
func locateOffset(largeBlockLength, smallBlockLength int64, datSize int64, offset int64) (blockIndex int, isLargeBlock bool, innerBlockOffset int64) {
largeRowSize := largeBlockLength * DataShardsCount
nLargeBlockRows := datSize / (largeBlockLength * DataShardsCount)
// if offset is within the large block area
if offset < nLargeBlockRows*largeRowSize {
isLargeBlock = true
blockIndex, innerBlockOffset = locateOffsetWithinBlocks(largeBlockLength, offset)
return
}
isLargeBlock = false
offset -= nLargeBlockRows * largeRowSize
blockIndex, innerBlockOffset = locateOffsetWithinBlocks(smallBlockLength, offset)
return
}
func locateOffsetWithinBlocks(blockLength int64, offset int64) (blockIndex int, innerBlockOffset int64) {
blockIndex = int(offset / blockLength)
innerBlockOffset = offset % blockLength
return
}
func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64) {
ecFileOffset := interval.InnerBlockOffset
rowIndex := interval.BlockIndex / DataShardsCount
if interval.IsLargeBlock {
ecFileOffset += int64(rowIndex) * largeBlockSize
} else {
ecFileOffset += int64(interval.LargeBlockRowsCount)*largeBlockSize + int64(rowIndex)*smallBlockSize
}
ecFileIndex := interval.BlockIndex % DataShardsCount
return ShardId(ecFileIndex), ecFileOffset
}
|
