diff options
Diffstat (limited to 'weed/storage/erasure_coding')
| -rw-r--r-- | weed/storage/erasure_coding/ec_locate.go | 12 | ||||
| -rw-r--r-- | weed/storage/erasure_coding/ec_shard.go | 6 | ||||
| -rw-r--r-- | weed/storage/erasure_coding/ec_test.go | 19 | ||||
| -rw-r--r-- | weed/storage/erasure_coding/ec_volume.go | 13 |
4 files changed, 29 insertions, 21 deletions
diff --git a/weed/storage/erasure_coding/ec_locate.go b/weed/storage/erasure_coding/ec_locate.go index ee8af3382..562966f8f 100644 --- a/weed/storage/erasure_coding/ec_locate.go +++ b/weed/storage/erasure_coding/ec_locate.go @@ -69,3 +69,15 @@ func locateOffsetWithinBlocks(blockLength int64, offset int64) (blockIndex int, innerBlockOffset = offset % blockLength return } + +func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64) { + ecFileOffset := interval.InnerBlockOffset + rowIndex := interval.BlockIndex / DataShardsCount + if interval.IsLargeBlock { + ecFileOffset += int64(rowIndex) * largeBlockSize + } else { + ecFileOffset += int64(interval.LargeBlockRowsCount)*largeBlockSize + int64(rowIndex)*smallBlockSize + } + ecFileIndex := interval.BlockIndex % DataShardsCount + return ShardId(ecFileIndex), ecFileOffset +} diff --git a/weed/storage/erasure_coding/ec_shard.go b/weed/storage/erasure_coding/ec_shard.go index 084d3f03b..fa4bfcecd 100644 --- a/weed/storage/erasure_coding/ec_shard.go +++ b/weed/storage/erasure_coding/ec_shard.go @@ -105,3 +105,9 @@ func (shard *EcVolumeShard) findNeedleFromEcx(needleId types.NeedleId) (offset t err = fmt.Errorf("needle id %d not found", needleId) return } + +func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error) { + + return shard.ecdFile.ReadAt(buf, offset) + +} diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go index 83b0bc23a..e2e872dbe 100644 --- a/weed/storage/erasure_coding/ec_test.go +++ b/weed/storage/erasure_coding/ec_test.go @@ -105,10 +105,8 @@ func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size uin intervals := LocateData(largeBlockSize, smallBlockSize, datSize, offset.ToAcutalOffset(), size) - nLargeBlockRows := int(datSize / (largeBlockSize * DataShardsCount)) - for i, interval := range intervals { - if d, e := readOneInterval(interval, ecFiles, nLargeBlockRows); e != nil { + if d, e := readOneInterval(interval, ecFiles); e != nil { return nil, e } else { if i == 0 { @@ -122,21 +120,14 @@ func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size uin return data, nil } -func readOneInterval(interval Interval, ecFiles []*os.File, nLargeBlockRows int) (data []byte, err error) { - ecFileOffset := interval.InnerBlockOffset - rowIndex := interval.BlockIndex / DataShardsCount - if interval.IsLargeBlock { - ecFileOffset += int64(rowIndex) * largeBlockSize - } else { - ecFileOffset += int64(nLargeBlockRows)*largeBlockSize + int64(rowIndex)*smallBlockSize - } +func readOneInterval(interval Interval, ecFiles []*os.File) (data []byte, err error) { - ecFileIndex := interval.BlockIndex % DataShardsCount + ecFileOffset, ecFileIndex := interval.ToShardIdAndOffset(largeBlockSize, smallBlockSize) data = make([]byte, interval.Size) err = readFromFile(ecFiles[ecFileIndex], data, ecFileOffset) { // do some ec testing - ecData, err := readFromOtherEcFiles(ecFiles, ecFileIndex, ecFileOffset, interval.Size) + ecData, err := readFromOtherEcFiles(ecFiles, int(ecFileIndex), ecFileOffset, interval.Size) if err != nil { return nil, fmt.Errorf("ec reconstruct error: %v", err) } @@ -198,7 +189,7 @@ func TestLocateData(t *testing.T) { if len(intervals) != 1 { t.Errorf("unexpected interval size %d", len(intervals)) } - if !intervals[0].sameAs(Interval{0, 0, 1, false}) { + if !intervals[0].sameAs(Interval{0, 0, 1, false, 1}) { t.Errorf("unexpected interval %+v", intervals[0]) } diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go index d57a28449..881e88d5f 100644 --- a/weed/storage/erasure_coding/ec_volume.go +++ b/weed/storage/erasure_coding/ec_volume.go @@ -6,6 +6,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/types" ) type EcVolumeShards []*EcVolumeShard @@ -71,19 +72,17 @@ func (shards *EcVolumeShards) ToVolumeEcShardInformationMessage() (messages []*m return } -func (shards *EcVolumeShards) ReadEcShardNeedle(n *needle.Needle) (int, error) { +func (shards *EcVolumeShards) LocateEcShardNeedle(n *needle.Needle) (offset types.Offset, size uint32, intervals []Interval, err error) { shard := (*shards)[0] // find the needle from ecx file - offset, size, err := shard.findNeedleFromEcx(n.Id) + offset, size, err = shard.findNeedleFromEcx(n.Id) if err != nil { - return 0, err + return types.Offset{}, 0, nil, err } // calculate the locations in the ec shards - intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shard.ecxFileSize, offset.ToAcutalOffset(), size) + intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shard.ecxFileSize, offset.ToAcutalOffset(), size) - // TODO read the intervals - - return len(intervals), nil + return } |
