1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
|
package erasure_coding
import (
"io"
"os"
"testing"
"github.com/klauspost/reedsolomon"
)
func TestEncodingDecoding(t *testing.T) {
largeBlockSize := int64(10000)
smallBlockSize := int64(100)
bufferSize := 50
file, err := os.OpenFile("1.dat", os.O_RDONLY, 0)
if err != nil {
t.Logf("failed to open dat file: %v", err)
}
fi, err := file.Stat()
if err != nil {
t.Logf("failed to stat dat file: %v", err)
}
remainingSize := fi.Size()
var processedSize int64
enc, err := reedsolomon.New(DataShardsCount, ParityShardsCount)
if err != nil {
t.Logf("failed to create encoder: %v", err)
}
buffers := make([][]byte, DataShardsCount+ParityShardsCount)
for i, _ := range buffers {
buffers[i] = make([]byte, bufferSize)
}
for remainingSize > largeBlockSize*DataShardsCount {
encodeData(file, enc, processedSize, largeBlockSize, buffers)
remainingSize -= largeBlockSize * DataShardsCount
processedSize += largeBlockSize * DataShardsCount
}
for remainingSize > 0 {
encodeData(file, enc, processedSize, smallBlockSize, buffers)
remainingSize -= smallBlockSize * DataShardsCount
processedSize += smallBlockSize * DataShardsCount
}
}
func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte) error {
// read data into buffers
for i := 0; i < DataShardsCount; i++ {
n, err := file.ReadAt(buffers[i], startOffset+blockSize*int64(i))
if err != nil {
if err != io.EOF {
return err
}
}
if n < len(buffers[i]) {
for t := len(buffers[i]) - 1; t >= n; t-- {
buffers[i][t] = 0
}
}
}
err := enc.Encode(buffers)
if err != nil {
return err
}
return nil
}
|
