1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
package storage
import (
"os"
"path/filepath"
"testing"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
// TestCalculateExpectedShardSizeWithRealEncoding validates our shard size calculation
// by actually running EC encoding on real files and comparing the results
func TestCalculateExpectedShardSizeWithRealEncoding(t *testing.T) {
tempDir := t.TempDir()
tests := []struct {
name string
datFileSize int64
description string
}{
{
name: "5MB file",
datFileSize: 5 * 1024 * 1024,
description: "Small file that needs 1 small block per shard",
},
{
name: "10MB file (exactly 10 small blocks)",
datFileSize: 10 * 1024 * 1024,
description: "Exactly fits in 1MB small blocks",
},
{
name: "15MB file",
datFileSize: 15 * 1024 * 1024,
description: "Requires 2 small blocks per shard",
},
{
name: "50MB file",
datFileSize: 50 * 1024 * 1024,
description: "Requires 5 small blocks per shard",
},
{
name: "100MB file",
datFileSize: 100 * 1024 * 1024,
description: "Requires 10 small blocks per shard",
},
{
name: "512MB file",
datFileSize: 512 * 1024 * 1024,
description: "Requires 52 small blocks per shard (rounded up)",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create a test .dat file with the specified size
baseFileName := filepath.Join(tempDir, "test_volume")
datFileName := baseFileName + ".dat"
// Create .dat file with random data pattern (so it's compressible but realistic)
datFile, err := os.Create(datFileName)
if err != nil {
t.Fatalf("Failed to create .dat file: %v", err)
}
// Write some pattern data (not all zeros, to be more realistic)
pattern := make([]byte, 4096)
for i := range pattern {
pattern[i] = byte(i % 256)
}
written := int64(0)
for written < tt.datFileSize {
toWrite := tt.datFileSize - written
if toWrite > int64(len(pattern)) {
toWrite = int64(len(pattern))
}
n, err := datFile.Write(pattern[:toWrite])
if err != nil {
t.Fatalf("Failed to write to .dat file: %v", err)
}
written += int64(n)
}
datFile.Close()
// Calculate expected shard size using our function
expectedShardSize := calculateExpectedShardSize(tt.datFileSize)
// Run actual EC encoding
err = erasure_coding.WriteEcFiles(baseFileName)
if err != nil {
t.Fatalf("Failed to encode EC files: %v", err)
}
// Measure actual shard sizes
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
shardFileName := baseFileName + erasure_coding.ToExt(i)
shardInfo, err := os.Stat(shardFileName)
if err != nil {
t.Fatalf("Failed to stat shard file %s: %v", shardFileName, err)
}
actualShardSize := shardInfo.Size()
// Verify actual size matches expected size
if actualShardSize != expectedShardSize {
t.Errorf("Shard %d size mismatch:\n"+
" .dat file size: %d bytes\n"+
" Expected shard size: %d bytes\n"+
" Actual shard size: %d bytes\n"+
" Difference: %d bytes\n"+
" %s",
i, tt.datFileSize, expectedShardSize, actualShardSize,
actualShardSize-expectedShardSize, tt.description)
}
}
// If we got here, all shards match!
t.Logf("✓ SUCCESS: .dat size %d → actual shard size %d matches calculated size (%s)",
tt.datFileSize, expectedShardSize, tt.description)
// Cleanup
os.Remove(datFileName)
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
os.Remove(baseFileName + erasure_coding.ToExt(i))
}
})
}
}
// TestCalculateExpectedShardSizeEdgeCases tests edge cases with real encoding
func TestCalculateExpectedShardSizeEdgeCases(t *testing.T) {
tempDir := t.TempDir()
tests := []struct {
name string
datFileSize int64
}{
{"1 byte file", 1},
{"1KB file", 1024},
{"10KB file", 10 * 1024},
{"1MB file (1 small block)", 1024 * 1024},
{"1MB + 1 byte", 1024*1024 + 1},
{"9.9MB (almost 1 small block per shard)", 9*1024*1024 + 900*1024},
{"10.1MB (just over 1 small block per shard)", 10*1024*1024 + 100*1024},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
baseFileName := filepath.Join(tempDir, tt.name)
datFileName := baseFileName + ".dat"
// Create .dat file
datFile, err := os.Create(datFileName)
if err != nil {
t.Fatalf("Failed to create .dat file: %v", err)
}
// Write exactly the specified number of bytes
data := make([]byte, tt.datFileSize)
for i := range data {
data[i] = byte(i % 256)
}
datFile.Write(data)
datFile.Close()
// Calculate expected
expectedShardSize := calculateExpectedShardSize(tt.datFileSize)
// Run actual EC encoding
err = erasure_coding.WriteEcFiles(baseFileName)
if err != nil {
t.Fatalf("Failed to encode EC files: %v", err)
}
// Check first shard (all should be same size)
shardFileName := baseFileName + erasure_coding.ToExt(0)
shardInfo, err := os.Stat(shardFileName)
if err != nil {
t.Fatalf("Failed to stat shard file: %v", err)
}
actualShardSize := shardInfo.Size()
if actualShardSize != expectedShardSize {
t.Errorf("File size %d: expected shard %d, got %d (diff: %d)",
tt.datFileSize, expectedShardSize, actualShardSize, actualShardSize-expectedShardSize)
} else {
t.Logf("✓ File size %d → shard size %d (correct)", tt.datFileSize, actualShardSize)
}
// Cleanup
os.Remove(datFileName)
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
os.Remove(baseFileName + erasure_coding.ToExt(i))
}
})
}
}
|
