diff options
Diffstat (limited to 'weed/storage')
| -rw-r--r-- | weed/storage/erasure_coding/ec_decoder.go | 22 | ||||
| -rw-r--r-- | weed/storage/erasure_coding/ec_decoder_test.go | 81 |
2 files changed, 103 insertions, 0 deletions
diff --git a/weed/storage/erasure_coding/ec_decoder.go b/weed/storage/erasure_coding/ec_decoder.go index a1d929f6c..429dd7ac4 100644 --- a/weed/storage/erasure_coding/ec_decoder.go +++ b/weed/storage/erasure_coding/ec_decoder.go @@ -14,6 +14,23 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util" ) +// EcNoLiveEntriesSubstring is used for server/client coordination when ec.decode determines that +// decoding should be a no-op (all entries are deleted). +const EcNoLiveEntriesSubstring = "has no live entries" + +// HasLiveNeedles returns whether the EC index (.ecx) contains at least one live (non-deleted) entry. +// This is used by ec.decode to avoid generating an empty normal volume when all entries were deleted. +func HasLiveNeedles(indexBaseFileName string) (hasLive bool, err error) { + err = iterateEcxFile(indexBaseFileName, func(_ types.NeedleId, _ types.Offset, size types.Size) error { + if !size.IsDeleted() { + hasLive = true + return io.EOF // stop early + } + return nil + }) + return +} + // write .idx file from .ecx and .ecj files func WriteIdxFileFromEcIndex(baseFileName string) (err error) { @@ -52,6 +69,11 @@ func FindDatFileSize(dataBaseFileName, indexBaseFileName string) (datSize int64, return 0, fmt.Errorf("read ec volume %s version: %v", dataBaseFileName, err) } + // Safety: ensure datSize is at least SuperBlockSize. While the caller typically + // checks HasLiveNeedles first, this protects against direct calls to FindDatFileSize + // when all needles are deleted (see issue #7748). + datSize = int64(super_block.SuperBlockSize) + err = iterateEcxFile(indexBaseFileName, func(key types.NeedleId, offset types.Offset, size types.Size) error { if size.IsDeleted() { diff --git a/weed/storage/erasure_coding/ec_decoder_test.go b/weed/storage/erasure_coding/ec_decoder_test.go new file mode 100644 index 000000000..625d55402 --- /dev/null +++ b/weed/storage/erasure_coding/ec_decoder_test.go @@ -0,0 +1,81 @@ +package erasure_coding_test + +import ( + "os" + "path/filepath" + "testing" + + erasure_coding "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +func TestHasLiveNeedles_AllDeletedIsFalse(t *testing.T) { + dir := t.TempDir() + + collection := "foo" + base := filepath.Join(dir, collection+"_1") + + // Build an ecx file with only deleted entries. + // ecx file entries are the same format as .idx entries. + ecx := makeNeedleMapEntry(types.NeedleId(1), types.Offset{}, types.TombstoneFileSize) + if err := os.WriteFile(base+".ecx", ecx, 0644); err != nil { + t.Fatalf("write ecx: %v", err) + } + + hasLive, err := erasure_coding.HasLiveNeedles(base) + if err != nil { + t.Fatalf("HasLiveNeedles: %v", err) + } + if hasLive { + t.Fatalf("expected no live entries") + } +} + +func TestHasLiveNeedles_WithLiveEntryIsTrue(t *testing.T) { + dir := t.TempDir() + + collection := "foo" + base := filepath.Join(dir, collection+"_1") + + // Build an ecx file containing at least one live entry. + // ecx file entries are the same format as .idx entries. + live := makeNeedleMapEntry(types.NeedleId(1), types.Offset{}, types.Size(1)) + if err := os.WriteFile(base+".ecx", live, 0644); err != nil { + t.Fatalf("write ecx: %v", err) + } + + hasLive, err := erasure_coding.HasLiveNeedles(base) + if err != nil { + t.Fatalf("HasLiveNeedles: %v", err) + } + if !hasLive { + t.Fatalf("expected live entries") + } +} + +func TestHasLiveNeedles_EmptyFileIsFalse(t *testing.T) { + dir := t.TempDir() + + base := filepath.Join(dir, "foo_1") + + // Create an empty ecx file. + if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil { + t.Fatalf("write ecx: %v", err) + } + + hasLive, err := erasure_coding.HasLiveNeedles(base) + if err != nil { + t.Fatalf("HasLiveNeedles: %v", err) + } + if hasLive { + t.Fatalf("expected no live entries for empty file") + } +} + +func makeNeedleMapEntry(key types.NeedleId, offset types.Offset, size types.Size) []byte { + b := make([]byte, types.NeedleIdSize+types.OffsetSize+types.SizeSize) + types.NeedleIdToBytes(b[0:types.NeedleIdSize], key) + types.OffsetToBytes(b[types.NeedleIdSize:types.NeedleIdSize+types.OffsetSize], offset) + types.SizeToBytes(b[types.NeedleIdSize+types.OffsetSize:types.NeedleIdSize+types.OffsetSize+types.SizeSize], size) + return b +} |
