Admin UI: include ec shard sizes into volume server info (#7071)

* show ec shards on dashboard, show max in its own column

* master collect shard size info

* master send shard size via VolumeList

* change to more efficient shard sizes slice

* include ec shard sizes into volume server info

* Eliminated Redundant gRPC Calls

* much more efficient

* Efficient Counting: bits.OnesCount32() uses CPU-optimized instructions to count set bits in O(1)

* avoid extra volume list call

* simplify

* preserve existing shard sizes

* avoid hard coded value

* Update weed/storage/erasure_coding/ec_volume_info.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/admin/dash/volume_management.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update ec_volume_info.go

* address comments

* avoid duplicated functions

* Update weed/admin/dash/volume_management.go

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* simplify

* refactoring

* fix compilation

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

5 files changed, 333 insertions, 18 deletions

diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go
index 5db65a2c8..eeeb156e6 100644
--- a/weed/storage/erasure_coding/ec_encoder.go
+++ b/weed/storage/erasure_coding/ec_encoder.go
@@ -18,6 +18,7 @@ const (
 	DataShardsCount             = 10
 	ParityShardsCount           = 4
 	TotalShardsCount            = DataShardsCount + ParityShardsCount
+	MinTotalDisks               = TotalShardsCount/ParityShardsCount + 1
 	ErasureCodingLargeBlockSize = 1024 * 1024 * 1024 // 1GB
 	ErasureCodingSmallBlockSize = 1024 * 1024        // 1MB
 )
diff --git a/weed/storage/erasure_coding/ec_shard_size_helper.go b/weed/storage/erasure_coding/ec_shard_size_helper.go
new file mode 100644
index 000000000..43d9a4f2f
--- /dev/null
+++ b/weed/storage/erasure_coding/ec_shard_size_helper.go
@@ -0,0 +1,68 @@
+package erasure_coding
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// GetShardSize returns the size of a specific shard from VolumeEcShardInformationMessage
+// Returns the size and true if the shard exists, 0 and false if not present
+func GetShardSize(msg *master_pb.VolumeEcShardInformationMessage, shardId ShardId) (size int64, found bool) {
+	if msg == nil || msg.ShardSizes == nil {
+		return 0, false
+	}
+
+	shardBits := ShardBits(msg.EcIndexBits)
+	index, found := shardBits.ShardIdToIndex(shardId)
+	if !found || index >= len(msg.ShardSizes) {
+		return 0, false
+	}
+
+	return msg.ShardSizes[index], true
+}
+
+// SetShardSize sets the size of a specific shard in VolumeEcShardInformationMessage
+// Returns true if successful, false if the shard is not present in EcIndexBits
+func SetShardSize(msg *master_pb.VolumeEcShardInformationMessage, shardId ShardId, size int64) bool {
+	if msg == nil {
+		return false
+	}
+
+	shardBits := ShardBits(msg.EcIndexBits)
+	index, found := shardBits.ShardIdToIndex(shardId)
+	if !found {
+		return false
+	}
+
+	// Initialize ShardSizes slice if needed
+	expectedLength := shardBits.ShardIdCount()
+	if msg.ShardSizes == nil {
+		msg.ShardSizes = make([]int64, expectedLength)
+	} else if len(msg.ShardSizes) != expectedLength {
+		// Resize the slice to match the expected length
+		newSizes := make([]int64, expectedLength)
+		copy(newSizes, msg.ShardSizes)
+		msg.ShardSizes = newSizes
+	}
+
+	if index >= len(msg.ShardSizes) {
+		return false
+	}
+
+	msg.ShardSizes[index] = size
+	return true
+}
+
+// InitializeShardSizes initializes the ShardSizes slice based on EcIndexBits
+// This ensures the slice has the correct length for all present shards
+func InitializeShardSizes(msg *master_pb.VolumeEcShardInformationMessage) {
+	if msg == nil {
+		return
+	}
+
+	shardBits := ShardBits(msg.EcIndexBits)
+	expectedLength := shardBits.ShardIdCount()
+
+	if msg.ShardSizes == nil || len(msg.ShardSizes) != expectedLength {
+		msg.ShardSizes = make([]int64, expectedLength)
+	}
+}
diff --git a/weed/storage/erasure_coding/ec_shard_size_helper_test.go b/weed/storage/erasure_coding/ec_shard_size_helper_test.go
new file mode 100644
index 000000000..2ef54c949
--- /dev/null
+++ b/weed/storage/erasure_coding/ec_shard_size_helper_test.go
@@ -0,0 +1,117 @@
+package erasure_coding
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func TestShardSizeHelpers(t *testing.T) {
+	// Create a message with shards 0, 2, and 5 present (EcIndexBits = 0b100101 = 37)
+	msg := &master_pb.VolumeEcShardInformationMessage{
+		Id:          123,
+		EcIndexBits: 37, // Binary: 100101, shards 0, 2, 5 are present
+	}
+
+	// Test SetShardSize
+	if !SetShardSize(msg, 0, 1000) {
+		t.Error("Failed to set size for shard 0")
+	}
+	if !SetShardSize(msg, 2, 2000) {
+		t.Error("Failed to set size for shard 2")
+	}
+	if !SetShardSize(msg, 5, 5000) {
+		t.Error("Failed to set size for shard 5")
+	}
+
+	// Test setting size for non-present shard should fail
+	if SetShardSize(msg, 1, 1500) {
+		t.Error("Should not be able to set size for non-present shard 1")
+	}
+
+	// Verify ShardSizes slice has correct length (3 shards)
+	if len(msg.ShardSizes) != 3 {
+		t.Errorf("Expected ShardSizes length 3, got %d", len(msg.ShardSizes))
+	}
+
+	// Test GetShardSize
+	if size, found := GetShardSize(msg, 0); !found || size != 1000 {
+		t.Errorf("Expected shard 0 size 1000, got %d (found: %v)", size, found)
+	}
+	if size, found := GetShardSize(msg, 2); !found || size != 2000 {
+		t.Errorf("Expected shard 2 size 2000, got %d (found: %v)", size, found)
+	}
+	if size, found := GetShardSize(msg, 5); !found || size != 5000 {
+		t.Errorf("Expected shard 5 size 5000, got %d (found: %v)", size, found)
+	}
+
+	// Test getting size for non-present shard
+	if size, found := GetShardSize(msg, 1); found {
+		t.Errorf("Should not find shard 1, but got size %d", size)
+	}
+
+	// Test direct slice access
+	if len(msg.ShardSizes) != 3 {
+		t.Errorf("Expected 3 shard sizes in slice, got %d", len(msg.ShardSizes))
+	}
+
+	expectedSizes := []int64{1000, 2000, 5000} // Ordered by shard ID: 0, 2, 5
+	for i, expectedSize := range expectedSizes {
+		if i < len(msg.ShardSizes) && msg.ShardSizes[i] != expectedSize {
+			t.Errorf("Expected ShardSizes[%d] = %d, got %d", i, expectedSize, msg.ShardSizes[i])
+		}
+	}
+}
+
+func TestShardBitsHelpers(t *testing.T) {
+	// Test with EcIndexBits = 37 (binary: 100101, shards 0, 2, 5)
+	shardBits := ShardBits(37)
+
+	// Test ShardIdToIndex
+	if index, found := shardBits.ShardIdToIndex(0); !found || index != 0 {
+		t.Errorf("Expected shard 0 at index 0, got %d (found: %v)", index, found)
+	}
+	if index, found := shardBits.ShardIdToIndex(2); !found || index != 1 {
+		t.Errorf("Expected shard 2 at index 1, got %d (found: %v)", index, found)
+	}
+	if index, found := shardBits.ShardIdToIndex(5); !found || index != 2 {
+		t.Errorf("Expected shard 5 at index 2, got %d (found: %v)", index, found)
+	}
+
+	// Test for non-present shard
+	if index, found := shardBits.ShardIdToIndex(1); found {
+		t.Errorf("Should not find shard 1, but got index %d", index)
+	}
+
+	// Test IndexToShardId
+	if shardId, found := shardBits.IndexToShardId(0); !found || shardId != 0 {
+		t.Errorf("Expected index 0 to be shard 0, got %d (found: %v)", shardId, found)
+	}
+	if shardId, found := shardBits.IndexToShardId(1); !found || shardId != 2 {
+		t.Errorf("Expected index 1 to be shard 2, got %d (found: %v)", shardId, found)
+	}
+	if shardId, found := shardBits.IndexToShardId(2); !found || shardId != 5 {
+		t.Errorf("Expected index 2 to be shard 5, got %d (found: %v)", shardId, found)
+	}
+
+	// Test for invalid index
+	if shardId, found := shardBits.IndexToShardId(3); found {
+		t.Errorf("Should not find shard for index 3, but got shard %d", shardId)
+	}
+
+	// Test EachSetIndex
+	var collectedShards []ShardId
+	shardBits.EachSetIndex(func(shardId ShardId) {
+		collectedShards = append(collectedShards, shardId)
+	})
+	expectedShards := []ShardId{0, 2, 5}
+	if len(collectedShards) != len(expectedShards) {
+		t.Errorf("Expected EachSetIndex to collect %v, got %v", expectedShards, collectedShards)
+	}
+	for i, expected := range expectedShards {
+		if i >= len(collectedShards) || collectedShards[i] != expected {
+			t.Errorf("Expected EachSetIndex to collect %v, got %v", expectedShards, collectedShards)
+			break
+		}
+	}
+}
diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go
index 33bc4ac7d..61057674f 100644
--- a/weed/storage/erasure_coding/ec_volume.go
+++ b/weed/storage/erasure_coding/ec_volume.go
@@ -227,6 +227,9 @@ func (ev *EcVolume) ToVolumeEcShardInformationMessage(diskId uint32) (messages [
 		}
 		prevVolumeId = s.VolumeId
 		m.EcIndexBits = uint32(ShardBits(m.EcIndexBits).AddShardId(s.ShardId))
+
+		// Add shard size information using the optimized format
+		SetShardSize(m, s.ShardId, s.Size())
 	}
 	return
 }
diff --git a/weed/storage/erasure_coding/ec_volume_info.go b/weed/storage/erasure_coding/ec_volume_info.go
index 787910b0c..53b352168 100644
--- a/weed/storage/erasure_coding/ec_volume_info.go
+++ b/weed/storage/erasure_coding/ec_volume_info.go
@@ -1,6 +1,8 @@
 package erasure_coding
 
 import (
+	"math/bits"
+
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 )
@@ -11,27 +13,51 @@ type EcVolumeInfo struct {
 	Collection  string
 	ShardBits   ShardBits
 	DiskType    string
-	DiskId      uint32 // ID of the disk this EC volume is on
-	ExpireAtSec uint64 // ec volume destroy time, calculated from the ec volume was created
-}
-
-func NewEcVolumeInfo(diskType string, collection string, vid needle.VolumeId, shardBits ShardBits, expireAtSec uint64, diskId uint32) *EcVolumeInfo {
-	return &EcVolumeInfo{
-		Collection:  collection,
-		VolumeId:    vid,
-		ShardBits:   shardBits,
-		DiskType:    diskType,
-		DiskId:      diskId,
-		ExpireAtSec: expireAtSec,
-	}
+	DiskId      uint32  // ID of the disk this EC volume is on
+	ExpireAtSec uint64  // ec volume destroy time, calculated from the ec volume was created
+	ShardSizes  []int64 // optimized: sizes for shards in order of set bits in ShardBits
 }
 
 func (ecInfo *EcVolumeInfo) AddShardId(id ShardId) {
+	oldBits := ecInfo.ShardBits
 	ecInfo.ShardBits = ecInfo.ShardBits.AddShardId(id)
+
+	// If shard was actually added, resize ShardSizes array
+	if oldBits != ecInfo.ShardBits {
+		ecInfo.resizeShardSizes(oldBits)
+	}
 }
 
 func (ecInfo *EcVolumeInfo) RemoveShardId(id ShardId) {
+	oldBits := ecInfo.ShardBits
 	ecInfo.ShardBits = ecInfo.ShardBits.RemoveShardId(id)
+
+	// If shard was actually removed, resize ShardSizes array
+	if oldBits != ecInfo.ShardBits {
+		ecInfo.resizeShardSizes(oldBits)
+	}
+}
+
+func (ecInfo *EcVolumeInfo) SetShardSize(id ShardId, size int64) {
+	ecInfo.ensureShardSizesInitialized()
+	if index, found := ecInfo.ShardBits.ShardIdToIndex(id); found && index < len(ecInfo.ShardSizes) {
+		ecInfo.ShardSizes[index] = size
+	}
+}
+
+func (ecInfo *EcVolumeInfo) GetShardSize(id ShardId) (int64, bool) {
+	if index, found := ecInfo.ShardBits.ShardIdToIndex(id); found && index < len(ecInfo.ShardSizes) {
+		return ecInfo.ShardSizes[index], true
+	}
+	return 0, false
+}
+
+func (ecInfo *EcVolumeInfo) GetTotalSize() int64 {
+	var total int64
+	for _, size := range ecInfo.ShardSizes {
+		total += size
+	}
+	return total
 }
 
 func (ecInfo *EcVolumeInfo) HasShardId(id ShardId) bool {
@@ -48,17 +74,33 @@ func (ecInfo *EcVolumeInfo) ShardIdCount() (count int) {
 
 func (ecInfo *EcVolumeInfo) Minus(other *EcVolumeInfo) *EcVolumeInfo {
 	ret := &EcVolumeInfo{
-		VolumeId:   ecInfo.VolumeId,
-		Collection: ecInfo.Collection,
-		ShardBits:  ecInfo.ShardBits.Minus(other.ShardBits),
-		DiskType:   ecInfo.DiskType,
+		VolumeId:    ecInfo.VolumeId,
+		Collection:  ecInfo.Collection,
+		ShardBits:   ecInfo.ShardBits.Minus(other.ShardBits),
+		DiskType:    ecInfo.DiskType,
+		DiskId:      ecInfo.DiskId,
+		ExpireAtSec: ecInfo.ExpireAtSec,
+	}
+
+	// Initialize optimized ShardSizes for the result
+	ret.ensureShardSizesInitialized()
+
+	// Copy shard sizes for remaining shards
+	retIndex := 0
+	for shardId := ShardId(0); shardId < TotalShardsCount && retIndex < len(ret.ShardSizes); shardId++ {
+		if ret.ShardBits.HasShardId(shardId) {
+			if size, exists := ecInfo.GetShardSize(shardId); exists {
+				ret.ShardSizes[retIndex] = size
+			}
+			retIndex++
+		}
 	}
 
 	return ret
 }
 
 func (ecInfo *EcVolumeInfo) ToVolumeEcShardInformationMessage() (ret *master_pb.VolumeEcShardInformationMessage) {
-	return &master_pb.VolumeEcShardInformationMessage{
+	t := &master_pb.VolumeEcShardInformationMessage{
 		Id:          uint32(ecInfo.VolumeId),
 		EcIndexBits: uint32(ecInfo.ShardBits),
 		Collection:  ecInfo.Collection,
@@ -66,6 +108,12 @@ func (ecInfo *EcVolumeInfo) ToVolumeEcShardInformationMessage() (ret *master_pb.
 		ExpireAtSec: ecInfo.ExpireAtSec,
 		DiskId:      ecInfo.DiskId,
 	}
+
+	// Directly set the optimized ShardSizes
+	t.ShardSizes = make([]int64, len(ecInfo.ShardSizes))
+	copy(t.ShardSizes, ecInfo.ShardSizes)
+
+	return t
 }
 
 type ShardBits uint32 // use bits to indicate the shard id, use 32 bits just for possible future extension
@@ -121,3 +169,81 @@ func (b ShardBits) MinusParityShards() ShardBits {
 	}
 	return b
 }
+
+// ShardIdToIndex converts a shard ID to its index position in the ShardSizes slice
+// Returns the index and true if the shard is present, -1 and false if not present
+func (b ShardBits) ShardIdToIndex(shardId ShardId) (index int, found bool) {
+	if !b.HasShardId(shardId) {
+		return -1, false
+	}
+
+	// Create a mask for bits before the shardId
+	mask := uint32((1 << shardId) - 1)
+	// Count set bits before the shardId using efficient bit manipulation
+	index = bits.OnesCount32(uint32(b) & mask)
+	return index, true
+}
+
+// EachSetIndex iterates over all set shard IDs and calls the provided function for each
+// This is highly efficient using bit manipulation - only iterates over actual set bits
+func (b ShardBits) EachSetIndex(fn func(shardId ShardId)) {
+	bitsValue := uint32(b)
+	for bitsValue != 0 {
+		// Find the position of the least significant set bit
+		shardId := ShardId(bits.TrailingZeros32(bitsValue))
+		fn(shardId)
+		// Clear the least significant set bit
+		bitsValue &= bitsValue - 1
+	}
+}
+
+// IndexToShardId converts an index position in ShardSizes slice to the corresponding shard ID
+// Returns the shard ID and true if valid index, -1 and false if invalid index
+func (b ShardBits) IndexToShardId(index int) (shardId ShardId, found bool) {
+	if index < 0 {
+		return 0, false
+	}
+
+	currentIndex := 0
+	for i := ShardId(0); i < TotalShardsCount; i++ {
+		if b.HasShardId(i) {
+			if currentIndex == index {
+				return i, true
+			}
+			currentIndex++
+		}
+	}
+	return 0, false // index out of range
+}
+
+// Helper methods for EcVolumeInfo to manage the optimized ShardSizes slice
+func (ecInfo *EcVolumeInfo) ensureShardSizesInitialized() {
+	expectedLength := ecInfo.ShardBits.ShardIdCount()
+	if ecInfo.ShardSizes == nil {
+		ecInfo.ShardSizes = make([]int64, expectedLength)
+	} else if len(ecInfo.ShardSizes) != expectedLength {
+		// Resize and preserve existing data
+		ecInfo.resizeShardSizes(ecInfo.ShardBits)
+	}
+}
+
+func (ecInfo *EcVolumeInfo) resizeShardSizes(prevShardBits ShardBits) {
+	expectedLength := ecInfo.ShardBits.ShardIdCount()
+	newSizes := make([]int64, expectedLength)
+
+	// Copy existing sizes to new positions based on current ShardBits
+	if len(ecInfo.ShardSizes) > 0 {
+		newIndex := 0
+		for shardId := ShardId(0); shardId < TotalShardsCount && newIndex < expectedLength; shardId++ {
+			if ecInfo.ShardBits.HasShardId(shardId) {
+				// Try to find the size for this shard in the old array using previous ShardBits
+				if oldIndex, found := prevShardBits.ShardIdToIndex(shardId); found && oldIndex < len(ecInfo.ShardSizes) {
+					newSizes[newIndex] = ecInfo.ShardSizes[oldIndex]
+				}
+				newIndex++
+			}
+		}
+	}
+
+	ecInfo.ShardSizes = newSizes
+}