Admin: misc improvements on admin server and workers. EC now works. (#7055)

* initial design

* added simulation as tests

* reorganized the codebase to move the simulation framework and tests into their own dedicated package

* integration test. ec worker task

* remove "enhanced" reference

* start master, volume servers, filer

Current Status
✅ Master: Healthy and running (port 9333)
✅ Filer: Healthy and running (port 8888)
✅ Volume Servers: All 6 servers running (ports 8080-8085)
🔄 Admin/Workers: Will start when dependencies are ready

* generate write load

* tasks are assigned

* admin start wtih grpc port. worker has its own working directory

* Update .gitignore

* working worker and admin. Task detection is not working yet.

* compiles, detection uses volumeSizeLimitMB from master

* compiles

* worker retries connecting to admin

* build and restart

* rendering pending tasks

* skip task ID column

* sticky worker id

* test canScheduleTaskNow

* worker reconnect to admin

* clean up logs

* worker register itself first

* worker can run ec work and report status

but:
1. one volume should not be repeatedly worked on.
2. ec shards needs to be distributed and source data should be deleted.

* move ec task logic

* listing ec shards

* local copy, ec. Need to distribute.

* ec is mostly working now

* distribution of ec shards needs improvement
* need configuration to enable ec

* show ec volumes

* interval field UI component

* rename

* integration test with vauuming

* garbage percentage threshold

* fix warning

* display ec shard sizes

* fix ec volumes list

* Update ui.go

* show default values

* ensure correct default value

* MaintenanceConfig use ConfigField

* use schema defined defaults

* config

* reduce duplication

* refactor to use BaseUIProvider

* each task register its schema

* checkECEncodingCandidate use ecDetector

* use vacuumDetector

* use volumeSizeLimitMB

* remove

remove

* remove unused

* refactor

* use new framework

* remove v2 reference

* refactor

* left menu can scroll now

* The maintenance manager was not being initialized when no data directory was configured for persistent storage.

* saving config

* Update task_config_schema_templ.go

* enable/disable tasks

* protobuf encoded task configurations

* fix system settings

* use ui component

* remove logs

* interface{} Reduction

* reduce interface{}

* reduce interface{}

* avoid from/to map

* reduce interface{}

* refactor

* keep it DRY

* added logging

* debug messages

* debug level

* debug

* show the log caller line

* use configured task policy

* log level

* handle admin heartbeat response

* Update worker.go

* fix EC rack and dc count

* Report task status to admin server

* fix task logging, simplify interface checking, use erasure_coding constants

* factor in empty volume server during task planning

* volume.list adds disk id

* track disk id also

* fix locking scheduled and manual scanning

* add active topology

* simplify task detector

* ec task completed, but shards are not showing up

* implement ec in ec_typed.go

* adjust log level

* dedup

* implementing ec copying shards and only ecx files

* use disk id when distributing ec shards

🎯 Planning: ActiveTopology creates DestinationPlan with specific TargetDisk
📦 Task Creation: maintenance_integration.go creates ECDestination with DiskId
🚀 Task Execution: EC task passes DiskId in VolumeEcShardsCopyRequest
💾 Volume Server: Receives disk_id and stores shards on specific disk (vs.store.Locations[req.DiskId])
📂 File System: EC shards and metadata land in the exact disk directory planned

* Delete original volume from all locations

* clean up existing shard locations

* local encoding and distributing

* Update docker/admin_integration/EC-TESTING-README.md

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* check volume id range

* simplify

* fix tests

* fix types

* clean up logs and tests

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

1 files changed, 124 insertions, 0 deletions

diff --git a/weed/admin/maintenance/config_verification.go b/weed/admin/maintenance/config_verification.go
new file mode 100644
index 000000000..0ac40aad1
--- /dev/null
+++ b/weed/admin/maintenance/config_verification.go
@@ -0,0 +1,124 @@
+package maintenance
+
+import (
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+)
+
+// VerifyProtobufConfig demonstrates that the protobuf configuration system is working
+func VerifyProtobufConfig() error {
+	// Create configuration manager
+	configManager := NewMaintenanceConfigManager()
+	config := configManager.GetConfig()
+
+	// Verify basic configuration
+	if !config.Enabled {
+		return fmt.Errorf("expected config to be enabled by default")
+	}
+
+	if config.ScanIntervalSeconds != 30*60 {
+		return fmt.Errorf("expected scan interval to be 1800 seconds, got %d", config.ScanIntervalSeconds)
+	}
+
+	// Verify policy configuration
+	if config.Policy == nil {
+		return fmt.Errorf("expected policy to be configured")
+	}
+
+	if config.Policy.GlobalMaxConcurrent != 4 {
+		return fmt.Errorf("expected global max concurrent to be 4, got %d", config.Policy.GlobalMaxConcurrent)
+	}
+
+	// Verify task policies
+	vacuumPolicy := config.Policy.TaskPolicies["vacuum"]
+	if vacuumPolicy == nil {
+		return fmt.Errorf("expected vacuum policy to be configured")
+	}
+
+	if !vacuumPolicy.Enabled {
+		return fmt.Errorf("expected vacuum policy to be enabled")
+	}
+
+	// Verify typed configuration access
+	vacuumConfig := vacuumPolicy.GetVacuumConfig()
+	if vacuumConfig == nil {
+		return fmt.Errorf("expected vacuum config to be accessible")
+	}
+
+	if vacuumConfig.GarbageThreshold != 0.3 {
+		return fmt.Errorf("expected garbage threshold to be 0.3, got %f", vacuumConfig.GarbageThreshold)
+	}
+
+	// Verify helper functions work
+	if !IsTaskEnabled(config.Policy, "vacuum") {
+		return fmt.Errorf("expected vacuum task to be enabled via helper function")
+	}
+
+	maxConcurrent := GetMaxConcurrent(config.Policy, "vacuum")
+	if maxConcurrent != 2 {
+		return fmt.Errorf("expected vacuum max concurrent to be 2, got %d", maxConcurrent)
+	}
+
+	// Verify erasure coding configuration
+	ecPolicy := config.Policy.TaskPolicies["erasure_coding"]
+	if ecPolicy == nil {
+		return fmt.Errorf("expected EC policy to be configured")
+	}
+
+	ecConfig := ecPolicy.GetErasureCodingConfig()
+	if ecConfig == nil {
+		return fmt.Errorf("expected EC config to be accessible")
+	}
+
+	// Verify configurable EC fields only
+	if ecConfig.FullnessRatio <= 0 || ecConfig.FullnessRatio > 1 {
+		return fmt.Errorf("expected EC config to have valid fullness ratio (0-1), got %f", ecConfig.FullnessRatio)
+	}
+
+	return nil
+}
+
+// GetProtobufConfigSummary returns a summary of the current protobuf configuration
+func GetProtobufConfigSummary() string {
+	configManager := NewMaintenanceConfigManager()
+	config := configManager.GetConfig()
+
+	summary := fmt.Sprintf("SeaweedFS Protobuf Maintenance Configuration:\n")
+	summary += fmt.Sprintf("  Enabled: %v\n", config.Enabled)
+	summary += fmt.Sprintf("  Scan Interval: %d seconds\n", config.ScanIntervalSeconds)
+	summary += fmt.Sprintf("  Max Retries: %d\n", config.MaxRetries)
+	summary += fmt.Sprintf("  Global Max Concurrent: %d\n", config.Policy.GlobalMaxConcurrent)
+	summary += fmt.Sprintf("  Task Policies: %d configured\n", len(config.Policy.TaskPolicies))
+
+	for taskType, policy := range config.Policy.TaskPolicies {
+		summary += fmt.Sprintf("    %s: enabled=%v, max_concurrent=%d\n",
+			taskType, policy.Enabled, policy.MaxConcurrent)
+	}
+
+	return summary
+}
+
+// CreateCustomConfig demonstrates creating a custom protobuf configuration
+func CreateCustomConfig() *worker_pb.MaintenanceConfig {
+	return &worker_pb.MaintenanceConfig{
+		Enabled:             true,
+		ScanIntervalSeconds: 60 * 60, // 1 hour
+		MaxRetries:          5,
+		Policy: &worker_pb.MaintenancePolicy{
+			GlobalMaxConcurrent: 8,
+			TaskPolicies: map[string]*worker_pb.TaskPolicy{
+				"custom_vacuum": {
+					Enabled:       true,
+					MaxConcurrent: 4,
+					TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
+						VacuumConfig: &worker_pb.VacuumTaskConfig{
+							GarbageThreshold:  0.5,
+							MinVolumeAgeHours: 48,
+						},
+					},
+				},
+			},
+		},
+	}
+}