Admin: misc improvements on admin server and workers. EC now works. (#7055)

* initial design

* added simulation as tests

* reorganized the codebase to move the simulation framework and tests into their own dedicated package

* integration test. ec worker task

* remove "enhanced" reference

* start master, volume servers, filer

Current Status
✅ Master: Healthy and running (port 9333)
✅ Filer: Healthy and running (port 8888)
✅ Volume Servers: All 6 servers running (ports 8080-8085)
🔄 Admin/Workers: Will start when dependencies are ready

* generate write load

* tasks are assigned

* admin start wtih grpc port. worker has its own working directory

* Update .gitignore

* working worker and admin. Task detection is not working yet.

* compiles, detection uses volumeSizeLimitMB from master

* compiles

* worker retries connecting to admin

* build and restart

* rendering pending tasks

* skip task ID column

* sticky worker id

* test canScheduleTaskNow

* worker reconnect to admin

* clean up logs

* worker register itself first

* worker can run ec work and report status

but:
1. one volume should not be repeatedly worked on.
2. ec shards needs to be distributed and source data should be deleted.

* move ec task logic

* listing ec shards

* local copy, ec. Need to distribute.

* ec is mostly working now

* distribution of ec shards needs improvement
* need configuration to enable ec

* show ec volumes

* interval field UI component

* rename

* integration test with vauuming

* garbage percentage threshold

* fix warning

* display ec shard sizes

* fix ec volumes list

* Update ui.go

* show default values

* ensure correct default value

* MaintenanceConfig use ConfigField

* use schema defined defaults

* config

* reduce duplication

* refactor to use BaseUIProvider

* each task register its schema

* checkECEncodingCandidate use ecDetector

* use vacuumDetector

* use volumeSizeLimitMB

* remove

remove

* remove unused

* refactor

* use new framework

* remove v2 reference

* refactor

* left menu can scroll now

* The maintenance manager was not being initialized when no data directory was configured for persistent storage.

* saving config

* Update task_config_schema_templ.go

* enable/disable tasks

* protobuf encoded task configurations

* fix system settings

* use ui component

* remove logs

* interface{} Reduction

* reduce interface{}

* reduce interface{}

* avoid from/to map

* reduce interface{}

* refactor

* keep it DRY

* added logging

* debug messages

* debug level

* debug

* show the log caller line

* use configured task policy

* log level

* handle admin heartbeat response

* Update worker.go

* fix EC rack and dc count

* Report task status to admin server

* fix task logging, simplify interface checking, use erasure_coding constants

* factor in empty volume server during task planning

* volume.list adds disk id

* track disk id also

* fix locking scheduled and manual scanning

* add active topology

* simplify task detector

* ec task completed, but shards are not showing up

* implement ec in ec_typed.go

* adjust log level

* dedup

* implementing ec copying shards and only ecx files

* use disk id when distributing ec shards

🎯 Planning: ActiveTopology creates DestinationPlan with specific TargetDisk
📦 Task Creation: maintenance_integration.go creates ECDestination with DiskId
🚀 Task Execution: EC task passes DiskId in VolumeEcShardsCopyRequest
💾 Volume Server: Receives disk_id and stores shards on specific disk (vs.store.Locations[req.DiskId])
📂 File System: EC shards and metadata land in the exact disk directory planned

* Delete original volume from all locations

* clean up existing shard locations

* local encoding and distributing

* Update docker/admin_integration/EC-TESTING-README.md

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* check volume id range

* simplify

* fix tests

* fix types

* clean up logs and tests

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

1 files changed, 140 insertions, 0 deletions

diff --git a/weed/worker/tasks/erasure_coding/detection.go b/weed/worker/tasks/erasure_coding/detection.go
new file mode 100644
index 000000000..1a2558396
--- /dev/null
+++ b/weed/worker/tasks/erasure_coding/detection.go
@@ -0,0 +1,140 @@
+package erasure_coding
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
+	"github.com/seaweedfs/seaweedfs/weed/worker/types"
+)
+
+// Detection implements the detection logic for erasure coding tasks
+func Detection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig) ([]*types.TaskDetectionResult, error) {
+	if !config.IsEnabled() {
+		return nil, nil
+	}
+
+	ecConfig := config.(*Config)
+	var results []*types.TaskDetectionResult
+	now := time.Now()
+	quietThreshold := time.Duration(ecConfig.QuietForSeconds) * time.Second
+	minSizeBytes := uint64(ecConfig.MinSizeMB) * 1024 * 1024 // Configurable minimum
+
+	debugCount := 0
+	skippedAlreadyEC := 0
+	skippedTooSmall := 0
+	skippedCollectionFilter := 0
+	skippedQuietTime := 0
+	skippedFullness := 0
+
+	for _, metric := range metrics {
+		// Skip if already EC volume
+		if metric.IsECVolume {
+			skippedAlreadyEC++
+			continue
+		}
+
+		// Check minimum size requirement
+		if metric.Size < minSizeBytes {
+			skippedTooSmall++
+			continue
+		}
+
+		// Check collection filter if specified
+		if ecConfig.CollectionFilter != "" {
+			// Parse comma-separated collections
+			allowedCollections := make(map[string]bool)
+			for _, collection := range strings.Split(ecConfig.CollectionFilter, ",") {
+				allowedCollections[strings.TrimSpace(collection)] = true
+			}
+			// Skip if volume's collection is not in the allowed list
+			if !allowedCollections[metric.Collection] {
+				skippedCollectionFilter++
+				continue
+			}
+		}
+
+		// Check quiet duration and fullness criteria
+		if metric.Age >= quietThreshold && metric.FullnessRatio >= ecConfig.FullnessRatio {
+			result := &types.TaskDetectionResult{
+				TaskType:   types.TaskTypeErasureCoding,
+				VolumeID:   metric.VolumeID,
+				Server:     metric.Server,
+				Collection: metric.Collection,
+				Priority:   types.TaskPriorityLow, // EC is not urgent
+				Reason: fmt.Sprintf("Volume meets EC criteria: quiet for %.1fs (>%ds), fullness=%.1f%% (>%.1f%%), size=%.1fMB (>100MB)",
+					metric.Age.Seconds(), ecConfig.QuietForSeconds, metric.FullnessRatio*100, ecConfig.FullnessRatio*100,
+					float64(metric.Size)/(1024*1024)),
+				ScheduleAt: now,
+			}
+			results = append(results, result)
+		} else {
+			// Count debug reasons
+			if debugCount < 5 { // Limit to avoid spam
+				if metric.Age < quietThreshold {
+					skippedQuietTime++
+				}
+				if metric.FullnessRatio < ecConfig.FullnessRatio {
+					skippedFullness++
+				}
+			}
+			debugCount++
+		}
+	}
+
+	// Log debug summary if no tasks were created
+	if len(results) == 0 && len(metrics) > 0 {
+		totalVolumes := len(metrics)
+		glog.V(1).Infof("EC detection: No tasks created for %d volumes (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full)",
+			totalVolumes, skippedAlreadyEC, skippedTooSmall, skippedCollectionFilter, skippedQuietTime, skippedFullness)
+
+		// Show details for first few volumes
+		for i, metric := range metrics {
+			if i >= 3 || metric.IsECVolume { // Limit to first 3 non-EC volumes
+				continue
+			}
+			sizeMB := float64(metric.Size) / (1024 * 1024)
+			glog.Infof("ERASURE CODING: Volume %d: size=%.1fMB (need ≥%dMB), age=%s (need ≥%s), fullness=%.1f%% (need ≥%.1f%%)",
+				metric.VolumeID, sizeMB, ecConfig.MinSizeMB, metric.Age.Truncate(time.Minute), quietThreshold.Truncate(time.Minute),
+				metric.FullnessRatio*100, ecConfig.FullnessRatio*100)
+		}
+	}
+
+	return results, nil
+}
+
+// Scheduling implements the scheduling logic for erasure coding tasks
+func Scheduling(task *types.Task, runningTasks []*types.Task, availableWorkers []*types.Worker, config base.TaskConfig) bool {
+	ecConfig := config.(*Config)
+
+	// Check if we have available workers
+	if len(availableWorkers) == 0 {
+		return false
+	}
+
+	// Count running EC tasks
+	runningCount := 0
+	for _, runningTask := range runningTasks {
+		if runningTask.Type == types.TaskTypeErasureCoding {
+			runningCount++
+		}
+	}
+
+	// Check concurrency limit
+	if runningCount >= ecConfig.MaxConcurrent {
+		return false
+	}
+
+	// Check if any worker can handle EC tasks
+	for _, worker := range availableWorkers {
+		for _, capability := range worker.Capabilities {
+			if capability == types.TaskTypeErasureCoding {
+				return true
+			}
+		}
+	}
+
+	return false
+}