[master] process grow request with must grow (#5999)

process grow request with must grow

2 files changed, 43 insertions, 27 deletions

diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go
index e40f054e0..6d5ee0385 100644
--- a/weed/server/master_grpc_server_volume.go
+++ b/weed/server/master_grpc_server_volume.go
@@ -3,6 +3,7 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"math/rand/v2"
 	"strings"
 	"sync"
@@ -20,6 +21,10 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )
 
+const (
+	volumeGrowStepCount = 2
+)
+
 func (ms *MasterServer) DoAutomaticVolumeGrow(req *topology.VolumeGrowRequest) {
 	glog.V(1).Infoln("starting automatic volume grow")
 	start := time.Now()
@@ -36,38 +41,48 @@ func (ms *MasterServer) DoAutomaticVolumeGrow(req *topology.VolumeGrowRequest) {
 
 func (ms *MasterServer) ProcessGrowRequest() {
 	go func() {
+		ctx := context.Background()
 		for {
 			if !ms.Topo.IsLeader() {
 				continue
 			}
 			dcs := ms.Topo.ListDataCenters()
+			var err error
 			for _, vlc := range ms.Topo.ListVolumeLayoutCollections() {
 				vl := vlc.VolumeLayout
+				lastGrowCount := vl.GetLastGrowCount()
 				if vl.HasGrowRequest() {
 					continue
 				}
-				if vl.ShouldGrowVolumes(vlc.Collection) {
-					vl.AddGrowRequest()
-					ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{
-						Option: vlc.ToGrowOption(),
-						Count:  vl.GetLastGrowCount(),
-						Reason: "collection autogrow",
-					}
-				} else {
+				writable, crowded := vl.GetWritableVolumeCount()
+				mustGrow := int(lastGrowCount) - writable
+				vgr := vlc.ToVolumeGrowRequest()
+				stats.MasterVolumeLayoutWritable.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(writable))
+				stats.MasterVolumeLayoutCrowded.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(crowded))
+
+				switch {
+				case mustGrow > 0:
+					vgr.WritableVolumeCount = uint32(mustGrow)
+					_, err = ms.VolumeGrow(ctx, vgr)
+				case crowded+volumeGrowStepCount >= writable:
+					vgr.WritableVolumeCount = volumeGrowStepCount
+					_, err = ms.VolumeGrow(ctx, vgr)
+				default:
 					for _, dc := range dcs {
 						if vl.ShouldGrowVolumesByDataNode("DataCenter", dc) {
-							vl.AddGrowRequest()
-							volumeGrowOption := vlc.ToGrowOption()
-							volumeGrowOption.DataCenter = dc
-							ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{
-								Option: volumeGrowOption,
-								Count:  vl.GetLastGrowCount(),
-								Force:  true,
-								Reason: "per-dc autogrow",
+							vgr.DataCenter = dc
+							if lastGrowCount > 0 {
+								vgr.WritableVolumeCount = uint32(int(lastGrowCount) / len(dcs))
+							} else {
+								vgr.WritableVolumeCount = volumeGrowStepCount
 							}
+							_, err = ms.VolumeGrow(ctx, vgr)
 						}
 					}
 				}
+				if err != nil {
+					glog.V(0).Infof("volume grow request failed: %+v", err)
+				}
 			}
 			time.Sleep(14*time.Minute + time.Duration(120*rand.Float32())*time.Second)
 		}
@@ -101,7 +116,7 @@ func (ms *MasterServer) ProcessGrowRequest() {
 			})
 
 			// not atomic but it's okay
-			if found || (!req.Force && !vl.ShouldGrowVolumes(req.Option.Collection)) {
+			if found || (!req.Force && !vl.ShouldGrowVolumes()) {
 				glog.V(4).Infoln("discard volume grow request")
 				time.Sleep(time.Millisecond * 211)
 				vl.DoneGrowRequest()
@@ -302,6 +317,9 @@ func (ms *MasterServer) VolumeGrow(ctx context.Context, req *master_pb.VolumeGro
 	if err != nil {
 		return nil, err
 	}
+	if req.DataCenter != "" && !ms.Topo.DataCenterExists(req.DataCenter) {
+		return nil, fmt.Errorf("data center not exists")
+	}
 	volumeGrowOption := topology.VolumeGrowOption{
 		Collection:         req.Collection,
 		ReplicaPlacement:   replicaPlacement,
diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go
index 0951da60e..610502769 100644
--- a/weed/topology/volume_layout.go
+++ b/weed/topology/volume_layout.go
@@ -2,7 +2,7 @@ package topology
 
 import (
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"math/rand"
 	"sync"
 	"sync/atomic"
@@ -359,10 +359,8 @@ func (vl *VolumeLayout) GetLastGrowCount() uint32 {
 	return vl.lastGrowCount.Load()
 }
 
-func (vl *VolumeLayout) ShouldGrowVolumes(collection string) bool {
+func (vl *VolumeLayout) ShouldGrowVolumes() bool {
 	writable, crowded := vl.GetWritableVolumeCount()
-	stats.MasterVolumeLayoutWritable.WithLabelValues(collection, vl.diskType.String(), vl.rp.String(), vl.ttl.String()).Set(float64(writable))
-	stats.MasterVolumeLayoutCrowded.WithLabelValues(collection, vl.diskType.String(), vl.rp.String(), vl.ttl.String()).Set(float64(crowded))
 	return writable <= crowded
 }
 
@@ -544,12 +542,12 @@ func (vl *VolumeLayout) ToInfo() (info VolumeLayoutInfo) {
 	return
 }
 
-func (vlc *VolumeLayoutCollection) ToGrowOption() (option *VolumeGrowOption) {
-	return &VolumeGrowOption{
-		Collection:       vlc.Collection,
-		ReplicaPlacement: vlc.VolumeLayout.rp,
-		Ttl:              vlc.VolumeLayout.ttl,
-		DiskType:         vlc.VolumeLayout.diskType,
+func (vlc *VolumeLayoutCollection) ToVolumeGrowRequest() *master_pb.VolumeGrowRequest {
+	return &master_pb.VolumeGrowRequest{
+		Collection:  vlc.Collection,
+		Replication: vlc.VolumeLayout.rp.String(),
+		Ttl:         vlc.VolumeLayout.ttl.String(),
+		DiskType:    vlc.VolumeLayout.diskType.String(),
 	}
 }