1 files changed, 33 insertions, 14 deletions
diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go
index 8fc06bdb6..eb0d43705 100644
--- a/weed/server/master_grpc_server_volume.go
+++ b/weed/server/master_grpc_server_volume.go
@@ -27,7 +27,7 @@ func (ms *MasterServer) DoAutomaticVolumeGrow(req *topology.VolumeGrowRequest) {
 	newVidLocations, err := ms.vg.AutomaticGrowByType(req.Option, ms.grpcDialOption, ms.Topo, req.Count)
 	glog.V(1).Infoln("finished automatic volume grow, cost ", time.Now().Sub(start))
 	if err != nil {
-		glog.Warningf("automatic volume grow %s: %+v", req.Option, err)
+		glog.V(1).Infof("automatic volume grow failed: %+v", err)
 		return
 	}
 	for _, newVidLocation := range newVidLocations {
@@ -38,19 +38,37 @@ func (ms *MasterServer) DoAutomaticVolumeGrow(req *topology.VolumeGrowRequest) {
 func (ms *MasterServer) ProcessGrowRequest() {
 	go func() {
 		for {
-			time.Sleep(14*time.Minute + time.Duration(120*rand.Float32())*time.Second)
 			if !ms.Topo.IsLeader() {
 				continue
 			}
-			for _, vl := range ms.Topo.ListVolumeLayouts() {
-				if !vl.HasGrowRequest() && vl.ShouldGrowVolumes(&topology.VolumeGrowOption{}) {
+			dcs := ms.Topo.ListDataCenters()
+			for _, vlc := range ms.Topo.ListVolumeLayoutCollections() {
+				vl := vlc.VolumeLayout
+				if vl.HasGrowRequest() {
+					continue
+				}
+				if vl.ShouldGrowVolumes(vlc.Collection) {
 					vl.AddGrowRequest()
 					ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{
-						Option: vl.ToGrowOption(),
+						Option: vlc.ToGrowOption(),
 						Count:  vl.GetLastGrowCount(),
 					}
+				} else {
+					for _, dc := range dcs {
+						if vl.ShouldGrowVolumesByDataNode("DataCenter", dc) {
+							vl.AddGrowRequest()
+							volumeGrowOption := vlc.ToGrowOption()
+							volumeGrowOption.DataCenter = dc
+							ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{
+								Option: volumeGrowOption,
+								Count:  vl.GetLastGrowCount(),
+								Force:  true,
+							}
+						}
+					}
 				}
 			}
+			time.Sleep(14*time.Minute + time.Duration(120*rand.Float32())*time.Second)
 		}
 	}()
 	go func() {
@@ -81,19 +99,20 @@ func (ms *MasterServer) ProcessGrowRequest() {
 			})
 
 			// not atomic but it's okay
-			if !found && vl.ShouldGrowVolumes(option) {
-				filter.Store(req, nil)
-				// we have lock called inside vg
-				go func(req *topology.VolumeGrowRequest, vl *topology.VolumeLayout) {
-					ms.DoAutomaticVolumeGrow(req)
-					vl.DoneGrowRequest()
-					filter.Delete(req)
-				}(req, vl)
-			} else {
+			if found || (!req.Force && !vl.ShouldGrowVolumes(req.Option.Collection)) {
 				glog.V(4).Infoln("discard volume grow request")
 				time.Sleep(time.Millisecond * 211)
 				vl.DoneGrowRequest()
+				continue
 			}
+
+			filter.Store(req, nil)
+			// we have lock called inside vg
+			go func(req *topology.VolumeGrowRequest, vl *topology.VolumeLayout) {
+				ms.DoAutomaticVolumeGrow(req)
+				vl.DoneGrowRequest()
+				filter.Delete(req)
+			}(req, vl)
 		}
 	}()
 }