Set timeout for master and volume non-streaming rpc

8 files changed, 46 insertions, 10 deletions

diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go
index 924b29c0e..00e1caad5 100644
--- a/weed/operation/assign_file_id.go
+++ b/weed/operation/assign_file_id.go
@@ -3,6 +3,7 @@ package operation
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
@@ -40,6 +41,9 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 		}
 
 		lastError = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+			defer cancel()
+
 			req := &master_pb.AssignRequest{
 				Count:       primaryRequest.Count,
 				Replication: primaryRequest.Replication,
@@ -49,7 +53,7 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 				Rack:        primaryRequest.Rack,
 				DataNode:    primaryRequest.DataNode,
 			}
-			resp, grpcErr := masterClient.Assign(context.Background(), req)
+			resp, grpcErr := masterClient.Assign(ctx, req)
 			if grpcErr != nil {
 				return grpcErr
 			}
diff --git a/weed/operation/delete_content.go b/weed/operation/delete_content.go
index e15859d15..3e468e1a3 100644
--- a/weed/operation/delete_content.go
+++ b/weed/operation/delete_content.go
@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 )
@@ -108,12 +109,14 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin
 func DeleteFilesAtOneVolumeServer(volumeServer string, fileIds []string) (ret []*volume_server_pb.DeleteResult, err error) {
 
 	err = WithVolumeServerClient(volumeServer, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+		defer cancel()
 
 		req := &volume_server_pb.BatchDeleteRequest{
 			FileIds: fileIds,
 		}
 
-		resp, err := volumeServerClient.BatchDelete(context.Background(), req)
+		resp, err := volumeServerClient.BatchDelete(ctx, req)
 
 		// fmt.Printf("deleted %v %v: %v\n", fileIds, err, resp)
 
diff --git a/weed/operation/lookup.go b/weed/operation/lookup.go
index c495af1af..562a11580 100644
--- a/weed/operation/lookup.go
+++ b/weed/operation/lookup.go
@@ -99,10 +99,13 @@ func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, err
 	//only query unknown_vids
 
 	err := withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+		defer cancel()
+
 		req := &master_pb.LookupVolumeRequest{
 			VolumeIds: unknown_vids,
 		}
-		resp, grpcErr := masterClient.LookupVolume(context.Background(), req)
+		resp, grpcErr := masterClient.LookupVolume(ctx, req)
 		if grpcErr != nil {
 			return grpcErr
 		}
diff --git a/weed/operation/stats.go b/weed/operation/stats.go
index 277b0e835..364727272 100644
--- a/weed/operation/stats.go
+++ b/weed/operation/stats.go
@@ -2,6 +2,7 @@ package operation
 
 import (
 	"context"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
@@ -9,7 +10,10 @@ import (
 func Statistics(server string, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) {
 
 	err = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
-		grpcResponse, grpcErr := masterClient.Statistics(context.Background(), req)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+		defer cancel()
+
+		grpcResponse, grpcErr := masterClient.Statistics(ctx, req)
 		if grpcErr != nil {
 			return grpcErr
 		}
diff --git a/weed/operation/sync_volume.go b/weed/operation/sync_volume.go
index ac2e2bf79..e40c7de41 100644
--- a/weed/operation/sync_volume.go
+++ b/weed/operation/sync_volume.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
@@ -13,7 +14,10 @@ import (
 func GetVolumeSyncStatus(server string, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
 
 	WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
-		resp, err = client.VolumeSyncStatus(context.Background(), &volume_server_pb.VolumeSyncStatusRequest{
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+		defer cancel()
+
+		resp, err = client.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{
 			VolumdId: vid,
 		})
 		return nil
diff --git a/weed/server/master_server_handlers_admin.go b/weed/server/master_server_handlers_admin.go
index 490693d97..3a2662908 100644
--- a/weed/server/master_server_handlers_admin.go
+++ b/weed/server/master_server_handlers_admin.go
@@ -7,6 +7,7 @@ import (
 	"math/rand"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
@@ -24,7 +25,10 @@ func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.R
 	}
 	for _, server := range collection.ListVolumeServers() {
 		err := operation.WithVolumeServerClient(server.Url(), func(client volume_server_pb.VolumeServerClient) error {
-			_, deleteErr := client.DeleteCollection(context.Background(), &volume_server_pb.DeleteCollectionRequest{
+			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+			defer cancel()
+
+			_, deleteErr := client.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{
 				Collection: collection.Name,
 			})
 			return deleteErr
diff --git a/weed/topology/allocate_volume.go b/weed/topology/allocate_volume.go
index 61fc9d479..55796ab43 100644
--- a/weed/topology/allocate_volume.go
+++ b/weed/topology/allocate_volume.go
@@ -2,6 +2,7 @@ package topology
 
 import (
 	"context"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
@@ -15,7 +16,10 @@ type AllocateVolumeResult struct {
 func AllocateVolume(dn *DataNode, vid storage.VolumeId, option *VolumeGrowOption) error {
 
 	return operation.WithVolumeServerClient(dn.Url(), func(client volume_server_pb.VolumeServerClient) error {
-		_, deleteErr := client.AssignVolume(context.Background(), &volume_server_pb.AssignVolumeRequest{
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+		defer cancel()
+
+		_, deleteErr := client.AssignVolume(ctx, &volume_server_pb.AssignVolumeRequest{
 			VolumdId:    uint32(vid),
 			Collection:  option.Collection,
 			Replication: option.ReplicaPlacement.String(),
diff --git a/weed/topology/topology_vacuum.go b/weed/topology/topology_vacuum.go
index e386dabdb..95d9376d5 100644
--- a/weed/topology/topology_vacuum.go
+++ b/weed/topology/topology_vacuum.go
@@ -15,7 +15,10 @@ func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist
 	for index, dn := range locationlist.list {
 		go func(index int, url string, vid storage.VolumeId) {
 			err := operation.WithVolumeServerClient(url, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-				resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
+				ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+				defer cancel()
+
+				resp, err := volumeServerClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{
 					VolumdId: uint32(vid),
 				})
 				if err != nil {
@@ -48,6 +51,7 @@ func batchVacuumVolumeCompact(vl *VolumeLayout, vid storage.VolumeId, locationli
 	ch := make(chan bool, locationlist.Length())
 	for index, dn := range locationlist.list {
 		go func(index int, url string, vid storage.VolumeId) {
+			//	TODO: set timeout according to actual compact duration
 			glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
 			err := operation.WithVolumeServerClient(url, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
 				_, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
@@ -81,7 +85,10 @@ func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlis
 	for _, dn := range locationlist.list {
 		glog.V(0).Infoln("Start Commiting vacuum", vid, "on", dn.Url())
 		err := operation.WithVolumeServerClient(dn.Url(), func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-			_, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
+			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+			defer cancel()
+
+			_, err := volumeServerClient.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{
 				VolumdId: uint32(vid),
 			})
 			return err
@@ -102,7 +109,10 @@ func batchVacuumVolumeCleanup(vl *VolumeLayout, vid storage.VolumeId, locationli
 	for _, dn := range locationlist.list {
 		glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url())
 		err := operation.WithVolumeServerClient(dn.Url(), func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-			_, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
+			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+			defer cancel()
+
+			_, err := volumeServerClient.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{
 				VolumdId: uint32(vid),
 			})
 			return err