shell: cluster.check prints out clock delta and network latency

1 files changed, 42 insertions, 30 deletions

diff --git a/weed/shell/command_cluster_check.go b/weed/shell/command_cluster_check.go
index c8fe110ba..616669b6d 100644
--- a/weed/shell/command_cluster_check.go
+++ b/weed/shell/command_cluster_check.go
@@ -7,10 +7,9 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/cluster"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"io"
-
-	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
 
 func init() {
@@ -98,15 +97,16 @@ func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer i
 		for _, volumeServer := range volumeServers {
 			fmt.Fprintf(writer, "checking master %s to volume server %s ... ", string(master), string(volumeServer))
 			err := pb.WithMasterClient(false, master, commandEnv.option.GrpcDialOption, func(client master_pb.SeaweedClient) error {
-				_, err := client.Ping(context.Background(), &master_pb.PingRequest{
+				pong, err := client.Ping(context.Background(), &master_pb.PingRequest{
 					Target:     string(volumeServer),
 					TargetType: cluster.VolumeServerType,
 				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
 				return err
 			})
-			if err == nil {
-				fmt.Fprintf(writer, "ok\n")
-			} else {
+			if err != nil {
 				fmt.Fprintf(writer, "%v\n", err)
 			}
 		}
@@ -120,15 +120,16 @@ func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer i
 			}
 			fmt.Fprintf(writer, "checking master %s to %s ... ", string(sourceMaster), string(targetMaster))
 			err := pb.WithMasterClient(false, sourceMaster, commandEnv.option.GrpcDialOption, func(client master_pb.SeaweedClient) error {
-				_, err := client.Ping(context.Background(), &master_pb.PingRequest{
+				pong, err := client.Ping(context.Background(), &master_pb.PingRequest{
 					Target:     string(targetMaster),
 					TargetType: cluster.MasterType,
 				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
 				return err
 			})
-			if err == nil {
-				fmt.Fprintf(writer, "ok\n")
-			} else {
+			if err != nil {
 				fmt.Fprintf(writer, "%v\n", err)
 			}
 		}
@@ -139,15 +140,16 @@ func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer i
 		for _, master := range masters {
 			fmt.Fprintf(writer, "checking volume server %s to master %s ... ", string(volumeServer), string(master))
 			err := pb.WithVolumeServerClient(false, volumeServer, commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
-				_, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{
+				pong, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{
 					Target:     string(master),
 					TargetType: cluster.MasterType,
 				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
 				return err
 			})
-			if err == nil {
-				fmt.Fprintf(writer, "ok\n")
-			} else {
+			if err != nil {
 				fmt.Fprintf(writer, "%v\n", err)
 			}
 		}
@@ -158,15 +160,16 @@ func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer i
 		for _, master := range masters {
 			fmt.Fprintf(writer, "checking filer %s to master %s ... ", string(filer), string(master))
 			err := pb.WithFilerClient(false, filer, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-				_, err := client.Ping(context.Background(), &filer_pb.PingRequest{
+				pong, err := client.Ping(context.Background(), &filer_pb.PingRequest{
 					Target:     string(master),
 					TargetType: cluster.MasterType,
 				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
 				return err
 			})
-			if err == nil {
-				fmt.Fprintf(writer, "ok\n")
-			} else {
+			if err != nil {
 				fmt.Fprintf(writer, "%v\n", err)
 			}
 		}
@@ -177,15 +180,16 @@ func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer i
 		for _, volumeServer := range volumeServers {
 			fmt.Fprintf(writer, "checking filer %s to volume server %s ... ", string(filer), string(volumeServer))
 			err := pb.WithFilerClient(false, filer, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-				_, err := client.Ping(context.Background(), &filer_pb.PingRequest{
+				pong, err := client.Ping(context.Background(), &filer_pb.PingRequest{
 					Target:     string(volumeServer),
 					TargetType: cluster.VolumeServerType,
 				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
 				return err
 			})
-			if err == nil {
-				fmt.Fprintf(writer, "ok\n")
-			} else {
+			if err != nil {
 				fmt.Fprintf(writer, "%v\n", err)
 			}
 		}
@@ -199,15 +203,16 @@ func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer i
 			}
 			fmt.Fprintf(writer, "checking volume server %s to %s ... ", string(sourceVolumeServer), string(targetVolumeServer))
 			err := pb.WithVolumeServerClient(false, sourceVolumeServer, commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
-				_, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{
+				pong, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{
 					Target:     string(targetVolumeServer),
 					TargetType: cluster.VolumeServerType,
 				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
 				return err
 			})
-			if err == nil {
-				fmt.Fprintf(writer, "ok\n")
-			} else {
+			if err != nil {
 				fmt.Fprintf(writer, "%v\n", err)
 			}
 		}
@@ -218,15 +223,16 @@ func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer i
 		for _, targetFiler := range filers {
 			fmt.Fprintf(writer, "checking filer %s to %s ... ", string(sourceFiler), string(targetFiler))
 			err := pb.WithFilerClient(false, sourceFiler, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-				_, err := client.Ping(context.Background(), &filer_pb.PingRequest{
+				pong, err := client.Ping(context.Background(), &filer_pb.PingRequest{
 					Target:     string(targetFiler),
 					TargetType: cluster.FilerType,
 				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
 				return err
 			})
-			if err == nil {
-				fmt.Fprintf(writer, "ok\n")
-			} else {
+			if err != nil {
 				fmt.Fprintf(writer, "%v\n", err)
 			}
 		}
@@ -234,3 +240,9 @@ func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer i
 
 	return nil
 }
+
+func printTiming(writer io.Writer, startNs, remoteNs, stopNs int64) {
+	roundTripTimeMs := float32(stopNs-startNs) / 1000000
+	deltaTimeMs := float32(remoteNs-(startNs+stopNs)/2) / 1000000
+	fmt.Fprintf(writer, "ok round trip %.3fms clock delta %.3fms\n", roundTripTimeMs, deltaTimeMs)
+}