configure raft metrics (#3798)

2 files changed, 37 insertions, 20 deletions

diff --git a/weed/server/raft_hashicorp.go b/weed/server/raft_hashicorp.go
index 8f81924d1..76468b5e1 100644
--- a/weed/server/raft_hashicorp.go
+++ b/weed/server/raft_hashicorp.go
@@ -6,10 +6,13 @@ package weed_server
 import (
 	"fmt"
 	transport "github.com/Jille/raft-grpc-transport"
+	"github.com/armon/go-metrics"
+	"github.com/armon/go-metrics/prometheus"
 	"github.com/hashicorp/raft"
 	boltdb "github.com/hashicorp/raft-boltdb/v2"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"google.golang.org/grpc"
 	"math/rand"
 	"os"
@@ -182,5 +185,18 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) {
 		}()
 	}
 
+	// Configure a prometheus sink as the raft metrics sink
+	if sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{
+		Registerer: stats.Gather,
+	}); err != nil {
+		return nil, fmt.Errorf("NewPrometheusSink: %v", err)
+	} else {
+		metricsConf := metrics.DefaultConfig(stats.Namespace)
+		metricsConf.EnableRuntimeMetrics = false
+		if _, err = metrics.NewGlobal(metricsConf, sink); err != nil {
+			return nil, fmt.Errorf("metrics.NewGlobal: %v", err)
+		}
+	}
+
 	return s, nil
 }
diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go
index 38c5590b3..9947e2316 100644
--- a/weed/stats/metrics.go
+++ b/weed/stats/metrics.go
@@ -19,6 +19,7 @@ import (
 
 // Readonly volume types
 const (
+	Namespace        = "SeaweedFS"
 	IsReadOnly       = "IsReadOnly"
 	NoWriteOrDelete  = "noWriteOrDelete"
 	NoWriteCanDelete = "noWriteCanDelete"
@@ -32,7 +33,7 @@ var (
 
 	MasterClientConnectCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "wdclient",
 			Name:      "connect_updates",
 			Help:      "Counter of master client leader updates.",
@@ -40,7 +41,7 @@ var (
 
 	MasterRaftIsleader = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "master",
 			Name:      "is_leader",
 			Help:      "is leader",
@@ -48,7 +49,7 @@ var (
 
 	MasterReceivedHeartbeatCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "master",
 			Name:      "received_heartbeats",
 			Help:      "Counter of master received heartbeat.",
@@ -56,7 +57,7 @@ var (
 
 	MasterReplicaPlacementMismatch = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "master",
 			Name:      "replica_placement_mismatch",
 			Help:      "replica placement mismatch",
@@ -64,7 +65,7 @@ var (
 
 	MasterLeaderChangeCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "master",
 			Name:      "leader_changes",
 			Help:      "Counter of master leader changes.",
@@ -72,7 +73,7 @@ var (
 
 	FilerRequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filer",
 			Name:      "request_total",
 			Help:      "Counter of filer requests.",
@@ -80,7 +81,7 @@ var (
 
 	FilerRequestHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filer",
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of filer request processing time.",
@@ -89,7 +90,7 @@ var (
 
 	FilerServerLastSendTsOfSubscribeGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filer",
 			Name:      "last_send_timestamp_of_subscribe",
 			Help:      "The last send timestamp of the filer subscription.",
@@ -97,7 +98,7 @@ var (
 
 	FilerStoreCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filerStore",
 			Name:      "request_total",
 			Help:      "Counter of filer store requests.",
@@ -105,7 +106,7 @@ var (
 
 	FilerStoreHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filerStore",
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of filer store request processing time.",
@@ -114,7 +115,7 @@ var (
 
 	FilerSyncOffsetGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filerSync",
 			Name:      "sync_offset",
 			Help:      "The offset of the filer synchronization service.",
@@ -122,7 +123,7 @@ var (
 
 	VolumeServerRequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "request_total",
 			Help:      "Counter of volume server requests.",
@@ -130,7 +131,7 @@ var (
 
 	VolumeServerRequestHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of volume server request processing time.",
@@ -139,7 +140,7 @@ var (
 
 	VolumeServerVolumeCounter = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "volumes",
 			Help:      "Number of volumes or shards.",
@@ -147,7 +148,7 @@ var (
 
 	VolumeServerReadOnlyVolumeGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "read_only_volumes",
 			Help:      "Number of read only volumes.",
@@ -155,7 +156,7 @@ var (
 
 	VolumeServerMaxVolumeCounter = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "max_volumes",
 			Help:      "Maximum number of volumes.",
@@ -163,7 +164,7 @@ var (
 
 	VolumeServerDiskSizeGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "total_disk_size",
 			Help:      "Actual disk size used by volumes.",
@@ -171,7 +172,7 @@ var (
 
 	VolumeServerResourceGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "resource",
 			Help:      "Resource usage",
@@ -179,7 +180,7 @@ var (
 
 	S3RequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "s3",
 			Name:      "request_total",
 			Help:      "Counter of s3 requests.",
@@ -187,7 +188,7 @@ var (
 
 	S3RequestHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "s3",
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of s3 request processing time.",