6 files changed, 105 insertions, 3 deletions
diff --git a/weed/stats/disk_notsupported.go b/weed/stats/disk_notsupported.go
index 3d99e6ce7..2bc24a2f0 100644
--- a/weed/stats/disk_notsupported.go
+++ b/weed/stats/disk_notsupported.go
@@ -1,3 +1,4 @@
+//go:build openbsd || netbsd || plan9 || solaris
 // +build openbsd netbsd plan9 solaris
 
 package stats
diff --git a/weed/stats/disk_supported.go b/weed/stats/disk_supported.go
index dff580b5b..e698b0095 100644
--- a/weed/stats/disk_supported.go
+++ b/weed/stats/disk_supported.go
@@ -1,3 +1,4 @@
+//go:build !windows && !openbsd && !netbsd && !plan9 && !solaris
 // +build !windows,!openbsd,!netbsd,!plan9,!solaris
 
 package stats
diff --git a/weed/stats/memory_notsupported.go b/weed/stats/memory_notsupported.go
index 2bed95266..2508b561b 100644
--- a/weed/stats/memory_notsupported.go
+++ b/weed/stats/memory_notsupported.go
@@ -1,3 +1,4 @@
+//go:build !linux
 // +build !linux
 
 package stats
diff --git a/weed/stats/memory_supported.go b/weed/stats/memory_supported.go
index 91fdd005b..55ad4c6bb 100644
--- a/weed/stats/memory_supported.go
+++ b/weed/stats/memory_supported.go
@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package stats
diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go
index 48d964418..f0b810608 100644
--- a/weed/stats/metrics.go
+++ b/weed/stats/metrics.go
@@ -3,8 +3,10 @@ package stats
 import (
 	"fmt"
 	"log"
+	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -18,6 +20,46 @@ import (
 var (
 	Gather = prometheus.NewRegistry()
 
+	MasterClientConnectCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "wdclient",
+			Name:      "connect_updates",
+			Help:      "Counter of master client leader updates.",
+		}, []string{"type"})
+
+	MasterRaftIsleader = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "master",
+			Name:      "is_leader",
+			Help:      "is leader",
+		})
+
+	MasterReceivedHeartbeatCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "master",
+			Name:      "received_heartbeats",
+			Help:      "Counter of master received heartbeat.",
+		}, []string{"type"})
+
+	MasterReplicaPlacementMismatch = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "master",
+			Name:      "replica_placement_mismatch",
+			Help:      "replica placement mismatch",
+		}, []string{"collection", "id"})
+
+	MasterLeaderChangeCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "master",
+			Name:      "leader_changes",
+			Help:      "Counter of master leader changes.",
+		}, []string{"type"})
+
 	FilerRequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "SeaweedFS",
@@ -35,6 +77,14 @@ var (
 			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 24),
 		}, []string{"type"})
 
+	FilerServerLastSendTsOfSubscribeGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "filer",
+			Name:      "last_send_timestamp_of_subscribe",
+			Help:      "The last send timestamp of the filer subscription.",
+		}, []string{"sourceFiler", "clientName", "path"})
+
 	FilerStoreCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "SeaweedFS",
@@ -52,6 +102,14 @@ var (
 			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 24),
 		}, []string{"store", "type"})
 
+	FilerSyncOffsetGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "filerSync",
+			Name:      "sync_offset",
+			Help:      "The offset of the filer synchronization service.",
+		}, []string{"sourceFiler", "targetFiler", "clientName", "path"})
+
 	VolumeServerRequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "SeaweedFS",
@@ -115,7 +173,8 @@ var (
 			Subsystem: "s3",
 			Name:      "request_total",
 			Help:      "Counter of s3 requests.",
-		}, []string{"type", "code"})
+		}, []string{"type", "code", "bucket"})
+
 	S3RequestHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: "SeaweedFS",
@@ -123,14 +182,22 @@ var (
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of s3 request processing time.",
 			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 24),
-		}, []string{"type"})
+		}, []string{"type", "bucket"})
 )
 
 func init() {
+	Gather.MustRegister(MasterClientConnectCounter)
+	Gather.MustRegister(MasterRaftIsleader)
+	Gather.MustRegister(MasterReceivedHeartbeatCounter)
+	Gather.MustRegister(MasterLeaderChangeCounter)
+	Gather.MustRegister(MasterReplicaPlacementMismatch)
+
 	Gather.MustRegister(FilerRequestCounter)
 	Gather.MustRegister(FilerRequestHistogram)
 	Gather.MustRegister(FilerStoreCounter)
 	Gather.MustRegister(FilerStoreHistogram)
+	Gather.MustRegister(FilerSyncOffsetGauge)
+	Gather.MustRegister(FilerServerLastSendTsOfSubscribeGauge)
 	Gather.MustRegister(collectors.NewGoCollector())
 	Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
 
@@ -180,5 +247,5 @@ func SourceName(port uint32) string {
 	if err != nil {
 		return "unknown"
 	}
-	return fmt.Sprintf("%s:%d", hostname, port)
+	return net.JoinHostPort(hostname, strconv.Itoa(int(port)))
 }
diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go
new file mode 100644
index 000000000..59ae022aa
--- /dev/null
+++ b/weed/stats/metrics_names.go
@@ -0,0 +1,31 @@
+package stats
+
+// This file contains metric names for all errors
+// The naming convention is ErrorSomeThing = "error.some.thing"
+const (
+	// volume server
+	ErrorSizeMismatchOffsetSize = "errorSizeMismatchOffsetSize"
+	ErrorSizeMismatch           = "errorSizeMismatch"
+	ErrorCRC                    = "errorCRC"
+	ErrorIndexOutOfRange        = "errorIndexOutOfRange"
+
+	// master topology
+	ErrorWriteToLocalDisk = "errorWriteToLocalDisk"
+	ErrorUnmarshalPairs   = "errorUnmarshalPairs"
+	ErrorWriteToReplicas  = "errorWriteToReplicas"
+
+	// master client
+	FailedToKeepConnected = "failedToKeepConnected"
+	FailedToSend          = "failedToSend"
+	FailedToReceive       = "failedToReceive"
+	RedirectedToleader    = "redirectedToleader"
+	OnPeerUpdate          = "onPeerUpdate"
+	Failed                = "failed"
+
+	// filer handler
+	ErrorReadNotFound = "read.notfound"
+	ErrorReadInternal = "read.internalerror"
+	ErrorWriteEntry   = "write.entry.failed"
+	ErrorReadCache    = "read.cache.failed"
+	ErrorReadStream   = "read.stream.failed"
+)