filer: adds basic metrics pushing to Prometheus gateway

6 files changed, 88 insertions, 2 deletions

diff --git a/weed/command/filer.go b/weed/command/filer.go
index 2aa022cd0..83e9df20f 100644
--- a/weed/command/filer.go
+++ b/weed/command/filer.go
@@ -34,6 +34,8 @@ type FilerOptions struct {
 	dataCenter              *string
 	enableNotification      *bool
 	disableHttp             *bool
+	metricsAddress          *string
+	metricsIntervalSec      *int
 
 	// default leveldb directory, used in "weed server" mode
 	defaultLevelDbDirectory *string
@@ -53,6 +55,8 @@ func init() {
 	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
 	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
 	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
+	f.metricsAddress = cmdFiler.Flag.String("metrics.address", "", "Prometheus gateway address")
+	f.metricsIntervalSec = cmdFiler.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
 }
 
 var cmdFiler = &Command{
@@ -110,6 +114,8 @@ func (fo *FilerOptions) startFiler() {
 		DataCenter:         *fo.dataCenter,
 		DefaultLevelDbDir:  defaultLevelDbDirectory,
 		DisableHttp:        *fo.disableHttp,
+		MetricsAddress:     *fo.metricsAddress,
+		MetricsIntervalSec: *fo.metricsIntervalSec,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
diff --git a/weed/command/server.go b/weed/command/server.go
index 437b0ad83..30618c452 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -24,8 +24,10 @@ import (
 )
 
 type ServerOptions struct {
-	cpuprofile *string
-	v          VolumeServerOptions
+	cpuprofile         *string
+	metricsAddress     *string
+	metricsIntervalSec *int
+	v                  VolumeServerOptions
 }
 
 var (
@@ -81,6 +83,8 @@ var (
 
 func init() {
 	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
+	serverOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
+	serverOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
 	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
@@ -143,6 +147,9 @@ func runServer(cmd *Command, args []string) bool {
 	filerOptions.dataCenter = serverDataCenter
 	filerOptions.disableHttp = serverDisableHttp
 
+	filerOptions.metricsAddress = serverOptions.metricsAddress
+	filerOptions.metricsIntervalSec = serverOptions.metricsIntervalSec
+
 	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
 	s3Options.filer = &filerAddress
 
diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go
index f65723b47..0046035b0 100644
--- a/weed/server/filer_server.go
+++ b/weed/server/filer_server.go
@@ -3,8 +3,11 @@ package weed_server
 import (
 	"net/http"
 	"os"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/push"
 	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
@@ -36,6 +39,8 @@ type FilerOption struct {
 	DataCenter         string
 	DefaultLevelDbDir  string
 	DisableHttp        bool
+	MetricsAddress     string
+	MetricsIntervalSec int
 }
 
 type FilerServer struct {
@@ -83,5 +88,29 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 		readonlyMux.HandleFunc("/", fs.readonlyFilerHandler)
 	}
 
+	startPushingMetric(option.MetricsAddress, option.MetricsIntervalSec)
+
 	return fs, nil
 }
+
+func startPushingMetric(addr string, intervalSeconds int) {
+	if intervalSeconds == 0 || addr == "" {
+		glog.V(0).Info("disable metrics reporting")
+		return
+	}
+	glog.V(0).Infof("push metrics to %s every %d seconds", addr, intervalSeconds)
+	go loopPushMetrics(addr, intervalSeconds)
+}
+
+func loopPushMetrics(addr string, intervalSeconds int) {
+
+	pusher := push.New(addr, "filer").Gatherer(prometheus.DefaultGatherer)
+
+	for {
+		err := pusher.Push()
+		if err != nil {
+			glog.V(0).Infof("could not push metrics to prometheus push gateway %s: %v", addr, err)
+		}
+		time.Sleep(time.Duration(intervalSeconds) * time.Second)
+	}
+}
diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go
index 663bf4b58..ca6269fd1 100644
--- a/weed/server/filer_server_handlers_read.go
+++ b/weed/server/filer_server_handlers_read.go
@@ -11,6 +11,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -18,6 +19,11 @@ import (
 )
 
 func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) {
+
+	filerRequestCounter.WithLabelValues("get").Inc()
+	start := time.Now()
+	defer func() { filerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }()
+
 	path := r.URL.Path
 	if strings.HasSuffix(path, "/") && len(path) > 1 {
 		path = path[:len(path)-1]
@@ -30,6 +36,8 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 			return
 		}
 		glog.V(1).Infof("Not found %s: %v", path, err)
+
+		filerRequestCounter.WithLabelValues("read.notfound").Inc()
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
@@ -45,6 +53,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 
 	if len(entry.Chunks) == 0 {
 		glog.V(1).Infof("no file chunks for %s, attr=%+v", path, entry.Attr)
+		filerRequestCounter.WithLabelValues("read.nocontent").Inc()
 		w.WriteHeader(http.StatusNoContent)
 		return
 	}
diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go
index e6fe9af76..214a77102 100644
--- a/weed/server/filer_server_handlers_write.go
+++ b/weed/server/filer_server_handlers_write.go
@@ -70,6 +70,10 @@ func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request,
 
 func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 
+	filerRequestCounter.WithLabelValues("post").Inc()
+	start := time.Now()
+	defer func() { filerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) }()
+
 	ctx := context.Background()
 
 	query := r.URL.Query()
@@ -228,6 +232,10 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 // curl -X DELETE http://localhost:8888/path/to?recursive=true
 func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
 
+	filerRequestCounter.WithLabelValues("delete").Inc()
+	start := time.Now()
+	defer func() { filerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds()) }()
+
 	isRecursive := r.FormValue("recursive") == "true"
 
 	err := fs.filer.DeleteEntryMetaAndData(context.Background(), filer2.FullPath(r.URL.Path), isRecursive, true)
diff --git a/weed/server/metrics.go b/weed/server/metrics.go
new file mode 100644
index 000000000..89a7d2a96
--- /dev/null
+++ b/weed/server/metrics.go
@@ -0,0 +1,27 @@
+package weed_server
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	filerRequestCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "filer",
+			Name:      "request_total",
+			Help:      "Counter of filer requests.",
+		}, []string{"type"})
+
+	filerRequestHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "filer",
+			Name:      "request_seconds",
+			Help:      "Bucketed histogram of filer request processing time.",
+			Buckets:   prometheus.ExponentialBuckets(0.0005, 2, 18),
+		}, []string{"type"})
+)
+
+func init() {
+	prometheus.MustRegister(filerRequestCounter)
+	prometheus.MustRegister(filerRequestHistogram)
+}