diff options
| -rw-r--r-- | other/metrics/grafana_seaweedfs.json | 333 | ||||
| -rw-r--r-- | weed/s3api/s3api_circuit_breaker.go | 22 | ||||
| -rw-r--r-- | weed/server/filer_server_handlers.go | 14 | ||||
| -rw-r--r-- | weed/stats/metrics.go | 36 |
4 files changed, 392 insertions, 13 deletions
diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index e8c98ae21..147677294 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -459,6 +459,170 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 86, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "SeaweedFS_filer_in_flight_upload_count", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "uploads", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer Concurrent Uploads", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 87, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "SeaweedFS_filer_in_flight_upload_bytes", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer Concurrent Upload Bytes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } ], "repeat": null, @@ -479,7 +643,172 @@ "y": 1 }, "id": 70, - "panels": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 88, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "SeaweedFS_s3_in_flight_upload_count", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "uploads", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "S3 Concurrent Uploads", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 89, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "SeaweedFS_s3_in_flight_upload_bytes", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "S3 Concurrent Upload Bytes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], "repeat": null, "title": "S3 Gateway", "type": "row" @@ -2693,4 +3022,4 @@ "title": "SeaweedFS", "uid": "nh02dOVnz", "version": 2 -} +}
\ No newline at end of file diff --git a/weed/s3api/s3api_circuit_breaker.go b/weed/s3api/s3api_circuit_breaker.go index 3c4f55a23..e19caf91b 100644 --- a/weed/s3api/s3api_circuit_breaker.go +++ b/weed/s3api/s3api_circuit_breaker.go @@ -3,6 +3,10 @@ package s3api import ( "errors" "fmt" + "net/http" + "sync" + "sync/atomic" + "github.com/gorilla/mux" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -11,9 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" - "net/http" - "sync" - "sync/atomic" + "github.com/seaweedfs/seaweedfs/weed/stats" ) type CircuitBreaker struct { @@ -121,12 +123,18 @@ func (cb *CircuitBreaker) Limit(f func(w http.ResponseWriter, r *http.Request), cb.s3a.inFlightDataLimitCond.L.Unlock() // Increment counters - atomic.AddInt64(&cb.s3a.inFlightUploads, 1) - atomic.AddInt64(&cb.s3a.inFlightDataSize, contentLength) + newUploads := atomic.AddInt64(&cb.s3a.inFlightUploads, 1) + newSize := atomic.AddInt64(&cb.s3a.inFlightDataSize, contentLength) + // Update metrics + stats.S3InFlightUploadCountGauge.Set(float64(newUploads)) + stats.S3InFlightUploadBytesGauge.Set(float64(newSize)) defer func() { // Decrement counters - atomic.AddInt64(&cb.s3a.inFlightUploads, -1) - atomic.AddInt64(&cb.s3a.inFlightDataSize, -contentLength) + newUploads := atomic.AddInt64(&cb.s3a.inFlightUploads, -1) + newSize := atomic.AddInt64(&cb.s3a.inFlightDataSize, -contentLength) + // Update metrics + stats.S3InFlightUploadCountGauge.Set(float64(newUploads)) + stats.S3InFlightUploadBytesGauge.Set(float64(newSize)) cb.s3a.inFlightDataLimitCond.Signal() }() } diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index a2eab9365..f068579ec 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -112,12 +112,18 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { fs.inFlightDataLimitCond.L.Unlock() // Increment counters - atomic.AddInt64(&fs.inFlightUploads, 1) - atomic.AddInt64(&fs.inFlightDataSize, contentLength) + newUploads := atomic.AddInt64(&fs.inFlightUploads, 1) + newSize := atomic.AddInt64(&fs.inFlightDataSize, contentLength) + // Update metrics + stats.FilerInFlightUploadCountGauge.Set(float64(newUploads)) + stats.FilerInFlightUploadBytesGauge.Set(float64(newSize)) defer func() { // Decrement counters - atomic.AddInt64(&fs.inFlightUploads, -1) - atomic.AddInt64(&fs.inFlightDataSize, -contentLength) + newUploads := atomic.AddInt64(&fs.inFlightUploads, -1) + newSize := atomic.AddInt64(&fs.inFlightDataSize, -contentLength) + // Update metrics + stats.FilerInFlightUploadCountGauge.Set(float64(newUploads)) + stats.FilerInFlightUploadBytesGauge.Set(float64(newSize)) fs.inFlightDataLimitCond.Signal() }() diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 95491abf1..914d59d28 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -167,6 +167,22 @@ var ( Help: "Current number of in-flight requests being handled by filer.", }, []string{"type"}) + FilerInFlightUploadBytesGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "filer", + Name: "in_flight_upload_bytes", + Help: "Current number of bytes being uploaded to filer.", + }) + + FilerInFlightUploadCountGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "filer", + Name: "in_flight_upload_count", + Help: "Current number of uploads in progress to filer.", + }) + FilerServerLastSendTsOfSubscribeGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: Namespace, @@ -371,6 +387,22 @@ var ( Help: "Current number of in-flight requests being handled by s3.", }, []string{"type"}) + S3InFlightUploadBytesGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "s3", + Name: "in_flight_upload_bytes", + Help: "Current number of bytes being uploaded to S3.", + }) + + S3InFlightUploadCountGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "s3", + Name: "in_flight_upload_count", + Help: "Current number of uploads in progress to S3.", + }) + S3BucketTrafficReceivedBytesCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: Namespace, @@ -422,6 +454,8 @@ func init() { Gather.MustRegister(FilerHandlerCounter) Gather.MustRegister(FilerRequestHistogram) Gather.MustRegister(FilerInFlightRequestsGauge) + Gather.MustRegister(FilerInFlightUploadBytesGauge) + Gather.MustRegister(FilerInFlightUploadCountGauge) Gather.MustRegister(FilerStoreCounter) Gather.MustRegister(FilerStoreHistogram) Gather.MustRegister(FilerSyncOffsetGauge) @@ -450,6 +484,8 @@ func init() { Gather.MustRegister(S3HandlerCounter) Gather.MustRegister(S3RequestHistogram) Gather.MustRegister(S3InFlightRequestsGauge) + Gather.MustRegister(S3InFlightUploadBytesGauge) + Gather.MustRegister(S3InFlightUploadCountGauge) Gather.MustRegister(S3TimeToFirstByteHistogram) Gather.MustRegister(S3BucketTrafficReceivedBytesCounter) Gather.MustRegister(S3BucketTrafficSentBytesCounter) |