diff options
Diffstat (limited to 'test/kafka/kafka-client-loadtest/monitoring')
5 files changed, 245 insertions, 0 deletions
diff --git a/test/kafka/kafka-client-loadtest/monitoring/grafana/dashboards/kafka-loadtest.json b/test/kafka/kafka-client-loadtest/monitoring/grafana/dashboards/kafka-loadtest.json new file mode 100644 index 000000000..3ea04fb68 --- /dev/null +++ b/test/kafka/kafka-client-loadtest/monitoring/grafana/dashboards/kafka-loadtest.json @@ -0,0 +1,106 @@ +{ + "dashboard": { + "id": null, + "title": "Kafka Client Load Test Dashboard", + "tags": ["kafka", "loadtest", "seaweedfs"], + "timezone": "browser", + "panels": [ + { + "id": 1, + "title": "Messages Produced/Consumed", + "type": "stat", + "targets": [ + { + "expr": "rate(kafka_loadtest_messages_produced_total[5m])", + "legendFormat": "Produced/sec" + }, + { + "expr": "rate(kafka_loadtest_messages_consumed_total[5m])", + "legendFormat": "Consumed/sec" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0} + }, + { + "id": 2, + "title": "Message Latency", + "type": "graph", + "targets": [ + { + "expr": "histogram_quantile(0.95, kafka_loadtest_message_latency_seconds)", + "legendFormat": "95th percentile" + }, + { + "expr": "histogram_quantile(0.99, kafka_loadtest_message_latency_seconds)", + "legendFormat": "99th percentile" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0} + }, + { + "id": 3, + "title": "Error Rates", + "type": "graph", + "targets": [ + { + "expr": "rate(kafka_loadtest_producer_errors_total[5m])", + "legendFormat": "Producer Errors/sec" + }, + { + "expr": "rate(kafka_loadtest_consumer_errors_total[5m])", + "legendFormat": "Consumer Errors/sec" + } + ], + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 8} + }, + { + "id": 4, + "title": "Throughput (MB/s)", + "type": "graph", + "targets": [ + { + "expr": "rate(kafka_loadtest_bytes_produced_total[5m]) / 1024 / 1024", + "legendFormat": "Produced MB/s" + }, + { + "expr": "rate(kafka_loadtest_bytes_consumed_total[5m]) / 1024 / 1024", + "legendFormat": "Consumed MB/s" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16} + }, + { + "id": 5, + "title": "Active Clients", + "type": "stat", + "targets": [ + { + "expr": "kafka_loadtest_active_producers", + "legendFormat": "Producers" + }, + { + "expr": "kafka_loadtest_active_consumers", + "legendFormat": "Consumers" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16} + }, + { + "id": 6, + "title": "Consumer Lag", + "type": "graph", + "targets": [ + { + "expr": "kafka_loadtest_consumer_lag_messages", + "legendFormat": "{{consumer_group}}-{{topic}}-{{partition}}" + } + ], + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 24} + } + ], + "time": {"from": "now-30m", "to": "now"}, + "refresh": "5s", + "schemaVersion": 16, + "version": 0 + } +} diff --git a/test/kafka/kafka-client-loadtest/monitoring/grafana/dashboards/seaweedfs.json b/test/kafka/kafka-client-loadtest/monitoring/grafana/dashboards/seaweedfs.json new file mode 100644 index 000000000..4c2261f22 --- /dev/null +++ b/test/kafka/kafka-client-loadtest/monitoring/grafana/dashboards/seaweedfs.json @@ -0,0 +1,62 @@ +{ + "dashboard": { + "id": null, + "title": "SeaweedFS Cluster Dashboard", + "tags": ["seaweedfs", "storage"], + "timezone": "browser", + "panels": [ + { + "id": 1, + "title": "Master Status", + "type": "stat", + "targets": [ + { + "expr": "up{job=\"seaweedfs-master\"}", + "legendFormat": "Master Up" + } + ], + "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0} + }, + { + "id": 2, + "title": "Volume Status", + "type": "stat", + "targets": [ + { + "expr": "up{job=\"seaweedfs-volume\"}", + "legendFormat": "Volume Up" + } + ], + "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0} + }, + { + "id": 3, + "title": "Filer Status", + "type": "stat", + "targets": [ + { + "expr": "up{job=\"seaweedfs-filer\"}", + "legendFormat": "Filer Up" + } + ], + "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0} + }, + { + "id": 4, + "title": "MQ Broker Status", + "type": "stat", + "targets": [ + { + "expr": "up{job=\"seaweedfs-mq-broker\"}", + "legendFormat": "MQ Broker Up" + } + ], + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0} + } + ], + "time": {"from": "now-30m", "to": "now"}, + "refresh": "10s", + "schemaVersion": 16, + "version": 0 + } +} diff --git a/test/kafka/kafka-client-loadtest/monitoring/grafana/provisioning/dashboards/dashboard.yml b/test/kafka/kafka-client-loadtest/monitoring/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 000000000..0bcf3d818 --- /dev/null +++ b/test/kafka/kafka-client-loadtest/monitoring/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards diff --git a/test/kafka/kafka-client-loadtest/monitoring/grafana/provisioning/datasources/datasource.yml b/test/kafka/kafka-client-loadtest/monitoring/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 000000000..fb78be722 --- /dev/null +++ b/test/kafka/kafka-client-loadtest/monitoring/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: true + editable: true + version: 1 diff --git a/test/kafka/kafka-client-loadtest/monitoring/prometheus/prometheus.yml b/test/kafka/kafka-client-loadtest/monitoring/prometheus/prometheus.yml new file mode 100644 index 000000000..f62091d52 --- /dev/null +++ b/test/kafka/kafka-client-loadtest/monitoring/prometheus/prometheus.yml @@ -0,0 +1,54 @@ +# Prometheus configuration for Kafka Load Test monitoring + +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + # - "first_rules.yml" + # - "second_rules.yml" + +scrape_configs: + # Scrape Prometheus itself + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + # Scrape load test metrics + - job_name: 'kafka-loadtest' + static_configs: + - targets: ['kafka-client-loadtest-runner:8080'] + scrape_interval: 5s + metrics_path: '/metrics' + + # Scrape SeaweedFS Master metrics + - job_name: 'seaweedfs-master' + static_configs: + - targets: ['seaweedfs-master:9333'] + metrics_path: '/metrics' + + # Scrape SeaweedFS Volume metrics + - job_name: 'seaweedfs-volume' + static_configs: + - targets: ['seaweedfs-volume:8080'] + metrics_path: '/metrics' + + # Scrape SeaweedFS Filer metrics + - job_name: 'seaweedfs-filer' + static_configs: + - targets: ['seaweedfs-filer:8888'] + metrics_path: '/metrics' + + # Scrape SeaweedFS MQ Broker metrics (if available) + - job_name: 'seaweedfs-mq-broker' + static_configs: + - targets: ['seaweedfs-mq-broker:17777'] + metrics_path: '/metrics' + scrape_interval: 10s + + # Scrape Kafka Gateway metrics (if available) + - job_name: 'kafka-gateway' + static_configs: + - targets: ['kafka-gateway:9093'] + metrics_path: '/metrics' + scrape_interval: 10s |