aboutsummaryrefslogtreecommitdiff
path: root/test/kafka/kafka-client-loadtest/scripts/run-loadtest.sh
diff options
context:
space:
mode:
Diffstat (limited to 'test/kafka/kafka-client-loadtest/scripts/run-loadtest.sh')
-rwxr-xr-xtest/kafka/kafka-client-loadtest/scripts/run-loadtest.sh480
1 files changed, 480 insertions, 0 deletions
diff --git a/test/kafka/kafka-client-loadtest/scripts/run-loadtest.sh b/test/kafka/kafka-client-loadtest/scripts/run-loadtest.sh
new file mode 100755
index 000000000..7f6ddc79a
--- /dev/null
+++ b/test/kafka/kafka-client-loadtest/scripts/run-loadtest.sh
@@ -0,0 +1,480 @@
+#!/bin/bash
+
+# Kafka Client Load Test Runner Script
+# This script helps run various load test scenarios against SeaweedFS Kafka Gateway
+
+set -euo pipefail
+
+# Default configuration
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
+DOCKER_COMPOSE_FILE="$PROJECT_DIR/docker-compose.yml"
+CONFIG_FILE="$PROJECT_DIR/config/loadtest.yaml"
+
+# Default test parameters
+TEST_MODE="comprehensive"
+TEST_DURATION="300s"
+PRODUCER_COUNT=10
+CONSUMER_COUNT=5
+MESSAGE_RATE=1000
+MESSAGE_SIZE=1024
+TOPIC_COUNT=5
+PARTITIONS_PER_TOPIC=3
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Function to print colored output
+log_info() {
+ echo -e "${BLUE}[INFO]${NC} $1"
+}
+
+log_success() {
+ echo -e "${GREEN}[SUCCESS]${NC} $1"
+}
+
+log_warning() {
+ echo -e "${YELLOW}[WARNING]${NC} $1"
+}
+
+log_error() {
+ echo -e "${RED}[ERROR]${NC} $1"
+}
+
+# Function to show usage
+show_usage() {
+ cat << EOF
+Kafka Client Load Test Runner
+
+Usage: $0 [OPTIONS] [COMMAND]
+
+Commands:
+ start Start the load test infrastructure and run tests
+ stop Stop all services
+ restart Restart all services
+ status Show service status
+ logs Show logs from all services
+ clean Clean up all resources (volumes, networks, etc.)
+ monitor Start monitoring stack (Prometheus + Grafana)
+ scenarios Run predefined test scenarios
+
+Options:
+ -m, --mode MODE Test mode: producer, consumer, comprehensive (default: comprehensive)
+ -d, --duration DURATION Test duration (default: 300s)
+ -p, --producers COUNT Number of producers (default: 10)
+ -c, --consumers COUNT Number of consumers (default: 5)
+ -r, --rate RATE Messages per second per producer (default: 1000)
+ -s, --size SIZE Message size in bytes (default: 1024)
+ -t, --topics COUNT Number of topics (default: 5)
+ --partitions COUNT Partitions per topic (default: 3)
+ --config FILE Configuration file (default: config/loadtest.yaml)
+ --monitoring Enable monitoring stack
+ --wait-ready Wait for services to be ready before starting tests
+ -v, --verbose Verbose output
+ -h, --help Show this help message
+
+Examples:
+ # Run comprehensive test for 5 minutes
+ $0 start -m comprehensive -d 5m
+
+ # Run producer-only test with high throughput
+ $0 start -m producer -p 20 -r 2000 -d 10m
+
+ # Run consumer-only test
+ $0 start -m consumer -c 10
+
+ # Run with monitoring
+ $0 start --monitoring -d 15m
+
+ # Clean up everything
+ $0 clean
+
+Predefined Scenarios:
+ quick Quick smoke test (1 min, low load)
+ standard Standard load test (5 min, medium load)
+ stress Stress test (10 min, high load)
+ endurance Endurance test (30 min, sustained load)
+ burst Burst test (variable load)
+
+EOF
+}
+
+# Parse command line arguments
+parse_args() {
+ while [[ $# -gt 0 ]]; do
+ case $1 in
+ -m|--mode)
+ TEST_MODE="$2"
+ shift 2
+ ;;
+ -d|--duration)
+ TEST_DURATION="$2"
+ shift 2
+ ;;
+ -p|--producers)
+ PRODUCER_COUNT="$2"
+ shift 2
+ ;;
+ -c|--consumers)
+ CONSUMER_COUNT="$2"
+ shift 2
+ ;;
+ -r|--rate)
+ MESSAGE_RATE="$2"
+ shift 2
+ ;;
+ -s|--size)
+ MESSAGE_SIZE="$2"
+ shift 2
+ ;;
+ -t|--topics)
+ TOPIC_COUNT="$2"
+ shift 2
+ ;;
+ --partitions)
+ PARTITIONS_PER_TOPIC="$2"
+ shift 2
+ ;;
+ --config)
+ CONFIG_FILE="$2"
+ shift 2
+ ;;
+ --monitoring)
+ ENABLE_MONITORING=1
+ shift
+ ;;
+ --wait-ready)
+ WAIT_READY=1
+ shift
+ ;;
+ -v|--verbose)
+ VERBOSE=1
+ shift
+ ;;
+ -h|--help)
+ show_usage
+ exit 0
+ ;;
+ -*)
+ log_error "Unknown option: $1"
+ show_usage
+ exit 1
+ ;;
+ *)
+ if [[ -z "${COMMAND:-}" ]]; then
+ COMMAND="$1"
+ else
+ log_error "Multiple commands specified"
+ show_usage
+ exit 1
+ fi
+ shift
+ ;;
+ esac
+ done
+}
+
+# Check if Docker and Docker Compose are available
+check_dependencies() {
+ if ! command -v docker &> /dev/null; then
+ log_error "Docker is not installed or not in PATH"
+ exit 1
+ fi
+
+ if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then
+ log_error "Docker Compose is not installed or not in PATH"
+ exit 1
+ fi
+
+ # Use docker compose if available, otherwise docker-compose
+ if docker compose version &> /dev/null; then
+ DOCKER_COMPOSE="docker compose"
+ else
+ DOCKER_COMPOSE="docker-compose"
+ fi
+}
+
+# Wait for services to be ready
+wait_for_services() {
+ log_info "Waiting for services to be ready..."
+
+ local timeout=300 # 5 minutes timeout
+ local elapsed=0
+ local check_interval=5
+
+ while [[ $elapsed -lt $timeout ]]; do
+ if $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" ps --format table | grep -q "healthy"; then
+ if check_service_health; then
+ log_success "All services are ready!"
+ return 0
+ fi
+ fi
+
+ sleep $check_interval
+ elapsed=$((elapsed + check_interval))
+ log_info "Waiting... ($elapsed/${timeout}s)"
+ done
+
+ log_error "Services did not become ready within $timeout seconds"
+ return 1
+}
+
+# Check health of critical services
+check_service_health() {
+ # Check Kafka Gateway
+ if ! curl -s http://localhost:9093 >/dev/null 2>&1; then
+ return 1
+ fi
+
+ # Check Schema Registry
+ if ! curl -s http://localhost:8081/subjects >/dev/null 2>&1; then
+ return 1
+ fi
+
+ return 0
+}
+
+# Start the load test infrastructure
+start_services() {
+ log_info "Starting SeaweedFS Kafka load test infrastructure..."
+
+ # Set environment variables
+ export TEST_MODE="$TEST_MODE"
+ export TEST_DURATION="$TEST_DURATION"
+ export PRODUCER_COUNT="$PRODUCER_COUNT"
+ export CONSUMER_COUNT="$CONSUMER_COUNT"
+ export MESSAGE_RATE="$MESSAGE_RATE"
+ export MESSAGE_SIZE="$MESSAGE_SIZE"
+ export TOPIC_COUNT="$TOPIC_COUNT"
+ export PARTITIONS_PER_TOPIC="$PARTITIONS_PER_TOPIC"
+
+ # Start core services
+ $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" up -d \
+ seaweedfs-master \
+ seaweedfs-volume \
+ seaweedfs-filer \
+ seaweedfs-mq-broker \
+ kafka-gateway \
+ schema-registry
+
+ # Start monitoring if enabled
+ if [[ "${ENABLE_MONITORING:-0}" == "1" ]]; then
+ log_info "Starting monitoring stack..."
+ $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" --profile monitoring up -d
+ fi
+
+ # Wait for services to be ready if requested
+ if [[ "${WAIT_READY:-0}" == "1" ]]; then
+ wait_for_services
+ fi
+
+ log_success "Infrastructure started successfully"
+}
+
+# Run the load test
+run_loadtest() {
+ log_info "Starting Kafka client load test..."
+ log_info "Mode: $TEST_MODE, Duration: $TEST_DURATION"
+ log_info "Producers: $PRODUCER_COUNT, Consumers: $CONSUMER_COUNT"
+ log_info "Message Rate: $MESSAGE_RATE msgs/sec, Size: $MESSAGE_SIZE bytes"
+
+ # Run the load test
+ $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" --profile loadtest up --abort-on-container-exit kafka-client-loadtest
+
+ # Show test results
+ show_results
+}
+
+# Show test results
+show_results() {
+ log_info "Load test completed! Gathering results..."
+
+ # Get final metrics from the load test container
+ if $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" ps kafka-client-loadtest-runner &>/dev/null; then
+ log_info "Final test statistics:"
+ $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" exec -T kafka-client-loadtest-runner curl -s http://localhost:8080/stats || true
+ fi
+
+ # Show Prometheus metrics if monitoring is enabled
+ if [[ "${ENABLE_MONITORING:-0}" == "1" ]]; then
+ log_info "Monitoring dashboards available at:"
+ log_info " Prometheus: http://localhost:9090"
+ log_info " Grafana: http://localhost:3000 (admin/admin)"
+ fi
+
+ # Show where results are stored
+ if [[ -d "$PROJECT_DIR/test-results" ]]; then
+ log_info "Test results saved to: $PROJECT_DIR/test-results/"
+ fi
+}
+
+# Stop services
+stop_services() {
+ log_info "Stopping all services..."
+ $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" --profile loadtest --profile monitoring down
+ log_success "Services stopped"
+}
+
+# Show service status
+show_status() {
+ log_info "Service status:"
+ $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" ps
+}
+
+# Show logs
+show_logs() {
+ $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" logs -f "${1:-}"
+}
+
+# Clean up all resources
+clean_all() {
+ log_warning "This will remove all volumes, networks, and containers. Are you sure? (y/N)"
+ read -r response
+ if [[ "$response" =~ ^[Yy]$ ]]; then
+ log_info "Cleaning up all resources..."
+ $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" --profile loadtest --profile monitoring down -v --remove-orphans
+
+ # Remove any remaining volumes
+ docker volume ls -q | grep -E "(kafka-client-loadtest|seaweedfs)" | xargs -r docker volume rm
+
+ # Remove networks
+ docker network ls -q | grep -E "kafka-client-loadtest" | xargs -r docker network rm
+
+ log_success "Cleanup completed"
+ else
+ log_info "Cleanup cancelled"
+ fi
+}
+
+# Run predefined scenarios
+run_scenario() {
+ local scenario="$1"
+
+ case "$scenario" in
+ quick)
+ TEST_MODE="comprehensive"
+ TEST_DURATION="1m"
+ PRODUCER_COUNT=2
+ CONSUMER_COUNT=2
+ MESSAGE_RATE=100
+ MESSAGE_SIZE=512
+ TOPIC_COUNT=2
+ ;;
+ standard)
+ TEST_MODE="comprehensive"
+ TEST_DURATION="5m"
+ PRODUCER_COUNT=5
+ CONSUMER_COUNT=3
+ MESSAGE_RATE=500
+ MESSAGE_SIZE=1024
+ TOPIC_COUNT=3
+ ;;
+ stress)
+ TEST_MODE="comprehensive"
+ TEST_DURATION="10m"
+ PRODUCER_COUNT=20
+ CONSUMER_COUNT=10
+ MESSAGE_RATE=2000
+ MESSAGE_SIZE=2048
+ TOPIC_COUNT=10
+ ;;
+ endurance)
+ TEST_MODE="comprehensive"
+ TEST_DURATION="30m"
+ PRODUCER_COUNT=10
+ CONSUMER_COUNT=5
+ MESSAGE_RATE=1000
+ MESSAGE_SIZE=1024
+ TOPIC_COUNT=5
+ ;;
+ burst)
+ TEST_MODE="comprehensive"
+ TEST_DURATION="10m"
+ PRODUCER_COUNT=10
+ CONSUMER_COUNT=5
+ MESSAGE_RATE=1000
+ MESSAGE_SIZE=1024
+ TOPIC_COUNT=5
+ # Note: Burst behavior would be configured in the load test config
+ ;;
+ *)
+ log_error "Unknown scenario: $scenario"
+ log_info "Available scenarios: quick, standard, stress, endurance, burst"
+ exit 1
+ ;;
+ esac
+
+ log_info "Running $scenario scenario..."
+ start_services
+ if [[ "${WAIT_READY:-0}" == "1" ]]; then
+ wait_for_services
+ fi
+ run_loadtest
+}
+
+# Main execution
+main() {
+ if [[ $# -eq 0 ]]; then
+ show_usage
+ exit 0
+ fi
+
+ parse_args "$@"
+ check_dependencies
+
+ case "${COMMAND:-}" in
+ start)
+ start_services
+ run_loadtest
+ ;;
+ stop)
+ stop_services
+ ;;
+ restart)
+ stop_services
+ start_services
+ ;;
+ status)
+ show_status
+ ;;
+ logs)
+ show_logs
+ ;;
+ clean)
+ clean_all
+ ;;
+ monitor)
+ ENABLE_MONITORING=1
+ $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" --profile monitoring up -d
+ log_success "Monitoring stack started"
+ log_info "Prometheus: http://localhost:9090"
+ log_info "Grafana: http://localhost:3000 (admin/admin)"
+ ;;
+ scenarios)
+ if [[ -n "${2:-}" ]]; then
+ run_scenario "$2"
+ else
+ log_error "Please specify a scenario"
+ log_info "Available scenarios: quick, standard, stress, endurance, burst"
+ exit 1
+ fi
+ ;;
+ *)
+ log_error "Unknown command: ${COMMAND:-}"
+ show_usage
+ exit 1
+ ;;
+ esac
+}
+
+# Set default values
+ENABLE_MONITORING=0
+WAIT_READY=0
+VERBOSE=0
+
+# Run main function
+main "$@"
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 04:11:57 +0000