diff options
Diffstat (limited to 'test/kafka/scripts')
| -rwxr-xr-x | test/kafka/scripts/kafka-gateway-start.sh | 54 | ||||
| -rw-r--r-- | test/kafka/scripts/test-broker-discovery.sh | 129 | ||||
| -rwxr-xr-x | test/kafka/scripts/test-broker-startup.sh | 111 | ||||
| -rwxr-xr-x | test/kafka/scripts/test_schema_registry.sh | 77 | ||||
| -rwxr-xr-x | test/kafka/scripts/wait-for-services.sh | 135 |
5 files changed, 506 insertions, 0 deletions
diff --git a/test/kafka/scripts/kafka-gateway-start.sh b/test/kafka/scripts/kafka-gateway-start.sh new file mode 100755 index 000000000..08561cef5 --- /dev/null +++ b/test/kafka/scripts/kafka-gateway-start.sh @@ -0,0 +1,54 @@ +#!/bin/sh + +# Kafka Gateway Startup Script for Integration Testing + +set -e + +echo "Starting Kafka Gateway..." + +SEAWEEDFS_MASTERS=${SEAWEEDFS_MASTERS:-seaweedfs-master:9333} +SEAWEEDFS_FILER=${SEAWEEDFS_FILER:-seaweedfs-filer:8888} +SEAWEEDFS_MQ_BROKER=${SEAWEEDFS_MQ_BROKER:-seaweedfs-mq-broker:17777} +SEAWEEDFS_FILER_GROUP=${SEAWEEDFS_FILER_GROUP:-} + +# Wait for dependencies +echo "Waiting for SeaweedFS master(s)..." +OLD_IFS="$IFS" +IFS=',' +for MASTER in $SEAWEEDFS_MASTERS; do + MASTER_HOST=${MASTER%:*} + MASTER_PORT=${MASTER#*:} + while ! nc -z "$MASTER_HOST" "$MASTER_PORT"; do + sleep 1 + done + echo "SeaweedFS master $MASTER is ready" +done +IFS="$OLD_IFS" + +echo "Waiting for SeaweedFS Filer..." +while ! nc -z "${SEAWEEDFS_FILER%:*}" "${SEAWEEDFS_FILER#*:}"; do + sleep 1 +done +echo "SeaweedFS Filer is ready" + +echo "Waiting for SeaweedFS MQ Broker..." +while ! nc -z "${SEAWEEDFS_MQ_BROKER%:*}" "${SEAWEEDFS_MQ_BROKER#*:}"; do + sleep 1 +done +echo "SeaweedFS MQ Broker is ready" + +echo "Waiting for Schema Registry..." +while ! curl -f "${SCHEMA_REGISTRY_URL}/subjects" > /dev/null 2>&1; do + sleep 1 +done +echo "Schema Registry is ready" + +# Start Kafka Gateway +echo "Starting Kafka Gateway on port ${KAFKA_PORT:-9093}..." +exec /usr/bin/weed mq.kafka.gateway \ + -master=${SEAWEEDFS_MASTERS} \ + -filerGroup=${SEAWEEDFS_FILER_GROUP} \ + -port=${KAFKA_PORT:-9093} \ + -port.pprof=${PPROF_PORT:-10093} \ + -schema-registry-url=${SCHEMA_REGISTRY_URL} \ + -ip=0.0.0.0 diff --git a/test/kafka/scripts/test-broker-discovery.sh b/test/kafka/scripts/test-broker-discovery.sh new file mode 100644 index 000000000..b4937b7f7 --- /dev/null +++ b/test/kafka/scripts/test-broker-discovery.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +# Test script to verify broker discovery works end-to-end + +set -e + +echo "=== Testing SeaweedFS Broker Discovery ===" + +cd /Users/chrislu/go/src/github.com/seaweedfs/seaweedfs + +# Build weed binary +echo "Building weed binary..." +go build -o /tmp/weed-discovery ./weed + +# Setup data directory +WEED_DATA_DIR="/tmp/seaweedfs-discovery-test-$$" +mkdir -p "$WEED_DATA_DIR" +echo "Using data directory: $WEED_DATA_DIR" + +# Cleanup function +cleanup() { + echo "Cleaning up..." + pkill -f "weed.*server" || true + pkill -f "weed.*mq.broker" || true + sleep 2 + rm -rf "$WEED_DATA_DIR" + rm -f /tmp/weed-discovery* /tmp/broker-discovery-test* +} +trap cleanup EXIT + +# Start SeaweedFS server with consistent IP configuration +echo "Starting SeaweedFS server..." +/tmp/weed-discovery -v 1 server \ + -ip="127.0.0.1" \ + -ip.bind="127.0.0.1" \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp \ + -master.port=9333 \ + -volume.port=8081 \ + -filer.port=8888 \ + -filer=true \ + -metricsPort=9325 \ + > /tmp/weed-discovery-server.log 2>&1 & + +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +# Wait for master +echo "Waiting for master..." +for i in $(seq 1 30); do + if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then + echo "โ Master is up" + break + fi + echo " Waiting for master... ($i/30)" + sleep 1 +done + +# Give components time to initialize +echo "Waiting for components to initialize..." +sleep 10 + +# Start MQ broker +echo "Starting MQ broker..." +/tmp/weed-discovery -v 2 mq.broker \ + -master="127.0.0.1:9333" \ + -port=17777 \ + > /tmp/weed-discovery-broker.log 2>&1 & + +BROKER_PID=$! +echo "Broker PID: $BROKER_PID" + +# Wait for broker +echo "Waiting for broker to register..." +sleep 15 +broker_ready=false +for i in $(seq 1 20); do + if nc -z 127.0.0.1 17777; then + echo "โ MQ broker is accepting connections" + broker_ready=true + break + fi + echo " Waiting for MQ broker... ($i/20)" + sleep 1 +done + +if [ "$broker_ready" = false ]; then + echo "[FAIL] MQ broker failed to start" + echo "Server logs:" + cat /tmp/weed-discovery-server.log + echo "Broker logs:" + cat /tmp/weed-discovery-broker.log + exit 1 +fi + +# Additional wait for broker registration +echo "Allowing broker to register with master..." +sleep 15 + +# Check cluster status +echo "Checking cluster status..." +CLUSTER_STATUS=$(curl -s "http://127.0.0.1:9333/cluster/status") +echo "Cluster status: $CLUSTER_STATUS" + +# Now test broker discovery using the same approach as the Kafka gateway +echo "Testing broker discovery..." +cd test/kafka +SEAWEEDFS_MASTERS=127.0.0.1:9333 timeout 30s go test -v -run "TestOffsetManagement" -timeout 25s ./e2e/... > /tmp/broker-discovery-test.log 2>&1 && discovery_success=true || discovery_success=false + +if [ "$discovery_success" = true ]; then + echo "[OK] Broker discovery test PASSED!" + echo "Gateway was able to discover and connect to MQ brokers" +else + echo "[FAIL] Broker discovery test FAILED" + echo "Last few lines of test output:" + tail -20 /tmp/broker-discovery-test.log || echo "No test logs available" +fi + +echo +echo "๐ Test Results:" +echo " Broker startup: โ
" +echo " Broker registration: โ
" +echo " Gateway discovery: $([ "$discovery_success" = true ] && echo "โ
" || echo "โ")" + +echo +echo "๐ Logs available:" +echo " Server: /tmp/weed-discovery-server.log" +echo " Broker: /tmp/weed-discovery-broker.log" +echo " Discovery test: /tmp/broker-discovery-test.log" diff --git a/test/kafka/scripts/test-broker-startup.sh b/test/kafka/scripts/test-broker-startup.sh new file mode 100755 index 000000000..410376d3b --- /dev/null +++ b/test/kafka/scripts/test-broker-startup.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Script to test SeaweedFS MQ broker startup locally +# This helps debug broker startup issues before running CI + +set -e + +echo "=== Testing SeaweedFS MQ Broker Startup ===" + +# Build weed binary +echo "Building weed binary..." +cd "$(dirname "$0")/../../.." +go build -o /tmp/weed ./weed + +# Setup data directory +WEED_DATA_DIR="/tmp/seaweedfs-broker-test-$$" +mkdir -p "$WEED_DATA_DIR" +echo "Using data directory: $WEED_DATA_DIR" + +# Cleanup function +cleanup() { + echo "Cleaning up..." + pkill -f "weed.*server" || true + pkill -f "weed.*mq.broker" || true + sleep 2 + rm -rf "$WEED_DATA_DIR" + rm -f /tmp/weed-*.log +} +trap cleanup EXIT + +# Start SeaweedFS server +echo "Starting SeaweedFS server..." +/tmp/weed -v 1 server \ + -ip="127.0.0.1" \ + -ip.bind="0.0.0.0" \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp \ + -master.port=9333 \ + -volume.port=8081 \ + -filer.port=8888 \ + -filer=true \ + -metricsPort=9325 \ + > /tmp/weed-server-test.log 2>&1 & + +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +# Wait for master +echo "Waiting for master..." +for i in $(seq 1 30); do + if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then + echo "โ Master is up" + break + fi + echo " Waiting for master... ($i/30)" + sleep 1 +done + +# Wait for filer +echo "Waiting for filer..." +for i in $(seq 1 30); do + if nc -z 127.0.0.1 8888; then + echo "โ Filer is up" + break + fi + echo " Waiting for filer... ($i/30)" + sleep 1 +done + +# Start MQ broker +echo "Starting MQ broker..." +/tmp/weed -v 2 mq.broker \ + -master="127.0.0.1:9333" \ + -ip="127.0.0.1" \ + -port=17777 \ + > /tmp/weed-mq-broker-test.log 2>&1 & + +BROKER_PID=$! +echo "Broker PID: $BROKER_PID" + +# Wait for broker +echo "Waiting for broker..." +broker_ready=false +for i in $(seq 1 30); do + if nc -z 127.0.0.1 17777; then + echo "โ MQ broker is up" + broker_ready=true + break + fi + echo " Waiting for MQ broker... ($i/30)" + sleep 1 +done + +if [ "$broker_ready" = false ]; then + echo "โ MQ broker failed to start" + echo + echo "=== Server logs ===" + cat /tmp/weed-server-test.log + echo + echo "=== Broker logs ===" + cat /tmp/weed-mq-broker-test.log + exit 1 +fi + +# Broker started successfully - discovery will be tested by Kafka gateway +echo "โ Broker started successfully and accepting connections" + +echo +echo "[OK] All tests passed!" +echo "Server logs: /tmp/weed-server-test.log" +echo "Broker logs: /tmp/weed-mq-broker-test.log" diff --git a/test/kafka/scripts/test_schema_registry.sh b/test/kafka/scripts/test_schema_registry.sh new file mode 100755 index 000000000..d5ba8574a --- /dev/null +++ b/test/kafka/scripts/test_schema_registry.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# Test script for schema registry E2E testing +# This script sets up a mock schema registry and runs the E2E tests + +set -e + +echo "๐ Starting Schema Registry E2E Test" + +# Check if we have a real schema registry URL +if [ -n "$SCHEMA_REGISTRY_URL" ]; then + echo "๐ก Using real Schema Registry: $SCHEMA_REGISTRY_URL" +else + echo "๐ง No SCHEMA_REGISTRY_URL set, using mock registry" + # For now, we'll skip the test if no real registry is available + # In the future, we could start a mock registry here + export SCHEMA_REGISTRY_URL="http://localhost:8081" + echo "โ ๏ธ Mock registry not implemented yet, test will be skipped" +fi + +# Start SeaweedFS infrastructure +echo "๐ฑ Starting SeaweedFS infrastructure..." +cd /Users/chrislu/go/src/github.com/seaweedfs/seaweedfs + +# Clean up any existing processes +pkill -f "weed server" || true +pkill -f "weed mq.broker" || true +sleep 2 + +# Start SeaweedFS server +echo "๐๏ธ Starting SeaweedFS server..." +/tmp/weed server -dir=/tmp/seaweedfs-test -master.port=9333 -volume.port=8080 -filer.port=8888 -ip=localhost > /tmp/seaweed-server.log 2>&1 & +SERVER_PID=$! + +# Wait for server to be ready +sleep 5 + +# Start MQ broker +echo "๐จ Starting SeaweedMQ broker..." +/tmp/weed mq.broker -master=localhost:9333 -port=17777 > /tmp/seaweed-broker.log 2>&1 & +BROKER_PID=$! + +# Wait for broker to be ready +sleep 3 + +# Check if services are running +if ! curl -s http://localhost:9333/cluster/status > /dev/null; then + echo "[FAIL] SeaweedFS server not ready" + exit 1 +fi + +echo "[OK] SeaweedFS infrastructure ready" + +# Run the schema registry E2E tests +echo "๐งช Running Schema Registry E2E tests..." +cd /Users/chrislu/go/src/github.com/seaweedfs/seaweedfs/test/kafka + +export SEAWEEDFS_MASTERS=127.0.0.1:9333 + +# Run the tests +if go test -v ./integration -run TestSchemaRegistryE2E -timeout 5m; then + echo "[OK] Schema Registry E2E tests PASSED!" + TEST_RESULT=0 +else + echo "[FAIL] Schema Registry E2E tests FAILED!" + TEST_RESULT=1 +fi + +# Cleanup +echo "๐งน Cleaning up..." +kill $BROKER_PID $SERVER_PID 2>/dev/null || true +sleep 2 +pkill -f "weed server" || true +pkill -f "weed mq.broker" || true + +echo "๐ Schema Registry E2E Test completed" +exit $TEST_RESULT diff --git a/test/kafka/scripts/wait-for-services.sh b/test/kafka/scripts/wait-for-services.sh new file mode 100755 index 000000000..8f1a965f5 --- /dev/null +++ b/test/kafka/scripts/wait-for-services.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +# Wait for Services Script for Kafka Integration Tests + +set -e + +echo "Waiting for services to be ready..." + +# Configuration +KAFKA_HOST=${KAFKA_HOST:-localhost} +KAFKA_PORT=${KAFKA_PORT:-9092} +SCHEMA_REGISTRY_URL=${SCHEMA_REGISTRY_URL:-http://localhost:8081} +KAFKA_GATEWAY_HOST=${KAFKA_GATEWAY_HOST:-localhost} +KAFKA_GATEWAY_PORT=${KAFKA_GATEWAY_PORT:-9093} +SEAWEEDFS_MASTER_URL=${SEAWEEDFS_MASTER_URL:-http://localhost:9333} +MAX_WAIT=${MAX_WAIT:-300} # 5 minutes + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Helper function to wait for a service +wait_for_service() { + local service_name=$1 + local check_command=$2 + local timeout=${3:-60} + + echo -e "${BLUE}Waiting for ${service_name}...${NC}" + + local count=0 + while [ $count -lt $timeout ]; do + if eval "$check_command" > /dev/null 2>&1; then + echo -e "${GREEN}[OK] ${service_name} is ready${NC}" + return 0 + fi + + if [ $((count % 10)) -eq 0 ]; then + echo -e "${YELLOW}Still waiting for ${service_name}... (${count}s)${NC}" + fi + + sleep 1 + count=$((count + 1)) + done + + echo -e "${RED}[FAIL] ${service_name} failed to start within ${timeout} seconds${NC}" + return 1 +} + +# Wait for Zookeeper +echo "=== Checking Zookeeper ===" +wait_for_service "Zookeeper" "nc -z localhost 2181" 30 + +# Wait for Kafka +echo "=== Checking Kafka ===" +wait_for_service "Kafka" "nc -z ${KAFKA_HOST} ${KAFKA_PORT}" 60 + +# Test Kafka broker API +echo "=== Testing Kafka API ===" +wait_for_service "Kafka API" "timeout 5 kafka-broker-api-versions --bootstrap-server ${KAFKA_HOST}:${KAFKA_PORT}" 30 + +# Wait for Schema Registry +echo "=== Checking Schema Registry ===" +wait_for_service "Schema Registry" "curl -f ${SCHEMA_REGISTRY_URL}/subjects" 60 + +# Wait for SeaweedFS Master +echo "=== Checking SeaweedFS Master ===" +wait_for_service "SeaweedFS Master" "curl -f ${SEAWEEDFS_MASTER_URL}/cluster/status" 30 + +# Wait for SeaweedFS Volume +echo "=== Checking SeaweedFS Volume ===" +wait_for_service "SeaweedFS Volume" "curl -f http://localhost:8080/status" 30 + +# Wait for SeaweedFS Filer +echo "=== Checking SeaweedFS Filer ===" +wait_for_service "SeaweedFS Filer" "curl -f http://localhost:8888/" 30 + +# Wait for SeaweedFS MQ Broker +echo "=== Checking SeaweedFS MQ Broker ===" +wait_for_service "SeaweedFS MQ Broker" "nc -z localhost 17777" 30 + +# Wait for SeaweedFS MQ Agent +echo "=== Checking SeaweedFS MQ Agent ===" +wait_for_service "SeaweedFS MQ Agent" "nc -z localhost 16777" 30 + +# Wait for Kafka Gateway +echo "=== Checking Kafka Gateway ===" +wait_for_service "Kafka Gateway" "nc -z ${KAFKA_GATEWAY_HOST} ${KAFKA_GATEWAY_PORT}" 60 + +# Final verification +echo "=== Final Verification ===" + +# Test Kafka topic creation +echo "Testing Kafka topic operations..." +TEST_TOPIC="health-check-$(date +%s)" +if kafka-topics --create --topic "$TEST_TOPIC" --bootstrap-server "${KAFKA_HOST}:${KAFKA_PORT}" --partitions 1 --replication-factor 1 > /dev/null 2>&1; then + echo -e "${GREEN}[OK] Kafka topic creation works${NC}" + kafka-topics --delete --topic "$TEST_TOPIC" --bootstrap-server "${KAFKA_HOST}:${KAFKA_PORT}" > /dev/null 2>&1 || true +else + echo -e "${RED}[FAIL] Kafka topic creation failed${NC}" + exit 1 +fi + +# Test Schema Registry +echo "Testing Schema Registry..." +if curl -f "${SCHEMA_REGISTRY_URL}/subjects" > /dev/null 2>&1; then + echo -e "${GREEN}[OK] Schema Registry is accessible${NC}" +else + echo -e "${RED}[FAIL] Schema Registry is not accessible${NC}" + exit 1 +fi + +# Test Kafka Gateway connectivity +echo "Testing Kafka Gateway..." +if nc -z "${KAFKA_GATEWAY_HOST}" "${KAFKA_GATEWAY_PORT}"; then + echo -e "${GREEN}[OK] Kafka Gateway is accessible${NC}" +else + echo -e "${RED}[FAIL] Kafka Gateway is not accessible${NC}" + exit 1 +fi + +echo -e "${GREEN}All services are ready!${NC}" +echo "" +echo "Service endpoints:" +echo " Kafka: ${KAFKA_HOST}:${KAFKA_PORT}" +echo " Schema Registry: ${SCHEMA_REGISTRY_URL}" +echo " Kafka Gateway: ${KAFKA_GATEWAY_HOST}:${KAFKA_GATEWAY_PORT}" +echo " SeaweedFS Master: ${SEAWEEDFS_MASTER_URL}" +echo " SeaweedFS Filer: http://localhost:8888" +echo " SeaweedFS MQ Broker: localhost:17777" +echo " SeaweedFS MQ Agent: localhost:16777" +echo "" +echo "Ready to run integration tests!" |