diff options
Diffstat (limited to 'seaweedfs-rdma-sidecar/scripts/run-performance-tests.sh')
| -rwxr-xr-x | seaweedfs-rdma-sidecar/scripts/run-performance-tests.sh | 338 |
1 files changed, 338 insertions, 0 deletions
diff --git a/seaweedfs-rdma-sidecar/scripts/run-performance-tests.sh b/seaweedfs-rdma-sidecar/scripts/run-performance-tests.sh new file mode 100755 index 000000000..4475365aa --- /dev/null +++ b/seaweedfs-rdma-sidecar/scripts/run-performance-tests.sh @@ -0,0 +1,338 @@ +#!/bin/bash + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Configuration +MOUNT_POINT=${MOUNT_POINT:-"/mnt/seaweedfs"} +RDMA_SIDECAR_ADDR=${RDMA_SIDECAR_ADDR:-"rdma-sidecar:8081"} +PERFORMANCE_RESULTS_DIR=${PERFORMANCE_RESULTS_DIR:-"/performance-results"} + +# Create results directory +mkdir -p "$PERFORMANCE_RESULTS_DIR" + +# Log file +LOG_FILE="$PERFORMANCE_RESULTS_DIR/performance-test.log" +exec > >(tee -a "$LOG_FILE") +exec 2>&1 + +echo -e "${BLUE}ð SEAWEEDFS RDMA MOUNT PERFORMANCE TESTS${NC}" +echo "===========================================" +echo "Mount Point: $MOUNT_POINT" +echo "RDMA Sidecar: $RDMA_SIDECAR_ADDR" +echo "Results Directory: $PERFORMANCE_RESULTS_DIR" +echo "Log File: $LOG_FILE" +echo "" + +# Function to wait for mount to be ready +wait_for_mount() { + local max_attempts=30 + local attempt=1 + + echo -e "${BLUE}âģ Waiting for mount to be ready...${NC}" + + while [[ $attempt -le $max_attempts ]]; do + if mountpoint -q "$MOUNT_POINT" 2>/dev/null && ls "$MOUNT_POINT" >/dev/null 2>&1; then + echo -e "${GREEN}â
Mount is ready${NC}" + return 0 + fi + echo " Attempt $attempt/$max_attempts..." + sleep 2 + ((attempt++)) + done + + echo -e "${RED}â Mount failed to be ready${NC}" + return 1 +} + +# Function to get RDMA statistics +get_rdma_stats() { + curl -s "http://$RDMA_SIDECAR_ADDR/stats" 2>/dev/null || echo "{}" +} + +# Function to run dd performance test +run_dd_test() { + local test_name=$1 + local file_size_mb=$2 + local block_size=$3 + local operation=$4 # "write" or "read" + + local test_file="$MOUNT_POINT/perf_test_${test_name}.dat" + local result_file="$PERFORMANCE_RESULTS_DIR/dd_${test_name}.json" + + echo -e "${BLUE}ðŽ Running DD test: $test_name${NC}" + echo " Size: ${file_size_mb}MB, Block Size: $block_size, Operation: $operation" + + local start_time end_time duration_ms throughput_mbps + + if [[ "$operation" == "write" ]]; then + start_time=$(date +%s%N) + dd if=/dev/zero of="$test_file" bs="$block_size" count=$((file_size_mb * 1024 * 1024 / $(numfmt --from=iec "$block_size"))) 2>/dev/null + end_time=$(date +%s%N) + else + # Create file first if it doesn't exist + if [[ ! -f "$test_file" ]]; then + dd if=/dev/zero of="$test_file" bs=1M count="$file_size_mb" 2>/dev/null + fi + start_time=$(date +%s%N) + dd if="$test_file" of=/dev/null bs="$block_size" 2>/dev/null + end_time=$(date +%s%N) + fi + + duration_ms=$(((end_time - start_time) / 1000000)) + throughput_mbps=$(bc <<< "scale=2; $file_size_mb * 1000 / $duration_ms") + + # Save results + cat > "$result_file" << EOF +{ + "test_name": "$test_name", + "operation": "$operation", + "file_size_mb": $file_size_mb, + "block_size": "$block_size", + "duration_ms": $duration_ms, + "throughput_mbps": $throughput_mbps, + "timestamp": "$(date -Iseconds)" +} +EOF + + echo " Duration: ${duration_ms}ms" + echo " Throughput: ${throughput_mbps} MB/s" + echo "" + + # Cleanup write test files + if [[ "$operation" == "write" ]]; then + rm -f "$test_file" + fi +} + +# Function to run FIO performance test +run_fio_test() { + local test_name=$1 + local rw_type=$2 # "read", "write", "randread", "randwrite" + local block_size=$3 + local file_size=$4 + local iodepth=$5 + + local test_file="$MOUNT_POINT/fio_test_${test_name}.dat" + local result_file="$PERFORMANCE_RESULTS_DIR/fio_${test_name}.json" + + echo -e "${BLUE}ðŽ Running FIO test: $test_name${NC}" + echo " Type: $rw_type, Block Size: $block_size, File Size: $file_size, IO Depth: $iodepth" + + # Run FIO test + fio --name="$test_name" \ + --filename="$test_file" \ + --rw="$rw_type" \ + --bs="$block_size" \ + --size="$file_size" \ + --iodepth="$iodepth" \ + --direct=1 \ + --runtime=30 \ + --time_based \ + --group_reporting \ + --output-format=json \ + --output="$result_file" \ + 2>/dev/null + + # Extract key metrics + if [[ -f "$result_file" ]]; then + local iops throughput_kbps latency_us + iops=$(jq -r '.jobs[0].'"$rw_type"'.iops // 0' "$result_file" 2>/dev/null || echo "0") + throughput_kbps=$(jq -r '.jobs[0].'"$rw_type"'.bw // 0' "$result_file" 2>/dev/null || echo "0") + latency_us=$(jq -r '.jobs[0].'"$rw_type"'.lat_ns.mean // 0' "$result_file" 2>/dev/null || echo "0") + latency_us=$(bc <<< "scale=2; $latency_us / 1000" 2>/dev/null || echo "0") + + echo " IOPS: $iops" + echo " Throughput: $(bc <<< "scale=2; $throughput_kbps / 1024") MB/s" + echo " Average Latency: ${latency_us} Ξs" + else + echo " FIO test failed or no results" + fi + echo "" + + # Cleanup + rm -f "$test_file" +} + +# Function to run concurrent access test +run_concurrent_test() { + local num_processes=$1 + local file_size_mb=$2 + + echo -e "${BLUE}ðŽ Running concurrent access test${NC}" + echo " Processes: $num_processes, File Size per Process: ${file_size_mb}MB" + + local start_time end_time duration_ms total_throughput + local pids=() + + start_time=$(date +%s%N) + + # Start concurrent processes + for i in $(seq 1 "$num_processes"); do + ( + local test_file="$MOUNT_POINT/concurrent_test_$i.dat" + dd if=/dev/zero of="$test_file" bs=1M count="$file_size_mb" 2>/dev/null + dd if="$test_file" of=/dev/null bs=1M 2>/dev/null + rm -f "$test_file" + ) & + pids+=($!) + done + + # Wait for all processes to complete + for pid in "${pids[@]}"; do + wait "$pid" + done + + end_time=$(date +%s%N) + duration_ms=$(((end_time - start_time) / 1000000)) + total_throughput=$(bc <<< "scale=2; $num_processes * $file_size_mb * 2 * 1000 / $duration_ms") + + # Save results + cat > "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" << EOF +{ + "test_name": "concurrent_access", + "num_processes": $num_processes, + "file_size_mb_per_process": $file_size_mb, + "total_data_mb": $((num_processes * file_size_mb * 2)), + "duration_ms": $duration_ms, + "total_throughput_mbps": $total_throughput, + "timestamp": "$(date -Iseconds)" +} +EOF + + echo " Duration: ${duration_ms}ms" + echo " Total Throughput: ${total_throughput} MB/s" + echo "" +} + +# Function to generate performance report +generate_report() { + local report_file="$PERFORMANCE_RESULTS_DIR/performance_report.html" + + echo -e "${BLUE}ð Generating performance report...${NC}" + + cat > "$report_file" << 'EOF' +<!DOCTYPE html> +<html> +<head> + <title>SeaweedFS RDMA Mount Performance Report</title> + <style> + body { font-family: Arial, sans-serif; margin: 20px; } + .header { background-color: #f0f0f0; padding: 20px; border-radius: 5px; } + .test-section { margin: 20px 0; padding: 15px; border: 1px solid #ddd; border-radius: 5px; } + .metric { margin: 5px 0; } + .good { color: green; font-weight: bold; } + .warning { color: orange; font-weight: bold; } + .error { color: red; font-weight: bold; } + table { border-collapse: collapse; width: 100%; } + th, td { border: 1px solid #ddd; padding: 8px; text-align: left; } + th { background-color: #f2f2f2; } + </style> +</head> +<body> + <div class="header"> + <h1>ð SeaweedFS RDMA Mount Performance Report</h1> + <p>Generated: $(date)</p> + <p>Mount Point: $MOUNT_POINT</p> + <p>RDMA Sidecar: $RDMA_SIDECAR_ADDR</p> + </div> +EOF + + # Add DD test results + echo '<div class="test-section"><h2>DD Performance Tests</h2><table><tr><th>Test</th><th>Operation</th><th>Size</th><th>Block Size</th><th>Throughput (MB/s)</th><th>Duration (ms)</th></tr>' >> "$report_file" + + for result_file in "$PERFORMANCE_RESULTS_DIR"/dd_*.json; do + if [[ -f "$result_file" ]]; then + local test_name operation file_size_mb block_size throughput_mbps duration_ms + test_name=$(jq -r '.test_name' "$result_file" 2>/dev/null || echo "unknown") + operation=$(jq -r '.operation' "$result_file" 2>/dev/null || echo "unknown") + file_size_mb=$(jq -r '.file_size_mb' "$result_file" 2>/dev/null || echo "0") + block_size=$(jq -r '.block_size' "$result_file" 2>/dev/null || echo "unknown") + throughput_mbps=$(jq -r '.throughput_mbps' "$result_file" 2>/dev/null || echo "0") + duration_ms=$(jq -r '.duration_ms' "$result_file" 2>/dev/null || echo "0") + + echo "<tr><td>$test_name</td><td>$operation</td><td>${file_size_mb}MB</td><td>$block_size</td><td>$throughput_mbps</td><td>$duration_ms</td></tr>" >> "$report_file" + fi + done + + echo '</table></div>' >> "$report_file" + + # Add FIO test results + echo '<div class="test-section"><h2>FIO Performance Tests</h2>' >> "$report_file" + echo '<p>Detailed FIO results are available in individual JSON files.</p></div>' >> "$report_file" + + # Add concurrent test results + if [[ -f "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" ]]; then + echo '<div class="test-section"><h2>Concurrent Access Test</h2>' >> "$report_file" + local num_processes total_throughput duration_ms + num_processes=$(jq -r '.num_processes' "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" 2>/dev/null || echo "0") + total_throughput=$(jq -r '.total_throughput_mbps' "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" 2>/dev/null || echo "0") + duration_ms=$(jq -r '.duration_ms' "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" 2>/dev/null || echo "0") + + echo "<p>Processes: $num_processes</p>" >> "$report_file" + echo "<p>Total Throughput: $total_throughput MB/s</p>" >> "$report_file" + echo "<p>Duration: $duration_ms ms</p>" >> "$report_file" + echo '</div>' >> "$report_file" + fi + + echo '</body></html>' >> "$report_file" + + echo " Report saved to: $report_file" +} + +# Main test execution +main() { + echo -e "${BLUE}ð Starting performance tests...${NC}" + echo "" + + # Wait for mount to be ready + if ! wait_for_mount; then + echo -e "${RED}â Mount is not ready, aborting tests${NC}" + exit 1 + fi + + # Get initial RDMA stats + echo -e "${BLUE}ð Initial RDMA Statistics:${NC}" + get_rdma_stats | jq . 2>/dev/null || get_rdma_stats + echo "" + + # Run DD performance tests + echo -e "${BLUE}ð Running DD Performance Tests...${NC}" + run_dd_test "small_write" 10 "4k" "write" + run_dd_test "small_read" 10 "4k" "read" + run_dd_test "medium_write" 100 "64k" "write" + run_dd_test "medium_read" 100 "64k" "read" + run_dd_test "large_write" 500 "1M" "write" + run_dd_test "large_read" 500 "1M" "read" + + # Run FIO performance tests + echo -e "${BLUE}ð Running FIO Performance Tests...${NC}" + run_fio_test "seq_read" "read" "64k" "100M" 1 + run_fio_test "seq_write" "write" "64k" "100M" 1 + run_fio_test "rand_read" "randread" "4k" "100M" 16 + run_fio_test "rand_write" "randwrite" "4k" "100M" 16 + + # Run concurrent access test + echo -e "${BLUE}ð Running Concurrent Access Test...${NC}" + run_concurrent_test 4 50 + + # Get final RDMA stats + echo -e "${BLUE}ð Final RDMA Statistics:${NC}" + get_rdma_stats | jq . 2>/dev/null || get_rdma_stats + echo "" + + # Generate performance report + generate_report + + echo -e "${GREEN}ð Performance tests completed!${NC}" + echo "Results saved to: $PERFORMANCE_RESULTS_DIR" +} + +# Run main function +main "$@" |