1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
|
# Kafka Client Load Test Configuration
# Test execution settings
test_mode: "comprehensive" # producer, consumer, comprehensive
duration: "60s" # Test duration (0 = run indefinitely) - producers will stop at this time, consumers get +120s to drain
# Kafka cluster configuration
kafka:
bootstrap_servers:
- "kafka-gateway:9093"
# Security settings (if needed)
security_protocol: "PLAINTEXT" # PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL
sasl_mechanism: "" # PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
sasl_username: ""
sasl_password: ""
# Schema Registry configuration
schema_registry:
url: "http://schema-registry:8081"
auth:
username: ""
password: ""
# Producer configuration
producers:
count: 10 # Number of producer instances
message_rate: 1000 # Messages per second per producer
message_size: 1024 # Message size in bytes
batch_size: 100 # Batch size for batching
linger_ms: 5 # Time to wait for batching
compression_type: "snappy" # none, gzip, snappy, lz4, zstd
acks: "all" # 0, 1, all
retries: 3
retry_backoff_ms: 100
request_timeout_ms: 30000
delivery_timeout_ms: 120000
# Message generation settings
key_distribution: "random" # random, sequential, uuid
value_type: "avro" # json, avro, protobuf, binary
schema_format: "" # AVRO, JSON, PROTOBUF - schema registry format (when schemas enabled)
# Leave empty to auto-distribute formats across topics for testing:
# topic-0: AVRO, topic-1: JSON, topic-2: PROTOBUF, topic-3: AVRO, topic-4: JSON
# Set to specific format (e.g. "AVRO") to use same format for all topics
include_timestamp: true
include_headers: true
# Consumer configuration
consumers:
count: 5 # Number of consumer instances
group_prefix: "loadtest-group" # Consumer group prefix
auto_offset_reset: "earliest" # earliest, latest
enable_auto_commit: true
auto_commit_interval_ms: 100 # Reduced from 1000ms to 100ms to minimize duplicate window
session_timeout_ms: 30000
heartbeat_interval_ms: 3000
max_poll_records: 500
max_poll_interval_ms: 300000
fetch_min_bytes: 1
fetch_max_bytes: 52428800 # 50MB
fetch_max_wait_ms: 100 # 100ms - very fast polling for concurrent fetches and quick drain
# Topic configuration
topics:
count: 5 # Number of topics to create/use
prefix: "loadtest-topic" # Topic name prefix
partitions: 4 # Partitions per topic (default: 4)
replication_factor: 1 # Replication factor
cleanup_policy: "delete" # delete, compact
retention_ms: 604800000 # 7 days
segment_ms: 86400000 # 1 day
# Schema configuration (for Avro/Protobuf tests)
schemas:
enabled: true
registry_timeout_ms: 10000
# Test schemas
user_event:
type: "avro"
schema: |
{
"type": "record",
"name": "UserEvent",
"namespace": "com.seaweedfs.test",
"fields": [
{"name": "user_id", "type": "string"},
{"name": "event_type", "type": "string"},
{"name": "timestamp", "type": "long"},
{"name": "properties", "type": {"type": "map", "values": "string"}}
]
}
transaction:
type: "avro"
schema: |
{
"type": "record",
"name": "Transaction",
"namespace": "com.seaweedfs.test",
"fields": [
{"name": "transaction_id", "type": "string"},
{"name": "amount", "type": "double"},
{"name": "currency", "type": "string"},
{"name": "merchant_id", "type": "string"},
{"name": "timestamp", "type": "long"}
]
}
# Metrics and monitoring
metrics:
enabled: true
collection_interval: "10s"
prometheus_port: 8080
# What to measure
track_latency: true
track_throughput: true
track_errors: true
track_consumer_lag: true
# Latency percentiles to track
latency_percentiles: [50, 90, 95, 99, 99.9]
# Load test scenarios
scenarios:
# Steady state load test
steady_load:
producer_rate: 1000 # messages/sec per producer
ramp_up_time: "30s"
steady_duration: "240s"
ramp_down_time: "30s"
# Burst load test
burst_load:
base_rate: 500
burst_rate: 5000
burst_duration: "10s"
burst_interval: "60s"
# Gradual ramp test
ramp_test:
start_rate: 100
end_rate: 2000
ramp_duration: "300s"
step_duration: "30s"
# Error injection (for resilience testing)
chaos:
enabled: false
producer_failure_rate: 0.01 # 1% of producers fail randomly
consumer_failure_rate: 0.01 # 1% of consumers fail randomly
network_partition_probability: 0.001 # Network issues
broker_restart_interval: "0s" # Restart brokers periodically (0s = disabled)
# Output and reporting
output:
results_dir: "/test-results"
export_prometheus: true
export_csv: true
export_json: true
real_time_stats: true
stats_interval: "30s"
# Logging
logging:
level: "info" # debug, info, warn, error
format: "text" # text, json
enable_kafka_logs: false # Enable Kafka client debug logs
|
