1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
services:
# SeaweedFS Master Server
seaweedfs-master:
image: chrislusf/seaweedfs:latest
container_name: seaweedfs-master
command: master -ip=seaweedfs-master -port=9333 -mdir=/data
ports:
- "9333:9333"
volumes:
- master-data:/data
networks:
- seaweedfs-rdma
healthcheck:
test: ["CMD", "pgrep", "-f", "weed"]
interval: 15s
timeout: 10s
retries: 5
start_period: 30s
# SeaweedFS Volume Server
seaweedfs-volume:
image: chrislusf/seaweedfs:latest
container_name: seaweedfs-volume
command: volume -mserver=seaweedfs-master:9333 -ip=seaweedfs-volume -port=8080 -dir=/data
ports:
- "8080:8080"
volumes:
- volume-data:/data
depends_on:
seaweedfs-master:
condition: service_healthy
networks:
- seaweedfs-rdma
healthcheck:
test: ["CMD", "pgrep", "-f", "weed"]
interval: 15s
timeout: 10s
retries: 5
start_period: 30s
# RDMA Simulation Environment
rdma-simulation:
build:
context: .
dockerfile: docker/Dockerfile.rdma-simulation
container_name: rdma-simulation
privileged: true # Required for RDMA kernel module loading
environment:
- RDMA_DEVICE=rxe0
- UCX_TLS=rc_verbs,ud_verbs,tcp
- UCX_LOG_LEVEL=info
volumes:
- /lib/modules:/lib/modules:ro # Host kernel modules
- /sys:/sys # Required for sysfs access
- rdma-simulation-data:/opt/rdma-sim/data
networks:
- seaweedfs-rdma
ports:
- "18515:18515" # RDMA application port
- "4791:4791" # RDMA CM port
- "4792:4792" # Additional RDMA port
command: |
bash -c "
echo '🚀 Setting up RDMA simulation environment...'
sudo /opt/rdma-sim/setup-soft-roce.sh || echo 'RDMA setup failed, continuing...'
echo '📋 RDMA environment status:'
/opt/rdma-sim/test-rdma.sh || true
echo '🔧 UCX information:'
/opt/rdma-sim/ucx-info.sh || true
echo '✅ RDMA simulation ready - keeping container alive...'
tail -f /dev/null
"
healthcheck:
test: ["CMD", "test", "-f", "/opt/rdma-sim/setup-soft-roce.sh"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
# Rust RDMA Engine (with RDMA simulation support)
rdma-engine:
build:
context: .
dockerfile: Dockerfile.rdma-engine
container_name: rdma-engine
environment:
- RUST_LOG=debug
- RDMA_SOCKET_PATH=/tmp/rdma-engine.sock
# UCX configuration for real RDMA
- UCX_TLS=rc_verbs,ud_verbs,tcp,shm
- UCX_NET_DEVICES=all
- UCX_LOG_LEVEL=info
- UCX_RNDV_SCHEME=put_zcopy
- UCX_RNDV_THRESH=8192
volumes:
- rdma-socket:/tmp
# Share network namespace with RDMA simulation for device access
network_mode: "container:rdma-simulation"
depends_on:
rdma-simulation:
condition: service_healthy
command: ["./rdma-engine-server", "--debug", "--ipc-socket", "/tmp/rdma-engine.sock"]
healthcheck:
test: ["CMD", "test", "-S", "/tmp/rdma-engine.sock"]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
# Go RDMA Sidecar / Demo Server
rdma-sidecar:
build:
context: .
dockerfile: Dockerfile.sidecar
container_name: rdma-sidecar
ports:
- "8081:8081"
environment:
- RDMA_SOCKET_PATH=/tmp/rdma-engine.sock
- VOLUME_SERVER_URL=http://seaweedfs-volume:8080
- DEBUG=true
volumes:
- rdma-socket:/tmp
depends_on:
rdma-engine:
condition: service_healthy
seaweedfs-volume:
condition: service_healthy
networks:
- seaweedfs-rdma
command: [
"./demo-server",
"--port", "8081",
"--rdma-socket", "/tmp/rdma-engine.sock",
"--volume-server", "http://seaweedfs-volume:8080",
"--enable-rdma",
"--debug"
]
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8081/health"]
interval: 10s
timeout: 5s
retries: 3
start_period: 20s
# Test Client for Integration Testing
test-client:
build:
context: .
dockerfile: Dockerfile.test-client
container_name: test-client
environment:
- RDMA_SOCKET_PATH=/tmp/rdma-engine.sock
- SIDECAR_URL=http://rdma-sidecar:8081
- SEAWEEDFS_MASTER=http://seaweedfs-master:9333
- SEAWEEDFS_VOLUME=http://seaweedfs-volume:8080
volumes:
- rdma-socket:/tmp
depends_on:
rdma-sidecar:
condition: service_healthy
networks:
- seaweedfs-rdma
profiles:
- testing
command: ["tail", "-f", "/dev/null"] # Keep container running for manual testing
# Integration Test Runner with RDMA
integration-tests-rdma:
build:
context: .
dockerfile: Dockerfile.test-client
container_name: integration-tests-rdma
environment:
- RDMA_SOCKET_PATH=/tmp/rdma-engine.sock
- SIDECAR_URL=http://rdma-sidecar:8081
- SEAWEEDFS_MASTER=http://seaweedfs-master:9333
- SEAWEEDFS_VOLUME=http://seaweedfs-volume:8080
- RDMA_SIMULATION=true
volumes:
- rdma-socket:/tmp
- ./tests:/tests
depends_on:
rdma-sidecar:
condition: service_healthy
rdma-simulation:
condition: service_healthy
networks:
- seaweedfs-rdma
profiles:
- testing
command: ["/tests/run-integration-tests.sh"]
volumes:
master-data:
driver: local
volume-data:
driver: local
rdma-socket:
driver: local
rdma-simulation-data:
driver: local
networks:
seaweedfs-rdma:
driver: bridge
ipam:
config:
- subnet: 172.20.0.0/16
|
