diff options
Diffstat (limited to 'seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml')
| -rw-r--r-- | seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml | 269 |
1 files changed, 269 insertions, 0 deletions
diff --git a/seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml b/seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml new file mode 100644 index 000000000..39eef0048 --- /dev/null +++ b/seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml @@ -0,0 +1,269 @@ +version: '3.8' + +services: + # SeaweedFS Master + seaweedfs-master: + image: chrislusf/seaweedfs:latest + container_name: seaweedfs-master + ports: + - "9333:9333" + - "19333:19333" + command: > + master + -port=9333 + -mdir=/data + -volumeSizeLimitMB=1024 + -defaultReplication=000 + volumes: + - seaweedfs_master_data:/data + networks: + - seaweedfs-rdma + healthcheck: + test: ["CMD", "wget", "--timeout=10", "--quiet", "--tries=1", "--spider", "http://127.0.0.1:9333/cluster/status"] + interval: 10s + timeout: 10s + retries: 6 + start_period: 60s + + # SeaweedFS Volume Server + seaweedfs-volume: + image: chrislusf/seaweedfs:latest + container_name: seaweedfs-volume + ports: + - "8080:8080" + - "18080:18080" + command: > + volume + -mserver=seaweedfs-master:9333 + -port=8080 + -dir=/data + -max=100 + volumes: + - seaweedfs_volume_data:/data + networks: + - seaweedfs-rdma + depends_on: + seaweedfs-master: + condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "pgrep weed && netstat -tln | grep :8080"] + interval: 10s + timeout: 10s + retries: 6 + start_period: 30s + + # SeaweedFS Filer + seaweedfs-filer: + image: chrislusf/seaweedfs:latest + container_name: seaweedfs-filer + ports: + - "8888:8888" + - "18888:18888" + command: > + filer + -master=seaweedfs-master:9333 + -port=8888 + -defaultReplicaPlacement=000 + networks: + - seaweedfs-rdma + depends_on: + seaweedfs-master: + condition: service_healthy + seaweedfs-volume: + condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "pgrep weed && netstat -tln | grep :8888"] + interval: 10s + timeout: 10s + retries: 6 + start_period: 45s + + # RDMA Engine (Rust) + rdma-engine: + build: + context: . + dockerfile: Dockerfile.rdma-engine + container_name: rdma-engine + volumes: + - rdma_socket:/tmp/rdma + networks: + - seaweedfs-rdma + environment: + - RUST_LOG=debug + - RDMA_SOCKET_PATH=/tmp/rdma/rdma-engine.sock + - RDMA_DEVICE=auto + - RDMA_PORT=18515 + - RDMA_GID_INDEX=0 + - DEBUG=true + command: > + ./rdma-engine-server + --ipc-socket ${RDMA_SOCKET_PATH} + --device ${RDMA_DEVICE} + --port ${RDMA_PORT} + --debug + healthcheck: + test: ["CMD", "sh", "-c", "pgrep rdma-engine-server >/dev/null && test -S /tmp/rdma/rdma-engine.sock"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 10s + + # RDMA Sidecar (Go) + rdma-sidecar: + build: + context: . + dockerfile: Dockerfile.sidecar + container_name: rdma-sidecar + ports: + - "8081:8081" + volumes: + - rdma_socket:/tmp/rdma + networks: + - seaweedfs-rdma + environment: + - RDMA_SOCKET_PATH=/tmp/rdma/rdma-engine.sock + - VOLUME_SERVER_URL=http://seaweedfs-volume:8080 + - SIDECAR_PORT=8081 + - ENABLE_RDMA=true + - ENABLE_ZEROCOPY=true + - ENABLE_POOLING=true + - MAX_CONNECTIONS=10 + - MAX_IDLE_TIME=5m + - DEBUG=true + command: > + ./demo-server + --port ${SIDECAR_PORT} + --rdma-socket ${RDMA_SOCKET_PATH} + --volume-server ${VOLUME_SERVER_URL} + --enable-rdma + --enable-zerocopy + --enable-pooling + --max-connections ${MAX_CONNECTIONS} + --max-idle-time ${MAX_IDLE_TIME} + --debug + depends_on: + rdma-engine: + condition: service_healthy + seaweedfs-volume: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8081/health"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s + + # SeaweedFS Mount with RDMA + seaweedfs-mount: + build: + context: . + dockerfile: Dockerfile.mount-rdma + platform: linux/amd64 + container_name: seaweedfs-mount + privileged: true # Required for FUSE + devices: + - /dev/fuse:/dev/fuse + cap_add: + - SYS_ADMIN + volumes: + - seaweedfs_mount:/mnt/seaweedfs + - /tmp/seaweedfs-mount-logs:/var/log/seaweedfs + networks: + - seaweedfs-rdma + environment: + - FILER_ADDR=seaweedfs-filer:8888 + - RDMA_SIDECAR_ADDR=rdma-sidecar:8081 + - MOUNT_POINT=/mnt/seaweedfs + - RDMA_ENABLED=true + - RDMA_FALLBACK=true + - RDMA_MAX_CONCURRENT=64 + - RDMA_TIMEOUT_MS=5000 + - DEBUG=true + command: /usr/local/bin/mount-helper.sh + depends_on: + seaweedfs-filer: + condition: service_healthy + rdma-sidecar: + condition: service_healthy + healthcheck: + test: ["CMD", "mountpoint", "-q", "/mnt/seaweedfs"] + interval: 15s + timeout: 10s + retries: 3 + start_period: 45s + + # Integration Test Runner + integration-test: + build: + context: . + dockerfile: Dockerfile.integration-test + container_name: integration-test + volumes: + - seaweedfs_mount:/mnt/seaweedfs + - ./test-results:/test-results + networks: + - seaweedfs-rdma + environment: + - MOUNT_POINT=/mnt/seaweedfs + - FILER_ADDR=seaweedfs-filer:8888 + - RDMA_SIDECAR_ADDR=rdma-sidecar:8081 + - TEST_RESULTS_DIR=/test-results + depends_on: + seaweedfs-mount: + condition: service_healthy + command: > + sh -c " + echo 'Starting RDMA Mount Integration Tests...' && + sleep 10 && + /usr/local/bin/run-integration-tests.sh + " + profiles: + - test + + # Performance Test Runner + performance-test: + build: + context: . + dockerfile: Dockerfile.performance-test + container_name: performance-test + volumes: + - seaweedfs_mount:/mnt/seaweedfs + - ./performance-results:/performance-results + networks: + - seaweedfs-rdma + environment: + - MOUNT_POINT=/mnt/seaweedfs + - RDMA_SIDECAR_ADDR=rdma-sidecar:8081 + - PERFORMANCE_RESULTS_DIR=/performance-results + depends_on: + seaweedfs-mount: + condition: service_healthy + command: > + sh -c " + echo 'Starting RDMA Mount Performance Tests...' && + sleep 10 && + /usr/local/bin/run-performance-tests.sh + " + profiles: + - performance + +volumes: + seaweedfs_master_data: + driver: local + seaweedfs_volume_data: + driver: local + seaweedfs_mount: + driver: local + driver_opts: + type: tmpfs + device: tmpfs + o: size=1g + rdma_socket: + driver: local + +networks: + seaweedfs-rdma: + driver: bridge + ipam: + config: + - subnet: 172.20.0.0/16 |
