aboutsummaryrefslogtreecommitdiff
path: root/seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml
diff options
context:
space:
mode:
Diffstat (limited to 'seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml')
-rw-r--r--seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml269
1 files changed, 269 insertions, 0 deletions
diff --git a/seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml b/seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml
new file mode 100644
index 000000000..39eef0048
--- /dev/null
+++ b/seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml
@@ -0,0 +1,269 @@
+version: '3.8'
+
+services:
+ # SeaweedFS Master
+ seaweedfs-master:
+ image: chrislusf/seaweedfs:latest
+ container_name: seaweedfs-master
+ ports:
+ - "9333:9333"
+ - "19333:19333"
+ command: >
+ master
+ -port=9333
+ -mdir=/data
+ -volumeSizeLimitMB=1024
+ -defaultReplication=000
+ volumes:
+ - seaweedfs_master_data:/data
+ networks:
+ - seaweedfs-rdma
+ healthcheck:
+ test: ["CMD", "wget", "--timeout=10", "--quiet", "--tries=1", "--spider", "http://127.0.0.1:9333/cluster/status"]
+ interval: 10s
+ timeout: 10s
+ retries: 6
+ start_period: 60s
+
+ # SeaweedFS Volume Server
+ seaweedfs-volume:
+ image: chrislusf/seaweedfs:latest
+ container_name: seaweedfs-volume
+ ports:
+ - "8080:8080"
+ - "18080:18080"
+ command: >
+ volume
+ -mserver=seaweedfs-master:9333
+ -port=8080
+ -dir=/data
+ -max=100
+ volumes:
+ - seaweedfs_volume_data:/data
+ networks:
+ - seaweedfs-rdma
+ depends_on:
+ seaweedfs-master:
+ condition: service_healthy
+ healthcheck:
+ test: ["CMD", "sh", "-c", "pgrep weed && netstat -tln | grep :8080"]
+ interval: 10s
+ timeout: 10s
+ retries: 6
+ start_period: 30s
+
+ # SeaweedFS Filer
+ seaweedfs-filer:
+ image: chrislusf/seaweedfs:latest
+ container_name: seaweedfs-filer
+ ports:
+ - "8888:8888"
+ - "18888:18888"
+ command: >
+ filer
+ -master=seaweedfs-master:9333
+ -port=8888
+ -defaultReplicaPlacement=000
+ networks:
+ - seaweedfs-rdma
+ depends_on:
+ seaweedfs-master:
+ condition: service_healthy
+ seaweedfs-volume:
+ condition: service_healthy
+ healthcheck:
+ test: ["CMD", "sh", "-c", "pgrep weed && netstat -tln | grep :8888"]
+ interval: 10s
+ timeout: 10s
+ retries: 6
+ start_period: 45s
+
+ # RDMA Engine (Rust)
+ rdma-engine:
+ build:
+ context: .
+ dockerfile: Dockerfile.rdma-engine
+ container_name: rdma-engine
+ volumes:
+ - rdma_socket:/tmp/rdma
+ networks:
+ - seaweedfs-rdma
+ environment:
+ - RUST_LOG=debug
+ - RDMA_SOCKET_PATH=/tmp/rdma/rdma-engine.sock
+ - RDMA_DEVICE=auto
+ - RDMA_PORT=18515
+ - RDMA_GID_INDEX=0
+ - DEBUG=true
+ command: >
+ ./rdma-engine-server
+ --ipc-socket ${RDMA_SOCKET_PATH}
+ --device ${RDMA_DEVICE}
+ --port ${RDMA_PORT}
+ --debug
+ healthcheck:
+ test: ["CMD", "sh", "-c", "pgrep rdma-engine-server >/dev/null && test -S /tmp/rdma/rdma-engine.sock"]
+ interval: 5s
+ timeout: 3s
+ retries: 5
+ start_period: 10s
+
+ # RDMA Sidecar (Go)
+ rdma-sidecar:
+ build:
+ context: .
+ dockerfile: Dockerfile.sidecar
+ container_name: rdma-sidecar
+ ports:
+ - "8081:8081"
+ volumes:
+ - rdma_socket:/tmp/rdma
+ networks:
+ - seaweedfs-rdma
+ environment:
+ - RDMA_SOCKET_PATH=/tmp/rdma/rdma-engine.sock
+ - VOLUME_SERVER_URL=http://seaweedfs-volume:8080
+ - SIDECAR_PORT=8081
+ - ENABLE_RDMA=true
+ - ENABLE_ZEROCOPY=true
+ - ENABLE_POOLING=true
+ - MAX_CONNECTIONS=10
+ - MAX_IDLE_TIME=5m
+ - DEBUG=true
+ command: >
+ ./demo-server
+ --port ${SIDECAR_PORT}
+ --rdma-socket ${RDMA_SOCKET_PATH}
+ --volume-server ${VOLUME_SERVER_URL}
+ --enable-rdma
+ --enable-zerocopy
+ --enable-pooling
+ --max-connections ${MAX_CONNECTIONS}
+ --max-idle-time ${MAX_IDLE_TIME}
+ --debug
+ depends_on:
+ rdma-engine:
+ condition: service_healthy
+ seaweedfs-volume:
+ condition: service_healthy
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:8081/health"]
+ interval: 10s
+ timeout: 5s
+ retries: 3
+ start_period: 15s
+
+ # SeaweedFS Mount with RDMA
+ seaweedfs-mount:
+ build:
+ context: .
+ dockerfile: Dockerfile.mount-rdma
+ platform: linux/amd64
+ container_name: seaweedfs-mount
+ privileged: true # Required for FUSE
+ devices:
+ - /dev/fuse:/dev/fuse
+ cap_add:
+ - SYS_ADMIN
+ volumes:
+ - seaweedfs_mount:/mnt/seaweedfs
+ - /tmp/seaweedfs-mount-logs:/var/log/seaweedfs
+ networks:
+ - seaweedfs-rdma
+ environment:
+ - FILER_ADDR=seaweedfs-filer:8888
+ - RDMA_SIDECAR_ADDR=rdma-sidecar:8081
+ - MOUNT_POINT=/mnt/seaweedfs
+ - RDMA_ENABLED=true
+ - RDMA_FALLBACK=true
+ - RDMA_MAX_CONCURRENT=64
+ - RDMA_TIMEOUT_MS=5000
+ - DEBUG=true
+ command: /usr/local/bin/mount-helper.sh
+ depends_on:
+ seaweedfs-filer:
+ condition: service_healthy
+ rdma-sidecar:
+ condition: service_healthy
+ healthcheck:
+ test: ["CMD", "mountpoint", "-q", "/mnt/seaweedfs"]
+ interval: 15s
+ timeout: 10s
+ retries: 3
+ start_period: 45s
+
+ # Integration Test Runner
+ integration-test:
+ build:
+ context: .
+ dockerfile: Dockerfile.integration-test
+ container_name: integration-test
+ volumes:
+ - seaweedfs_mount:/mnt/seaweedfs
+ - ./test-results:/test-results
+ networks:
+ - seaweedfs-rdma
+ environment:
+ - MOUNT_POINT=/mnt/seaweedfs
+ - FILER_ADDR=seaweedfs-filer:8888
+ - RDMA_SIDECAR_ADDR=rdma-sidecar:8081
+ - TEST_RESULTS_DIR=/test-results
+ depends_on:
+ seaweedfs-mount:
+ condition: service_healthy
+ command: >
+ sh -c "
+ echo 'Starting RDMA Mount Integration Tests...' &&
+ sleep 10 &&
+ /usr/local/bin/run-integration-tests.sh
+ "
+ profiles:
+ - test
+
+ # Performance Test Runner
+ performance-test:
+ build:
+ context: .
+ dockerfile: Dockerfile.performance-test
+ container_name: performance-test
+ volumes:
+ - seaweedfs_mount:/mnt/seaweedfs
+ - ./performance-results:/performance-results
+ networks:
+ - seaweedfs-rdma
+ environment:
+ - MOUNT_POINT=/mnt/seaweedfs
+ - RDMA_SIDECAR_ADDR=rdma-sidecar:8081
+ - PERFORMANCE_RESULTS_DIR=/performance-results
+ depends_on:
+ seaweedfs-mount:
+ condition: service_healthy
+ command: >
+ sh -c "
+ echo 'Starting RDMA Mount Performance Tests...' &&
+ sleep 10 &&
+ /usr/local/bin/run-performance-tests.sh
+ "
+ profiles:
+ - performance
+
+volumes:
+ seaweedfs_master_data:
+ driver: local
+ seaweedfs_volume_data:
+ driver: local
+ seaweedfs_mount:
+ driver: local
+ driver_opts:
+ type: tmpfs
+ device: tmpfs
+ o: size=1g
+ rdma_socket:
+ driver: local
+
+networks:
+ seaweedfs-rdma:
+ driver: bridge
+ ipam:
+ config:
+ - subnet: 172.20.0.0/16