aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorchrislu <chris.lu@gmail.com>2025-12-01 13:01:35 -0800
committerchrislu <chris.lu@gmail.com>2025-12-01 13:28:21 -0800
commite0d74fffea14cec0e23160c08dbe64dbf4a94531 (patch)
tree94183aeba11abd98d3bdaf4ce60a9d673f1af24e
parent2ff0cc80a7495dd75ef7c5ae2942b0e8dddfb1dc (diff)
downloadseaweedfs-e0d74fffea14cec0e23160c08dbe64dbf4a94531.tar.xz
seaweedfs-e0d74fffea14cec0e23160c08dbe64dbf4a94531.zip
Improve TUS integration test setup
Add comprehensive Makefile for TUS tests with targets: - test-with-server: Run tests with automatic server management - test-basic/chunked/resume/errors: Specific test categories - manual-start/stop: For development testing - debug-logs/status: For debugging - ci-test: For CI/CD pipelines Update README.md with: - Detailed TUS protocol documentation - All endpoint descriptions with headers - Usage examples with curl commands - Architecture diagram - Comparison with S3 multipart uploads Follows the pattern established by other tests in test/ folder.
-rw-r--r--test/tus/Makefile226
-rw-r--r--test/tus/README.md241
2 files changed, 427 insertions, 40 deletions
diff --git a/test/tus/Makefile b/test/tus/Makefile
new file mode 100644
index 000000000..fd0b6d6b7
--- /dev/null
+++ b/test/tus/Makefile
@@ -0,0 +1,226 @@
+# Makefile for TUS Protocol Integration Tests
+# This Makefile provides targets for running TUS (resumable upload) integration tests
+
+# Default values
+SEAWEEDFS_BINARY ?= weed
+FILER_PORT ?= 18888
+VOLUME_PORT ?= 18080
+MASTER_PORT ?= 19333
+TEST_TIMEOUT ?= 10m
+VOLUME_MAX_SIZE_MB ?= 50
+VOLUME_MAX_COUNT ?= 100
+
+# Test directory
+TEST_DIR := $(shell pwd)
+SEAWEEDFS_ROOT := $(shell cd ../.. && pwd)
+
+# Colors for output
+RED := \033[0;31m
+GREEN := \033[0;32m
+YELLOW := \033[1;33m
+NC := \033[0m # No Color
+
+.PHONY: all test clean start-seaweedfs stop-seaweedfs check-binary build-weed help test-basic test-chunked test-resume test-errors test-with-server
+
+all: test
+
+# Build SeaweedFS binary
+build-weed:
+ @echo "Building SeaweedFS binary..."
+ @cd $(SEAWEEDFS_ROOT)/weed && go build -o weed
+ @echo "$(GREEN)SeaweedFS binary built successfully$(NC)"
+
+help:
+ @echo "SeaweedFS TUS Protocol Integration Tests"
+ @echo ""
+ @echo "Available targets:"
+ @echo " test - Run all TUS integration tests"
+ @echo " test-basic - Run basic TUS upload tests"
+ @echo " test-chunked - Run chunked upload tests"
+ @echo " test-resume - Run upload resume tests"
+ @echo " test-errors - Run error handling tests"
+ @echo " test-with-server - Run tests with automatic server management"
+ @echo " start-seaweedfs - Start SeaweedFS server for testing"
+ @echo " stop-seaweedfs - Stop SeaweedFS server"
+ @echo " clean - Clean up test artifacts"
+ @echo " check-binary - Check if SeaweedFS binary exists"
+ @echo " build-weed - Build SeaweedFS binary"
+ @echo ""
+ @echo "Configuration:"
+ @echo " SEAWEEDFS_BINARY=$(SEAWEEDFS_BINARY)"
+ @echo " FILER_PORT=$(FILER_PORT)"
+ @echo " VOLUME_PORT=$(VOLUME_PORT)"
+ @echo " MASTER_PORT=$(MASTER_PORT)"
+ @echo " TEST_TIMEOUT=$(TEST_TIMEOUT)"
+
+check-binary:
+ @if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1 && [ ! -f "$(SEAWEEDFS_ROOT)/weed/weed" ]; then \
+ echo "$(RED)Error: SeaweedFS binary not found$(NC)"; \
+ echo "Please build SeaweedFS first: make build-weed"; \
+ exit 1; \
+ fi
+ @echo "$(GREEN)SeaweedFS binary found$(NC)"
+
+start-seaweedfs: check-binary
+ @echo "$(YELLOW)Starting SeaweedFS server for TUS testing...$(NC)"
+ @# Clean up any existing processes on our test ports
+ @lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM 2>/dev/null || true
+ @lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM 2>/dev/null || true
+ @lsof -ti :$(FILER_PORT) | xargs -r kill -TERM 2>/dev/null || true
+ @sleep 2
+
+ # Create necessary directories
+ @mkdir -p /tmp/seaweedfs-test-tus-master
+ @mkdir -p /tmp/seaweedfs-test-tus-volume
+ @mkdir -p /tmp/seaweedfs-test-tus-filer
+
+ # Determine binary path
+ $(eval WEED_BIN := $(shell command -v $(SEAWEEDFS_BINARY) 2>/dev/null || echo "$(SEAWEEDFS_ROOT)/weed/weed"))
+
+ # Start master server
+ @echo "Starting master server..."
+ @nohup $(WEED_BIN) master \
+ -port=$(MASTER_PORT) \
+ -mdir=/tmp/seaweedfs-test-tus-master \
+ -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) \
+ -ip=127.0.0.1 \
+ > /tmp/seaweedfs-tus-master.log 2>&1 &
+ @sleep 3
+
+ # Start volume server
+ @echo "Starting volume server..."
+ @nohup $(WEED_BIN) volume \
+ -port=$(VOLUME_PORT) \
+ -mserver=127.0.0.1:$(MASTER_PORT) \
+ -dir=/tmp/seaweedfs-test-tus-volume \
+ -max=$(VOLUME_MAX_COUNT) \
+ -ip=127.0.0.1 \
+ > /tmp/seaweedfs-tus-volume.log 2>&1 &
+ @sleep 3
+
+ # Start filer server
+ @echo "Starting filer server..."
+ @nohup $(WEED_BIN) filer \
+ -port=$(FILER_PORT) \
+ -master=127.0.0.1:$(MASTER_PORT) \
+ -ip=127.0.0.1 \
+ > /tmp/seaweedfs-tus-filer.log 2>&1 &
+ @sleep 5
+
+ # Wait for filer to be ready
+ @echo "$(YELLOW)Waiting for filer to be ready...$(NC)"
+ @for i in $$(seq 1 30); do \
+ if curl -s -f http://127.0.0.1:$(FILER_PORT)/ > /dev/null 2>&1; then \
+ echo "$(GREEN)Filer is ready$(NC)"; \
+ break; \
+ fi; \
+ if [ $$i -eq 30 ]; then \
+ echo "$(RED)Filer failed to start within 30 seconds$(NC)"; \
+ $(MAKE) debug-logs; \
+ exit 1; \
+ fi; \
+ echo "Waiting for filer... ($$i/30)"; \
+ sleep 1; \
+ done
+
+ @echo "$(GREEN)SeaweedFS server started successfully for TUS testing$(NC)"
+ @echo "Master: http://localhost:$(MASTER_PORT)"
+ @echo "Volume: http://localhost:$(VOLUME_PORT)"
+ @echo "Filer: http://localhost:$(FILER_PORT)"
+ @echo "TUS Endpoint: http://localhost:$(FILER_PORT)/.tus/"
+
+stop-seaweedfs:
+ @echo "$(YELLOW)Stopping SeaweedFS server...$(NC)"
+ @lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM 2>/dev/null || true
+ @lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM 2>/dev/null || true
+ @lsof -ti :$(FILER_PORT) | xargs -r kill -TERM 2>/dev/null || true
+ @sleep 2
+ @echo "$(GREEN)SeaweedFS server stopped$(NC)"
+
+clean:
+ @echo "$(YELLOW)Cleaning up TUS test artifacts...$(NC)"
+ @rm -rf /tmp/seaweedfs-test-tus-*
+ @rm -f /tmp/seaweedfs-tus-*.log
+ @echo "$(GREEN)TUS test cleanup completed$(NC)"
+
+# Run all tests
+test: check-binary
+ @echo "$(YELLOW)Running all TUS integration tests...$(NC)"
+ @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/...
+ @echo "$(GREEN)All TUS tests completed$(NC)"
+
+# Run basic upload tests
+test-basic: check-binary
+ @echo "$(YELLOW)Running basic TUS upload tests...$(NC)"
+ @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusBasicUpload|TestTusOptionsHandler" ./test/tus/...
+ @echo "$(GREEN)Basic TUS tests completed$(NC)"
+
+# Run chunked upload tests
+test-chunked: check-binary
+ @echo "$(YELLOW)Running chunked TUS upload tests...$(NC)"
+ @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusChunkedUpload" ./test/tus/...
+ @echo "$(GREEN)Chunked TUS tests completed$(NC)"
+
+# Run resume tests
+test-resume: check-binary
+ @echo "$(YELLOW)Running TUS upload resume tests...$(NC)"
+ @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusResumeAfterInterruption|TestTusHeadRequest" ./test/tus/...
+ @echo "$(GREEN)TUS resume tests completed$(NC)"
+
+# Run error handling tests
+test-errors: check-binary
+ @echo "$(YELLOW)Running TUS error handling tests...$(NC)"
+ @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusInvalidOffset|TestTusUploadNotFound|TestTusDeleteUpload" ./test/tus/...
+ @echo "$(GREEN)TUS error tests completed$(NC)"
+
+# Run tests with automatic server management
+test-with-server: build-weed
+ @echo "$(YELLOW)Running TUS tests with automatic server management...$(NC)"
+ @trap '$(MAKE) stop-seaweedfs clean || true' EXIT; \
+ $(MAKE) start-seaweedfs && \
+ sleep 3 && \
+ cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/... && \
+ echo "$(GREEN)All TUS tests passed!$(NC)"
+
+# Debug targets
+debug-logs:
+ @echo "$(YELLOW)=== Master Log ===$(NC)"
+ @tail -n 50 /tmp/seaweedfs-tus-master.log 2>/dev/null || echo "No master log found"
+ @echo "$(YELLOW)=== Volume Log ===$(NC)"
+ @tail -n 50 /tmp/seaweedfs-tus-volume.log 2>/dev/null || echo "No volume log found"
+ @echo "$(YELLOW)=== Filer Log ===$(NC)"
+ @tail -n 50 /tmp/seaweedfs-tus-filer.log 2>/dev/null || echo "No filer log found"
+
+debug-status:
+ @echo "$(YELLOW)=== Process Status ===$(NC)"
+ @ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
+ @echo "$(YELLOW)=== Port Status ===$(NC)"
+ @lsof -i :$(MASTER_PORT) -i :$(VOLUME_PORT) -i :$(FILER_PORT) 2>/dev/null || echo "No ports in use"
+
+# Manual testing targets
+manual-start: start-seaweedfs
+ @echo "$(GREEN)SeaweedFS is now running for manual TUS testing$(NC)"
+ @echo ""
+ @echo "TUS Endpoints:"
+ @echo " OPTIONS /.tus/ - Capability discovery"
+ @echo " POST /.tus/{path} - Create upload"
+ @echo " HEAD /.tus/.uploads/{id} - Get offset"
+ @echo " PATCH /.tus/.uploads/{id} - Upload data"
+ @echo " DELETE /.tus/.uploads/{id} - Cancel upload"
+ @echo ""
+ @echo "Example curl commands:"
+ @echo " curl -X OPTIONS http://localhost:$(FILER_PORT)/.tus/ -H 'Tus-Resumable: 1.0.0'"
+ @echo ""
+ @echo "Run 'make manual-stop' when finished"
+
+manual-stop: stop-seaweedfs clean
+
+# CI targets
+ci-test: test-with-server
+
+# Skip integration tests (short mode)
+test-short:
+ @echo "$(YELLOW)Running TUS tests in short mode (skipping integration tests)...$(NC)"
+ @cd $(SEAWEEDFS_ROOT) && go test -v -short ./test/tus/...
+ @echo "$(GREEN)Short tests completed$(NC)"
+
diff --git a/test/tus/README.md b/test/tus/README.md
index 7477168d0..e8913d82f 100644
--- a/test/tus/README.md
+++ b/test/tus/README.md
@@ -2,64 +2,225 @@
This directory contains integration tests for the TUS (resumable upload) protocol support in SeaweedFS Filer.
-## TUS Protocol Overview
+## Overview
-TUS is an open protocol for resumable file uploads. It allows clients to upload files in chunks and resume uploads after network failures.
+TUS is an open protocol for resumable file uploads over HTTP. It allows clients to upload files in chunks and resume uploads after network failures or interruptions.
-Key endpoints:
-- `OPTIONS /tus/` - Server capability discovery
-- `POST /tus/{path}` - Create new upload session
-- `HEAD /tus/{upload-id}` - Get current upload offset
-- `PATCH /tus/{upload-id}` - Upload data at offset
-- `DELETE /tus/{upload-id}` - Cancel upload
+### Why TUS?
-## Prerequisites
+- **Resumable uploads**: Resume interrupted uploads without re-sending data
+- **Chunked uploads**: Upload large files in smaller pieces
+- **Simple protocol**: Standard HTTP methods with custom headers
+- **Wide client support**: Libraries available for JavaScript, Python, Go, and more
-1. Build the weed binary:
+## TUS Protocol Endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| `OPTIONS` | `/.tus/` | Server capability discovery |
+| `POST` | `/.tus/{path}` | Create new upload session |
+| `HEAD` | `/.tus/.uploads/{id}` | Get current upload offset |
+| `PATCH` | `/.tus/.uploads/{id}` | Upload data at offset |
+| `DELETE` | `/.tus/.uploads/{id}` | Cancel upload |
+
+### TUS Headers
+
+**Request Headers:**
+- `Tus-Resumable: 1.0.0` - Protocol version (required)
+- `Upload-Length` - Total file size in bytes (required on POST)
+- `Upload-Offset` - Current byte offset (required on PATCH)
+- `Upload-Metadata` - Base64-encoded key-value pairs (optional)
+- `Content-Type: application/offset+octet-stream` (required on PATCH)
+
+**Response Headers:**
+- `Tus-Resumable` - Protocol version
+- `Tus-Version` - Supported versions
+- `Tus-Extension` - Supported extensions
+- `Tus-Max-Size` - Maximum upload size
+- `Upload-Offset` - Current byte offset
+- `Location` - Upload URL (on POST)
+
+## Test Structure
+
+### Integration Tests
+
+The tests cover:
+
+1. **Basic Functionality**
+ - `TestTusOptionsHandler` - Capability discovery
+ - `TestTusBasicUpload` - Simple complete upload
+ - `TestTusCreationWithUpload` - Creation-with-upload extension
+
+2. **Chunked Uploads**
+ - `TestTusChunkedUpload` - Upload in multiple chunks
+
+3. **Resumable Uploads**
+ - `TestTusHeadRequest` - Offset tracking
+ - `TestTusResumeAfterInterruption` - Resume after failure
+
+4. **Error Handling**
+ - `TestTusInvalidOffset` - Offset mismatch (409 Conflict)
+ - `TestTusUploadNotFound` - Missing upload (404 Not Found)
+ - `TestTusDeleteUpload` - Upload cancellation
+
+## Running Tests
+
+### Prerequisites
+
+1. **Build SeaweedFS**:
```bash
-cd ../../weed
-go build
+make build-weed
+# or
+cd ../../weed && go build -o weed
```
-2. The tests will automatically start required servers (master, volume, filer).
+### Using Makefile
-## Running Tests
+```bash
+# Show available targets
+make help
+
+# Run all tests with automatic server management
+make test-with-server
+
+# Run all tests (requires running server)
+make test
+
+# Run specific test categories
+make test-basic # Basic upload tests
+make test-chunked # Chunked upload tests
+make test-resume # Resume/HEAD tests
+make test-errors # Error handling tests
+
+# Manual testing
+make manual-start # Start SeaweedFS for manual testing
+make manual-stop # Stop and cleanup
+```
+
+### Using Go Test Directly
-### Run all TUS tests:
```bash
+# Run all TUS tests
go test -v ./test/tus/...
+
+# Run specific test
+go test -v ./test/tus -run TestTusBasicUpload
+
+# Skip integration tests (short mode)
+go test -v -short ./test/tus/...
```
-### Run specific test:
+### Debug
+
```bash
-go test -v ./test/tus -run TestTusBasicUpload
+# View server logs
+make debug-logs
+
+# Check process and port status
+make debug-status
```
-### Skip integration tests (short mode):
+## Test Environment
+
+Each test run:
+1. Starts a SeaweedFS cluster (master, volume, filer)
+2. Creates uploads using TUS protocol
+3. Verifies files are stored correctly
+4. Cleans up test data
+
+### Default Ports
+
+| Service | Port |
+|---------|------|
+| Master | 19333 |
+| Volume | 18080 |
+| Filer | 18888 |
+
+### Configuration
+
+Override defaults via environment or Makefile variables:
```bash
-go test -v -short ./test/tus/...
+FILER_PORT=8889 MASTER_PORT=9334 make test
```
-## Test Coverage
+## Example Usage
-The tests cover:
-- Basic upload creation and completion
-- Chunked/resumable uploads
-- Upload offset tracking (HEAD requests)
-- Upload cancellation (DELETE requests)
-- Error handling (invalid offsets, missing uploads)
-- Large file uploads with multiple chunks
-- Concurrent uploads
-- Metadata handling
-
-## TUS Protocol Headers
-
-Required headers for TUS requests:
-- `Tus-Resumable: 1.0.0` - Protocol version
-- `Upload-Length` - Total file size (on creation)
-- `Upload-Offset` - Current byte offset (on PATCH)
-- `Content-Type: application/offset+octet-stream` (on PATCH)
-
-Optional headers:
-- `Upload-Metadata` - Base64-encoded key-value pairs
+### Create Upload
+
+```bash
+curl -X POST http://localhost:18888/.tus/mydir/file.txt \
+ -H "Tus-Resumable: 1.0.0" \
+ -H "Upload-Length: 1000" \
+ -H "Upload-Metadata: filename dGVzdC50eHQ="
+```
+
+### Upload Data
+
+```bash
+curl -X PATCH http://localhost:18888/.tus/.uploads/{upload-id} \
+ -H "Tus-Resumable: 1.0.0" \
+ -H "Upload-Offset: 0" \
+ -H "Content-Type: application/offset+octet-stream" \
+ --data-binary @file.txt
+```
+
+### Check Offset
+
+```bash
+curl -I http://localhost:18888/.tus/.uploads/{upload-id} \
+ -H "Tus-Resumable: 1.0.0"
+```
+
+### Cancel Upload
+
+```bash
+curl -X DELETE http://localhost:18888/.tus/.uploads/{upload-id} \
+ -H "Tus-Resumable: 1.0.0"
+```
+
+## TUS Extensions Supported
+
+- **creation**: Create new uploads with POST
+- **creation-with-upload**: Send data in creation request
+- **termination**: Cancel uploads with DELETE
+
+## Architecture
+
+```
+Client Filer Volume Servers
+ | | |
+ |-- POST /.tus/path/file.mp4 ->| |
+ | |-- Create session dir ------->|
+ |<-- 201 Location: /.../{id} --| |
+ | | |
+ |-- PATCH /.tus/.uploads/{id} >| |
+ | Upload-Offset: 0 |-- Assign volume ------------>|
+ | [chunk data] |-- Upload chunk ------------->|
+ |<-- 204 Upload-Offset: N -----| |
+ | | |
+ | (network failure) | |
+ | | |
+ |-- HEAD /.tus/.uploads/{id} ->| |
+ |<-- Upload-Offset: N ---------| |
+ | | |
+ |-- PATCH (resume) ----------->|-- Upload remaining -------->|
+ |<-- 204 (complete) -----------|-- Assemble final file ----->|
+```
+
+## Comparison with S3 Multipart
+
+| Feature | TUS | S3 Multipart |
+|---------|-----|--------------|
+| Protocol | Custom HTTP headers | S3 API |
+| Session Init | POST with Upload-Length | CreateMultipartUpload |
+| Upload Data | PATCH with offset | UploadPart with partNumber |
+| Resume | HEAD to get offset | ListParts |
+| Complete | Automatic at final offset | CompleteMultipartUpload |
+| Ordering | Sequential (offset-based) | Parallel (part numbers) |
+
+## Related Resources
+- [TUS Protocol Specification](https://tus.io/protocols/resumable-upload)
+- [tus-js-client](https://github.com/tus/tus-js-client) - JavaScript client
+- [go-tus](https://github.com/eventials/go-tus) - Go client
+- [SeaweedFS S3 API](../../weed/s3api) - Alternative multipart upload