diff options
| -rw-r--r-- | .github/workflows/s3-go-tests.yml | 90 | ||||
| -rw-r--r-- | test/s3/tagging/Makefile | 322 |
2 files changed, 324 insertions, 88 deletions
diff --git a/.github/workflows/s3-go-tests.yml b/.github/workflows/s3-go-tests.yml index 74ca57269..54023fa78 100644 --- a/.github/workflows/s3-go-tests.yml +++ b/.github/workflows/s3-go-tests.yml @@ -433,86 +433,24 @@ jobs: - name: Run S3 Tagging Tests timeout-minutes: 15 + working-directory: test/s3/tagging run: | set -x - # Create clean data directory for this test run - export WEED_DATA_DIR="/tmp/seaweedfs-tagging-test-$(date +%s)" - mkdir -p "$WEED_DATA_DIR" - weed server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \ - -dir="$WEED_DATA_DIR" \ - -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ - -volume.max=100 -volume.preStopSeconds=1 \ - -master.port=9338 -volume.port=8085 -filer.port=8893 -s3.port=8006 -metricsPort=9329 \ - -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none & - pid=$! - - # Wait for all SeaweedFS components to be ready - echo "Waiting for SeaweedFS components to start..." - for i in {1..30}; do - if curl -s http://localhost:9338/cluster/status > /dev/null 2>&1; then - echo "Master server is ready" - break - fi - echo "Waiting for master server... ($i/30)" - sleep 2 - done - - for i in {1..30}; do - if curl -s http://localhost:8085/status > /dev/null 2>&1; then - echo "Volume server is ready" - break - fi - echo "Waiting for volume server... ($i/30)" - sleep 2 - done - - for i in {1..30}; do - if curl -s http://localhost:8893/ > /dev/null 2>&1; then - echo "Filer is ready" - break - fi - echo "Waiting for filer... ($i/30)" - sleep 2 - done - - for i in {1..30}; do - if curl -s http://localhost:8006/ > /dev/null 2>&1; then - echo "S3 server is ready" - break - fi - echo "Waiting for S3 server... ($i/30)" - sleep 2 - done - - echo "All SeaweedFS components are ready!" - cd ../test/s3/tagging - - # Set environment variables for the test - export S3_ENDPOINT="http://localhost:8006" - export S3_ACCESS_KEY="0555b35654ad1656d804" - export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==" - - # Additional wait for S3-Filer integration to be fully ready - echo "Waiting additional 10 seconds for S3-Filer integration..." - sleep 10 + echo "=== System Information ===" + uname -a + free -h - # Test S3 connection before running tests - echo "Testing S3 connection..." - for i in {1..10}; do - if curl -s -f http://localhost:8006/ > /dev/null 2>&1; then - echo "S3 connection test successful" - break + # Run the specific test that is equivalent to AWS S3 tagging behavior + make test-with-server || { + echo "❌ Test failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -100 weed-test.log fi - echo "S3 connection test failed, retrying... ($i/10)" - sleep 2 - done - - echo "✅ S3 server is responding, starting tests..." - - go test -v ./... - kill -9 $pid || true - # Clean up data directory - rm -rf "$WEED_DATA_DIR" || true + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } - name: Upload test logs on failure if: failure() diff --git a/test/s3/tagging/Makefile b/test/s3/tagging/Makefile index 6d3343a70..6e7b3fb88 100644 --- a/test/s3/tagging/Makefile +++ b/test/s3/tagging/Makefile @@ -1,23 +1,321 @@ # S3 Object Tagging Tests # Tests for GitHub issue #7589: S3 object Tags query comes back empty -.PHONY: test +.PHONY: help build-weed setup-server start-server stop-server test-tagging test-tagging-quick test-tagging-comprehensive test-all clean logs check-deps test-with-server -# Run all tagging tests -test: - go test -v ./... +# Configuration +WEED_BINARY := ../../../weed/weed_binary +S3_PORT := 8006 +MASTER_PORT := 9338 +VOLUME_PORT := 8085 +FILER_PORT := 8893 +TEST_TIMEOUT := 10m +TEST_PATTERN := TestObjectTagging -# Run specific test +# Default target +help: + @echo "S3 Object Tagging Tests Makefile" + @echo "" + @echo "Available targets:" + @echo " help - Show this help message" + @echo " build-weed - Build the SeaweedFS binary" + @echo " check-deps - Check dependencies and build binary if needed" + @echo " start-server - Start SeaweedFS server for testing" + @echo " stop-server - Stop SeaweedFS server" + @echo " test-tagging - Run all tagging tests" + @echo " test-tagging-quick - Run core tagging tests only" + @echo " test-tagging-comprehensive - Run comprehensive tagging tests" + @echo " test-with-server - Start server, run tests, stop server" + @echo " logs - Show server logs" + @echo " clean - Clean up test artifacts and stop server" + @echo " health-check - Check if server is accessible" + @echo "" + @echo "Configuration:" + @echo " S3_PORT=${S3_PORT}" + @echo " TEST_TIMEOUT=${TEST_TIMEOUT}" + +# Build the SeaweedFS binary +build-weed: + @echo "Building SeaweedFS binary..." + @cd ../../../weed && go build -o weed_binary . + @chmod +x $(WEED_BINARY) + @echo "✅ SeaweedFS binary built at $(WEED_BINARY)" + +check-deps: build-weed + @echo "Checking dependencies..." + @echo "🔍 DEBUG: Checking Go installation..." + @command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1) + @echo "🔍 DEBUG: Go version: $$(go version)" + @echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)..." + @test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1) + @echo "🔍 DEBUG: Binary size: $$(ls -lh $(WEED_BINARY) | awk '{print $$5}')" + @echo "🔍 DEBUG: Binary permissions: $$(ls -la $(WEED_BINARY) | awk '{print $$1}')" + @echo "🔍 DEBUG: Checking Go module dependencies..." + @go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1) + @go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1) + @echo "✅ All dependencies are available" + +# Start SeaweedFS server for testing +start-server: check-deps + @echo "Starting SeaweedFS server..." + @echo "🔍 DEBUG: Current working directory: $$(pwd)" + @echo "🔍 DEBUG: Checking for existing weed processes..." + @ps aux | grep weed | grep -v grep || echo "No existing weed processes found" + @echo "🔍 DEBUG: Cleaning up any existing PID file..." + @rm -f weed-server.pid + @echo "🔍 DEBUG: Checking for port conflicts..." + @if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \ + echo "⚠️ Port $(S3_PORT) is already in use, trying to find the process..."; \ + netstat -tlnp 2>/dev/null | grep $(S3_PORT) || true; \ + else \ + echo "✅ Port $(S3_PORT) is available"; \ + fi + @echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)" + @ls -la $(WEED_BINARY) || (echo "❌ Binary not found!" && exit 1) + @echo "🔍 DEBUG: Checking config file at ../../../docker/compose/s3.json" + @ls -la ../../../docker/compose/s3.json || echo "⚠️ Config file not found, continuing without it" + @echo "🔍 DEBUG: Creating volume directory..." + @mkdir -p ./test-volume-data + @echo "🔍 DEBUG: Launching SeaweedFS server in background..." + @echo "🔍 DEBUG: Command: $(WEED_BINARY) server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 -dir=./test-volume-data -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 -volume.max=100 -volume.preStopSeconds=1 -master.port=$(MASTER_PORT) -volume.port=$(VOLUME_PORT) -filer.port=$(FILER_PORT) -s3.port=$(S3_PORT) -metricsPort=9329 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -master.peers=none" + @$(WEED_BINARY) server \ + -filer \ + -filer.maxMB=64 \ + -s3 \ + -ip.bind 0.0.0.0 \ + -dir=./test-volume-data \ + -master.raftHashicorp \ + -master.electionTimeout 1s \ + -master.volumeSizeLimitMB=100 \ + -volume.max=100 \ + -volume.preStopSeconds=1 \ + -master.port=$(MASTER_PORT) \ + -volume.port=$(VOLUME_PORT) \ + -filer.port=$(FILER_PORT) \ + -s3.port=$(S3_PORT) \ + -metricsPort=9329 \ + -s3.allowEmptyFolder=false \ + -s3.allowDeleteBucketNotEmpty=true \ + -s3.config=../../../docker/compose/s3.json \ + -master.peers=none \ + > weed-test.log 2>&1 & echo $$! > weed-server.pid + @echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')" + @echo "🔍 DEBUG: Checking if PID is still running..." + @sleep 2 + @if [ -f weed-server.pid ]; then \ + SERVER_PID=$$(cat weed-server.pid); \ + ps -p $$SERVER_PID || echo "⚠️ Server PID $$SERVER_PID not found after 2 seconds"; \ + else \ + echo "⚠️ PID file not found"; \ + fi + @echo "🔍 DEBUG: Waiting for server to start (up to 90 seconds)..." + @for i in $$(seq 1 90); do \ + echo "🔍 DEBUG: Attempt $$i/90 - checking port $(S3_PORT)"; \ + if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \ + echo "✅ SeaweedFS server started successfully on port $(S3_PORT) after $$i seconds"; \ + exit 0; \ + fi; \ + if [ $$i -eq 5 ]; then \ + echo "🔍 DEBUG: After 5 seconds, checking process and logs..."; \ + ps aux | grep weed | grep -v grep || echo "No weed processes found"; \ + if [ -f weed-test.log ]; then \ + echo "=== First server logs ==="; \ + head -20 weed-test.log; \ + fi; \ + fi; \ + if [ $$i -eq 15 ]; then \ + echo "🔍 DEBUG: After 15 seconds, checking port bindings..."; \ + netstat -tlnp 2>/dev/null | grep $(S3_PORT) || echo "Port $(S3_PORT) not bound"; \ + netstat -tlnp 2>/dev/null | grep $(MASTER_PORT) || echo "Port $(MASTER_PORT) not bound"; \ + netstat -tlnp 2>/dev/null | grep $(VOLUME_PORT) || echo "Port $(VOLUME_PORT) not bound"; \ + fi; \ + if [ $$i -eq 30 ]; then \ + echo "⚠️ Server taking longer than expected (30s), checking logs..."; \ + if [ -f weed-test.log ]; then \ + echo "=== Recent server logs ==="; \ + tail -20 weed-test.log; \ + fi; \ + fi; \ + sleep 1; \ + done; \ + echo "❌ Server failed to start within 90 seconds"; \ + echo "🔍 DEBUG: Final process check:"; \ + ps aux | grep weed | grep -v grep || echo "No weed processes found"; \ + echo "🔍 DEBUG: Final port check:"; \ + netstat -tlnp 2>/dev/null | grep -E "($(S3_PORT)|$(MASTER_PORT)|$(VOLUME_PORT))" || echo "No ports bound"; \ + echo "=== Full server logs ==="; \ + if [ -f weed-test.log ]; then \ + cat weed-test.log; \ + else \ + echo "No log file found"; \ + fi; \ + exit 1 + +# Stop SeaweedFS server +stop-server: + @echo "Stopping SeaweedFS server..." + @if [ -f weed-server.pid ]; then \ + SERVER_PID=$$(cat weed-server.pid); \ + echo "Killing server PID $$SERVER_PID"; \ + if ps -p $$SERVER_PID >/dev/null 2>&1; then \ + kill -TERM $$SERVER_PID 2>/dev/null || true; \ + sleep 2; \ + if ps -p $$SERVER_PID >/dev/null 2>&1; then \ + echo "Process still running, sending KILL signal..."; \ + kill -KILL $$SERVER_PID 2>/dev/null || true; \ + sleep 1; \ + fi; \ + else \ + echo "Process $$SERVER_PID not found (already stopped)"; \ + fi; \ + rm -f weed-server.pid; \ + else \ + echo "No PID file found, checking for running processes..."; \ + echo "⚠️ Skipping automatic process cleanup to avoid CI issues"; \ + echo "Note: Any remaining weed processes should be cleaned up by the CI environment"; \ + fi + @echo "✅ SeaweedFS server stopped" + +# Show server logs +logs: + @if test -f weed-test.log; then \ + echo "=== SeaweedFS Server Logs ==="; \ + tail -f weed-test.log; \ + else \ + echo "No log file found. Server may not be running."; \ + fi + +# Core tagging tests (basic functionality) +test-tagging-quick: check-deps + @echo "Running core tagging tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUpload|TestPutObjectTaggingAPI" . + @echo "✅ Core tagging tests completed" + +# All tagging tests (comprehensive) +test-tagging: check-deps + @echo "Running all tagging tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" . + @echo "✅ All tagging tests completed" + +# Comprehensive tagging tests (all features) +test-tagging-comprehensive: check-deps + @echo "Running comprehensive tagging tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTagging" . + @echo "✅ Comprehensive tagging tests completed" + +# All tests without server management +test-tagging-simple: check-deps + @echo "Running tagging tests (assuming server is already running)..." + @go test -v -timeout=$(TEST_TIMEOUT) . + @echo "✅ All tagging tests completed" + +# Start server, run tests, stop server +test-with-server: start-server + @echo "Running tagging tests with managed server..." + @sleep 5 # Give server time to fully start + @make test-tagging-comprehensive || (echo "Tests failed, stopping server..." && make stop-server && exit 1) + @make stop-server + @echo "✅ All tests completed with managed server" + +# Health check +health-check: + @echo "Checking server health..." + @if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \ + echo "✅ Server is accessible on port $(S3_PORT)"; \ + else \ + echo "❌ Server is not accessible on port $(S3_PORT)"; \ + exit 1; \ + fi + +# Clean up +clean: + @echo "Cleaning up test artifacts..." + @make stop-server + @rm -f weed-test.log + @rm -f weed-server.pid + @rm -rf ./test-volume-data + @rm -f tagging.test + @go clean -testcache + @echo "✅ Cleanup completed" + +# Individual test targets for specific functionality test-upload: - go test -v -run TestObjectTaggingOnUpload + @echo "Running upload tagging tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUpload" . test-special-chars: - go test -v -run TestObjectTaggingOnUploadWithSpecialCharacters + @echo "Running special characters tagging tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUploadWithSpecialCharacters" . test-api: - go test -v -run TestPutObjectTaggingAPI + @echo "Running API tagging tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestPutObjectTaggingAPI" . + +test-get: + @echo "Running get tagging tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestGetObjectTaggingAPI" . + +test-delete: + @echo "Running delete tagging tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestDeleteObjectTaggingAPI" . + +# Development targets +dev-start: start-server + @echo "Development server started. Access S3 API at http://localhost:$(S3_PORT)" + @echo "To stop: make stop-server" + +dev-test: check-deps + @echo "Running tests in development mode..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUpload" . + +# CI targets +ci-test: check-deps + @echo "Running tests in CI mode..." + @go test -v -timeout=$(TEST_TIMEOUT) -race . + +# All targets +test-all: test-tagging test-tagging-comprehensive + @echo "✅ All tagging tests completed" + +# Benchmark targets +benchmark-tagging: + @echo "Running tagging performance benchmarks..." + @go test -v -timeout=$(TEST_TIMEOUT) -bench=. -benchmem . + +# Coverage targets +coverage: + @echo "Running tests with coverage..." + @go test -v -timeout=$(TEST_TIMEOUT) -coverprofile=coverage.out . + @go tool cover -html=coverage.out -o coverage.html + @echo "Coverage report generated: coverage.html" + +# Format and lint +fmt: + @echo "Formatting Go code..." + @go fmt . + +lint: + @echo "Running linter..." + @golint . || echo "golint not available, skipping..." + +# Install dependencies for development +install-deps: + @echo "Installing Go dependencies..." + @go mod tidy + @go mod download + +# Show current configuration +show-config: + @echo "Current configuration:" + @echo " WEED_BINARY: $(WEED_BINARY)" + @echo " S3_PORT: $(S3_PORT)" + @echo " TEST_TIMEOUT: $(TEST_TIMEOUT)" + @echo " TEST_PATTERN: $(TEST_PATTERN)" -# Test configuration (override these for different environments) -S3_ENDPOINT ?= http://localhost:8333 -S3_ACCESS_KEY ?= some_access_key1 -S3_SECRET_KEY ?= some_secret_key1 +# Legacy targets for backward compatibility +test: test-with-server +test-verbose: test-tagging-comprehensive +test-single: test-upload +test-clean: clean +build: check-deps +setup: check-deps |