diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/s3/retention/Makefile | 360 | ||||
| -rw-r--r-- | test/s3/retention/README.md | 264 | ||||
| -rw-r--r-- | test/s3/retention/go.mod | 31 | ||||
| -rw-r--r-- | test/s3/retention/go.sum | 62 | ||||
| -rw-r--r-- | test/s3/retention/s3_retention_test.go | 694 | ||||
| -rw-r--r-- | test/s3/retention/s3_worm_integration_test.go | 519 | ||||
| -rw-r--r-- | test/s3/retention/test_config.json | 9 |
7 files changed, 1939 insertions, 0 deletions
diff --git a/test/s3/retention/Makefile b/test/s3/retention/Makefile new file mode 100644 index 000000000..092d2caac --- /dev/null +++ b/test/s3/retention/Makefile @@ -0,0 +1,360 @@ +# S3 API Retention Test Makefile +# This Makefile provides comprehensive targets for running S3 retention tests + +.PHONY: help build-weed setup-server start-server stop-server test-retention test-retention-quick test-retention-comprehensive test-retention-worm test-all clean logs check-deps + +# Configuration +WEED_BINARY := ../../../weed/weed_binary +S3_PORT := 8333 +MASTER_PORT := 9333 +VOLUME_PORT := 8080 +FILER_PORT := 8888 +TEST_TIMEOUT := 15m +TEST_PATTERN := TestRetention + +# Default target +help: + @echo "S3 API Retention Test Makefile" + @echo "" + @echo "Available targets:" + @echo " help - Show this help message" + @echo " build-weed - Build the SeaweedFS binary" + @echo " check-deps - Check dependencies and build binary if needed" + @echo " start-server - Start SeaweedFS server for testing" + @echo " start-server-simple - Start server without process cleanup (for CI)" + @echo " stop-server - Stop SeaweedFS server" + @echo " test-retention - Run all retention tests" + @echo " test-retention-quick - Run core retention tests only" + @echo " test-retention-simple - Run tests without server management" + @echo " test-retention-comprehensive - Run comprehensive retention tests" + @echo " test-retention-worm - Run WORM integration tests" + @echo " test-all - Run all S3 API retention tests" + @echo " test-with-server - Start server, run tests, stop server" + @echo " logs - Show server logs" + @echo " clean - Clean up test artifacts and stop server" + @echo " health-check - Check if server is accessible" + @echo "" + @echo "Configuration:" + @echo " S3_PORT=${S3_PORT}" + @echo " TEST_TIMEOUT=${TEST_TIMEOUT}" + +# Build the SeaweedFS binary +build-weed: + @echo "Building SeaweedFS binary..." + @cd ../../../weed && go build -o weed_binary . + @chmod +x $(WEED_BINARY) + @echo "✅ SeaweedFS binary built at $(WEED_BINARY)" + +check-deps: build-weed + @echo "Checking dependencies..." + @echo "🔍 DEBUG: Checking Go installation..." + @command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1) + @echo "🔍 DEBUG: Go version: $$(go version)" + @echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)..." + @test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1) + @echo "🔍 DEBUG: Binary size: $$(ls -lh $(WEED_BINARY) | awk '{print $$5}')" + @echo "🔍 DEBUG: Binary permissions: $$(ls -la $(WEED_BINARY) | awk '{print $$1}')" + @echo "🔍 DEBUG: Checking Go module dependencies..." + @go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1) + @go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1) + @echo "✅ All dependencies are available" + +# Start SeaweedFS server for testing +start-server: check-deps + @echo "Starting SeaweedFS server..." + @echo "🔍 DEBUG: Current working directory: $$(pwd)" + @echo "🔍 DEBUG: Checking for existing weed processes..." + @ps aux | grep weed | grep -v grep || echo "No existing weed processes found" + @echo "🔍 DEBUG: Cleaning up any existing PID file..." + @rm -f weed-server.pid + @echo "🔍 DEBUG: Checking for port conflicts..." + @if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \ + echo "⚠️ Port $(S3_PORT) is already in use, trying to find the process..."; \ + netstat -tlnp 2>/dev/null | grep $(S3_PORT) || true; \ + else \ + echo "✅ Port $(S3_PORT) is available"; \ + fi + @echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)" + @ls -la $(WEED_BINARY) || (echo "❌ Binary not found!" && exit 1) + @echo "🔍 DEBUG: Checking config file at ../../../docker/compose/s3.json" + @ls -la ../../../docker/compose/s3.json || echo "⚠️ Config file not found, continuing without it" + @echo "🔍 DEBUG: Creating volume directory..." + @mkdir -p ./test-volume-data + @echo "🔍 DEBUG: Launching SeaweedFS server in background..." + @echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324" + @$(WEED_BINARY) server \ + -debug \ + -s3 \ + -s3.port=$(S3_PORT) \ + -s3.allowEmptyFolder=false \ + -s3.allowDeleteBucketNotEmpty=true \ + -s3.config=../../../docker/compose/s3.json \ + -filer \ + -filer.maxMB=64 \ + -master.volumeSizeLimitMB=50 \ + -volume.max=100 \ + -dir=./test-volume-data \ + -volume.preStopSeconds=1 \ + -metricsPort=9324 \ + > weed-test.log 2>&1 & echo $$! > weed-server.pid + @echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')" + @echo "🔍 DEBUG: Checking if PID is still running..." + @sleep 2 + @if [ -f weed-server.pid ]; then \ + SERVER_PID=$$(cat weed-server.pid); \ + ps -p $$SERVER_PID || echo "⚠️ Server PID $$SERVER_PID not found after 2 seconds"; \ + else \ + echo "⚠️ PID file not found"; \ + fi + @echo "🔍 DEBUG: Waiting for server to start (up to 90 seconds)..." + @for i in $$(seq 1 90); do \ + echo "🔍 DEBUG: Attempt $$i/90 - checking port $(S3_PORT)"; \ + if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \ + echo "✅ SeaweedFS server started successfully on port $(S3_PORT) after $$i seconds"; \ + exit 0; \ + fi; \ + if [ $$i -eq 5 ]; then \ + echo "🔍 DEBUG: After 5 seconds, checking process and logs..."; \ + ps aux | grep weed | grep -v grep || echo "No weed processes found"; \ + if [ -f weed-test.log ]; then \ + echo "=== First server logs ==="; \ + head -20 weed-test.log; \ + fi; \ + fi; \ + if [ $$i -eq 15 ]; then \ + echo "🔍 DEBUG: After 15 seconds, checking port bindings..."; \ + netstat -tlnp 2>/dev/null | grep $(S3_PORT) || echo "Port $(S3_PORT) not bound"; \ + netstat -tlnp 2>/dev/null | grep 9333 || echo "Port 9333 not bound"; \ + netstat -tlnp 2>/dev/null | grep 8080 || echo "Port 8080 not bound"; \ + fi; \ + if [ $$i -eq 30 ]; then \ + echo "⚠️ Server taking longer than expected (30s), checking logs..."; \ + if [ -f weed-test.log ]; then \ + echo "=== Recent server logs ==="; \ + tail -20 weed-test.log; \ + fi; \ + fi; \ + sleep 1; \ + done; \ + echo "❌ Server failed to start within 90 seconds"; \ + echo "🔍 DEBUG: Final process check:"; \ + ps aux | grep weed | grep -v grep || echo "No weed processes found"; \ + echo "🔍 DEBUG: Final port check:"; \ + netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080)" || echo "No ports bound"; \ + echo "=== Full server logs ==="; \ + if [ -f weed-test.log ]; then \ + cat weed-test.log; \ + else \ + echo "No log file found"; \ + fi; \ + exit 1 + +# Stop SeaweedFS server +stop-server: + @echo "Stopping SeaweedFS server..." + @if [ -f weed-server.pid ]; then \ + SERVER_PID=$$(cat weed-server.pid); \ + echo "Killing server PID $$SERVER_PID"; \ + if ps -p $$SERVER_PID >/dev/null 2>&1; then \ + kill -TERM $$SERVER_PID 2>/dev/null || true; \ + sleep 2; \ + if ps -p $$SERVER_PID >/dev/null 2>&1; then \ + echo "Process still running, sending KILL signal..."; \ + kill -KILL $$SERVER_PID 2>/dev/null || true; \ + sleep 1; \ + fi; \ + else \ + echo "Process $$SERVER_PID not found (already stopped)"; \ + fi; \ + rm -f weed-server.pid; \ + else \ + echo "No PID file found, checking for running processes..."; \ + echo "⚠️ Skipping automatic process cleanup to avoid CI issues"; \ + echo "Note: Any remaining weed processes should be cleaned up by the CI environment"; \ + fi + @echo "✅ SeaweedFS server stopped" + +# Show server logs +logs: + @if test -f weed-test.log; then \ + echo "=== SeaweedFS Server Logs ==="; \ + tail -f weed-test.log; \ + else \ + echo "No log file found. Server may not be running."; \ + fi + +# Core retention tests (basic functionality) +test-retention-quick: check-deps + @echo "Running core S3 retention tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestBasicRetentionWorkflow|TestRetentionModeCompliance|TestLegalHoldWorkflow" . + @echo "✅ Core retention tests completed" + +# All retention tests (comprehensive) +test-retention: check-deps + @echo "Running all S3 retention tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" . + @echo "✅ All retention tests completed" + +# WORM integration tests +test-retention-worm: check-deps + @echo "Running WORM integration tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestWORM|TestRetentionExtendedAttributes|TestRetentionConcurrentOperations" . + @echo "✅ WORM integration tests completed" + +# Comprehensive retention tests (all features) +test-retention-comprehensive: check-deps + @echo "Running comprehensive S3 retention tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetention|TestObjectLock|TestLegalHold|TestWORM" . + @echo "✅ Comprehensive retention tests completed" + +# All tests without server management +test-retention-simple: check-deps + @echo "Running retention tests (assuming server is already running)..." + @go test -v -timeout=$(TEST_TIMEOUT) . + @echo "✅ All retention tests completed" + +# Start server, run tests, stop server +test-with-server: start-server + @echo "Running retention tests with managed server..." + @sleep 5 # Give server time to fully start + @make test-retention-comprehensive || (echo "Tests failed, stopping server..." && make stop-server && exit 1) + @make stop-server + @echo "✅ All tests completed with managed server" + +# Health check +health-check: + @echo "Checking server health..." + @if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \ + echo "✅ Server is accessible on port $(S3_PORT)"; \ + else \ + echo "❌ Server is not accessible on port $(S3_PORT)"; \ + exit 1; \ + fi + +# Clean up +clean: + @echo "Cleaning up test artifacts..." + @make stop-server + @rm -f weed-test.log + @rm -f weed-server.pid + @rm -rf ./test-volume-data + @echo "✅ Cleanup completed" + +# Individual test targets for specific functionality +test-basic-retention: + @echo "Running basic retention tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestBasicRetentionWorkflow" . + +test-compliance-retention: + @echo "Running compliance retention tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionModeCompliance" . + +test-legal-hold: + @echo "Running legal hold tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestLegalHoldWorkflow" . + +test-object-lock-config: + @echo "Running object lock configuration tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectLockConfiguration" . + +test-retention-versions: + @echo "Running retention with versions tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionWithVersions" . + +test-retention-combination: + @echo "Running retention and legal hold combination tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionAndLegalHoldCombination" . + +test-expired-retention: + @echo "Running expired retention tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestExpiredRetention" . + +test-retention-errors: + @echo "Running retention error case tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionErrorCases" . + +# WORM-specific test targets +test-worm-integration: + @echo "Running WORM integration tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestWORMRetentionIntegration" . + +test-worm-legacy: + @echo "Running WORM legacy compatibility tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestWORMLegacyCompatibility" . + +test-retention-overwrite: + @echo "Running retention overwrite protection tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionOverwriteProtection" . + +test-retention-bulk: + @echo "Running retention bulk operations tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionBulkOperations" . + +test-retention-multipart: + @echo "Running retention multipart upload tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionWithMultipartUpload" . + +test-retention-extended-attrs: + @echo "Running retention extended attributes tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionExtendedAttributes" . + +test-retention-defaults: + @echo "Running retention bucket defaults tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionBucketDefaults" . + +test-retention-concurrent: + @echo "Running retention concurrent operations tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionConcurrentOperations" . + +# Development targets +dev-start: start-server + @echo "Development server started. Access S3 API at http://localhost:$(S3_PORT)" + @echo "To stop: make stop-server" + +dev-test: check-deps + @echo "Running tests in development mode..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestBasicRetentionWorkflow" . + +# CI targets +ci-test: check-deps + @echo "Running tests in CI mode..." + @go test -v -timeout=$(TEST_TIMEOUT) -race . + +# All targets +test-all: test-retention test-retention-worm + @echo "✅ All S3 retention tests completed" + +# Benchmark targets +benchmark-retention: + @echo "Running retention performance benchmarks..." + @go test -v -timeout=$(TEST_TIMEOUT) -bench=. -benchmem . + +# Coverage targets +coverage: + @echo "Running tests with coverage..." + @go test -v -timeout=$(TEST_TIMEOUT) -coverprofile=coverage.out . + @go tool cover -html=coverage.out -o coverage.html + @echo "Coverage report generated: coverage.html" + +# Format and lint +fmt: + @echo "Formatting Go code..." + @go fmt . + +lint: + @echo "Running linter..." + @golint . || echo "golint not available, skipping..." + +# Install dependencies for development +install-deps: + @echo "Installing Go dependencies..." + @go mod tidy + @go mod download + +# Show current configuration +show-config: + @echo "Current configuration:" + @echo " WEED_BINARY: $(WEED_BINARY)" + @echo " S3_PORT: $(S3_PORT)" + @echo " TEST_TIMEOUT: $(TEST_TIMEOUT)" + @echo " TEST_PATTERN: $(TEST_PATTERN)"
\ No newline at end of file diff --git a/test/s3/retention/README.md b/test/s3/retention/README.md new file mode 100644 index 000000000..7d92646e6 --- /dev/null +++ b/test/s3/retention/README.md @@ -0,0 +1,264 @@ +# SeaweedFS S3 Object Retention Tests + +This directory contains comprehensive tests for SeaweedFS S3 Object Retention functionality, including Object Lock, Legal Hold, and WORM (Write Once Read Many) capabilities. + +## Overview + +The test suite validates AWS S3-compatible object retention features including: + +- **Object Retention**: GOVERNANCE and COMPLIANCE modes with retain-until-date +- **Legal Hold**: Independent protection that can be applied/removed +- **Object Lock Configuration**: Bucket-level default retention policies +- **WORM Integration**: Compatibility with legacy WORM functionality +- **Version-specific Retention**: Different retention policies per object version +- **Enforcement**: Protection against deletion and overwriting + +## Test Files + +- `s3_retention_test.go` - Core retention functionality tests +- `s3_worm_integration_test.go` - WORM integration and advanced scenarios +- `test_config.json` - Test configuration (endpoints, credentials) +- `Makefile` - Comprehensive test automation +- `go.mod` - Go module dependencies + +## Prerequisites + +- Go 1.21 or later +- SeaweedFS binary built (`make build-weed`) +- AWS SDK Go v2 +- Testify testing framework + +## Quick Start + +### 1. Build and Start Server +```bash +# Build SeaweedFS and start test server +make start-server +``` + +### 2. Run Tests +```bash +# Run core retention tests +make test-retention-quick + +# Run all retention tests +make test-retention + +# Run WORM integration tests +make test-retention-worm + +# Run all tests with managed server +make test-with-server +``` + +### 3. Cleanup +```bash +make clean +``` + +## Test Categories + +### Core Retention Tests +- `TestBasicRetentionWorkflow` - Basic GOVERNANCE mode retention +- `TestRetentionModeCompliance` - COMPLIANCE mode (immutable) +- `TestLegalHoldWorkflow` - Legal hold on/off functionality +- `TestObjectLockConfiguration` - Bucket object lock settings + +### Advanced Tests +- `TestRetentionWithVersions` - Version-specific retention policies +- `TestRetentionAndLegalHoldCombination` - Multiple protection types +- `TestExpiredRetention` - Post-expiration behavior +- `TestRetentionErrorCases` - Error handling and edge cases + +### WORM Integration Tests +- `TestWORMRetentionIntegration` - New retention + legacy WORM +- `TestWORMLegacyCompatibility` - Backward compatibility +- `TestRetentionOverwriteProtection` - Prevent overwrites +- `TestRetentionBulkOperations` - Bulk delete with retention +- `TestRetentionWithMultipartUpload` - Multipart upload retention +- `TestRetentionExtendedAttributes` - Extended attribute storage +- `TestRetentionBucketDefaults` - Default retention application +- `TestRetentionConcurrentOperations` - Concurrent operation safety + +## Individual Test Targets + +Run specific test categories: + +```bash +# Basic functionality +make test-basic-retention +make test-compliance-retention +make test-legal-hold + +# Advanced features +make test-retention-versions +make test-retention-combination +make test-expired-retention + +# WORM integration +make test-worm-integration +make test-worm-legacy +make test-retention-bulk +``` + +## Configuration + +### Server Configuration +The tests use these default settings: +- S3 Port: 8333 +- Test timeout: 15 minutes +- Volume directory: `./test-volume-data` + +### Test Configuration (`test_config.json`) +```json +{ + "endpoint": "http://localhost:8333", + "access_key": "some_access_key1", + "secret_key": "some_secret_key1", + "region": "us-east-1", + "bucket_prefix": "test-retention-", + "use_ssl": false, + "skip_verify_ssl": true +} +``` + +## Expected Behavior + +### GOVERNANCE Mode +- Objects protected until retain-until-date +- Can be bypassed with `x-amz-bypass-governance-retention` header +- Supports time extension (not reduction) + +### COMPLIANCE Mode +- Objects immutably protected until retain-until-date +- Cannot be bypassed or shortened +- Strictest protection level + +### Legal Hold +- Independent ON/OFF protection +- Can coexist with retention policies +- Must be explicitly removed to allow deletion + +### Version Support +- Each object version can have individual retention +- Applies to both versioned and non-versioned buckets +- Version-specific retention retrieval + +## Development + +### Running in Development Mode +```bash +# Start server for development +make dev-start + +# Run quick test +make dev-test +``` + +### Code Quality +```bash +# Format code +make fmt + +# Run linter +make lint + +# Generate coverage report +make coverage +``` + +### Performance Testing +```bash +# Run benchmarks +make benchmark-retention +``` + +## Troubleshooting + +### Server Won't Start +```bash +# Check if port is in use +netstat -tlnp | grep 8333 + +# View server logs +make logs + +# Force cleanup +make clean +``` + +### Test Failures +```bash +# Run with verbose output +go test -v -timeout=15m . + +# Run specific test +go test -v -run TestBasicRetentionWorkflow . + +# Check server health +make health-check +``` + +### Dependencies +```bash +# Install/update dependencies +make install-deps + +# Check dependency status +make check-deps +``` + +## Integration with SeaweedFS + +These tests validate the retention implementation in: +- `weed/s3api/s3api_object_retention.go` - Core retention logic +- `weed/s3api/s3api_object_handlers_retention.go` - HTTP handlers +- `weed/s3api/s3_constants/extend_key.go` - Extended attribute keys +- `weed/s3api/s3err/s3api_errors.go` - Error definitions +- `weed/s3api/s3api_object_handlers_delete.go` - Deletion enforcement +- `weed/s3api/s3api_object_handlers_put.go` - Upload enforcement + +## AWS CLI Compatibility + +The retention implementation supports standard AWS CLI commands: + +```bash +# Set object retention +aws s3api put-object-retention \ + --bucket mybucket \ + --key myobject \ + --retention Mode=GOVERNANCE,RetainUntilDate=2024-12-31T23:59:59Z + +# Get object retention +aws s3api get-object-retention \ + --bucket mybucket \ + --key myobject + +# Set legal hold +aws s3api put-object-legal-hold \ + --bucket mybucket \ + --key myobject \ + --legal-hold Status=ON + +# Configure bucket object lock +aws s3api put-object-lock-configuration \ + --bucket mybucket \ + --object-lock-configuration ObjectLockEnabled=Enabled,Rule='{DefaultRetention={Mode=GOVERNANCE,Days=30}}' +``` + +## Contributing + +When adding new retention tests: + +1. Follow existing test patterns +2. Use descriptive test names +3. Include both positive and negative test cases +4. Test error conditions +5. Update this README with new test descriptions +6. Add appropriate Makefile targets for new test categories + +## References + +- [AWS S3 Object Lock Documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html) +- [AWS S3 API Reference - Object Retention](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html) +- [SeaweedFS S3 API Documentation](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API)
\ No newline at end of file diff --git a/test/s3/retention/go.mod b/test/s3/retention/go.mod new file mode 100644 index 000000000..3d0c0095d --- /dev/null +++ b/test/s3/retention/go.mod @@ -0,0 +1,31 @@ +module github.com/seaweedfs/seaweedfs/test/s3/retention + +go 1.21 + +require ( + github.com/aws/aws-sdk-go-v2 v1.21.2 + github.com/aws/aws-sdk-go-v2/config v1.18.45 + github.com/aws/aws-sdk-go-v2/credentials v1.13.43 + github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0 + github.com/stretchr/testify v1.8.4 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect + github.com/aws/smithy-go v1.15.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/test/s3/retention/go.sum b/test/s3/retention/go.sum new file mode 100644 index 000000000..f31ba829d --- /dev/null +++ b/test/s3/retention/go.sum @@ -0,0 +1,62 @@ +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= +github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= +github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 h1:wmGLw2i8ZTlHLw7a9ULGfQbuccw8uIiNr6sol5bFzc8= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6/go.mod h1:Q0Hq2X/NuL7z8b1Dww8rmOFl+jzusKEcyvkKspwdpyc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 h1:7R8uRYyXzdD71KWVCL78lJZltah6VVznXBazvKjfH58= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15/go.mod h1:26SQUPcTNgV1Tapwdt4a1rOsYRsnBsJHLMPoxK2b0d8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 h1:skaFGzv+3kA+v2BPKhuekeb1Hbb105+44r8ASC+q5SE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38/go.mod h1:epIZoRSSbRIwLPJU5F+OldHhwZPBdpDeQkRdCeY3+00= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 h1:9ulSU5ClouoPIYhDQdg9tpl83d5Yb91PXTKK+17q+ow= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6/go.mod h1:lnc2taBsR9nTlz9meD+lhFZZ9EWY712QHrRflWpTcOA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0 h1:wl5dxN1NONhTDQD9uaEvNsDRX29cBmGED/nl0jkWlt4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/test/s3/retention/s3_retention_test.go b/test/s3/retention/s3_retention_test.go new file mode 100644 index 000000000..e3f3222c9 --- /dev/null +++ b/test/s3/retention/s3_retention_test.go @@ -0,0 +1,694 @@ +package s3api + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// S3TestConfig holds configuration for S3 tests +type S3TestConfig struct { + Endpoint string + AccessKey string + SecretKey string + Region string + BucketPrefix string + UseSSL bool + SkipVerifySSL bool +} + +// Default test configuration - should match test_config.json +var defaultConfig = &S3TestConfig{ + Endpoint: "http://localhost:8333", // Default SeaweedFS S3 port + AccessKey: "some_access_key1", + SecretKey: "some_secret_key1", + Region: "us-east-1", + BucketPrefix: "test-retention-", + UseSSL: false, + SkipVerifySSL: true, +} + +// getS3Client creates an AWS S3 client for testing +func getS3Client(t *testing.T) *s3.Client { + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(defaultConfig.Region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + defaultConfig.AccessKey, + defaultConfig.SecretKey, + "", + )), + config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc( + func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: defaultConfig.Endpoint, + SigningRegion: defaultConfig.Region, + HostnameImmutable: true, + }, nil + })), + ) + require.NoError(t, err) + + return s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = true // Important for SeaweedFS + }) +} + +// getNewBucketName generates a unique bucket name +func getNewBucketName() string { + timestamp := time.Now().UnixNano() + return fmt.Sprintf("%s%d", defaultConfig.BucketPrefix, timestamp) +} + +// createBucket creates a new bucket for testing +func createBucket(t *testing.T, client *s3.Client, bucketName string) { + _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) +} + +// deleteBucket deletes a bucket and all its contents +func deleteBucket(t *testing.T, client *s3.Client, bucketName string) { + // First, try to delete all objects and versions + err := deleteAllObjectVersions(t, client, bucketName) + if err != nil { + t.Logf("Warning: failed to delete all object versions in first attempt: %v", err) + // Try once more in case of transient errors + time.Sleep(500 * time.Millisecond) + err = deleteAllObjectVersions(t, client, bucketName) + if err != nil { + t.Logf("Warning: failed to delete all object versions in second attempt: %v", err) + } + } + + // Wait a bit for eventual consistency + time.Sleep(100 * time.Millisecond) + + // Try to delete the bucket multiple times in case of eventual consistency issues + for retries := 0; retries < 3; retries++ { + _, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + if err == nil { + t.Logf("Successfully deleted bucket %s", bucketName) + return + } + + t.Logf("Warning: failed to delete bucket %s (attempt %d): %v", bucketName, retries+1, err) + if retries < 2 { + time.Sleep(200 * time.Millisecond) + } + } +} + +// deleteAllObjectVersions deletes all object versions in a bucket +func deleteAllObjectVersions(t *testing.T, client *s3.Client, bucketName string) error { + // List all object versions + paginator := s3.NewListObjectVersionsPaginator(client, &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucketName), + }) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(context.TODO()) + if err != nil { + return err + } + + var objectsToDelete []types.ObjectIdentifier + + // Add versions - first try to remove retention/legal hold + for _, version := range page.Versions { + // Try to remove legal hold if present + _, err := client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: version.Key, + VersionId: version.VersionId, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOff, + }, + }) + if err != nil { + // Legal hold might not be set, ignore error + t.Logf("Note: could not remove legal hold for %s@%s: %v", *version.Key, *version.VersionId, err) + } + + objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{ + Key: version.Key, + VersionId: version.VersionId, + }) + } + + // Add delete markers + for _, deleteMarker := range page.DeleteMarkers { + objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{ + Key: deleteMarker.Key, + VersionId: deleteMarker.VersionId, + }) + } + + // Delete objects in batches with bypass governance retention + if len(objectsToDelete) > 0 { + _, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{ + Bucket: aws.String(bucketName), + BypassGovernanceRetention: true, + Delete: &types.Delete{ + Objects: objectsToDelete, + Quiet: true, + }, + }) + if err != nil { + t.Logf("Warning: batch delete failed, trying individual deletion: %v", err) + // Try individual deletion for each object + for _, obj := range objectsToDelete { + _, delErr := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: obj.Key, + VersionId: obj.VersionId, + BypassGovernanceRetention: true, + }) + if delErr != nil { + t.Logf("Warning: failed to delete object %s@%s: %v", *obj.Key, *obj.VersionId, delErr) + } + } + } + } + } + + return nil +} + +// enableVersioning enables versioning on a bucket +func enableVersioning(t *testing.T, client *s3.Client, bucketName string) { + _, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucketName), + VersioningConfiguration: &types.VersioningConfiguration{ + Status: types.BucketVersioningStatusEnabled, + }, + }) + require.NoError(t, err) +} + +// putObject puts an object into a bucket +func putObject(t *testing.T, client *s3.Client, bucketName, key, content string) *s3.PutObjectOutput { + resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: strings.NewReader(content), + }) + require.NoError(t, err) + return resp +} + +// cleanupAllTestBuckets cleans up any leftover test buckets +func cleanupAllTestBuckets(t *testing.T, client *s3.Client) { + // List all buckets + listResp, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{}) + if err != nil { + t.Logf("Warning: failed to list buckets for cleanup: %v", err) + return + } + + // Delete buckets that match our test prefix + for _, bucket := range listResp.Buckets { + if bucket.Name != nil && strings.HasPrefix(*bucket.Name, defaultConfig.BucketPrefix) { + t.Logf("Cleaning up leftover test bucket: %s", *bucket.Name) + deleteBucket(t, client, *bucket.Name) + } + } +} + +// TestBasicRetentionWorkflow tests the basic retention functionality +func TestBasicRetentionWorkflow(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + + // Enable versioning (required for retention) + enableVersioning(t, client, bucketName) + + // Create object + key := "test-object" + content := "test content for retention" + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Set retention with GOVERNANCE mode + retentionUntil := time.Now().Add(24 * time.Hour) + _, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + // Get retention and verify it was set correctly + retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) + assert.Equal(t, types.ObjectLockRetentionModeGovernance, retentionResp.Retention.Mode) + assert.WithinDuration(t, retentionUntil, *retentionResp.Retention.RetainUntilDate, time.Second) + + // Try to delete object without bypass - should fail + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.Error(t, err) + + // Delete object with bypass governance - should succeed + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + BypassGovernanceRetention: true, + }) + require.NoError(t, err) +} + +// TestRetentionModeCompliance tests COMPLIANCE mode retention +func TestRetentionModeCompliance(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create object + key := "compliance-test-object" + content := "compliance test content" + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Set retention with COMPLIANCE mode + retentionUntil := time.Now().Add(1 * time.Hour) + _, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + // Get retention and verify + retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) + assert.Equal(t, types.ObjectLockRetentionModeCompliance, retentionResp.Retention.Mode) + + // Try to delete object with bypass - should still fail (compliance mode) + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + BypassGovernanceRetention: true, + }) + require.Error(t, err) + + // Try to delete object without bypass - should also fail + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.Error(t, err) +} + +// TestLegalHoldWorkflow tests legal hold functionality +func TestLegalHoldWorkflow(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create object + key := "legal-hold-test-object" + content := "legal hold test content" + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Set legal hold ON + _, err := client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + require.NoError(t, err) + + // Get legal hold and verify + legalHoldResp, err := client.GetObjectLegalHold(context.TODO(), &s3.GetObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) + assert.Equal(t, types.ObjectLockLegalHoldStatusOn, legalHoldResp.LegalHold.Status) + + // Try to delete object - should fail due to legal hold + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.Error(t, err) + + // Remove legal hold + _, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOff, + }, + }) + require.NoError(t, err) + + // Verify legal hold is off + legalHoldResp, err = client.GetObjectLegalHold(context.TODO(), &s3.GetObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) + assert.Equal(t, types.ObjectLockLegalHoldStatusOff, legalHoldResp.LegalHold.Status) + + // Now delete should succeed + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) +} + +// TestObjectLockConfiguration tests bucket object lock configuration +func TestObjectLockConfiguration(t *testing.T) { + client := getS3Client(t) + // Use a more unique bucket name to avoid conflicts + bucketName := fmt.Sprintf("object-lock-config-%d-%d", time.Now().UnixNano(), time.Now().UnixMilli()%10000) + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Set object lock configuration + _, err := client.PutObjectLockConfiguration(context.TODO(), &s3.PutObjectLockConfigurationInput{ + Bucket: aws.String(bucketName), + ObjectLockConfiguration: &types.ObjectLockConfiguration{ + ObjectLockEnabled: types.ObjectLockEnabledEnabled, + Rule: &types.ObjectLockRule{ + DefaultRetention: &types.DefaultRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + Days: 30, + }, + }, + }, + }) + if err != nil { + t.Logf("PutObjectLockConfiguration failed (may not be supported): %v", err) + t.Skip("Object lock configuration not supported, skipping test") + return + } + + // Get object lock configuration and verify + configResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + assert.Equal(t, types.ObjectLockEnabledEnabled, configResp.ObjectLockConfiguration.ObjectLockEnabled) + assert.Equal(t, types.ObjectLockRetentionModeGovernance, configResp.ObjectLockConfiguration.Rule.DefaultRetention.Mode) + assert.Equal(t, int32(30), configResp.ObjectLockConfiguration.Rule.DefaultRetention.Days) +} + +// TestRetentionWithVersions tests retention with specific object versions +func TestRetentionWithVersions(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create multiple versions of the same object + key := "versioned-retention-test" + content1 := "version 1 content" + content2 := "version 2 content" + + putResp1 := putObject(t, client, bucketName, key, content1) + require.NotNil(t, putResp1.VersionId) + + putResp2 := putObject(t, client, bucketName, key, content2) + require.NotNil(t, putResp2.VersionId) + + // Set retention on first version only + retentionUntil := time.Now().Add(1 * time.Hour) + _, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: putResp1.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + // Get retention for first version + retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: putResp1.VersionId, + }) + require.NoError(t, err) + assert.Equal(t, types.ObjectLockRetentionModeGovernance, retentionResp.Retention.Mode) + + // Try to get retention for second version - should fail (no retention set) + _, err = client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: putResp2.VersionId, + }) + require.Error(t, err) + + // Delete second version should succeed (no retention) + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: putResp2.VersionId, + }) + require.NoError(t, err) + + // Delete first version should fail (has retention) + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: putResp1.VersionId, + }) + require.Error(t, err) + + // Delete first version with bypass should succeed + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: putResp1.VersionId, + BypassGovernanceRetention: true, + }) + require.NoError(t, err) +} + +// TestRetentionAndLegalHoldCombination tests retention and legal hold together +func TestRetentionAndLegalHoldCombination(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create object + key := "combined-protection-test" + content := "combined protection test content" + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Set both retention and legal hold + retentionUntil := time.Now().Add(1 * time.Hour) + + // Set retention + _, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + // Set legal hold + _, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + require.NoError(t, err) + + // Try to delete with bypass governance - should still fail due to legal hold + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + BypassGovernanceRetention: true, + }) + require.Error(t, err) + + // Remove legal hold + _, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOff, + }, + }) + require.NoError(t, err) + + // Now delete with bypass governance should succeed + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + BypassGovernanceRetention: true, + }) + require.NoError(t, err) +} + +// TestExpiredRetention tests that objects can be deleted after retention expires +func TestExpiredRetention(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create object + key := "expired-retention-test" + content := "expired retention test content" + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Set retention for a very short time (2 seconds) + retentionUntil := time.Now().Add(2 * time.Second) + _, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + // Try to delete immediately - should fail + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.Error(t, err) + + // Wait for retention to expire + time.Sleep(3 * time.Second) + + // Now delete should succeed + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) +} + +// TestRetentionErrorCases tests various error conditions +func TestRetentionErrorCases(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Test setting retention on non-existent object + _, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String("non-existent-key"), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(time.Now().Add(1 * time.Hour)), + }, + }) + require.Error(t, err) + + // Test getting retention on non-existent object + _, err = client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String("non-existent-key"), + }) + require.Error(t, err) + + // Test setting legal hold on non-existent object + _, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String("non-existent-key"), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + require.Error(t, err) + + // Test getting legal hold on non-existent object + _, err = client.GetObjectLegalHold(context.TODO(), &s3.GetObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String("non-existent-key"), + }) + require.Error(t, err) + + // Test setting retention with past date + key := "retention-past-date-test" + content := "test content" + putObject(t, client, bucketName, key, content) + + pastDate := time.Now().Add(-1 * time.Hour) + _, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(pastDate), + }, + }) + require.Error(t, err) +} diff --git a/test/s3/retention/s3_worm_integration_test.go b/test/s3/retention/s3_worm_integration_test.go new file mode 100644 index 000000000..31217001f --- /dev/null +++ b/test/s3/retention/s3_worm_integration_test.go @@ -0,0 +1,519 @@ +package s3api + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestWORMRetentionIntegration tests that both retention and legacy WORM work together +func TestWORMRetentionIntegration(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create object + key := "worm-retention-integration-test" + content := "worm retention integration test content" + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Set retention (new system) + retentionUntil := time.Now().Add(1 * time.Hour) + _, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + // Try to delete - should fail due to retention + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.Error(t, err) + + // Delete with bypass should succeed + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + BypassGovernanceRetention: true, + }) + require.NoError(t, err) +} + +// TestWORMLegacyCompatibility tests that legacy WORM functionality still works +func TestWORMLegacyCompatibility(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create object with legacy WORM headers (if supported) + key := "legacy-worm-test" + content := "legacy worm test content" + + // Try to create object with legacy WORM TTL header + putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: strings.NewReader(content), + // Add legacy WORM headers if supported + Metadata: map[string]string{ + "x-amz-meta-worm-ttl": fmt.Sprintf("%d", time.Now().Add(1*time.Hour).Unix()), + }, + }) + require.NoError(t, err) + require.NotNil(t, putResp.VersionId) + + // Object should be created successfully + resp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) + assert.NotNil(t, resp.Metadata) +} + +// TestRetentionOverwriteProtection tests that retention prevents overwrites +func TestRetentionOverwriteProtection(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create object + key := "overwrite-protection-test" + content := "original content" + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Verify object exists before setting retention + _, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err, "Object should exist before setting retention") + + // Set retention with specific version ID + retentionUntil := time.Now().Add(1 * time.Hour) + _, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: putResp.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + // Try to overwrite object - should fail in non-versioned bucket context + content2 := "new content" + _, err = client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: strings.NewReader(content2), + }) + // Note: In a real scenario, this might fail or create a new version + // The actual behavior depends on the implementation + if err != nil { + t.Logf("Expected behavior: overwrite blocked due to retention: %v", err) + } else { + t.Logf("Overwrite allowed, likely created new version") + } +} + +// TestRetentionBulkOperations tests retention with bulk operations +func TestRetentionBulkOperations(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create multiple objects with retention + var objectsToDelete []types.ObjectIdentifier + retentionUntil := time.Now().Add(1 * time.Hour) + + for i := 0; i < 3; i++ { + key := fmt.Sprintf("bulk-test-object-%d", i) + content := fmt.Sprintf("bulk test content %d", i) + + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Set retention on each object with version ID + _, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: putResp.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{ + Key: aws.String(key), + VersionId: putResp.VersionId, + }) + } + + // Try bulk delete without bypass - should fail or have errors + deleteResp, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{ + Bucket: aws.String(bucketName), + Delete: &types.Delete{ + Objects: objectsToDelete, + Quiet: false, + }, + }) + + // Check if operation failed or returned errors for protected objects + if err != nil { + t.Logf("Expected: bulk delete failed due to retention: %v", err) + } else if deleteResp != nil && len(deleteResp.Errors) > 0 { + t.Logf("Expected: bulk delete returned %d errors due to retention", len(deleteResp.Errors)) + for _, delErr := range deleteResp.Errors { + t.Logf("Delete error: %s - %s", *delErr.Code, *delErr.Message) + } + } else { + t.Logf("Warning: bulk delete succeeded - retention may not be enforced for bulk operations") + } + + // Try bulk delete with bypass - should succeed + _, err = client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{ + Bucket: aws.String(bucketName), + BypassGovernanceRetention: true, + Delete: &types.Delete{ + Objects: objectsToDelete, + Quiet: false, + }, + }) + if err != nil { + t.Logf("Bulk delete with bypass failed (may not be supported): %v", err) + } else { + t.Logf("Bulk delete with bypass succeeded") + } +} + +// TestRetentionWithMultipartUpload tests retention with multipart uploads +func TestRetentionWithMultipartUpload(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Start multipart upload + key := "multipart-retention-test" + createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) + uploadId := createResp.UploadId + + // Upload a part + partContent := "This is a test part for multipart upload" + uploadResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + PartNumber: 1, + UploadId: uploadId, + Body: strings.NewReader(partContent), + }) + require.NoError(t, err) + + // Complete multipart upload + completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + UploadId: uploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + { + ETag: uploadResp.ETag, + PartNumber: 1, + }, + }, + }, + }) + require.NoError(t, err) + + // Add a small delay to ensure the object is fully created + time.Sleep(500 * time.Millisecond) + + // Verify object exists after multipart upload - retry if needed + var headErr error + for retries := 0; retries < 10; retries++ { + _, headErr = client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + if headErr == nil { + break + } + t.Logf("HeadObject attempt %d failed: %v", retries+1, headErr) + time.Sleep(200 * time.Millisecond) + } + + if headErr != nil { + t.Logf("Object not found after multipart upload completion, checking if multipart upload is fully supported") + // Check if the object exists by trying to list it + listResp, listErr := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketName), + Prefix: aws.String(key), + }) + if listErr != nil || len(listResp.Contents) == 0 { + t.Skip("Multipart upload may not be fully supported, skipping test") + return + } + // If object exists in listing but not accessible via HeadObject, skip test + t.Skip("Object exists in listing but not accessible via HeadObject, multipart upload may not be fully supported") + return + } + + require.NoError(t, headErr, "Object should exist after multipart upload") + + // Set retention on the completed multipart object with version ID + retentionUntil := time.Now().Add(1 * time.Hour) + _, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: completeResp.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + // Try to delete - should fail + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.Error(t, err) +} + +// TestRetentionExtendedAttributes tests that retention uses extended attributes correctly +func TestRetentionExtendedAttributes(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create object + key := "extended-attrs-test" + content := "extended attributes test content" + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Set retention + retentionUntil := time.Now().Add(1 * time.Hour) + _, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: putResp.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + // Set legal hold + _, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + VersionId: putResp.VersionId, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + require.NoError(t, err) + + // Get object metadata to verify extended attributes are set + resp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) + + // Check that the object has metadata (may be empty in some implementations) + // Note: The actual metadata keys depend on the implementation + if resp.Metadata != nil && len(resp.Metadata) > 0 { + t.Logf("Object metadata: %+v", resp.Metadata) + } else { + t.Logf("Object metadata: empty (extended attributes may be stored internally)") + } + + // Verify retention can be retrieved + retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) + assert.Equal(t, types.ObjectLockRetentionModeGovernance, retentionResp.Retention.Mode) + + // Verify legal hold can be retrieved + legalHoldResp, err := client.GetObjectLegalHold(context.TODO(), &s3.GetObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err) + assert.Equal(t, types.ObjectLockLegalHoldStatusOn, legalHoldResp.LegalHold.Status) +} + +// TestRetentionBucketDefaults tests object lock configuration defaults +func TestRetentionBucketDefaults(t *testing.T) { + client := getS3Client(t) + // Use a very unique bucket name to avoid conflicts + bucketName := fmt.Sprintf("bucket-defaults-%d-%d", time.Now().UnixNano(), time.Now().UnixMilli()%10000) + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Set bucket object lock configuration with default retention + _, err := client.PutObjectLockConfiguration(context.TODO(), &s3.PutObjectLockConfigurationInput{ + Bucket: aws.String(bucketName), + ObjectLockConfiguration: &types.ObjectLockConfiguration{ + ObjectLockEnabled: types.ObjectLockEnabledEnabled, + Rule: &types.ObjectLockRule{ + DefaultRetention: &types.DefaultRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + Days: 1, // 1 day default + }, + }, + }, + }) + if err != nil { + t.Logf("PutObjectLockConfiguration failed (may not be supported): %v", err) + t.Skip("Object lock configuration not supported, skipping test") + return + } + + // Create object (should inherit default retention) + key := "bucket-defaults-test" + content := "bucket defaults test content" + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Check if object has default retention applied + // Note: This depends on the implementation - some S3 services apply + // default retention automatically, others require explicit setting + retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + if err != nil { + t.Logf("No automatic default retention applied: %v", err) + } else { + t.Logf("Default retention applied: %+v", retentionResp.Retention) + assert.Equal(t, types.ObjectLockRetentionModeGovernance, retentionResp.Retention.Mode) + } +} + +// TestRetentionConcurrentOperations tests concurrent retention operations +func TestRetentionConcurrentOperations(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + + // Create bucket and enable versioning + createBucket(t, client, bucketName) + defer deleteBucket(t, client, bucketName) + enableVersioning(t, client, bucketName) + + // Create object + key := "concurrent-ops-test" + content := "concurrent operations test content" + putResp := putObject(t, client, bucketName, key, content) + require.NotNil(t, putResp.VersionId) + + // Test concurrent retention and legal hold operations + retentionUntil := time.Now().Add(1 * time.Hour) + + // Set retention and legal hold concurrently + errChan := make(chan error, 2) + + go func() { + _, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + errChan <- err + }() + + go func() { + _, err := client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + errChan <- err + }() + + // Wait for both operations to complete + for i := 0; i < 2; i++ { + err := <-errChan + if err != nil { + t.Logf("Concurrent operation failed: %v", err) + } + } + + // Verify both settings are applied + retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + if err == nil { + assert.Equal(t, types.ObjectLockRetentionModeGovernance, retentionResp.Retention.Mode) + } + + legalHoldResp, err := client.GetObjectLegalHold(context.TODO(), &s3.GetObjectLegalHoldInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + if err == nil { + assert.Equal(t, types.ObjectLockLegalHoldStatusOn, legalHoldResp.LegalHold.Status) + } +} diff --git a/test/s3/retention/test_config.json b/test/s3/retention/test_config.json new file mode 100644 index 000000000..b3281778b --- /dev/null +++ b/test/s3/retention/test_config.json @@ -0,0 +1,9 @@ +{ + "endpoint": "http://localhost:8333", + "access_key": "some_access_key1", + "secret_key": "some_secret_key1", + "region": "us-east-1", + "bucket_prefix": "test-retention-", + "use_ssl": false, + "skip_verify_ssl": true +}
\ No newline at end of file |