diff options
Diffstat (limited to 'test/s3/iam/s3_iam_distributed_test.go')
| -rw-r--r-- | test/s3/iam/s3_iam_distributed_test.go | 426 |
1 files changed, 426 insertions, 0 deletions
diff --git a/test/s3/iam/s3_iam_distributed_test.go b/test/s3/iam/s3_iam_distributed_test.go new file mode 100644 index 000000000..545a56bcb --- /dev/null +++ b/test/s3/iam/s3_iam_distributed_test.go @@ -0,0 +1,426 @@ +package iam + +import ( + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestS3IAMDistributedTests tests IAM functionality across multiple S3 gateway instances +func TestS3IAMDistributedTests(t *testing.T) { + // Skip if not in distributed test mode + if os.Getenv("ENABLE_DISTRIBUTED_TESTS") != "true" { + t.Skip("Distributed tests not enabled. Set ENABLE_DISTRIBUTED_TESTS=true") + } + + framework := NewS3IAMTestFramework(t) + defer framework.Cleanup() + + t.Run("distributed_session_consistency", func(t *testing.T) { + // Test that sessions created on one instance are visible on others + // This requires filer-based session storage + + // Create S3 clients that would connect to different gateway instances + // In a real distributed setup, these would point to different S3 gateway ports + client1, err := framework.CreateS3ClientWithJWT("test-user", "TestAdminRole") + require.NoError(t, err) + + client2, err := framework.CreateS3ClientWithJWT("test-user", "TestAdminRole") + require.NoError(t, err) + + // Both clients should be able to perform operations + bucketName := "test-distributed-session" + + err = framework.CreateBucket(client1, bucketName) + require.NoError(t, err) + + // Client2 should see the bucket created by client1 + listResult, err := client2.ListBuckets(&s3.ListBucketsInput{}) + require.NoError(t, err) + + found := false + for _, bucket := range listResult.Buckets { + if *bucket.Name == bucketName { + found = true + break + } + } + assert.True(t, found, "Bucket should be visible across distributed instances") + + // Cleanup + _, err = client1.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + }) + + t.Run("distributed_role_consistency", func(t *testing.T) { + // Test that role definitions are consistent across instances + // This requires filer-based role storage + + // Create clients with different roles + adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole") + require.NoError(t, err) + + readOnlyClient, err := framework.CreateS3ClientWithJWT("readonly-user", "TestReadOnlyRole") + require.NoError(t, err) + + bucketName := "test-distributed-roles" + objectKey := "test-object.txt" + + // Admin should be able to create bucket + err = framework.CreateBucket(adminClient, bucketName) + require.NoError(t, err) + + // Admin should be able to put object + err = framework.PutTestObject(adminClient, bucketName, objectKey, "test content") + require.NoError(t, err) + + // Read-only user should be able to get object + content, err := framework.GetTestObject(readOnlyClient, bucketName, objectKey) + require.NoError(t, err) + assert.Equal(t, "test content", content) + + // Read-only user should NOT be able to put object + err = framework.PutTestObject(readOnlyClient, bucketName, "forbidden-object.txt", "forbidden content") + require.Error(t, err, "Read-only user should not be able to put objects") + + // Cleanup + err = framework.DeleteTestObject(adminClient, bucketName, objectKey) + require.NoError(t, err) + _, err = adminClient.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + }) + + t.Run("distributed_concurrent_operations", func(t *testing.T) { + // Test concurrent operations across distributed instances with robust retry mechanisms + // This approach implements proper retry logic instead of tolerating errors to catch real concurrency issues + const numGoroutines = 3 // Reduced concurrency for better CI reliability + const numOperationsPerGoroutine = 2 // Minimal operations per goroutine + const maxRetries = 3 // Maximum retry attempts for transient failures + const retryDelay = 200 * time.Millisecond // Increased delay for better stability + + var wg sync.WaitGroup + errors := make(chan error, numGoroutines*numOperationsPerGoroutine) + + // Helper function to determine if an error is retryable + isRetryableError := func(err error) bool { + if err == nil { + return false + } + errorMsg := err.Error() + return strings.Contains(errorMsg, "timeout") || + strings.Contains(errorMsg, "connection reset") || + strings.Contains(errorMsg, "temporary failure") || + strings.Contains(errorMsg, "TooManyRequests") || + strings.Contains(errorMsg, "ServiceUnavailable") || + strings.Contains(errorMsg, "InternalError") + } + + // Helper function to execute operations with retry logic + executeWithRetry := func(operation func() error, operationName string) error { + var lastErr error + for attempt := 0; attempt <= maxRetries; attempt++ { + if attempt > 0 { + time.Sleep(retryDelay * time.Duration(attempt)) // Linear backoff + } + + lastErr = operation() + if lastErr == nil { + return nil // Success + } + + if !isRetryableError(lastErr) { + // Non-retryable error - fail immediately + return fmt.Errorf("%s failed with non-retryable error: %w", operationName, lastErr) + } + + // Retryable error - continue to next attempt + if attempt < maxRetries { + t.Logf("Retrying %s (attempt %d/%d) after error: %v", operationName, attempt+1, maxRetries, lastErr) + } + } + + // All retries exhausted + return fmt.Errorf("%s failed after %d retries, last error: %w", operationName, maxRetries, lastErr) + } + + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func(goroutineID int) { + defer wg.Done() + + client, err := framework.CreateS3ClientWithJWT(fmt.Sprintf("user-%d", goroutineID), "TestAdminRole") + if err != nil { + errors <- fmt.Errorf("failed to create S3 client for goroutine %d: %w", goroutineID, err) + return + } + + for j := 0; j < numOperationsPerGoroutine; j++ { + bucketName := fmt.Sprintf("test-concurrent-%d-%d", goroutineID, j) + objectKey := "test-object.txt" + objectContent := fmt.Sprintf("content-%d-%d", goroutineID, j) + + // Execute full operation sequence with individual retries + operationFailed := false + + // 1. Create bucket with retry + if err := executeWithRetry(func() error { + return framework.CreateBucket(client, bucketName) + }, fmt.Sprintf("CreateBucket-%s", bucketName)); err != nil { + errors <- err + operationFailed = true + } + + if !operationFailed { + // 2. Put object with retry + if err := executeWithRetry(func() error { + return framework.PutTestObject(client, bucketName, objectKey, objectContent) + }, fmt.Sprintf("PutObject-%s/%s", bucketName, objectKey)); err != nil { + errors <- err + operationFailed = true + } + } + + if !operationFailed { + // 3. Get object with retry + if err := executeWithRetry(func() error { + _, err := framework.GetTestObject(client, bucketName, objectKey) + return err + }, fmt.Sprintf("GetObject-%s/%s", bucketName, objectKey)); err != nil { + errors <- err + operationFailed = true + } + } + + if !operationFailed { + // 4. Delete object with retry + if err := executeWithRetry(func() error { + return framework.DeleteTestObject(client, bucketName, objectKey) + }, fmt.Sprintf("DeleteObject-%s/%s", bucketName, objectKey)); err != nil { + errors <- err + operationFailed = true + } + } + + // 5. Always attempt bucket cleanup, even if previous operations failed + if err := executeWithRetry(func() error { + _, err := client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + return err + }, fmt.Sprintf("DeleteBucket-%s", bucketName)); err != nil { + // Only log cleanup failures, don't fail the test + t.Logf("Warning: Failed to cleanup bucket %s: %v", bucketName, err) + } + + // Increased delay between operation sequences to reduce server load and improve stability + time.Sleep(100 * time.Millisecond) + } + }(i) + } + + wg.Wait() + close(errors) + + // Collect and analyze errors - with retry logic, we should see very few errors + var errorList []error + for err := range errors { + errorList = append(errorList, err) + } + + totalOperations := numGoroutines * numOperationsPerGoroutine + + // Report results + if len(errorList) == 0 { + t.Logf("🎉 All %d concurrent operations completed successfully with retry mechanisms!", totalOperations) + } else { + t.Logf("Concurrent operations summary:") + t.Logf(" Total operations: %d", totalOperations) + t.Logf(" Failed operations: %d (%.1f%% error rate)", len(errorList), float64(len(errorList))/float64(totalOperations)*100) + + // Log first few errors for debugging + for i, err := range errorList { + if i >= 3 { // Limit to first 3 errors + t.Logf(" ... and %d more errors", len(errorList)-3) + break + } + t.Logf(" Error %d: %v", i+1, err) + } + } + + // With proper retry mechanisms, we should expect near-zero failures + // Any remaining errors likely indicate real concurrency issues or system problems + if len(errorList) > 0 { + t.Errorf("❌ %d operation(s) failed even after retry mechanisms (%.1f%% failure rate). This indicates potential system issues or race conditions that need investigation.", + len(errorList), float64(len(errorList))/float64(totalOperations)*100) + } + }) +} + +// TestS3IAMPerformanceTests tests IAM performance characteristics +func TestS3IAMPerformanceTests(t *testing.T) { + // Skip if not in performance test mode + if os.Getenv("ENABLE_PERFORMANCE_TESTS") != "true" { + t.Skip("Performance tests not enabled. Set ENABLE_PERFORMANCE_TESTS=true") + } + + framework := NewS3IAMTestFramework(t) + defer framework.Cleanup() + + t.Run("authentication_performance", func(t *testing.T) { + // Test authentication performance + const numRequests = 100 + + client, err := framework.CreateS3ClientWithJWT("perf-user", "TestAdminRole") + require.NoError(t, err) + + bucketName := "test-auth-performance" + err = framework.CreateBucket(client, bucketName) + require.NoError(t, err) + defer func() { + _, err := client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + }() + + start := time.Now() + + for i := 0; i < numRequests; i++ { + _, err := client.ListBuckets(&s3.ListBucketsInput{}) + require.NoError(t, err) + } + + duration := time.Since(start) + avgLatency := duration / numRequests + + t.Logf("Authentication performance: %d requests in %v (avg: %v per request)", + numRequests, duration, avgLatency) + + // Performance assertion - should be under 100ms per request on average + assert.Less(t, avgLatency, 100*time.Millisecond, + "Average authentication latency should be under 100ms") + }) + + t.Run("authorization_performance", func(t *testing.T) { + // Test authorization performance with different policy complexities + const numRequests = 50 + + client, err := framework.CreateS3ClientWithJWT("perf-user", "TestAdminRole") + require.NoError(t, err) + + bucketName := "test-authz-performance" + err = framework.CreateBucket(client, bucketName) + require.NoError(t, err) + defer func() { + _, err := client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + }() + + start := time.Now() + + for i := 0; i < numRequests; i++ { + objectKey := fmt.Sprintf("perf-object-%d.txt", i) + err := framework.PutTestObject(client, bucketName, objectKey, "performance test content") + require.NoError(t, err) + + _, err = framework.GetTestObject(client, bucketName, objectKey) + require.NoError(t, err) + + err = framework.DeleteTestObject(client, bucketName, objectKey) + require.NoError(t, err) + } + + duration := time.Since(start) + avgLatency := duration / (numRequests * 3) // 3 operations per iteration + + t.Logf("Authorization performance: %d operations in %v (avg: %v per operation)", + numRequests*3, duration, avgLatency) + + // Performance assertion - should be under 50ms per operation on average + assert.Less(t, avgLatency, 50*time.Millisecond, + "Average authorization latency should be under 50ms") + }) +} + +// BenchmarkS3IAMAuthentication benchmarks JWT authentication +func BenchmarkS3IAMAuthentication(b *testing.B) { + if os.Getenv("ENABLE_PERFORMANCE_TESTS") != "true" { + b.Skip("Performance tests not enabled. Set ENABLE_PERFORMANCE_TESTS=true") + } + + framework := NewS3IAMTestFramework(&testing.T{}) + defer framework.Cleanup() + + client, err := framework.CreateS3ClientWithJWT("bench-user", "TestAdminRole") + require.NoError(b, err) + + bucketName := "test-bench-auth" + err = framework.CreateBucket(client, bucketName) + require.NoError(b, err) + defer func() { + _, err := client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(b, err) + }() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := client.ListBuckets(&s3.ListBucketsInput{}) + if err != nil { + b.Error(err) + } + } + }) +} + +// BenchmarkS3IAMAuthorization benchmarks policy evaluation +func BenchmarkS3IAMAuthorization(b *testing.B) { + if os.Getenv("ENABLE_PERFORMANCE_TESTS") != "true" { + b.Skip("Performance tests not enabled. Set ENABLE_PERFORMANCE_TESTS=true") + } + + framework := NewS3IAMTestFramework(&testing.T{}) + defer framework.Cleanup() + + client, err := framework.CreateS3ClientWithJWT("bench-user", "TestAdminRole") + require.NoError(b, err) + + bucketName := "test-bench-authz" + err = framework.CreateBucket(client, bucketName) + require.NoError(b, err) + defer func() { + _, err := client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(b, err) + }() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + objectKey := fmt.Sprintf("bench-object-%d.txt", i) + err := framework.PutTestObject(client, bucketName, objectKey, "benchmark content") + if err != nil { + b.Error(err) + } + i++ + } + }) +} |