diff options
| -rw-r--r-- | weed/s3api/s3_constants/header.go | 10 | ||||
| -rw-r--r-- | weed/s3api/s3_sse_c_range_test.go | 3 | ||||
| -rw-r--r-- | weed/s3api/s3_sse_copy_test.go | 2 | ||||
| -rw-r--r-- | weed/s3api/s3_sse_metadata.go | 152 | ||||
| -rw-r--r-- | weed/s3api/s3_sse_s3.go | 25 | ||||
| -rw-r--r-- | weed/s3api/s3_sse_s3_integration_test.go | 325 | ||||
| -rw-r--r-- | weed/s3api/s3_sse_s3_test.go | 984 | ||||
| -rw-r--r-- | weed/s3api/s3_sse_test_utils_test.go | 2 | ||||
| -rw-r--r-- | weed/s3api/s3_validation_utils.go | 27 | ||||
| -rw-r--r-- | weed/s3api/s3api_key_rotation.go | 6 | ||||
| -rw-r--r-- | weed/s3api/s3api_object_handlers.go | 407 | ||||
| -rw-r--r-- | weed/s3api/s3api_object_handlers_copy.go | 8 | ||||
| -rw-r--r-- | weed/s3api/s3api_streaming_copy.go | 10 | ||||
| -rw-r--r-- | weed/s3api/s3err/s3api_errors.go | 6 | ||||
| -rw-r--r-- | weed/server/filer_server_handlers_read.go | 9 | ||||
| -rw-r--r-- | weed/server/filer_server_handlers_write_autochunk.go | 10 | ||||
| -rw-r--r-- | weed/topology/volume_growth_reservation_test.go | 32 |
17 files changed, 1762 insertions, 256 deletions
diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index 86863f257..82a270111 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -94,6 +94,9 @@ const ( AmzEncryptedDataKey = "x-amz-encrypted-data-key" AmzEncryptionContextMeta = "x-amz-encryption-context" + // SeaweedFS internal metadata prefix (used to filter internal headers from client responses) + SeaweedFSInternalPrefix = "x-seaweedfs-" + // SeaweedFS internal metadata keys for encryption (prefixed to avoid automatic HTTP header conversion) SeaweedFSSSEKMSKey = "x-seaweedfs-sse-kms-key" // Key for storing serialized SSE-KMS metadata SeaweedFSSSES3Key = "x-seaweedfs-sse-s3-key" // Key for storing serialized SSE-S3 metadata @@ -157,3 +160,10 @@ var PassThroughHeaders = map[string]string{ "response-content-type": "Content-Type", "response-expires": "Expires", } + +// IsSeaweedFSInternalHeader checks if a header key is a SeaweedFS internal header +// that should be filtered from client responses. +// Header names are case-insensitive in HTTP, so this function normalizes to lowercase. +func IsSeaweedFSInternalHeader(headerKey string) bool { + return strings.HasPrefix(strings.ToLower(headerKey), SeaweedFSInternalPrefix) +} diff --git a/weed/s3api/s3_sse_c_range_test.go b/weed/s3api/s3_sse_c_range_test.go index 318771d8c..b704c39af 100644 --- a/weed/s3api/s3_sse_c_range_test.go +++ b/weed/s3api/s3_sse_c_range_test.go @@ -56,7 +56,8 @@ func TestSSECRangeRequestsSupported(t *testing.T) { } rec := httptest.NewRecorder() w := recorderFlusher{rec} - statusCode, _ := s3a.handleSSECResponse(req, proxyResponse, w) + // Pass nil for entry since this test focuses on Range request handling + statusCode, _ := s3a.handleSSECResponse(req, proxyResponse, w, nil) // Range requests should now be allowed to proceed (will be handled by filer layer) // The exact status code depends on the object existence and filer response diff --git a/weed/s3api/s3_sse_copy_test.go b/weed/s3api/s3_sse_copy_test.go index 35839a704..b377b45a9 100644 --- a/weed/s3api/s3_sse_copy_test.go +++ b/weed/s3api/s3_sse_copy_test.go @@ -43,7 +43,7 @@ func TestSSECObjectCopy(t *testing.T) { // Test copy strategy determination sourceMetadata := make(map[string][]byte) - StoreIVInMetadata(sourceMetadata, iv) + StoreSSECIVInMetadata(sourceMetadata, iv) sourceMetadata[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256") sourceMetadata[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(sourceKey.KeyMD5) diff --git a/weed/s3api/s3_sse_metadata.go b/weed/s3api/s3_sse_metadata.go index 8b641f150..7cb695251 100644 --- a/weed/s3api/s3_sse_metadata.go +++ b/weed/s3api/s3_sse_metadata.go @@ -2,158 +2,28 @@ package s3api import ( "encoding/base64" - "encoding/json" "fmt" -) - -// SSE metadata keys for storing encryption information in entry metadata -const ( - // MetaSSEIV is the initialization vector used for encryption - MetaSSEIV = "X-SeaweedFS-Server-Side-Encryption-Iv" - - // MetaSSEAlgorithm is the encryption algorithm used - MetaSSEAlgorithm = "X-SeaweedFS-Server-Side-Encryption-Algorithm" - - // MetaSSECKeyMD5 is the MD5 hash of the SSE-C customer key - MetaSSECKeyMD5 = "X-SeaweedFS-Server-Side-Encryption-Customer-Key-MD5" - - // MetaSSEKMSKeyID is the KMS key ID used for encryption - MetaSSEKMSKeyID = "X-SeaweedFS-Server-Side-Encryption-KMS-Key-Id" - - // MetaSSEKMSEncryptedKey is the encrypted data key from KMS - MetaSSEKMSEncryptedKey = "X-SeaweedFS-Server-Side-Encryption-KMS-Encrypted-Key" - - // MetaSSEKMSContext is the encryption context for KMS - MetaSSEKMSContext = "X-SeaweedFS-Server-Side-Encryption-KMS-Context" - // MetaSSES3KeyID is the key ID for SSE-S3 encryption - MetaSSES3KeyID = "X-SeaweedFS-Server-Side-Encryption-S3-Key-Id" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" ) -// StoreIVInMetadata stores the IV in entry metadata as base64 encoded string -func StoreIVInMetadata(metadata map[string][]byte, iv []byte) { +// StoreSSECIVInMetadata stores the SSE-C IV in entry metadata as base64 encoded string +// Used by SSE-C for storing IV in entry.Extended +func StoreSSECIVInMetadata(metadata map[string][]byte, iv []byte) { if len(iv) > 0 { - metadata[MetaSSEIV] = []byte(base64.StdEncoding.EncodeToString(iv)) + metadata[s3_constants.SeaweedFSSSEIV] = []byte(base64.StdEncoding.EncodeToString(iv)) } } -// GetIVFromMetadata retrieves the IV from entry metadata -func GetIVFromMetadata(metadata map[string][]byte) ([]byte, error) { - if ivBase64, exists := metadata[MetaSSEIV]; exists { +// GetSSECIVFromMetadata retrieves the SSE-C IV from entry metadata +// Used by SSE-C for retrieving IV from entry.Extended +func GetSSECIVFromMetadata(metadata map[string][]byte) ([]byte, error) { + if ivBase64, exists := metadata[s3_constants.SeaweedFSSSEIV]; exists { iv, err := base64.StdEncoding.DecodeString(string(ivBase64)) if err != nil { - return nil, fmt.Errorf("failed to decode IV from metadata: %w", err) + return nil, fmt.Errorf("failed to decode SSE-C IV from metadata: %w", err) } return iv, nil } - return nil, fmt.Errorf("IV not found in metadata") -} - -// StoreSSECMetadata stores SSE-C related metadata -func StoreSSECMetadata(metadata map[string][]byte, iv []byte, keyMD5 string) { - StoreIVInMetadata(metadata, iv) - metadata[MetaSSEAlgorithm] = []byte("AES256") - if keyMD5 != "" { - metadata[MetaSSECKeyMD5] = []byte(keyMD5) - } -} - -// StoreSSEKMSMetadata stores SSE-KMS related metadata -func StoreSSEKMSMetadata(metadata map[string][]byte, iv []byte, keyID string, encryptedKey []byte, context map[string]string) { - StoreIVInMetadata(metadata, iv) - metadata[MetaSSEAlgorithm] = []byte("aws:kms") - if keyID != "" { - metadata[MetaSSEKMSKeyID] = []byte(keyID) - } - if len(encryptedKey) > 0 { - metadata[MetaSSEKMSEncryptedKey] = []byte(base64.StdEncoding.EncodeToString(encryptedKey)) - } - if len(context) > 0 { - // Marshal context to JSON to handle special characters correctly - contextBytes, err := json.Marshal(context) - if err == nil { - metadata[MetaSSEKMSContext] = contextBytes - } - // Note: json.Marshal for map[string]string should never fail, but we handle it gracefully - } -} - -// StoreSSES3Metadata stores SSE-S3 related metadata -func StoreSSES3Metadata(metadata map[string][]byte, iv []byte, keyID string) { - StoreIVInMetadata(metadata, iv) - metadata[MetaSSEAlgorithm] = []byte("AES256") - if keyID != "" { - metadata[MetaSSES3KeyID] = []byte(keyID) - } -} - -// GetSSECMetadata retrieves SSE-C metadata -func GetSSECMetadata(metadata map[string][]byte) (iv []byte, keyMD5 string, err error) { - iv, err = GetIVFromMetadata(metadata) - if err != nil { - return nil, "", err - } - - if keyMD5Bytes, exists := metadata[MetaSSECKeyMD5]; exists { - keyMD5 = string(keyMD5Bytes) - } - - return iv, keyMD5, nil -} - -// GetSSEKMSMetadata retrieves SSE-KMS metadata -func GetSSEKMSMetadata(metadata map[string][]byte) (iv []byte, keyID string, encryptedKey []byte, context map[string]string, err error) { - iv, err = GetIVFromMetadata(metadata) - if err != nil { - return nil, "", nil, nil, err - } - - if keyIDBytes, exists := metadata[MetaSSEKMSKeyID]; exists { - keyID = string(keyIDBytes) - } - - if encKeyBase64, exists := metadata[MetaSSEKMSEncryptedKey]; exists { - encryptedKey, err = base64.StdEncoding.DecodeString(string(encKeyBase64)) - if err != nil { - return nil, "", nil, nil, fmt.Errorf("failed to decode encrypted key: %w", err) - } - } - - // Parse context from JSON - if contextBytes, exists := metadata[MetaSSEKMSContext]; exists { - context = make(map[string]string) - if err := json.Unmarshal(contextBytes, &context); err != nil { - return nil, "", nil, nil, fmt.Errorf("failed to parse KMS context JSON: %w", err) - } - } - - return iv, keyID, encryptedKey, context, nil -} - -// GetSSES3Metadata retrieves SSE-S3 metadata -func GetSSES3Metadata(metadata map[string][]byte) (iv []byte, keyID string, err error) { - iv, err = GetIVFromMetadata(metadata) - if err != nil { - return nil, "", err - } - - if keyIDBytes, exists := metadata[MetaSSES3KeyID]; exists { - keyID = string(keyIDBytes) - } - - return iv, keyID, nil -} - -// IsSSEEncrypted checks if the metadata indicates any form of SSE encryption -func IsSSEEncrypted(metadata map[string][]byte) bool { - _, exists := metadata[MetaSSEIV] - return exists -} - -// GetSSEAlgorithm returns the SSE algorithm from metadata -func GetSSEAlgorithm(metadata map[string][]byte) string { - if alg, exists := metadata[MetaSSEAlgorithm]; exists { - return string(alg) - } - return "" + return nil, fmt.Errorf("SSE-C IV not found in metadata") } diff --git a/weed/s3api/s3_sse_s3.go b/weed/s3api/s3_sse_s3.go index bb563eee5..bc648205e 100644 --- a/weed/s3api/s3_sse_s3.go +++ b/weed/s3api/s3_sse_s3.go @@ -485,6 +485,31 @@ func GetSSES3KeyFromMetadata(metadata map[string][]byte, keyManager *SSES3KeyMan return DeserializeSSES3Metadata(keyData, keyManager) } +// GetSSES3IV extracts the IV for single-part SSE-S3 objects +// Priority: 1) object-level metadata (for inline/small files), 2) first chunk metadata +func GetSSES3IV(entry *filer_pb.Entry, sseS3Key *SSES3Key, keyManager *SSES3KeyManager) ([]byte, error) { + // First check if IV is in the object-level key (for small/inline files) + if len(sseS3Key.IV) > 0 { + return sseS3Key.IV, nil + } + + // Fallback: Get IV from first chunk's metadata (for chunked files) + if len(entry.GetChunks()) > 0 { + chunk := entry.GetChunks()[0] + if len(chunk.GetSseMetadata()) > 0 { + chunkKey, err := DeserializeSSES3Metadata(chunk.GetSseMetadata(), keyManager) + if err != nil { + return nil, fmt.Errorf("failed to deserialize chunk SSE-S3 metadata: %w", err) + } + if len(chunkKey.IV) > 0 { + return chunkKey.IV, nil + } + } + } + + return nil, fmt.Errorf("SSE-S3 IV not found in object or chunk metadata") +} + // CreateSSES3EncryptedReaderWithBaseIV creates an encrypted reader using a base IV for multipart upload consistency. // The returned IV is the offset-derived IV, calculated from the input baseIV and offset. func CreateSSES3EncryptedReaderWithBaseIV(reader io.Reader, key *SSES3Key, baseIV []byte, offset int64) (io.Reader, []byte /* derivedIV */, error) { diff --git a/weed/s3api/s3_sse_s3_integration_test.go b/weed/s3api/s3_sse_s3_integration_test.go new file mode 100644 index 000000000..8232aea7f --- /dev/null +++ b/weed/s3api/s3_sse_s3_integration_test.go @@ -0,0 +1,325 @@ +package s3api + +import ( + "bytes" + "io" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" +) + +// NOTE: These are integration tests that test the end-to-end encryption/decryption flow. +// Full HTTP handler tests (PUT -> GET) would require a complete mock server with filer, +// which is complex to set up. These tests focus on the critical decrypt path. + +// TestSSES3EndToEndSmallFile tests the complete encryption->storage->decryption cycle for small inline files +// This test would have caught the IV retrieval bug for inline files +func TestSSES3EndToEndSmallFile(t *testing.T) { + // Initialize global SSE-S3 key manager + globalSSES3KeyManager = NewSSES3KeyManager() + defer func() { + globalSSES3KeyManager = NewSSES3KeyManager() + }() + + // Set up the key manager with a super key for testing + keyManager := GetSSES3KeyManager() + keyManager.superKey = make([]byte, 32) + for i := range keyManager.superKey { + keyManager.superKey[i] = byte(i) + } + + testCases := []struct { + name string + data []byte + }{ + {"tiny file (10 bytes)", []byte("test12345")}, + {"small file (50 bytes)", []byte("This is a small test file for SSE-S3 encryption")}, + {"medium file (256 bytes)", bytes.Repeat([]byte("a"), 256)}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Step 1: Encrypt (simulates what happens during PUT) + sseS3Key, err := GenerateSSES3Key() + if err != nil { + t.Fatalf("Failed to generate SSE-S3 key: %v", err) + } + + encryptedReader, iv, err := CreateSSES3EncryptedReader(bytes.NewReader(tc.data), sseS3Key) + if err != nil { + t.Fatalf("Failed to create encrypted reader: %v", err) + } + + encryptedData, err := io.ReadAll(encryptedReader) + if err != nil { + t.Fatalf("Failed to read encrypted data: %v", err) + } + + // Store IV in the key (this is critical for inline files!) + sseS3Key.IV = iv + + // Serialize the metadata (this is stored in entry.Extended) + serializedMetadata, err := SerializeSSES3Metadata(sseS3Key) + if err != nil { + t.Fatalf("Failed to serialize SSE-S3 metadata: %v", err) + } + + // Step 2: Simulate storage (inline file - no chunks) + // For inline files, data is in Content, metadata in Extended + mockEntry := &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.SeaweedFSSSES3Key: serializedMetadata, + s3_constants.AmzServerSideEncryption: []byte("AES256"), + }, + Content: encryptedData, + Chunks: []*filer_pb.FileChunk{}, // Critical: inline files have NO chunks + } + + // Step 3: Decrypt (simulates what happens during GET) + // This tests the IV retrieval path for inline files + + // First, deserialize metadata from storage + retrievedKeyData := mockEntry.Extended[s3_constants.SeaweedFSSSES3Key] + retrievedKey, err := DeserializeSSES3Metadata(retrievedKeyData, keyManager) + if err != nil { + t.Fatalf("Failed to deserialize SSE-S3 metadata: %v", err) + } + + // CRITICAL TEST: For inline files, IV must be in object-level metadata + var retrievedIV []byte + if len(retrievedKey.IV) > 0 { + // Success path: IV found in object-level key + retrievedIV = retrievedKey.IV + } else if len(mockEntry.GetChunks()) > 0 { + // Fallback path: would check chunks (but inline files have no chunks) + t.Fatal("Inline file should have IV in object-level metadata, not chunks") + } + + if len(retrievedIV) == 0 { + // THIS IS THE BUG WE FIXED: inline files had no way to get IV! + t.Fatal("Failed to retrieve IV for inline file - this is the bug we fixed!") + } + + // Now decrypt with the retrieved IV + decryptedReader, err := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData), retrievedKey, retrievedIV) + if err != nil { + t.Fatalf("Failed to create decrypted reader: %v", err) + } + + decryptedData, err := io.ReadAll(decryptedReader) + if err != nil { + t.Fatalf("Failed to read decrypted data: %v", err) + } + + // Verify decrypted data matches original + if !bytes.Equal(decryptedData, tc.data) { + t.Errorf("Decrypted data doesn't match original.\nExpected: %q\nGot: %q", tc.data, decryptedData) + } + }) + } +} + +// TestSSES3EndToEndChunkedFile tests the complete flow for chunked files +func TestSSES3EndToEndChunkedFile(t *testing.T) { + // Initialize global SSE-S3 key manager + globalSSES3KeyManager = NewSSES3KeyManager() + defer func() { + globalSSES3KeyManager = NewSSES3KeyManager() + }() + + keyManager := GetSSES3KeyManager() + keyManager.superKey = make([]byte, 32) + for i := range keyManager.superKey { + keyManager.superKey[i] = byte(i) + } + + // Generate SSE-S3 key + sseS3Key, err := GenerateSSES3Key() + if err != nil { + t.Fatalf("Failed to generate SSE-S3 key: %v", err) + } + + // Create test data for two chunks + chunk1Data := []byte("This is chunk 1 data for SSE-S3 encryption test") + chunk2Data := []byte("This is chunk 2 data for SSE-S3 encryption test") + + // Encrypt chunk 1 + encryptedReader1, iv1, err := CreateSSES3EncryptedReader(bytes.NewReader(chunk1Data), sseS3Key) + if err != nil { + t.Fatalf("Failed to create encrypted reader for chunk 1: %v", err) + } + encryptedChunk1, _ := io.ReadAll(encryptedReader1) + + // Encrypt chunk 2 + encryptedReader2, iv2, err := CreateSSES3EncryptedReader(bytes.NewReader(chunk2Data), sseS3Key) + if err != nil { + t.Fatalf("Failed to create encrypted reader for chunk 2: %v", err) + } + encryptedChunk2, _ := io.ReadAll(encryptedReader2) + + // Create metadata for each chunk + chunk1Key := &SSES3Key{ + Key: sseS3Key.Key, + IV: iv1, + Algorithm: sseS3Key.Algorithm, + KeyID: sseS3Key.KeyID, + } + chunk2Key := &SSES3Key{ + Key: sseS3Key.Key, + IV: iv2, + Algorithm: sseS3Key.Algorithm, + KeyID: sseS3Key.KeyID, + } + + serializedChunk1Meta, _ := SerializeSSES3Metadata(chunk1Key) + serializedChunk2Meta, _ := SerializeSSES3Metadata(chunk2Key) + serializedObjMeta, _ := SerializeSSES3Metadata(sseS3Key) + + // Create mock entry with chunks + mockEntry := &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.SeaweedFSSSES3Key: serializedObjMeta, + s3_constants.AmzServerSideEncryption: []byte("AES256"), + }, + Chunks: []*filer_pb.FileChunk{ + { + FileId: "chunk1,123", + Offset: 0, + Size: uint64(len(encryptedChunk1)), + SseType: filer_pb.SSEType_SSE_S3, + SseMetadata: serializedChunk1Meta, + }, + { + FileId: "chunk2,456", + Offset: int64(len(chunk1Data)), + Size: uint64(len(encryptedChunk2)), + SseType: filer_pb.SSEType_SSE_S3, + SseMetadata: serializedChunk2Meta, + }, + }, + } + + // Verify multipart detection + sses3Chunks := 0 + for _, chunk := range mockEntry.GetChunks() { + if chunk.GetSseType() == filer_pb.SSEType_SSE_S3 && len(chunk.GetSseMetadata()) > 0 { + sses3Chunks++ + } + } + + isMultipart := sses3Chunks > 1 + if !isMultipart { + t.Error("Expected multipart SSE-S3 object detection") + } + + if sses3Chunks != 2 { + t.Errorf("Expected 2 SSE-S3 chunks, got %d", sses3Chunks) + } + + // Verify each chunk has valid metadata with IV + for i, chunk := range mockEntry.GetChunks() { + deserializedKey, err := DeserializeSSES3Metadata(chunk.GetSseMetadata(), keyManager) + if err != nil { + t.Errorf("Failed to deserialize chunk %d metadata: %v", i, err) + } + if len(deserializedKey.IV) == 0 { + t.Errorf("Chunk %d has no IV", i) + } + + // Decrypt this chunk to verify it works + var chunkData []byte + if i == 0 { + chunkData = encryptedChunk1 + } else { + chunkData = encryptedChunk2 + } + + decryptedReader, err := CreateSSES3DecryptedReader(bytes.NewReader(chunkData), deserializedKey, deserializedKey.IV) + if err != nil { + t.Errorf("Failed to decrypt chunk %d: %v", i, err) + continue + } + + decrypted, _ := io.ReadAll(decryptedReader) + var expectedData []byte + if i == 0 { + expectedData = chunk1Data + } else { + expectedData = chunk2Data + } + + if !bytes.Equal(decrypted, expectedData) { + t.Errorf("Chunk %d decryption failed", i) + } + } +} + +// TestSSES3EndToEndWithDetectPrimaryType tests that type detection works correctly for different scenarios +func TestSSES3EndToEndWithDetectPrimaryType(t *testing.T) { + s3a := &S3ApiServer{} + + testCases := []struct { + name string + entry *filer_pb.Entry + expectedType string + shouldBeSSES3 bool + }{ + { + name: "Inline SSE-S3 file (no chunks)", + entry: &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("AES256"), + }, + Attributes: &filer_pb.FuseAttributes{}, + Content: []byte("encrypted data"), + Chunks: []*filer_pb.FileChunk{}, + }, + expectedType: s3_constants.SSETypeS3, + shouldBeSSES3: true, + }, + { + name: "Single chunk SSE-S3 file", + entry: &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("AES256"), + }, + Attributes: &filer_pb.FuseAttributes{}, + Chunks: []*filer_pb.FileChunk{ + { + FileId: "1,123", + SseType: filer_pb.SSEType_SSE_S3, + SseMetadata: []byte("metadata"), + }, + }, + }, + expectedType: s3_constants.SSETypeS3, + shouldBeSSES3: true, + }, + { + name: "SSE-KMS file (has KMS key ID)", + entry: &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("AES256"), + s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("kms-key-123"), + }, + Attributes: &filer_pb.FuseAttributes{}, + Chunks: []*filer_pb.FileChunk{}, + }, + expectedType: s3_constants.SSETypeKMS, + shouldBeSSES3: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + detectedType := s3a.detectPrimarySSEType(tc.entry) + if detectedType != tc.expectedType { + t.Errorf("Expected type %s, got %s", tc.expectedType, detectedType) + } + if (detectedType == s3_constants.SSETypeS3) != tc.shouldBeSSES3 { + t.Errorf("SSE-S3 detection mismatch: expected %v, got %v", tc.shouldBeSSES3, detectedType == s3_constants.SSETypeS3) + } + }) + } +} diff --git a/weed/s3api/s3_sse_s3_test.go b/weed/s3api/s3_sse_s3_test.go new file mode 100644 index 000000000..391692921 --- /dev/null +++ b/weed/s3api/s3_sse_s3_test.go @@ -0,0 +1,984 @@ +package s3api + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" +) + +// TestSSES3EncryptionDecryption tests basic SSE-S3 encryption and decryption +func TestSSES3EncryptionDecryption(t *testing.T) { + // Generate SSE-S3 key + sseS3Key, err := GenerateSSES3Key() + if err != nil { + t.Fatalf("Failed to generate SSE-S3 key: %v", err) + } + + // Test data + testData := []byte("Hello, World! This is a test of SSE-S3 encryption.") + + // Create encrypted reader + dataReader := bytes.NewReader(testData) + encryptedReader, iv, err := CreateSSES3EncryptedReader(dataReader, sseS3Key) + if err != nil { + t.Fatalf("Failed to create encrypted reader: %v", err) + } + + // Read encrypted data + encryptedData, err := io.ReadAll(encryptedReader) + if err != nil { + t.Fatalf("Failed to read encrypted data: %v", err) + } + + // Verify data is actually encrypted (different from original) + if bytes.Equal(encryptedData, testData) { + t.Error("Data doesn't appear to be encrypted") + } + + // Create decrypted reader + encryptedReader2 := bytes.NewReader(encryptedData) + decryptedReader, err := CreateSSES3DecryptedReader(encryptedReader2, sseS3Key, iv) + if err != nil { + t.Fatalf("Failed to create decrypted reader: %v", err) + } + + // Read decrypted data + decryptedData, err := io.ReadAll(decryptedReader) + if err != nil { + t.Fatalf("Failed to read decrypted data: %v", err) + } + + // Verify decrypted data matches original + if !bytes.Equal(decryptedData, testData) { + t.Errorf("Decrypted data doesn't match original.\nOriginal: %s\nDecrypted: %s", testData, decryptedData) + } +} + +// TestSSES3IsRequestInternal tests detection of SSE-S3 requests +func TestSSES3IsRequestInternal(t *testing.T) { + testCases := []struct { + name string + headers map[string]string + expected bool + }{ + { + name: "Valid SSE-S3 request", + headers: map[string]string{ + s3_constants.AmzServerSideEncryption: "AES256", + }, + expected: true, + }, + { + name: "No SSE headers", + headers: map[string]string{}, + expected: false, + }, + { + name: "SSE-KMS request", + headers: map[string]string{ + s3_constants.AmzServerSideEncryption: "aws:kms", + }, + expected: false, + }, + { + name: "SSE-C request", + headers: map[string]string{ + s3_constants.AmzServerSideEncryptionCustomerAlgorithm: "AES256", + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req := &http.Request{Header: make(http.Header)} + for k, v := range tc.headers { + req.Header.Set(k, v) + } + + result := IsSSES3RequestInternal(req) + if result != tc.expected { + t.Errorf("Expected %v, got %v", tc.expected, result) + } + }) + } +} + +// TestSSES3MetadataSerialization tests SSE-S3 metadata serialization and deserialization +func TestSSES3MetadataSerialization(t *testing.T) { + // Initialize global key manager + globalSSES3KeyManager = NewSSES3KeyManager() + defer func() { + globalSSES3KeyManager = NewSSES3KeyManager() + }() + + // Set up the key manager with a super key for testing + keyManager := GetSSES3KeyManager() + keyManager.superKey = make([]byte, 32) + for i := range keyManager.superKey { + keyManager.superKey[i] = byte(i) + } + + // Generate SSE-S3 key + sseS3Key, err := GenerateSSES3Key() + if err != nil { + t.Fatalf("Failed to generate SSE-S3 key: %v", err) + } + + // Add IV to the key + sseS3Key.IV = make([]byte, 16) + for i := range sseS3Key.IV { + sseS3Key.IV[i] = byte(i * 2) + } + + // Serialize metadata + serialized, err := SerializeSSES3Metadata(sseS3Key) + if err != nil { + t.Fatalf("Failed to serialize SSE-S3 metadata: %v", err) + } + + if len(serialized) == 0 { + t.Error("Serialized metadata is empty") + } + + // Deserialize metadata + deserializedKey, err := DeserializeSSES3Metadata(serialized, keyManager) + if err != nil { + t.Fatalf("Failed to deserialize SSE-S3 metadata: %v", err) + } + + // Verify key matches + if !bytes.Equal(deserializedKey.Key, sseS3Key.Key) { + t.Error("Deserialized key doesn't match original key") + } + + // Verify IV matches + if !bytes.Equal(deserializedKey.IV, sseS3Key.IV) { + t.Error("Deserialized IV doesn't match original IV") + } + + // Verify algorithm matches + if deserializedKey.Algorithm != sseS3Key.Algorithm { + t.Errorf("Algorithm mismatch: expected %s, got %s", sseS3Key.Algorithm, deserializedKey.Algorithm) + } + + // Verify key ID matches + if deserializedKey.KeyID != sseS3Key.KeyID { + t.Errorf("Key ID mismatch: expected %s, got %s", sseS3Key.KeyID, deserializedKey.KeyID) + } +} + +// TestDetectPrimarySSETypeS3 tests detection of SSE-S3 as primary encryption type +func TestDetectPrimarySSETypeS3(t *testing.T) { + s3a := &S3ApiServer{} + + testCases := []struct { + name string + entry *filer_pb.Entry + expected string + }{ + { + name: "Single SSE-S3 chunk", + entry: &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("AES256"), + }, + Attributes: &filer_pb.FuseAttributes{}, + Chunks: []*filer_pb.FileChunk{ + { + FileId: "1,123", + Offset: 0, + Size: 1024, + SseType: filer_pb.SSEType_SSE_S3, + SseMetadata: []byte("metadata"), + }, + }, + }, + expected: s3_constants.SSETypeS3, + }, + { + name: "Multiple SSE-S3 chunks", + entry: &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("AES256"), + }, + Attributes: &filer_pb.FuseAttributes{}, + Chunks: []*filer_pb.FileChunk{ + { + FileId: "1,123", + Offset: 0, + Size: 1024, + SseType: filer_pb.SSEType_SSE_S3, + SseMetadata: []byte("metadata1"), + }, + { + FileId: "2,456", + Offset: 1024, + Size: 1024, + SseType: filer_pb.SSEType_SSE_S3, + SseMetadata: []byte("metadata2"), + }, + }, + }, + expected: s3_constants.SSETypeS3, + }, + { + name: "Mixed SSE-S3 and SSE-KMS chunks (SSE-S3 majority)", + entry: &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("AES256"), + }, + Attributes: &filer_pb.FuseAttributes{}, + Chunks: []*filer_pb.FileChunk{ + { + FileId: "1,123", + Offset: 0, + Size: 1024, + SseType: filer_pb.SSEType_SSE_S3, + SseMetadata: []byte("metadata1"), + }, + { + FileId: "2,456", + Offset: 1024, + Size: 1024, + SseType: filer_pb.SSEType_SSE_S3, + SseMetadata: []byte("metadata2"), + }, + { + FileId: "3,789", + Offset: 2048, + Size: 1024, + SseType: filer_pb.SSEType_SSE_KMS, + SseMetadata: []byte("metadata3"), + }, + }, + }, + expected: s3_constants.SSETypeS3, + }, + { + name: "No chunks, SSE-S3 metadata without KMS key ID", + entry: &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("AES256"), + }, + Attributes: &filer_pb.FuseAttributes{}, + Chunks: []*filer_pb.FileChunk{}, + }, + expected: s3_constants.SSETypeS3, + }, + { + name: "No chunks, SSE-KMS metadata with KMS key ID", + entry: &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("AES256"), + s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("test-key-id"), + }, + Attributes: &filer_pb.FuseAttributes{}, + Chunks: []*filer_pb.FileChunk{}, + }, + expected: s3_constants.SSETypeKMS, + }, + { + name: "SSE-C chunks", + entry: &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.AmzServerSideEncryptionCustomerAlgorithm: []byte("AES256"), + }, + Attributes: &filer_pb.FuseAttributes{}, + Chunks: []*filer_pb.FileChunk{ + { + FileId: "1,123", + Offset: 0, + Size: 1024, + SseType: filer_pb.SSEType_SSE_C, + SseMetadata: []byte("metadata"), + }, + }, + }, + expected: s3_constants.SSETypeC, + }, + { + name: "Unencrypted", + entry: &filer_pb.Entry{ + Extended: map[string][]byte{}, + Attributes: &filer_pb.FuseAttributes{}, + Chunks: []*filer_pb.FileChunk{ + { + FileId: "1,123", + Offset: 0, + Size: 1024, + }, + }, + }, + expected: "None", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := s3a.detectPrimarySSEType(tc.entry) + if result != tc.expected { + t.Errorf("Expected %s, got %s", tc.expected, result) + } + }) + } +} + +// TestAddSSES3HeadersToResponse tests that SSE-S3 headers are added to responses +func TestAddSSES3HeadersToResponse(t *testing.T) { + s3a := &S3ApiServer{} + + entry := &filer_pb.Entry{ + Extended: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("AES256"), + }, + Attributes: &filer_pb.FuseAttributes{}, + Chunks: []*filer_pb.FileChunk{ + { + FileId: "1,123", + Offset: 0, + Size: 1024, + SseType: filer_pb.SSEType_SSE_S3, + SseMetadata: []byte("metadata"), + }, + }, + } + + proxyResponse := &http.Response{ + Header: make(http.Header), + } + + s3a.addSSEHeadersToResponse(proxyResponse, entry) + + algorithm := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryption) + if algorithm != "AES256" { + t.Errorf("Expected SSE algorithm AES256, got %s", algorithm) + } + + // Should NOT have SSE-C or SSE-KMS specific headers + if proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm) != "" { + t.Error("Should not have SSE-C customer algorithm header") + } + + if proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionAwsKmsKeyId) != "" { + t.Error("Should not have SSE-KMS key ID header") + } +} + +// TestSSES3EncryptionWithBaseIV tests multipart encryption with base IV +func TestSSES3EncryptionWithBaseIV(t *testing.T) { + // Generate SSE-S3 key + sseS3Key, err := GenerateSSES3Key() + if err != nil { + t.Fatalf("Failed to generate SSE-S3 key: %v", err) + } + + // Generate base IV + baseIV := make([]byte, 16) + for i := range baseIV { + baseIV[i] = byte(i) + } + + // Test data for two parts + testData1 := []byte("Part 1 of multipart upload test.") + testData2 := []byte("Part 2 of multipart upload test.") + + // Encrypt part 1 at offset 0 + dataReader1 := bytes.NewReader(testData1) + encryptedReader1, iv1, err := CreateSSES3EncryptedReaderWithBaseIV(dataReader1, sseS3Key, baseIV, 0) + if err != nil { + t.Fatalf("Failed to create encrypted reader for part 1: %v", err) + } + + encryptedData1, err := io.ReadAll(encryptedReader1) + if err != nil { + t.Fatalf("Failed to read encrypted data for part 1: %v", err) + } + + // Encrypt part 2 at offset (simulating second part) + dataReader2 := bytes.NewReader(testData2) + offset2 := int64(len(testData1)) + encryptedReader2, iv2, err := CreateSSES3EncryptedReaderWithBaseIV(dataReader2, sseS3Key, baseIV, offset2) + if err != nil { + t.Fatalf("Failed to create encrypted reader for part 2: %v", err) + } + + encryptedData2, err := io.ReadAll(encryptedReader2) + if err != nil { + t.Fatalf("Failed to read encrypted data for part 2: %v", err) + } + + // IVs should be different (offset-based) + if bytes.Equal(iv1, iv2) { + t.Error("IVs should be different for different offsets") + } + + // Decrypt part 1 + decryptedReader1, err := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData1), sseS3Key, iv1) + if err != nil { + t.Fatalf("Failed to create decrypted reader for part 1: %v", err) + } + + decryptedData1, err := io.ReadAll(decryptedReader1) + if err != nil { + t.Fatalf("Failed to read decrypted data for part 1: %v", err) + } + + // Decrypt part 2 + decryptedReader2, err := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData2), sseS3Key, iv2) + if err != nil { + t.Fatalf("Failed to create decrypted reader for part 2: %v", err) + } + + decryptedData2, err := io.ReadAll(decryptedReader2) + if err != nil { + t.Fatalf("Failed to read decrypted data for part 2: %v", err) + } + + // Verify decrypted data matches original + if !bytes.Equal(decryptedData1, testData1) { + t.Errorf("Decrypted part 1 doesn't match original.\nOriginal: %s\nDecrypted: %s", testData1, decryptedData1) + } + + if !bytes.Equal(decryptedData2, testData2) { + t.Errorf("Decrypted part 2 doesn't match original.\nOriginal: %s\nDecrypted: %s", testData2, decryptedData2) + } +} + +// TestSSES3WrongKeyDecryption tests that wrong key fails decryption +func TestSSES3WrongKeyDecryption(t *testing.T) { + // Generate two different keys + sseS3Key1, err := GenerateSSES3Key() + if err != nil { + t.Fatalf("Failed to generate SSE-S3 key 1: %v", err) + } + + sseS3Key2, err := GenerateSSES3Key() + if err != nil { + t.Fatalf("Failed to generate SSE-S3 key 2: %v", err) + } + + // Test data + testData := []byte("Secret data encrypted with key 1") + + // Encrypt with key 1 + dataReader := bytes.NewReader(testData) + encryptedReader, iv, err := CreateSSES3EncryptedReader(dataReader, sseS3Key1) + if err != nil { + t.Fatalf("Failed to create encrypted reader: %v", err) + } + + encryptedData, err := io.ReadAll(encryptedReader) + if err != nil { + t.Fatalf("Failed to read encrypted data: %v", err) + } + + // Try to decrypt with key 2 (wrong key) + decryptedReader, err := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData), sseS3Key2, iv) + if err != nil { + t.Fatalf("Failed to create decrypted reader: %v", err) + } + + decryptedData, err := io.ReadAll(decryptedReader) + if err != nil { + t.Fatalf("Failed to read decrypted data: %v", err) + } + + // Decrypted data should NOT match original (wrong key produces garbage) + if bytes.Equal(decryptedData, testData) { + t.Error("Decryption with wrong key should not produce correct plaintext") + } +} + +// TestSSES3KeyGeneration tests SSE-S3 key generation +func TestSSES3KeyGeneration(t *testing.T) { + // Generate multiple keys + keys := make([]*SSES3Key, 10) + for i := range keys { + key, err := GenerateSSES3Key() + if err != nil { + t.Fatalf("Failed to generate SSE-S3 key %d: %v", i, err) + } + keys[i] = key + + // Verify key properties + if len(key.Key) != SSES3KeySize { + t.Errorf("Key %d has wrong size: expected %d, got %d", i, SSES3KeySize, len(key.Key)) + } + + if key.Algorithm != SSES3Algorithm { + t.Errorf("Key %d has wrong algorithm: expected %s, got %s", i, SSES3Algorithm, key.Algorithm) + } + + if key.KeyID == "" { + t.Errorf("Key %d has empty key ID", i) + } + } + + // Verify keys are unique + for i := 0; i < len(keys); i++ { + for j := i + 1; j < len(keys); j++ { + if bytes.Equal(keys[i].Key, keys[j].Key) { + t.Errorf("Keys %d and %d are identical (should be unique)", i, j) + } + if keys[i].KeyID == keys[j].KeyID { + t.Errorf("Key IDs %d and %d are identical (should be unique)", i, j) + } + } + } +} + +// TestSSES3VariousSizes tests SSE-S3 encryption/decryption with various data sizes +func TestSSES3VariousSizes(t *testing.T) { + sizes := []int{1, 15, 16, 17, 100, 1024, 4096, 1048576} + + for _, size := range sizes { + t.Run(fmt.Sprintf("size_%d", size), func(t *testing.T) { + // Generate test data + testData := make([]byte, size) + for i := range testData { + testData[i] = byte(i % 256) + } + + // Generate key + sseS3Key, err := GenerateSSES3Key() + if err != nil { + t.Fatalf("Failed to generate SSE-S3 key: %v", err) + } + + // Encrypt + dataReader := bytes.NewReader(testData) + encryptedReader, iv, err := CreateSSES3EncryptedReader(dataReader, sseS3Key) + if err != nil { + t.Fatalf("Failed to create encrypted reader: %v", err) + } + + encryptedData, err := io.ReadAll(encryptedReader) + if err != nil { + t.Fatalf("Failed to read encrypted data: %v", err) + } + + // Verify encrypted size matches original + if len(encryptedData) != size { + t.Errorf("Encrypted size mismatch: expected %d, got %d", size, len(encryptedData)) + } + + // Decrypt + decryptedReader, err := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData), sseS3Key, iv) + if err != nil { + t.Fatalf("Failed to create decrypted reader: %v", err) + } + + decryptedData, err := io.ReadAll(decryptedReader) + if err != nil { + t.Fatalf("Failed to read decrypted data: %v", err) + } + + // Verify + if !bytes.Equal(decryptedData, testData) { + t.Errorf("Decrypted data doesn't match original for size %d", size) + } + }) + } +} + +// TestSSES3ResponseHeaders tests that SSE-S3 response headers are set correctly +func TestSSES3ResponseHeaders(t *testing.T) { + w := httptest.NewRecorder() + + // Simulate setting SSE-S3 response headers + w.Header().Set(s3_constants.AmzServerSideEncryption, SSES3Algorithm) + + // Verify headers + algorithm := w.Header().Get(s3_constants.AmzServerSideEncryption) + if algorithm != "AES256" { + t.Errorf("Expected algorithm AES256, got %s", algorithm) + } + + // Should NOT have customer key headers + if w.Header().Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm) != "" { + t.Error("Should not have SSE-C customer algorithm header") + } + + if w.Header().Get(s3_constants.AmzServerSideEncryptionCustomerKeyMD5) != "" { + t.Error("Should not have SSE-C customer key MD5 header") + } + + // Should NOT have KMS key ID + if w.Header().Get(s3_constants.AmzServerSideEncryptionAwsKmsKeyId) != "" { + t.Error("Should not have SSE-KMS key ID header") + } +} + +// TestSSES3IsEncryptedInternal tests detection of SSE-S3 encryption from metadata +func TestSSES3IsEncryptedInternal(t *testing.T) { + testCases := []struct { + name string + metadata map[string][]byte + expected bool + }{ + { + name: "Empty metadata", + metadata: map[string][]byte{}, + expected: false, + }, + { + name: "Valid SSE-S3 metadata", + metadata: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("AES256"), + }, + expected: true, + }, + { + name: "SSE-KMS metadata", + metadata: map[string][]byte{ + s3_constants.AmzServerSideEncryption: []byte("aws:kms"), + }, + expected: false, + }, + { + name: "SSE-C metadata", + metadata: map[string][]byte{ + s3_constants.AmzServerSideEncryptionCustomerAlgorithm: []byte("AES256"), + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := IsSSES3EncryptedInternal(tc.metadata) + if result != tc.expected { + t.Errorf("Expected %v, got %v", tc.expected, result) + } + }) + } +} + +// TestSSES3InvalidMetadataDeserialization tests error handling for invalid metadata +func TestSSES3InvalidMetadataDeserialization(t *testing.T) { + keyManager := NewSSES3KeyManager() + keyManager.superKey = make([]byte, 32) + + testCases := []struct { + name string + metadata []byte + shouldError bool + }{ + { + name: "Empty metadata", + metadata: []byte{}, + shouldError: true, + }, + { + name: "Invalid JSON", + metadata: []byte("not valid json"), + shouldError: true, + }, + { + name: "Missing keyId", + metadata: []byte(`{"algorithm":"AES256"}`), + shouldError: true, + }, + { + name: "Invalid base64 encrypted DEK", + metadata: []byte(`{"keyId":"test","algorithm":"AES256","encryptedDEK":"not-valid-base64!","nonce":"dGVzdA=="}`), + shouldError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := DeserializeSSES3Metadata(tc.metadata, keyManager) + if tc.shouldError && err == nil { + t.Error("Expected error but got none") + } + if !tc.shouldError && err != nil { + t.Errorf("Unexpected error: %v", err) + } + }) + } +} + +// TestGetSSES3Headers tests SSE-S3 header generation +func TestGetSSES3Headers(t *testing.T) { + headers := GetSSES3Headers() + + if len(headers) == 0 { + t.Error("Expected headers to be non-empty") + } + + algorithm, exists := headers[s3_constants.AmzServerSideEncryption] + if !exists { + t.Error("Expected AmzServerSideEncryption header to exist") + } + + if algorithm != "AES256" { + t.Errorf("Expected algorithm AES256, got %s", algorithm) + } +} + +// TestProcessSSES3Request tests processing of SSE-S3 requests +func TestProcessSSES3Request(t *testing.T) { + // Initialize global key manager + globalSSES3KeyManager = NewSSES3KeyManager() + defer func() { + globalSSES3KeyManager = NewSSES3KeyManager() + }() + + // Set up the key manager with a super key for testing + keyManager := GetSSES3KeyManager() + keyManager.superKey = make([]byte, 32) + for i := range keyManager.superKey { + keyManager.superKey[i] = byte(i) + } + + // Create SSE-S3 request + req := httptest.NewRequest("PUT", "/bucket/object", nil) + req.Header.Set(s3_constants.AmzServerSideEncryption, "AES256") + + // Process request + metadata, err := ProcessSSES3Request(req) + if err != nil { + t.Fatalf("Failed to process SSE-S3 request: %v", err) + } + + if metadata == nil { + t.Fatal("Expected metadata to be non-nil") + } + + // Verify metadata contains SSE algorithm + if sseAlgo, exists := metadata[s3_constants.AmzServerSideEncryption]; !exists { + t.Error("Expected SSE algorithm in metadata") + } else if string(sseAlgo) != "AES256" { + t.Errorf("Expected AES256, got %s", string(sseAlgo)) + } + + // Verify metadata contains key data + if _, exists := metadata[s3_constants.SeaweedFSSSES3Key]; !exists { + t.Error("Expected SSE-S3 key data in metadata") + } +} + +// TestGetSSES3KeyFromMetadata tests extraction of SSE-S3 key from metadata +func TestGetSSES3KeyFromMetadata(t *testing.T) { + // Initialize global key manager + globalSSES3KeyManager = NewSSES3KeyManager() + defer func() { + globalSSES3KeyManager = NewSSES3KeyManager() + }() + + // Set up the key manager with a super key for testing + keyManager := GetSSES3KeyManager() + keyManager.superKey = make([]byte, 32) + for i := range keyManager.superKey { + keyManager.superKey[i] = byte(i) + } + + // Generate and serialize key + sseS3Key, err := GenerateSSES3Key() + if err != nil { + t.Fatalf("Failed to generate SSE-S3 key: %v", err) + } + + sseS3Key.IV = make([]byte, 16) + for i := range sseS3Key.IV { + sseS3Key.IV[i] = byte(i) + } + + serialized, err := SerializeSSES3Metadata(sseS3Key) + if err != nil { + t.Fatalf("Failed to serialize SSE-S3 metadata: %v", err) + } + + metadata := map[string][]byte{ + s3_constants.SeaweedFSSSES3Key: serialized, + } + + // Extract key + extractedKey, err := GetSSES3KeyFromMetadata(metadata, keyManager) + if err != nil { + t.Fatalf("Failed to get SSE-S3 key from metadata: %v", err) + } + + // Verify key matches + if !bytes.Equal(extractedKey.Key, sseS3Key.Key) { + t.Error("Extracted key doesn't match original key") + } + + if !bytes.Equal(extractedKey.IV, sseS3Key.IV) { + t.Error("Extracted IV doesn't match original IV") + } +} + +// TestSSES3EnvelopeEncryption tests that envelope encryption works correctly +func TestSSES3EnvelopeEncryption(t *testing.T) { + // Initialize key manager with a super key + keyManager := NewSSES3KeyManager() + keyManager.superKey = make([]byte, 32) + for i := range keyManager.superKey { + keyManager.superKey[i] = byte(i + 100) + } + + // Generate a DEK + dek := make([]byte, 32) + for i := range dek { + dek[i] = byte(i) + } + + // Encrypt DEK with super key + encryptedDEK, nonce, err := keyManager.encryptKeyWithSuperKey(dek) + if err != nil { + t.Fatalf("Failed to encrypt DEK: %v", err) + } + + if len(encryptedDEK) == 0 { + t.Error("Encrypted DEK is empty") + } + + if len(nonce) == 0 { + t.Error("Nonce is empty") + } + + // Decrypt DEK with super key + decryptedDEK, err := keyManager.decryptKeyWithSuperKey(encryptedDEK, nonce) + if err != nil { + t.Fatalf("Failed to decrypt DEK: %v", err) + } + + // Verify DEK matches + if !bytes.Equal(decryptedDEK, dek) { + t.Error("Decrypted DEK doesn't match original DEK") + } +} + +// TestValidateSSES3Key tests SSE-S3 key validation +func TestValidateSSES3Key(t *testing.T) { + testCases := []struct { + name string + key *SSES3Key + shouldError bool + errorMsg string + }{ + { + name: "Nil key", + key: nil, + shouldError: true, + errorMsg: "SSE-S3 key cannot be nil", + }, + { + name: "Valid key", + key: &SSES3Key{ + Key: make([]byte, 32), + KeyID: "test-key", + Algorithm: "AES256", + }, + shouldError: false, + }, + { + name: "Valid key with IV", + key: &SSES3Key{ + Key: make([]byte, 32), + KeyID: "test-key", + Algorithm: "AES256", + IV: make([]byte, 16), + }, + shouldError: false, + }, + { + name: "Invalid key size (too small)", + key: &SSES3Key{ + Key: make([]byte, 16), + KeyID: "test-key", + Algorithm: "AES256", + }, + shouldError: true, + errorMsg: "invalid SSE-S3 key size", + }, + { + name: "Invalid key size (too large)", + key: &SSES3Key{ + Key: make([]byte, 64), + KeyID: "test-key", + Algorithm: "AES256", + }, + shouldError: true, + errorMsg: "invalid SSE-S3 key size", + }, + { + name: "Nil key bytes", + key: &SSES3Key{ + Key: nil, + KeyID: "test-key", + Algorithm: "AES256", + }, + shouldError: true, + errorMsg: "SSE-S3 key bytes cannot be nil", + }, + { + name: "Empty key ID", + key: &SSES3Key{ + Key: make([]byte, 32), + KeyID: "", + Algorithm: "AES256", + }, + shouldError: true, + errorMsg: "SSE-S3 key ID cannot be empty", + }, + { + name: "Invalid algorithm", + key: &SSES3Key{ + Key: make([]byte, 32), + KeyID: "test-key", + Algorithm: "INVALID", + }, + shouldError: true, + errorMsg: "invalid SSE-S3 algorithm", + }, + { + name: "Invalid IV length", + key: &SSES3Key{ + Key: make([]byte, 32), + KeyID: "test-key", + Algorithm: "AES256", + IV: make([]byte, 8), // Wrong size + }, + shouldError: true, + errorMsg: "invalid SSE-S3 IV length", + }, + { + name: "Empty IV is allowed (set during encryption)", + key: &SSES3Key{ + Key: make([]byte, 32), + KeyID: "test-key", + Algorithm: "AES256", + IV: []byte{}, // Empty is OK + }, + shouldError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateSSES3Key(tc.key) + if tc.shouldError { + if err == nil { + t.Error("Expected error but got none") + } else if tc.errorMsg != "" && !strings.Contains(err.Error(), tc.errorMsg) { + t.Errorf("Expected error containing %q, got: %v", tc.errorMsg, err) + } + } else { + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + } + }) + } +} diff --git a/weed/s3api/s3_sse_test_utils_test.go b/weed/s3api/s3_sse_test_utils_test.go index 1c57be791..a4c52994a 100644 --- a/weed/s3api/s3_sse_test_utils_test.go +++ b/weed/s3api/s3_sse_test_utils_test.go @@ -115,7 +115,7 @@ func CreateTestMetadataWithSSEC(keyPair *TestKeyPair) map[string][]byte { for i := range iv { iv[i] = byte(i) } - StoreIVInMetadata(metadata, iv) + StoreSSECIVInMetadata(metadata, iv) return metadata } diff --git a/weed/s3api/s3_validation_utils.go b/weed/s3api/s3_validation_utils.go index da53342b1..e0e80d0a8 100644 --- a/weed/s3api/s3_validation_utils.go +++ b/weed/s3api/s3_validation_utils.go @@ -66,10 +66,35 @@ func ValidateSSECKey(customerKey *SSECustomerKey) error { return nil } -// ValidateSSES3Key validates that an SSE-S3 key is not nil +// ValidateSSES3Key validates that an SSE-S3 key has valid structure and contents func ValidateSSES3Key(sseKey *SSES3Key) error { if sseKey == nil { return fmt.Errorf("SSE-S3 key cannot be nil") } + + // Validate key bytes + if sseKey.Key == nil { + return fmt.Errorf("SSE-S3 key bytes cannot be nil") + } + if len(sseKey.Key) != SSES3KeySize { + return fmt.Errorf("invalid SSE-S3 key size: expected %d bytes, got %d", SSES3KeySize, len(sseKey.Key)) + } + + // Validate algorithm + if sseKey.Algorithm != SSES3Algorithm { + return fmt.Errorf("invalid SSE-S3 algorithm: expected %q, got %q", SSES3Algorithm, sseKey.Algorithm) + } + + // Validate key ID (should not be empty) + if sseKey.KeyID == "" { + return fmt.Errorf("SSE-S3 key ID cannot be empty") + } + + // IV validation is optional during key creation - it will be set during encryption + // If IV is set, validate its length + if len(sseKey.IV) > 0 && len(sseKey.IV) != s3_constants.AESBlockSize { + return fmt.Errorf("invalid SSE-S3 IV length: expected %d bytes, got %d", s3_constants.AESBlockSize, len(sseKey.IV)) + } + return nil } diff --git a/weed/s3api/s3api_key_rotation.go b/weed/s3api/s3api_key_rotation.go index e8d29ff7a..499505678 100644 --- a/weed/s3api/s3api_key_rotation.go +++ b/weed/s3api/s3api_key_rotation.go @@ -100,9 +100,9 @@ func (s3a *S3ApiServer) rotateSSEKMSMetadataOnly(entry *filer_pb.Entry, srcKeyID // rotateSSECChunks re-encrypts all chunks with new SSE-C key func (s3a *S3ApiServer) rotateSSECChunks(entry *filer_pb.Entry, sourceKey, destKey *SSECustomerKey) ([]*filer_pb.FileChunk, error) { // Get IV from entry metadata - iv, err := GetIVFromMetadata(entry.Extended) + iv, err := GetSSECIVFromMetadata(entry.Extended) if err != nil { - return nil, fmt.Errorf("get IV from metadata: %w", err) + return nil, fmt.Errorf("get SSE-C IV from metadata: %w", err) } var rotatedChunks []*filer_pb.FileChunk @@ -125,7 +125,7 @@ func (s3a *S3ApiServer) rotateSSECChunks(entry *filer_pb.Entry, sourceKey, destK if entry.Extended == nil { entry.Extended = make(map[string][]byte) } - StoreIVInMetadata(entry.Extended, newIV) + StoreSSECIVInMetadata(entry.Extended, newIV) entry.Extended[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256") entry.Extended[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(destKey.KeyMD5) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index f30522292..163633e22 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -278,11 +278,11 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) glog.V(1).Infof("GetObject: bucket %s, object %s, versioningConfigured=%v, versionId=%s", bucket, object, versioningConfigured, versionId) var destUrl string + var entry *filer_pb.Entry // Declare entry at function scope for SSE processing if versioningConfigured { // Handle versioned GET - all versions are stored in .versions directory var targetVersionId string - var entry *filer_pb.Entry if versionId != "" { // Request for specific version @@ -363,6 +363,14 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) } } + // Fetch the correct entry for SSE processing (respects versionId) + objectEntryForSSE, err := s3a.getObjectEntryForSSE(r, versioningConfigured, entry) + if err != nil { + glog.Errorf("GetObjectHandler: %v", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + s3a.proxyToFiler(w, r, destUrl, false, func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) { // Restore the original Range header for SSE processing if sseObject && originalRangeHeader != "" { @@ -371,14 +379,12 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) } // Add SSE metadata headers based on object metadata before SSE processing - bucket, object := s3_constants.GetBucketAndObject(r) - objectPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) - if objectEntry, err := s3a.getEntry("", objectPath); err == nil { - s3a.addSSEHeadersToResponse(proxyResponse, objectEntry) + if objectEntryForSSE != nil { + s3a.addSSEHeadersToResponse(proxyResponse, objectEntryForSSE) } // Handle SSE decryption (both SSE-C and SSE-KMS) if needed - return s3a.handleSSEResponse(r, proxyResponse, w) + return s3a.handleSSEResponse(r, proxyResponse, w, objectEntryForSSE) }) } @@ -422,11 +428,11 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request } var destUrl string + var entry *filer_pb.Entry // Declare entry at function scope for SSE processing if versioningConfigured { // Handle versioned HEAD - all versions are stored in .versions directory var targetVersionId string - var entry *filer_pb.Entry if versionId != "" { // Request for specific version @@ -488,9 +494,17 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request destUrl = s3a.toFilerUrl(bucket, object) } + // Fetch the correct entry for SSE processing (respects versionId) + objectEntryForSSE, err := s3a.getObjectEntryForSSE(r, versioningConfigured, entry) + if err != nil { + glog.Errorf("HeadObjectHandler: %v", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + s3a.proxyToFiler(w, r, destUrl, false, func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) { // Handle SSE validation (both SSE-C and SSE-KMS) for HEAD requests - return s3a.handleSSEResponse(r, proxyResponse, w) + return s3a.handleSSEResponse(r, proxyResponse, w, objectEntryForSSE) }) } @@ -646,20 +660,53 @@ func writeFinalResponse(w http.ResponseWriter, proxyResponse *http.Response, bod return statusCode, bytesTransferred } -func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) { - // Capture existing CORS headers that may have been set by middleware - capturedCORSHeaders := captureCORSHeaders(w, corsHeaders) +// getObjectEntryForSSE fetches the correct filer entry for SSE processing +// For versioned objects, it reuses the already-fetched entry +// For non-versioned objects, it fetches the entry from the filer +func (s3a *S3ApiServer) getObjectEntryForSSE(r *http.Request, versioningConfigured bool, versionedEntry *filer_pb.Entry) (*filer_pb.Entry, error) { + if versioningConfigured { + // For versioned objects, we already have the correct entry + return versionedEntry, nil + } - // Copy headers from proxy response + // For non-versioned objects, fetch the entry + bucket, object := s3_constants.GetBucketAndObject(r) + objectPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) + fetchedEntry, err := s3a.getEntry("", objectPath) + if err != nil && !errors.Is(err, filer_pb.ErrNotFound) { + return nil, fmt.Errorf("failed to get entry for SSE check %s: %w", objectPath, err) + } + return fetchedEntry, nil +} + +// copyResponseHeaders copies headers from proxy response to the response writer, +// excluding internal SeaweedFS headers and optionally excluding body-related headers +func copyResponseHeaders(w http.ResponseWriter, proxyResponse *http.Response, excludeBodyHeaders bool) { for k, v := range proxyResponse.Header { + // Always exclude internal SeaweedFS headers + if s3_constants.IsSeaweedFSInternalHeader(k) { + continue + } + // Optionally exclude body-related headers that might change after decryption + if excludeBodyHeaders && (k == "Content-Length" || k == "Content-Encoding") { + continue + } w.Header()[k] = v } +} + +func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) { + // Capture existing CORS headers that may have been set by middleware + capturedCORSHeaders := captureCORSHeaders(w, corsHeaders) + + // Copy headers from proxy response (excluding internal SeaweedFS headers) + copyResponseHeaders(w, proxyResponse, false) return writeFinalResponse(w, proxyResponse, proxyResponse.Body, capturedCORSHeaders) } // handleSSECResponse handles SSE-C decryption and response processing -func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) { +func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, entry *filer_pb.Entry) (statusCode int, bytesTransferred int64) { // Check if the object has SSE-C metadata sseAlgorithm := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm) sseKeyMD5 := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerKeyMD5) @@ -692,9 +739,8 @@ func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http. // Range requests will be handled by the filer layer with proper offset-based decryption // Check if this is a chunked or small content SSE-C object - bucket, object := s3_constants.GetBucketAndObject(r) - objectPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) - if entry, err := s3a.getEntry("", objectPath); err == nil { + // Use the entry parameter passed from the caller (avoids redundant lookup) + if entry != nil { // Check for SSE-C chunks sseCChunks := 0 for _, chunk := range entry.GetChunks() { @@ -716,10 +762,8 @@ func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http. // Capture existing CORS headers capturedCORSHeaders := captureCORSHeaders(w, corsHeaders) - // Copy headers from proxy response - for k, v := range proxyResponse.Header { - w.Header()[k] = v - } + // Copy headers from proxy response (excluding internal SeaweedFS headers) + copyResponseHeaders(w, proxyResponse, false) // Set proper headers for range requests rangeHeader := r.Header.Get("Range") @@ -785,12 +829,8 @@ func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http. // Capture existing CORS headers that may have been set by middleware capturedCORSHeaders := captureCORSHeaders(w, corsHeaders) - // Copy headers from proxy response (excluding body-related headers that might change) - for k, v := range proxyResponse.Header { - if k != "Content-Length" && k != "Content-Encoding" { - w.Header()[k] = v - } - } + // Copy headers from proxy response (excluding body-related headers that might change and internal SeaweedFS headers) + copyResponseHeaders(w, proxyResponse, true) // Set correct Content-Length for SSE-C (only for full object requests) // With IV stored in metadata, the encrypted length equals the original length @@ -821,29 +861,37 @@ func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http. } // handleSSEResponse handles both SSE-C and SSE-KMS decryption/validation and response processing -func (s3a *S3ApiServer) handleSSEResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) { +// The objectEntry parameter should be the correct entry for the requested version (if versioned) +func (s3a *S3ApiServer) handleSSEResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, objectEntry *filer_pb.Entry) (statusCode int, bytesTransferred int64) { // Check what the client is expecting based on request headers clientExpectsSSEC := IsSSECRequest(r) // Check what the stored object has in headers (may be conflicting after copy) kmsMetadataHeader := proxyResponse.Header.Get(s3_constants.SeaweedFSSSEKMSKeyHeader) - sseAlgorithm := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm) - // Get actual object state by examining chunks (most reliable for cross-encryption) - bucket, object := s3_constants.GetBucketAndObject(r) - objectPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) + // Detect actual object SSE type from the provided entry (respects versionId) actualObjectType := "Unknown" - if objectEntry, err := s3a.getEntry("", objectPath); err == nil { + if objectEntry != nil { actualObjectType = s3a.detectPrimarySSEType(objectEntry) } + // If objectEntry is nil, we cannot determine SSE type from chunks + // This should only happen for 404s which will be handled by the proxy + if objectEntry == nil { + glog.V(4).Infof("Object entry not available for SSE routing, passing through") + return passThroughResponse(proxyResponse, w) + } + // Route based on ACTUAL object type (from chunks) rather than conflicting headers if actualObjectType == s3_constants.SSETypeC && clientExpectsSSEC { // Object is SSE-C and client expects SSE-C → SSE-C handler - return s3a.handleSSECResponse(r, proxyResponse, w) + return s3a.handleSSECResponse(r, proxyResponse, w, objectEntry) } else if actualObjectType == s3_constants.SSETypeKMS && !clientExpectsSSEC { // Object is SSE-KMS and client doesn't expect SSE-C → SSE-KMS handler - return s3a.handleSSEKMSResponse(r, proxyResponse, w, kmsMetadataHeader) + return s3a.handleSSEKMSResponse(r, proxyResponse, w, objectEntry, kmsMetadataHeader) + } else if actualObjectType == s3_constants.SSETypeS3 && !clientExpectsSSEC { + // Object is SSE-S3 and client doesn't expect SSE-C → SSE-S3 handler + return s3a.handleSSES3Response(r, proxyResponse, w, objectEntry) } else if actualObjectType == "None" && !clientExpectsSSEC { // Object is unencrypted and client doesn't expect SSE-C → pass through return passThroughResponse(proxyResponse, w) @@ -855,24 +903,23 @@ func (s3a *S3ApiServer) handleSSEResponse(r *http.Request, proxyResponse *http.R // Object is SSE-KMS but client provides SSE-C headers → Error s3err.WriteErrorResponse(w, r, s3err.ErrSSECustomerKeyMissing) return http.StatusBadRequest, 0 + } else if actualObjectType == s3_constants.SSETypeS3 && clientExpectsSSEC { + // Object is SSE-S3 but client provides SSE-C headers → Error (mismatched encryption) + s3err.WriteErrorResponse(w, r, s3err.ErrSSEEncryptionTypeMismatch) + return http.StatusBadRequest, 0 } else if actualObjectType == "None" && clientExpectsSSEC { // Object is unencrypted but client provides SSE-C headers → Error s3err.WriteErrorResponse(w, r, s3err.ErrSSECustomerKeyMissing) return http.StatusBadRequest, 0 } - // Fallback for edge cases - use original logic with header-based detection - if clientExpectsSSEC && sseAlgorithm != "" { - return s3a.handleSSECResponse(r, proxyResponse, w) - } else if !clientExpectsSSEC && kmsMetadataHeader != "" { - return s3a.handleSSEKMSResponse(r, proxyResponse, w, kmsMetadataHeader) - } else { - return passThroughResponse(proxyResponse, w) - } + // Unknown state - pass through and let proxy handle it + glog.V(4).Infof("Unknown SSE state: objectType=%s, clientExpectsSSEC=%v", actualObjectType, clientExpectsSSEC) + return passThroughResponse(proxyResponse, w) } // handleSSEKMSResponse handles SSE-KMS decryption and response processing -func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, kmsMetadataHeader string) (statusCode int, bytesTransferred int64) { +func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, entry *filer_pb.Entry, kmsMetadataHeader string) (statusCode int, bytesTransferred int64) { // Deserialize SSE-KMS metadata kmsMetadataBytes, err := base64.StdEncoding.DecodeString(kmsMetadataHeader) if err != nil { @@ -893,10 +940,8 @@ func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *htt // Capture existing CORS headers that may have been set by middleware capturedCORSHeaders := captureCORSHeaders(w, corsHeaders) - // Copy headers from proxy response - for k, v := range proxyResponse.Header { - w.Header()[k] = v - } + // Copy headers from proxy response (excluding internal SeaweedFS headers) + copyResponseHeaders(w, proxyResponse, false) // Add SSE-KMS response headers AddSSEKMSResponseHeaders(w, sseKMSKey) @@ -908,20 +953,16 @@ func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *htt // We need to check the object structure to determine if it's multipart encrypted isMultipartSSEKMS := false - if sseKMSKey != nil { - // Get the object entry to check chunk structure - bucket, object := s3_constants.GetBucketAndObject(r) - objectPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) - if entry, err := s3a.getEntry("", objectPath); err == nil { - // Check for multipart SSE-KMS - sseKMSChunks := 0 - for _, chunk := range entry.GetChunks() { - if chunk.GetSseType() == filer_pb.SSEType_SSE_KMS && len(chunk.GetSseMetadata()) > 0 { - sseKMSChunks++ - } + if sseKMSKey != nil && entry != nil { + // Use the entry parameter passed from the caller (avoids redundant lookup) + // Check for multipart SSE-KMS + sseKMSChunks := 0 + for _, chunk := range entry.GetChunks() { + if chunk.GetSseType() == filer_pb.SSEType_SSE_KMS && len(chunk.GetSseMetadata()) > 0 { + sseKMSChunks++ } - isMultipartSSEKMS = sseKMSChunks > 1 } + isMultipartSSEKMS = sseKMSChunks > 1 } var decryptedReader io.Reader @@ -950,12 +991,8 @@ func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *htt // Capture existing CORS headers that may have been set by middleware capturedCORSHeaders := captureCORSHeaders(w, corsHeaders) - // Copy headers from proxy response (excluding body-related headers that might change) - for k, v := range proxyResponse.Header { - if k != "Content-Length" && k != "Content-Encoding" { - w.Header()[k] = v - } - } + // Copy headers from proxy response (excluding body-related headers that might change and internal SeaweedFS headers) + copyResponseHeaders(w, proxyResponse, true) // Set correct Content-Length for SSE-KMS if proxyResponse.Header.Get("Content-Range") == "" { @@ -971,6 +1008,99 @@ func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *htt return writeFinalResponse(w, proxyResponse, decryptedReader, capturedCORSHeaders) } +// handleSSES3Response handles SSE-S3 decryption and response processing +func (s3a *S3ApiServer) handleSSES3Response(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, entry *filer_pb.Entry) (statusCode int, bytesTransferred int64) { + + // For HEAD requests, we don't need to decrypt the body, just add response headers + if r.Method == "HEAD" { + // Capture existing CORS headers that may have been set by middleware + capturedCORSHeaders := captureCORSHeaders(w, corsHeaders) + + // Copy headers from proxy response (excluding internal SeaweedFS headers) + copyResponseHeaders(w, proxyResponse, false) + + // Add SSE-S3 response headers + w.Header().Set(s3_constants.AmzServerSideEncryption, SSES3Algorithm) + + return writeFinalResponse(w, proxyResponse, proxyResponse.Body, capturedCORSHeaders) + } + + // For GET requests, check if this is a multipart SSE-S3 object + isMultipartSSES3 := false + sses3Chunks := 0 + for _, chunk := range entry.GetChunks() { + if chunk.GetSseType() == filer_pb.SSEType_SSE_S3 && len(chunk.GetSseMetadata()) > 0 { + sses3Chunks++ + } + } + isMultipartSSES3 = sses3Chunks > 1 + + var decryptedReader io.Reader + if isMultipartSSES3 { + // Handle multipart SSE-S3 objects - each chunk needs independent decryption + multipartReader, decErr := s3a.createMultipartSSES3DecryptedReader(r, entry) + if decErr != nil { + glog.Errorf("Failed to create multipart SSE-S3 decrypted reader: %v", decErr) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return http.StatusInternalServerError, 0 + } + decryptedReader = multipartReader + glog.V(3).Infof("Using multipart SSE-S3 decryption for object") + } else { + // Handle single-part SSE-S3 objects + // Extract SSE-S3 key from metadata + keyManager := GetSSES3KeyManager() + if keyData, exists := entry.Extended[s3_constants.SeaweedFSSSES3Key]; !exists { + glog.Errorf("SSE-S3 key metadata not found in object entry") + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return http.StatusInternalServerError, 0 + } else { + sseS3Key, err := DeserializeSSES3Metadata(keyData, keyManager) + if err != nil { + glog.Errorf("Failed to deserialize SSE-S3 metadata: %v", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return http.StatusInternalServerError, 0 + } + + // Extract IV from metadata using helper function + iv, err := GetSSES3IV(entry, sseS3Key, keyManager) + if err != nil { + glog.Errorf("Failed to get SSE-S3 IV: %v", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return http.StatusInternalServerError, 0 + } + + singlePartReader, decErr := CreateSSES3DecryptedReader(proxyResponse.Body, sseS3Key, iv) + if decErr != nil { + glog.Errorf("Failed to create SSE-S3 decrypted reader: %v", decErr) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return http.StatusInternalServerError, 0 + } + decryptedReader = singlePartReader + glog.V(3).Infof("Using single-part SSE-S3 decryption for object") + } + } + + // Capture existing CORS headers that may have been set by middleware + capturedCORSHeaders := captureCORSHeaders(w, corsHeaders) + + // Copy headers from proxy response (excluding body-related headers that might change and internal SeaweedFS headers) + copyResponseHeaders(w, proxyResponse, true) + + // Set correct Content-Length for SSE-S3 + if proxyResponse.Header.Get("Content-Range") == "" { + // For full object requests, encrypted length equals original length + if contentLengthStr := proxyResponse.Header.Get("Content-Length"); contentLengthStr != "" { + w.Header().Set("Content-Length", contentLengthStr) + } + } + + // Add SSE-S3 response headers + w.Header().Set(s3_constants.AmzServerSideEncryption, SSES3Algorithm) + + return writeFinalResponse(w, proxyResponse, decryptedReader, capturedCORSHeaders) +} + // addObjectLockHeadersToResponse extracts object lock metadata from entry Extended attributes // and adds the appropriate S3 headers to the response func (s3a *S3ApiServer) addObjectLockHeadersToResponse(w http.ResponseWriter, entry *filer_pb.Entry) { @@ -1049,6 +1179,10 @@ func (s3a *S3ApiServer) addSSEHeadersToResponse(proxyResponse *http.Response, en proxyResponse.Header.Set(s3_constants.AmzServerSideEncryptionAwsKmsKeyId, string(kmsKeyID)) } + case s3_constants.SSETypeS3: + // Add only SSE-S3 headers + proxyResponse.Header.Set(s3_constants.AmzServerSideEncryption, SSES3Algorithm) + default: // Unencrypted or unknown - don't set any SSE headers } @@ -1063,10 +1197,26 @@ func (s3a *S3ApiServer) detectPrimarySSEType(entry *filer_pb.Entry) string { hasSSEC := entry.Extended[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] != nil hasSSEKMS := entry.Extended[s3_constants.AmzServerSideEncryption] != nil - if hasSSEC && !hasSSEKMS { + // Check for SSE-S3: algorithm is AES256 but no customer key + if hasSSEKMS && !hasSSEC { + // Distinguish SSE-S3 from SSE-KMS: check the algorithm value and the presence of a KMS key ID + sseAlgo := string(entry.Extended[s3_constants.AmzServerSideEncryption]) + switch sseAlgo { + case s3_constants.SSEAlgorithmAES256: + // Could be SSE-S3 or SSE-KMS, check for KMS key ID + if _, hasKMSKey := entry.Extended[s3_constants.AmzServerSideEncryptionAwsKmsKeyId]; hasKMSKey { + return s3_constants.SSETypeKMS + } + // No KMS key, this is SSE-S3 + return s3_constants.SSETypeS3 + case s3_constants.SSEAlgorithmKMS: + return s3_constants.SSETypeKMS + default: + // Unknown or unsupported algorithm + return "None" + } + } else if hasSSEC && !hasSSEKMS { return s3_constants.SSETypeC - } else if hasSSEKMS && !hasSSEC { - return s3_constants.SSETypeKMS } else if hasSSEC && hasSSEKMS { // Both present - this should only happen during cross-encryption copies // Use content to determine actual encryption state @@ -1084,24 +1234,39 @@ func (s3a *S3ApiServer) detectPrimarySSEType(entry *filer_pb.Entry) string { // Count chunk types to determine primary (multipart objects) ssecChunks := 0 ssekmsChunks := 0 + sses3Chunks := 0 for _, chunk := range entry.GetChunks() { switch chunk.GetSseType() { case filer_pb.SSEType_SSE_C: ssecChunks++ case filer_pb.SSEType_SSE_KMS: - ssekmsChunks++ + if len(chunk.GetSseMetadata()) > 0 { + ssekmsChunks++ + } + case filer_pb.SSEType_SSE_S3: + if len(chunk.GetSseMetadata()) > 0 { + sses3Chunks++ + } } } // Primary type is the one with more chunks - if ssecChunks > ssekmsChunks { + // Note: Tie-breaking follows precedence order SSE-C > SSE-KMS > SSE-S3 + // Mixed encryption in an object indicates potential corruption and should not occur in normal operation + if ssecChunks > ssekmsChunks && ssecChunks > sses3Chunks { return s3_constants.SSETypeC - } else if ssekmsChunks > ssecChunks { + } else if ssekmsChunks > ssecChunks && ssekmsChunks > sses3Chunks { return s3_constants.SSETypeKMS + } else if sses3Chunks > ssecChunks && sses3Chunks > ssekmsChunks { + return s3_constants.SSETypeS3 } else if ssecChunks > 0 { - // Equal number, prefer SSE-C (shouldn't happen in practice) + // Equal number or ties - precedence: SSE-C first return s3_constants.SSETypeC + } else if ssekmsChunks > 0 { + return s3_constants.SSETypeKMS + } else if sses3Chunks > 0 { + return s3_constants.SSETypeS3 } return "None" @@ -1150,21 +1315,9 @@ func (s3a *S3ApiServer) createMultipartSSEKMSDecryptedReader(r *http.Request, pr } } - // Fallback to object-level metadata (legacy support) - if chunkSSEKMSKey == nil { - objectMetadataHeader := proxyResponse.Header.Get(s3_constants.SeaweedFSSSEKMSKeyHeader) - if objectMetadataHeader != "" { - kmsMetadataBytes, decodeErr := base64.StdEncoding.DecodeString(objectMetadataHeader) - if decodeErr == nil { - kmsKey, _ := DeserializeSSEKMSMetadata(kmsMetadataBytes) - if kmsKey != nil { - // For object-level metadata (legacy), use absolute file offset as fallback - kmsKey.ChunkOffset = chunk.GetOffset() - chunkSSEKMSKey = kmsKey - } - } - } - } + // Note: No fallback to object-level metadata for multipart objects + // Each chunk in a multipart SSE-KMS object must have its own unique IV + // Falling back to object-level metadata could lead to IV reuse or incorrect decryption if chunkSSEKMSKey == nil { return nil, fmt.Errorf("no SSE-KMS metadata found for chunk %s in multipart object", chunk.GetFileIdString()) @@ -1189,6 +1342,86 @@ func (s3a *S3ApiServer) createMultipartSSEKMSDecryptedReader(r *http.Request, pr return multiReader, nil } +// createMultipartSSES3DecryptedReader creates a reader for multipart SSE-S3 objects +func (s3a *S3ApiServer) createMultipartSSES3DecryptedReader(r *http.Request, entry *filer_pb.Entry) (io.Reader, error) { + // Sort chunks by offset to ensure correct order + chunks := entry.GetChunks() + sort.Slice(chunks, func(i, j int) bool { + return chunks[i].GetOffset() < chunks[j].GetOffset() + }) + + // Create readers for each chunk, decrypting them independently + var readers []io.Reader + keyManager := GetSSES3KeyManager() + + for _, chunk := range chunks { + // Get this chunk's encrypted data + chunkReader, err := s3a.createEncryptedChunkReader(chunk) + if err != nil { + return nil, fmt.Errorf("failed to create chunk reader: %v", err) + } + + // Handle based on chunk's encryption type + if chunk.GetSseType() == filer_pb.SSEType_SSE_S3 { + var chunkSSES3Key *SSES3Key + + // Check if this chunk has per-chunk SSE-S3 metadata + if len(chunk.GetSseMetadata()) > 0 { + // Use the per-chunk SSE-S3 metadata + sseKey, err := DeserializeSSES3Metadata(chunk.GetSseMetadata(), keyManager) + if err != nil { + glog.Errorf("Failed to deserialize per-chunk SSE-S3 metadata for chunk %s: %v", chunk.GetFileIdString(), err) + chunkReader.Close() + return nil, fmt.Errorf("failed to deserialize SSE-S3 metadata: %v", err) + } + chunkSSES3Key = sseKey + } + + // Note: No fallback to object-level metadata for multipart objects + // Each chunk in a multipart SSE-S3 object must have its own unique IV + // Falling back to object-level metadata could lead to IV reuse or incorrect decryption + + if chunkSSES3Key == nil { + chunkReader.Close() + return nil, fmt.Errorf("no SSE-S3 metadata found for chunk %s in multipart object", chunk.GetFileIdString()) + } + + // Extract IV from chunk metadata + if len(chunkSSES3Key.IV) == 0 { + chunkReader.Close() + return nil, fmt.Errorf("no IV found in SSE-S3 metadata for chunk %s", chunk.GetFileIdString()) + } + + // Create decrypted reader for this chunk + decryptedChunkReader, decErr := CreateSSES3DecryptedReader(chunkReader, chunkSSES3Key, chunkSSES3Key.IV) + if decErr != nil { + chunkReader.Close() + return nil, fmt.Errorf("failed to decrypt chunk: %v", decErr) + } + + // Use the streaming decrypted reader directly, ensuring the underlying chunkReader can be closed + readers = append(readers, struct { + io.Reader + io.Closer + }{ + Reader: decryptedChunkReader, + Closer: chunkReader, + }) + glog.V(4).Infof("Added streaming decrypted reader for chunk %s in multipart SSE-S3 object", chunk.GetFileIdString()) + } else { + // Non-SSE-S3 chunk (unencrypted or other encryption type), use as-is + readers = append(readers, chunkReader) + glog.V(4).Infof("Added passthrough reader for non-SSE-S3 chunk %s (type: %v)", chunk.GetFileIdString(), chunk.GetSseType()) + } + } + + // Combine all decrypted chunk readers into a single stream + multiReader := NewMultipartSSEReader(readers) + glog.V(3).Infof("Created multipart SSE-S3 decrypted reader with %d chunks", len(readers)) + + return multiReader, nil +} + // createEncryptedChunkReader creates a reader for a single encrypted chunk func (s3a *S3ApiServer) createEncryptedChunkReader(chunk *filer_pb.FileChunk) (io.ReadCloser, error) { // Get chunk URL diff --git a/weed/s3api/s3api_object_handlers_copy.go b/weed/s3api/s3api_object_handlers_copy.go index a71b52a39..65de55d1e 100644 --- a/weed/s3api/s3api_object_handlers_copy.go +++ b/weed/s3api/s3api_object_handlers_copy.go @@ -1152,7 +1152,7 @@ func (s3a *S3ApiServer) copyMultipartSSECChunks(entry *filer_pb.Entry, copySourc dstMetadata := make(map[string][]byte) if destKey != nil && len(destIV) > 0 { // Store the IV and SSE-C headers for single-part compatibility - StoreIVInMetadata(dstMetadata, destIV) + StoreSSECIVInMetadata(dstMetadata, destIV) dstMetadata[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256") dstMetadata[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(destKey.KeyMD5) glog.V(2).Infof("Prepared multipart SSE-C destination metadata: %s", dstPath) @@ -1504,7 +1504,7 @@ func (s3a *S3ApiServer) copyMultipartCrossEncryption(entry *filer_pb.Entry, r *h if len(dstChunks) > 0 && dstChunks[0].GetSseType() == filer_pb.SSEType_SSE_C && len(dstChunks[0].GetSseMetadata()) > 0 { if ssecMetadata, err := DeserializeSSECMetadata(dstChunks[0].GetSseMetadata()); err == nil { if iv, ivErr := base64.StdEncoding.DecodeString(ssecMetadata.IV); ivErr == nil { - StoreIVInMetadata(dstMetadata, iv) + StoreSSECIVInMetadata(dstMetadata, iv) dstMetadata[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256") dstMetadata[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(destSSECKey.KeyMD5) } @@ -1772,7 +1772,7 @@ func (s3a *S3ApiServer) copyChunksWithSSEC(entry *filer_pb.Entry, r *http.Reques dstMetadata := make(map[string][]byte) if destKey != nil && len(destIV) > 0 { // Store the IV - StoreIVInMetadata(dstMetadata, destIV) + StoreSSECIVInMetadata(dstMetadata, destIV) // Store SSE-C algorithm and key MD5 for proper metadata dstMetadata[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256") @@ -1855,7 +1855,7 @@ func (s3a *S3ApiServer) copyChunkWithReencryption(chunk *filer_pb.FileChunk, cop // Decrypt if source is encrypted if copySourceKey != nil { // Get IV from source metadata - srcIV, err := GetIVFromMetadata(srcMetadata) + srcIV, err := GetSSECIVFromMetadata(srcMetadata) if err != nil { return nil, fmt.Errorf("failed to get IV from metadata: %w", err) } diff --git a/weed/s3api/s3api_streaming_copy.go b/weed/s3api/s3api_streaming_copy.go index 7c52a918c..49480b6ea 100644 --- a/weed/s3api/s3api_streaming_copy.go +++ b/weed/s3api/s3api_streaming_copy.go @@ -256,7 +256,7 @@ func (scm *StreamingCopyManager) createDecryptionReader(reader io.Reader, encSpe case EncryptionTypeSSEC: if sourceKey, ok := encSpec.SourceKey.(*SSECustomerKey); ok { // Get IV from metadata - iv, err := GetIVFromMetadata(encSpec.SourceMetadata) + iv, err := GetSSECIVFromMetadata(encSpec.SourceMetadata) if err != nil { return nil, fmt.Errorf("get IV from metadata: %w", err) } @@ -272,10 +272,10 @@ func (scm *StreamingCopyManager) createDecryptionReader(reader io.Reader, encSpe case EncryptionTypeSSES3: if sseKey, ok := encSpec.SourceKey.(*SSES3Key); ok { - // Get IV from metadata - iv, err := GetIVFromMetadata(encSpec.SourceMetadata) - if err != nil { - return nil, fmt.Errorf("get IV from metadata: %w", err) + // For SSE-S3, the IV is stored within the SSES3Key metadata, not as separate metadata + iv := sseKey.IV + if len(iv) == 0 { + return nil, fmt.Errorf("SSE-S3 key is missing IV for streaming copy") } return CreateSSES3DecryptedReader(reader, sseKey, iv) } diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index 24f8e1b56..0d354ee8c 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -129,6 +129,7 @@ const ( ErrSSECustomerKeyMD5Mismatch ErrSSECustomerKeyMissing ErrSSECustomerKeyNotNeeded + ErrSSEEncryptionTypeMismatch // SSE-KMS related errors ErrKMSKeyNotFound @@ -540,6 +541,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The object was not encrypted with customer provided keys.", HTTPStatusCode: http.StatusBadRequest, }, + ErrSSEEncryptionTypeMismatch: { + Code: "InvalidRequest", + Description: "The encryption method specified in the request does not match the encryption method used to encrypt the object.", + HTTPStatusCode: http.StatusBadRequest, + }, // SSE-KMS error responses ErrKMSKeyNotFound: { diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index ab474eef0..92aadcfc8 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -192,9 +192,9 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) // print out the header from extended properties for k, v := range entry.Extended { - if !strings.HasPrefix(k, "xattr-") && !strings.HasPrefix(k, "x-seaweedfs-") { + if !strings.HasPrefix(k, "xattr-") && !s3_constants.IsSeaweedFSInternalHeader(k) { // "xattr-" prefix is set in filesys.XATTR_PREFIX - // "x-seaweedfs-" prefix is for internal metadata that should not become HTTP headers + // IsSeaweedFSInternalHeader filters internal metadata that should not become HTTP headers w.Header().Set(k, string(v)) } } @@ -241,6 +241,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.Header().Set(s3_constants.SeaweedFSSSEKMSKeyHeader, kmsBase64) } + if _, exists := entry.Extended[s3_constants.SeaweedFSSSES3Key]; exists { + // Set standard S3 SSE-S3 response header (not the internal SeaweedFS header) + w.Header().Set(s3_constants.AmzServerSideEncryption, s3_constants.SSEAlgorithmAES256) + } + SetEtag(w, etag) filename := entry.Name() diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index ca36abcac..d2b3d8b52 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -377,6 +377,16 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa } } + if sseS3Header := r.Header.Get(s3_constants.SeaweedFSSSES3Key); sseS3Header != "" { + // Decode base64-encoded S3 metadata and store + if s3Data, err := base64.StdEncoding.DecodeString(sseS3Header); err == nil { + entry.Extended[s3_constants.SeaweedFSSSES3Key] = s3Data + glog.V(4).Infof("Stored SSE-S3 metadata for %s", entry.FullPath) + } else { + glog.Errorf("Failed to decode SSE-S3 metadata header for %s: %v", entry.FullPath, err) + } + } + dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, skipCheckParentDirEntry(r), so.MaxFileNameLength) // In test_bucket_listv2_delimiter_basic, the valid object key is the parent folder if dbErr != nil && strings.HasSuffix(dbErr.Error(), " is a file") && isS3Request(r) { diff --git a/weed/topology/volume_growth_reservation_test.go b/weed/topology/volume_growth_reservation_test.go index 442995b80..a29d924bd 100644 --- a/weed/topology/volume_growth_reservation_test.go +++ b/weed/topology/volume_growth_reservation_test.go @@ -181,23 +181,35 @@ func TestVolumeGrowth_ConcurrentAllocationPreventsRaceCondition(t *testing.T) { wg.Wait() - // With reservation system, only 5 requests should succeed (capacity limit) - // The rest should fail due to insufficient capacity - if successCount.Load() != 5 { - t.Errorf("Expected exactly 5 successful reservations, got %d", successCount.Load()) + // Collect results + successes := successCount.Load() + failures := failureCount.Load() + total := successes + failures + + if total != concurrentRequests { + t.Fatalf("Expected %d total attempts recorded, got %d", concurrentRequests, total) + } + + // At most the available capacity should succeed + const capacity = 5 + if successes > capacity { + t.Errorf("Expected no more than %d successful reservations, got %d", capacity, successes) } - if failureCount.Load() != 5 { - t.Errorf("Expected exactly 5 failed reservations, got %d", failureCount.Load()) + // We should see at least the remaining attempts fail + minExpectedFailures := concurrentRequests - capacity + if failures < int32(minExpectedFailures) { + t.Errorf("Expected at least %d failed reservations, got %d", minExpectedFailures, failures) } - // Verify final state + // Verify final state matches the number of successful allocations finalAvailable := dn.AvailableSpaceFor(option) - if finalAvailable != 0 { - t.Errorf("Expected 0 available space after all allocations, got %d", finalAvailable) + expectedAvailable := int64(capacity - successes) + if finalAvailable != expectedAvailable { + t.Errorf("Expected %d available space after allocations, got %d", expectedAvailable, finalAvailable) } - t.Logf("Concurrent test completed: %d successes, %d failures", successCount.Load(), failureCount.Load()) + t.Logf("Concurrent test completed: %d successes, %d failures", successes, failures) } func TestVolumeGrowth_ReservationFailureRollback(t *testing.T) { |