aboutsummaryrefslogtreecommitdiff
path: root/test/s3/copying/s3_copying_test.go
diff options
context:
space:
mode:
Diffstat (limited to 'test/s3/copying/s3_copying_test.go')
-rw-r--r--test/s3/copying/s3_copying_test.go1014
1 files changed, 1014 insertions, 0 deletions
diff --git a/test/s3/copying/s3_copying_test.go b/test/s3/copying/s3_copying_test.go
new file mode 100644
index 000000000..4bad01de4
--- /dev/null
+++ b/test/s3/copying/s3_copying_test.go
@@ -0,0 +1,1014 @@
+package copying_test
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "fmt"
+ "io"
+ mathrand "math/rand"
+ "net/url"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// S3TestConfig holds configuration for S3 tests
+type S3TestConfig struct {
+ Endpoint string
+ AccessKey string
+ SecretKey string
+ Region string
+ BucketPrefix string
+ UseSSL bool
+ SkipVerifySSL bool
+}
+
+// Default test configuration - should match test_config.json
+var defaultConfig = &S3TestConfig{
+ Endpoint: "http://127.0.0.1:8000", // Use explicit IPv4 address
+ AccessKey: "some_access_key1",
+ SecretKey: "some_secret_key1",
+ Region: "us-east-1",
+ BucketPrefix: "test-copying-",
+ UseSSL: false,
+ SkipVerifySSL: true,
+}
+
+// Initialize math/rand with current time to ensure randomness
+func init() {
+ mathrand.Seed(time.Now().UnixNano())
+}
+
+// getS3Client creates an AWS S3 client for testing
+func getS3Client(t *testing.T) *s3.Client {
+ cfg, err := config.LoadDefaultConfig(context.TODO(),
+ config.WithRegion(defaultConfig.Region),
+ config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+ defaultConfig.AccessKey,
+ defaultConfig.SecretKey,
+ "",
+ )),
+ config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+ func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+ return aws.Endpoint{
+ URL: defaultConfig.Endpoint,
+ SigningRegion: defaultConfig.Region,
+ HostnameImmutable: true,
+ }, nil
+ })),
+ )
+ require.NoError(t, err)
+
+ return s3.NewFromConfig(cfg, func(o *s3.Options) {
+ o.UsePathStyle = true // Important for SeaweedFS
+ })
+}
+
+// waitForS3Service waits for the S3 service to be ready
+func waitForS3Service(t *testing.T, client *s3.Client, timeout time.Duration) {
+ start := time.Now()
+ for time.Since(start) < timeout {
+ _, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{})
+ if err == nil {
+ return
+ }
+ t.Logf("Waiting for S3 service to be ready... (error: %v)", err)
+ time.Sleep(time.Second)
+ }
+ t.Fatalf("S3 service not ready after %v", timeout)
+}
+
+// getNewBucketName generates a unique bucket name
+func getNewBucketName() string {
+ timestamp := time.Now().UnixNano()
+ // Add random suffix to prevent collisions when tests run quickly
+ randomSuffix := mathrand.Intn(100000)
+ return fmt.Sprintf("%s%d-%d", defaultConfig.BucketPrefix, timestamp, randomSuffix)
+}
+
+// cleanupTestBuckets removes any leftover test buckets from previous runs
+func cleanupTestBuckets(t *testing.T, client *s3.Client) {
+ resp, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{})
+ if err != nil {
+ t.Logf("Warning: failed to list buckets for cleanup: %v", err)
+ return
+ }
+
+ for _, bucket := range resp.Buckets {
+ bucketName := *bucket.Name
+ // Only delete buckets that match our test prefix
+ if strings.HasPrefix(bucketName, defaultConfig.BucketPrefix) {
+ t.Logf("Cleaning up leftover test bucket: %s", bucketName)
+ deleteBucket(t, client, bucketName)
+ }
+ }
+}
+
+// createBucket creates a new bucket for testing
+func createBucket(t *testing.T, client *s3.Client, bucketName string) {
+ // First, try to delete the bucket if it exists (cleanup from previous failed tests)
+ deleteBucket(t, client, bucketName)
+
+ // Create the bucket
+ _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+ Bucket: aws.String(bucketName),
+ })
+ require.NoError(t, err)
+}
+
+// deleteBucket deletes a bucket and all its contents
+func deleteBucket(t *testing.T, client *s3.Client, bucketName string) {
+ // First, delete all objects
+ deleteAllObjects(t, client, bucketName)
+
+ // Then delete the bucket
+ _, err := client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
+ Bucket: aws.String(bucketName),
+ })
+ if err != nil {
+ // Only log warnings for actual errors, not "bucket doesn't exist"
+ if !strings.Contains(err.Error(), "NoSuchBucket") {
+ t.Logf("Warning: failed to delete bucket %s: %v", bucketName, err)
+ }
+ }
+}
+
+// deleteAllObjects deletes all objects in a bucket
+func deleteAllObjects(t *testing.T, client *s3.Client, bucketName string) {
+ // List all objects
+ paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
+ Bucket: aws.String(bucketName),
+ })
+
+ for paginator.HasMorePages() {
+ page, err := paginator.NextPage(context.TODO())
+ if err != nil {
+ // Only log warnings for actual errors, not "bucket doesn't exist"
+ if !strings.Contains(err.Error(), "NoSuchBucket") {
+ t.Logf("Warning: failed to list objects in bucket %s: %v", bucketName, err)
+ }
+ return
+ }
+
+ if len(page.Contents) == 0 {
+ break
+ }
+
+ var objectsToDelete []types.ObjectIdentifier
+ for _, obj := range page.Contents {
+ objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
+ Key: obj.Key,
+ })
+ }
+
+ // Delete objects in batches
+ if len(objectsToDelete) > 0 {
+ _, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
+ Bucket: aws.String(bucketName),
+ Delete: &types.Delete{
+ Objects: objectsToDelete,
+ Quiet: aws.Bool(true),
+ },
+ })
+ if err != nil {
+ t.Logf("Warning: failed to delete objects in bucket %s: %v", bucketName, err)
+ }
+ }
+ }
+}
+
+// putObject puts an object into a bucket
+func putObject(t *testing.T, client *s3.Client, bucketName, key, content string) *s3.PutObjectOutput {
+ resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(key),
+ Body: strings.NewReader(content),
+ })
+ require.NoError(t, err)
+ return resp
+}
+
+// putObjectWithMetadata puts an object with metadata into a bucket
+func putObjectWithMetadata(t *testing.T, client *s3.Client, bucketName, key, content string, metadata map[string]string, contentType string) *s3.PutObjectOutput {
+ input := &s3.PutObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(key),
+ Body: strings.NewReader(content),
+ }
+
+ if metadata != nil {
+ input.Metadata = metadata
+ }
+
+ if contentType != "" {
+ input.ContentType = aws.String(contentType)
+ }
+
+ resp, err := client.PutObject(context.TODO(), input)
+ require.NoError(t, err)
+ return resp
+}
+
+// getObject gets an object from a bucket
+func getObject(t *testing.T, client *s3.Client, bucketName, key string) *s3.GetObjectOutput {
+ resp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(key),
+ })
+ require.NoError(t, err)
+ return resp
+}
+
+// getObjectBody gets the body content of an object
+func getObjectBody(t *testing.T, resp *s3.GetObjectOutput) string {
+ body, err := io.ReadAll(resp.Body)
+ require.NoError(t, err)
+ resp.Body.Close()
+ return string(body)
+}
+
+// generateRandomData generates random data of specified size
+func generateRandomData(size int) []byte {
+ data := make([]byte, size)
+ _, err := rand.Read(data)
+ if err != nil {
+ panic(err)
+ }
+ return data
+}
+
+// createCopySource creates a properly URL-encoded copy source string
+func createCopySource(bucketName, key string) string {
+ // URL encode the key to handle special characters like spaces
+ encodedKey := url.PathEscape(key)
+ return fmt.Sprintf("%s/%s", bucketName, encodedKey)
+}
+
+// TestBasicPutGet tests basic S3 put and get operations
+func TestBasicPutGet(t *testing.T) {
+ client := getS3Client(t)
+ bucketName := getNewBucketName()
+
+ // Create bucket
+ createBucket(t, client, bucketName)
+ defer deleteBucket(t, client, bucketName)
+
+ // Test 1: Put and get a simple text object
+ t.Run("Simple text object", func(t *testing.T) {
+ key := "test-simple.txt"
+ content := "Hello, SeaweedFS S3!"
+
+ // Put object
+ putResp := putObject(t, client, bucketName, key, content)
+ assert.NotNil(t, putResp.ETag)
+
+ // Get object
+ getResp := getObject(t, client, bucketName, key)
+ body := getObjectBody(t, getResp)
+ assert.Equal(t, content, body)
+ assert.Equal(t, putResp.ETag, getResp.ETag)
+ })
+
+ // Test 2: Put and get an empty object
+ t.Run("Empty object", func(t *testing.T) {
+ key := "test-empty.txt"
+ content := ""
+
+ putResp := putObject(t, client, bucketName, key, content)
+ assert.NotNil(t, putResp.ETag)
+
+ getResp := getObject(t, client, bucketName, key)
+ body := getObjectBody(t, getResp)
+ assert.Equal(t, content, body)
+ assert.Equal(t, putResp.ETag, getResp.ETag)
+ })
+
+ // Test 3: Put and get a binary object
+ t.Run("Binary object", func(t *testing.T) {
+ key := "test-binary.bin"
+ content := string(generateRandomData(1024)) // 1KB of random data
+
+ putResp := putObject(t, client, bucketName, key, content)
+ assert.NotNil(t, putResp.ETag)
+
+ getResp := getObject(t, client, bucketName, key)
+ body := getObjectBody(t, getResp)
+ assert.Equal(t, content, body)
+ assert.Equal(t, putResp.ETag, getResp.ETag)
+ })
+
+ // Test 4: Put and get object with metadata
+ t.Run("Object with metadata", func(t *testing.T) {
+ key := "test-metadata.txt"
+ content := "Content with metadata"
+ metadata := map[string]string{
+ "author": "test",
+ "description": "test object with metadata",
+ }
+ contentType := "text/plain"
+
+ putResp := putObjectWithMetadata(t, client, bucketName, key, content, metadata, contentType)
+ assert.NotNil(t, putResp.ETag)
+
+ getResp := getObject(t, client, bucketName, key)
+ body := getObjectBody(t, getResp)
+ assert.Equal(t, content, body)
+ assert.Equal(t, putResp.ETag, getResp.ETag)
+ assert.Equal(t, contentType, *getResp.ContentType)
+ assert.Equal(t, metadata["author"], getResp.Metadata["author"])
+ assert.Equal(t, metadata["description"], getResp.Metadata["description"])
+ })
+}
+
+// TestBasicBucketOperations tests basic bucket operations
+func TestBasicBucketOperations(t *testing.T) {
+ client := getS3Client(t)
+ bucketName := getNewBucketName()
+
+ // Test 1: Create bucket
+ t.Run("Create bucket", func(t *testing.T) {
+ createBucket(t, client, bucketName)
+
+ // Verify bucket exists by listing buckets
+ resp, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{})
+ require.NoError(t, err)
+
+ found := false
+ for _, bucket := range resp.Buckets {
+ if *bucket.Name == bucketName {
+ found = true
+ break
+ }
+ }
+ assert.True(t, found, "Bucket should exist after creation")
+ })
+
+ // Test 2: Put objects and list them
+ t.Run("List objects", func(t *testing.T) {
+ // Put multiple objects
+ objects := []string{"test1.txt", "test2.txt", "dir/test3.txt"}
+ for _, key := range objects {
+ putObject(t, client, bucketName, key, fmt.Sprintf("content of %s", key))
+ }
+
+ // List objects
+ resp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+ Bucket: aws.String(bucketName),
+ })
+ require.NoError(t, err)
+
+ assert.Equal(t, len(objects), len(resp.Contents))
+
+ // Verify each object exists
+ for _, obj := range resp.Contents {
+ found := false
+ for _, expected := range objects {
+ if *obj.Key == expected {
+ found = true
+ break
+ }
+ }
+ assert.True(t, found, "Object %s should be in list", *obj.Key)
+ }
+ })
+
+ // Test 3: Delete bucket (cleanup)
+ t.Run("Delete bucket", func(t *testing.T) {
+ deleteBucket(t, client, bucketName)
+
+ // Verify bucket is deleted by trying to list its contents
+ _, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+ Bucket: aws.String(bucketName),
+ })
+ assert.Error(t, err, "Bucket should not exist after deletion")
+ })
+}
+
+// TestBasicLargeObject tests handling of larger objects (up to volume limit)
+func TestBasicLargeObject(t *testing.T) {
+ client := getS3Client(t)
+ bucketName := getNewBucketName()
+
+ createBucket(t, client, bucketName)
+ defer deleteBucket(t, client, bucketName)
+
+ // Test with progressively larger objects
+ sizes := []int{
+ 1024, // 1KB
+ 1024 * 10, // 10KB
+ 1024 * 100, // 100KB
+ 1024 * 1024, // 1MB
+ 1024 * 1024 * 5, // 5MB
+ 1024 * 1024 * 10, // 10MB
+ }
+
+ for _, size := range sizes {
+ t.Run(fmt.Sprintf("Size_%dMB", size/(1024*1024)), func(t *testing.T) {
+ key := fmt.Sprintf("large-object-%d.bin", size)
+ content := string(generateRandomData(size))
+
+ putResp := putObject(t, client, bucketName, key, content)
+ assert.NotNil(t, putResp.ETag)
+
+ getResp := getObject(t, client, bucketName, key)
+ body := getObjectBody(t, getResp)
+ assert.Equal(t, len(content), len(body))
+ assert.Equal(t, content, body)
+ assert.Equal(t, putResp.ETag, getResp.ETag)
+ })
+ }
+}
+
+// TestObjectCopySameBucket tests copying an object within the same bucket
+func TestObjectCopySameBucket(t *testing.T) {
+ client := getS3Client(t)
+
+ // Wait for S3 service to be ready
+ waitForS3Service(t, client, 30*time.Second)
+
+ bucketName := getNewBucketName()
+
+ // Create bucket
+ createBucket(t, client, bucketName)
+ defer deleteBucket(t, client, bucketName)
+
+ // Put source object
+ sourceKey := "foo123bar"
+ sourceContent := "foo"
+ putObject(t, client, bucketName, sourceKey, sourceContent)
+
+ // Copy object within the same bucket
+ destKey := "bar321foo"
+ copySource := createCopySource(bucketName, sourceKey)
+ _, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(destKey),
+ CopySource: aws.String(copySource),
+ })
+ require.NoError(t, err, "Failed to copy object within same bucket")
+
+ // Verify the copied object
+ resp := getObject(t, client, bucketName, destKey)
+ body := getObjectBody(t, resp)
+ assert.Equal(t, sourceContent, body)
+}
+
+// TestObjectCopyDiffBucket tests copying an object to a different bucket
+func TestObjectCopyDiffBucket(t *testing.T) {
+ client := getS3Client(t)
+ sourceBucketName := getNewBucketName()
+ destBucketName := getNewBucketName()
+
+ // Create buckets
+ createBucket(t, client, sourceBucketName)
+ defer deleteBucket(t, client, sourceBucketName)
+ createBucket(t, client, destBucketName)
+ defer deleteBucket(t, client, destBucketName)
+
+ // Put source object
+ sourceKey := "foo123bar"
+ sourceContent := "foo"
+ putObject(t, client, sourceBucketName, sourceKey, sourceContent)
+
+ // Copy object to different bucket
+ destKey := "bar321foo"
+ copySource := createCopySource(sourceBucketName, sourceKey)
+ _, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ CopySource: aws.String(copySource),
+ })
+ require.NoError(t, err)
+
+ // Verify the copied object
+ resp := getObject(t, client, destBucketName, destKey)
+ body := getObjectBody(t, resp)
+ assert.Equal(t, sourceContent, body)
+}
+
+// TestObjectCopyCannedAcl tests copying with ACL settings
+func TestObjectCopyCannedAcl(t *testing.T) {
+ client := getS3Client(t)
+ bucketName := getNewBucketName()
+
+ // Create bucket
+ createBucket(t, client, bucketName)
+ defer deleteBucket(t, client, bucketName)
+
+ // Put source object
+ sourceKey := "foo123bar"
+ sourceContent := "foo"
+ putObject(t, client, bucketName, sourceKey, sourceContent)
+
+ // Copy object with public-read ACL
+ destKey := "bar321foo"
+ copySource := createCopySource(bucketName, sourceKey)
+ _, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(destKey),
+ CopySource: aws.String(copySource),
+ ACL: types.ObjectCannedACLPublicRead,
+ })
+ require.NoError(t, err)
+
+ // Verify the copied object
+ resp := getObject(t, client, bucketName, destKey)
+ body := getObjectBody(t, resp)
+ assert.Equal(t, sourceContent, body)
+
+ // Test metadata replacement with ACL
+ metadata := map[string]string{"abc": "def"}
+ destKey2 := "foo123bar2"
+ copySource2 := createCopySource(bucketName, destKey)
+ _, err = client.CopyObject(context.TODO(), &s3.CopyObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(destKey2),
+ CopySource: aws.String(copySource2),
+ ACL: types.ObjectCannedACLPublicRead,
+ Metadata: metadata,
+ MetadataDirective: types.MetadataDirectiveReplace,
+ })
+ require.NoError(t, err)
+
+ // Verify the copied object with metadata
+ resp2 := getObject(t, client, bucketName, destKey2)
+ body2 := getObjectBody(t, resp2)
+ assert.Equal(t, sourceContent, body2)
+ assert.Equal(t, metadata, resp2.Metadata)
+}
+
+// TestObjectCopyRetainingMetadata tests copying while retaining metadata
+func TestObjectCopyRetainingMetadata(t *testing.T) {
+ client := getS3Client(t)
+ bucketName := getNewBucketName()
+
+ // Create bucket
+ createBucket(t, client, bucketName)
+ defer deleteBucket(t, client, bucketName)
+
+ // Test with different sizes
+ sizes := []int{3, 1024 * 1024} // 3 bytes and 1MB
+ for _, size := range sizes {
+ t.Run(fmt.Sprintf("size_%d", size), func(t *testing.T) {
+ sourceKey := fmt.Sprintf("foo123bar_%d", size)
+ sourceContent := string(generateRandomData(size))
+ contentType := "audio/ogg"
+ metadata := map[string]string{"key1": "value1", "key2": "value2"}
+
+ // Put source object with metadata
+ putObjectWithMetadata(t, client, bucketName, sourceKey, sourceContent, metadata, contentType)
+
+ // Copy object (should retain metadata)
+ destKey := fmt.Sprintf("bar321foo_%d", size)
+ copySource := createCopySource(bucketName, sourceKey)
+ _, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(destKey),
+ CopySource: aws.String(copySource),
+ })
+ require.NoError(t, err)
+
+ // Verify the copied object
+ resp := getObject(t, client, bucketName, destKey)
+ body := getObjectBody(t, resp)
+ assert.Equal(t, sourceContent, body)
+ assert.Equal(t, contentType, *resp.ContentType)
+ assert.Equal(t, metadata, resp.Metadata)
+ require.NotNil(t, resp.ContentLength)
+ assert.Equal(t, int64(size), *resp.ContentLength)
+ })
+ }
+}
+
+// TestMultipartCopySmall tests multipart copying of small files
+func TestMultipartCopySmall(t *testing.T) {
+ client := getS3Client(t)
+
+ // Clean up any leftover buckets from previous test runs
+ cleanupTestBuckets(t, client)
+
+ sourceBucketName := getNewBucketName()
+ destBucketName := getNewBucketName()
+
+ // Create buckets
+ createBucket(t, client, sourceBucketName)
+ defer deleteBucket(t, client, sourceBucketName)
+ createBucket(t, client, destBucketName)
+ defer deleteBucket(t, client, destBucketName)
+
+ // Put source object
+ sourceKey := "foo"
+ sourceContent := "x" // 1 byte
+ putObject(t, client, sourceBucketName, sourceKey, sourceContent)
+
+ // Create multipart upload
+ destKey := "mymultipart"
+ createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ })
+ require.NoError(t, err)
+ uploadID := *createResp.UploadId
+
+ // Upload part copy
+ copySource := createCopySource(sourceBucketName, sourceKey)
+ copyResp, err := client.UploadPartCopy(context.TODO(), &s3.UploadPartCopyInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ UploadId: aws.String(uploadID),
+ PartNumber: aws.Int32(1),
+ CopySource: aws.String(copySource),
+ CopySourceRange: aws.String("bytes=0-0"),
+ })
+ require.NoError(t, err)
+
+ // Complete multipart upload
+ _, err = client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ UploadId: aws.String(uploadID),
+ MultipartUpload: &types.CompletedMultipartUpload{
+ Parts: []types.CompletedPart{
+ {
+ ETag: copyResp.CopyPartResult.ETag,
+ PartNumber: aws.Int32(1),
+ },
+ },
+ },
+ })
+ require.NoError(t, err)
+
+ // Verify the copied object
+ resp := getObject(t, client, destBucketName, destKey)
+ body := getObjectBody(t, resp)
+ assert.Equal(t, sourceContent, body)
+ require.NotNil(t, resp.ContentLength)
+ assert.Equal(t, int64(1), *resp.ContentLength)
+}
+
+// TestMultipartCopyWithoutRange tests multipart copying without range specification
+func TestMultipartCopyWithoutRange(t *testing.T) {
+ client := getS3Client(t)
+
+ // Clean up any leftover buckets from previous test runs
+ cleanupTestBuckets(t, client)
+
+ sourceBucketName := getNewBucketName()
+ destBucketName := getNewBucketName()
+
+ // Create buckets
+ createBucket(t, client, sourceBucketName)
+ defer deleteBucket(t, client, sourceBucketName)
+ createBucket(t, client, destBucketName)
+ defer deleteBucket(t, client, destBucketName)
+
+ // Put source object
+ sourceKey := "source"
+ sourceContent := string(generateRandomData(10))
+ putObject(t, client, sourceBucketName, sourceKey, sourceContent)
+
+ // Create multipart upload
+ destKey := "mymultipartcopy"
+ createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ })
+ require.NoError(t, err)
+ uploadID := *createResp.UploadId
+
+ // Upload part copy without range (should copy entire object)
+ copySource := createCopySource(sourceBucketName, sourceKey)
+ copyResp, err := client.UploadPartCopy(context.TODO(), &s3.UploadPartCopyInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ UploadId: aws.String(uploadID),
+ PartNumber: aws.Int32(1),
+ CopySource: aws.String(copySource),
+ })
+ require.NoError(t, err)
+
+ // Complete multipart upload
+ _, err = client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ UploadId: aws.String(uploadID),
+ MultipartUpload: &types.CompletedMultipartUpload{
+ Parts: []types.CompletedPart{
+ {
+ ETag: copyResp.CopyPartResult.ETag,
+ PartNumber: aws.Int32(1),
+ },
+ },
+ },
+ })
+ require.NoError(t, err)
+
+ // Verify the copied object
+ resp := getObject(t, client, destBucketName, destKey)
+ body := getObjectBody(t, resp)
+ assert.Equal(t, sourceContent, body)
+ require.NotNil(t, resp.ContentLength)
+ assert.Equal(t, int64(10), *resp.ContentLength)
+}
+
+// TestMultipartCopySpecialNames tests multipart copying with special character names
+func TestMultipartCopySpecialNames(t *testing.T) {
+ client := getS3Client(t)
+
+ // Clean up any leftover buckets from previous test runs
+ cleanupTestBuckets(t, client)
+
+ sourceBucketName := getNewBucketName()
+ destBucketName := getNewBucketName()
+
+ // Create buckets
+ createBucket(t, client, sourceBucketName)
+ defer deleteBucket(t, client, sourceBucketName)
+ createBucket(t, client, destBucketName)
+ defer deleteBucket(t, client, destBucketName)
+
+ // Test with special key names
+ specialKeys := []string{" ", "_", "__", "?versionId"}
+ sourceContent := "x" // 1 byte
+ destKey := "mymultipart"
+
+ for i, sourceKey := range specialKeys {
+ t.Run(fmt.Sprintf("special_key_%d", i), func(t *testing.T) {
+ // Put source object
+ putObject(t, client, sourceBucketName, sourceKey, sourceContent)
+
+ // Create multipart upload
+ createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ })
+ require.NoError(t, err)
+ uploadID := *createResp.UploadId
+
+ // Upload part copy
+ copySource := createCopySource(sourceBucketName, sourceKey)
+ copyResp, err := client.UploadPartCopy(context.TODO(), &s3.UploadPartCopyInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ UploadId: aws.String(uploadID),
+ PartNumber: aws.Int32(1),
+ CopySource: aws.String(copySource),
+ CopySourceRange: aws.String("bytes=0-0"),
+ })
+ require.NoError(t, err)
+
+ // Complete multipart upload
+ _, err = client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ UploadId: aws.String(uploadID),
+ MultipartUpload: &types.CompletedMultipartUpload{
+ Parts: []types.CompletedPart{
+ {
+ ETag: copyResp.CopyPartResult.ETag,
+ PartNumber: aws.Int32(1),
+ },
+ },
+ },
+ })
+ require.NoError(t, err)
+
+ // Verify the copied object
+ resp := getObject(t, client, destBucketName, destKey)
+ body := getObjectBody(t, resp)
+ assert.Equal(t, sourceContent, body)
+ require.NotNil(t, resp.ContentLength)
+ assert.Equal(t, int64(1), *resp.ContentLength)
+ })
+ }
+}
+
+// TestMultipartCopyMultipleSizes tests multipart copying with various file sizes
+func TestMultipartCopyMultipleSizes(t *testing.T) {
+ client := getS3Client(t)
+
+ // Clean up any leftover buckets from previous test runs
+ cleanupTestBuckets(t, client)
+
+ sourceBucketName := getNewBucketName()
+ destBucketName := getNewBucketName()
+
+ // Create buckets
+ createBucket(t, client, sourceBucketName)
+ defer deleteBucket(t, client, sourceBucketName)
+ createBucket(t, client, destBucketName)
+ defer deleteBucket(t, client, destBucketName)
+
+ // Put source object (12MB)
+ sourceKey := "foo"
+ sourceSize := 12 * 1024 * 1024
+ sourceContent := generateRandomData(sourceSize)
+ _, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+ Bucket: aws.String(sourceBucketName),
+ Key: aws.String(sourceKey),
+ Body: bytes.NewReader(sourceContent),
+ })
+ require.NoError(t, err)
+
+ destKey := "mymultipart"
+ partSize := 5 * 1024 * 1024 // 5MB parts
+
+ // Test different copy sizes
+ testSizes := []int{
+ 5 * 1024 * 1024, // 5MB
+ 5*1024*1024 + 100*1024, // 5MB + 100KB
+ 5*1024*1024 + 600*1024, // 5MB + 600KB
+ 10*1024*1024 + 100*1024, // 10MB + 100KB
+ 10*1024*1024 + 600*1024, // 10MB + 600KB
+ 10 * 1024 * 1024, // 10MB
+ }
+
+ for _, size := range testSizes {
+ t.Run(fmt.Sprintf("size_%d", size), func(t *testing.T) {
+ // Create multipart upload
+ createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ })
+ require.NoError(t, err)
+ uploadID := *createResp.UploadId
+
+ // Upload parts
+ var parts []types.CompletedPart
+ copySource := createCopySource(sourceBucketName, sourceKey)
+
+ for i := 0; i < size; i += partSize {
+ partNum := int32(len(parts) + 1)
+ endOffset := i + partSize - 1
+ if endOffset >= size {
+ endOffset = size - 1
+ }
+
+ copyRange := fmt.Sprintf("bytes=%d-%d", i, endOffset)
+ copyResp, err := client.UploadPartCopy(context.TODO(), &s3.UploadPartCopyInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ UploadId: aws.String(uploadID),
+ PartNumber: aws.Int32(partNum),
+ CopySource: aws.String(copySource),
+ CopySourceRange: aws.String(copyRange),
+ })
+ require.NoError(t, err)
+
+ parts = append(parts, types.CompletedPart{
+ ETag: copyResp.CopyPartResult.ETag,
+ PartNumber: aws.Int32(partNum),
+ })
+ }
+
+ // Complete multipart upload
+ _, err = client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+ Bucket: aws.String(destBucketName),
+ Key: aws.String(destKey),
+ UploadId: aws.String(uploadID),
+ MultipartUpload: &types.CompletedMultipartUpload{
+ Parts: parts,
+ },
+ })
+ require.NoError(t, err)
+
+ // Verify the copied object
+ resp := getObject(t, client, destBucketName, destKey)
+ body, err := io.ReadAll(resp.Body)
+ require.NoError(t, err)
+ resp.Body.Close()
+
+ require.NotNil(t, resp.ContentLength)
+ assert.Equal(t, int64(size), *resp.ContentLength)
+ assert.Equal(t, sourceContent[:size], body)
+ })
+ }
+}
+
+// TestCopyObjectIfMatchGood tests copying with matching ETag condition
+func TestCopyObjectIfMatchGood(t *testing.T) {
+ client := getS3Client(t)
+ bucketName := getNewBucketName()
+
+ // Create bucket
+ createBucket(t, client, bucketName)
+ defer deleteBucket(t, client, bucketName)
+
+ // Put source object
+ sourceKey := "foo"
+ sourceContent := "bar"
+ putResp := putObject(t, client, bucketName, sourceKey, sourceContent)
+
+ // Copy object with matching ETag
+ destKey := "bar"
+ copySource := createCopySource(bucketName, sourceKey)
+ _, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(destKey),
+ CopySource: aws.String(copySource),
+ CopySourceIfMatch: putResp.ETag,
+ })
+ require.NoError(t, err)
+
+ // Verify the copied object
+ resp := getObject(t, client, bucketName, destKey)
+ body := getObjectBody(t, resp)
+ assert.Equal(t, sourceContent, body)
+}
+
+// TestCopyObjectIfNoneMatchFailed tests copying with non-matching ETag condition
+func TestCopyObjectIfNoneMatchFailed(t *testing.T) {
+ client := getS3Client(t)
+ bucketName := getNewBucketName()
+
+ // Create bucket
+ createBucket(t, client, bucketName)
+ defer deleteBucket(t, client, bucketName)
+
+ // Put source object
+ sourceKey := "foo"
+ sourceContent := "bar"
+ putObject(t, client, bucketName, sourceKey, sourceContent)
+
+ // Copy object with non-matching ETag (should succeed)
+ destKey := "bar"
+ copySource := createCopySource(bucketName, sourceKey)
+ _, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(destKey),
+ CopySource: aws.String(copySource),
+ CopySourceIfNoneMatch: aws.String("ABCORZ"),
+ })
+ require.NoError(t, err)
+
+ // Verify the copied object
+ resp := getObject(t, client, bucketName, destKey)
+ body := getObjectBody(t, resp)
+ assert.Equal(t, sourceContent, body)
+}
+
+// TestCopyObjectIfMatchFailed tests copying with non-matching ETag condition (should fail)
+func TestCopyObjectIfMatchFailed(t *testing.T) {
+ client := getS3Client(t)
+ bucketName := getNewBucketName()
+
+ // Create bucket
+ createBucket(t, client, bucketName)
+ defer deleteBucket(t, client, bucketName)
+
+ // Put source object
+ sourceKey := "foo"
+ sourceContent := "bar"
+ putObject(t, client, bucketName, sourceKey, sourceContent)
+
+ // Copy object with non-matching ETag (should fail)
+ destKey := "bar"
+ copySource := createCopySource(bucketName, sourceKey)
+ _, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(destKey),
+ CopySource: aws.String(copySource),
+ CopySourceIfMatch: aws.String("ABCORZ"),
+ })
+
+ // Should fail with precondition failed
+ require.Error(t, err)
+ // Note: We could check for specific error types, but SeaweedFS might return different error codes
+}
+
+// TestCopyObjectIfNoneMatchGood tests copying with matching ETag condition (should fail)
+func TestCopyObjectIfNoneMatchGood(t *testing.T) {
+ client := getS3Client(t)
+ bucketName := getNewBucketName()
+
+ // Create bucket
+ createBucket(t, client, bucketName)
+ defer deleteBucket(t, client, bucketName)
+
+ // Put source object
+ sourceKey := "foo"
+ sourceContent := "bar"
+ putResp := putObject(t, client, bucketName, sourceKey, sourceContent)
+
+ // Copy object with matching ETag for IfNoneMatch (should fail)
+ destKey := "bar"
+ copySource := createCopySource(bucketName, sourceKey)
+ _, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(destKey),
+ CopySource: aws.String(copySource),
+ CopySourceIfNoneMatch: putResp.ETag,
+ })
+
+ // Should fail with precondition failed
+ require.Error(t, err)
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 07:32:00 +0000