diff options
| author | chrislu <chris.lu@gmail.com> | 2025-07-21 09:15:43 -0700 |
|---|---|---|
| committer | chrislu <chris.lu@gmail.com> | 2025-07-21 09:15:43 -0700 |
| commit | 77ee3081d20a2ed53d46ee086f665457a782dec0 (patch) | |
| tree | e1c65d67f99e0d4c9c1b043ce3e367d2f086e689 | |
| parent | 6772a19b6db8bb5362e82553bb0d3368b9b19f5e (diff) | |
| download | seaweedfs-77ee3081d20a2ed53d46ee086f665457a782dec0.tar.xz seaweedfs-77ee3081d20a2ed53d46ee086f665457a782dec0.zip | |
fix isTruncated in listing
| -rw-r--r-- | test/s3/versioning/s3_directory_versioning_test.go | 81 | ||||
| -rwxr-xr-x | test/s3/versioning/versioning.test | bin | 0 -> 14917874 bytes | |||
| -rw-r--r-- | weed/s3api/s3api_object_handlers_list.go | 15 |
3 files changed, 94 insertions, 2 deletions
diff --git a/test/s3/versioning/s3_directory_versioning_test.go b/test/s3/versioning/s3_directory_versioning_test.go index 5a6652832..f36d37a86 100644 --- a/test/s3/versioning/s3_directory_versioning_test.go +++ b/test/s3/versioning/s3_directory_versioning_test.go @@ -877,6 +877,87 @@ func TestNextMarkerDelimiterFix(t *testing.T) { t.Logf("✅ NextMarker correctly includes trailing slash for CommonPrefixes") } +// TestIsTruncatedLogicFix tests the IsTruncated flag behavior for list operations +// This addresses the test failures where IsTruncated was incorrectly set to true +func TestIsTruncatedLogicFix(t *testing.T) { + s3Client := setupS3Client(t) + bucketName := "test-bucket-" + fmt.Sprintf("%d", time.Now().UnixNano()) + + // Create bucket + _, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + defer cleanupBucket(t, s3Client, bucketName) + + // Create test objects that match the failing test pattern + testObjects := []string{ + "asdf", + "boo/bar", + "boo/baz/xyzzy", + "cquux/thud", + "cquux/bla", + } + + for _, objectKey := range testObjects { + _, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Body: strings.NewReader("test content for " + objectKey), + }) + require.NoError(t, err) + } + + // Test ListObjectsV2 with the exact failing pattern from the s3tests + // With delimiter="/", this creates: 'asdf' (object), 'boo/' (CommonPrefix), 'cquux/' (CommonPrefix) + // So total of 3 "items" to return + + // First call: MaxKeys=1, should return 'asdf', IsTruncated=true (2 more items) + listResponse, err := s3Client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketName), + Delimiter: aws.String("/"), + MaxKeys: aws.Int32(1), + }) + require.NoError(t, err) + + assert.True(t, *listResponse.IsTruncated, "First response should be truncated") + assert.Equal(t, 1, listResponse.KeyCount, "Should return 1 item") + assert.Len(t, listResponse.Contents, 1, "Should return 1 object") + assert.Equal(t, "asdf", aws.ToString(listResponse.Contents[0].Key), "First object should be 'asdf'") + + // Second call: continue with NextContinuationToken, MaxKeys=1, should return 'boo/', IsTruncated=true (1 more item) + listResponse, err = s3Client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketName), + Delimiter: aws.String("/"), + MaxKeys: aws.Int32(1), + ContinuationToken: listResponse.NextContinuationToken, + }) + require.NoError(t, err) + + assert.True(t, *listResponse.IsTruncated, "Second response should be truncated") + assert.Equal(t, 1, listResponse.KeyCount, "Should return 1 item") + assert.Len(t, listResponse.CommonPrefixes, 1, "Should return 1 CommonPrefix") + assert.Equal(t, "boo/", aws.ToString(listResponse.CommonPrefixes[0].Prefix), "CommonPrefix should be 'boo/'") + + // Third call: continue with NextContinuationToken, MaxKeys=1, should return 'cquux/', IsTruncated=FALSE (no more items) + listResponse, err = s3Client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketName), + Delimiter: aws.String("/"), + MaxKeys: aws.Int32(1), + ContinuationToken: listResponse.NextContinuationToken, + }) + require.NoError(t, err) + + // This is the critical assertion that was failing before the fix + assert.False(t, *listResponse.IsTruncated, "Third response should NOT be truncated (this was the bug)") + assert.Equal(t, 1, listResponse.KeyCount, "Should return 1 item") + assert.Len(t, listResponse.CommonPrefixes, 1, "Should return 1 CommonPrefix") + assert.Equal(t, "cquux/", aws.ToString(listResponse.CommonPrefixes[0].Prefix), "CommonPrefix should be 'cquux/'") + assert.Nil(t, listResponse.NextContinuationToken, "NextContinuationToken should be nil when not truncated") + + t.Logf("✅ IsTruncated logic correctly identifies when there are no more results") +} + // Helper function to setup S3 client func setupS3Client(t *testing.T) *s3.Client { // S3TestConfig holds configuration for S3 tests diff --git a/test/s3/versioning/versioning.test b/test/s3/versioning/versioning.test Binary files differnew file mode 100755 index 000000000..0b7e16d28 --- /dev/null +++ b/test/s3/versioning/versioning.test diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index 23aacd638..0e9e58c40 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -391,8 +391,18 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d } } if cursor.maxKeys <= 0 { - cursor.isTruncated = true - continue + // Check if there are more entries available by trying to peek at the next one + _, nextRecvErr := stream.Recv() + if nextRecvErr == nil { + // There is another entry available, so we're truncated + cursor.isTruncated = true + } else if nextRecvErr != io.EOF { + // Some other error occurred + err = fmt.Errorf("peeking next entry: %v", nextRecvErr) + return + } + // If nextRecvErr == io.EOF, there are no more entries, so isTruncated remains false + break } entry := resp.Entry nextMarker = entry.Name @@ -462,6 +472,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d // Create logical entries for objects that have .versions directories for _, versionsDir := range versionsDirs { if cursor.maxKeys <= 0 { + // We have versioned objects remaining but can't process them due to limit cursor.isTruncated = true break } |
