aboutsummaryrefslogtreecommitdiff
path: root/weed/s3api
diff options
context:
space:
mode:
Diffstat (limited to 'weed/s3api')
-rw-r--r--weed/s3api/filer_util.go17
-rw-r--r--weed/s3api/s3api_objects_list_handlers.go17
2 files changed, 30 insertions, 4 deletions
diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go
index b93b603e2..ed9612d35 100644
--- a/weed/s3api/filer_util.go
+++ b/weed/s3api/filer_util.go
@@ -3,6 +3,7 @@ package s3api
import (
"context"
"fmt"
+ "io"
"os"
"strings"
"time"
@@ -89,13 +90,25 @@ func (s3a *S3ApiServer) list(ctx context.Context, parentDirectoryPath, prefix, s
}
glog.V(4).Infof("read directory: %v", request)
- resp, err := client.ListEntries(ctx, request)
+ stream, err := client.ListEntries(ctx, request)
if err != nil {
glog.V(0).Infof("read directory %v: %v", request, err)
return fmt.Errorf("list dir %v: %v", parentDirectoryPath, err)
}
- entries = resp.Entries
+ for {
+ resp, recvErr := stream.Recv()
+ if recvErr != nil {
+ if recvErr == io.EOF {
+ break
+ } else {
+ return recvErr
+ }
+ }
+
+ entries = append(entries, resp.Entry)
+
+ }
return nil
})
diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go
index 1fc8b6b37..aa6849cbd 100644
--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -3,6 +3,7 @@ package s3api
import (
"context"
"fmt"
+ "io"
"net/http"
"net/url"
"path/filepath"
@@ -107,7 +108,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
InclusiveStartFrom: false,
}
- resp, err := client.ListEntries(ctx, request)
+ stream, err := client.ListEntries(ctx, request)
if err != nil {
return fmt.Errorf("list buckets: %v", err)
}
@@ -117,7 +118,18 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
var counter int
var lastEntryName string
var isTruncated bool
- for _, entry := range resp.Entries {
+
+ for {
+ resp, recvErr := stream.Recv()
+ if recvErr != nil {
+ if recvErr == io.EOF {
+ break
+ } else {
+ return recvErr
+ }
+ }
+
+ entry := resp.Entry
counter++
if counter > maxKeys {
isTruncated = true
@@ -143,6 +155,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
StorageClass: "STANDARD",
})
}
+
}
response = ListBucketResult{