aboutsummaryrefslogtreecommitdiff
path: root/weed/s3api
diff options
context:
space:
mode:
Diffstat (limited to 'weed/s3api')
-rw-r--r--weed/s3api/filer_multipart.go75
-rw-r--r--weed/s3api/filer_multipart_test.go23
-rw-r--r--weed/s3api/filer_util.go7
-rw-r--r--weed/s3api/s3api_bucket_handlers.go2
-rw-r--r--weed/s3api/s3api_object_copy_handlers.go6
-rw-r--r--weed/s3api/s3api_objects_list_handlers.go4
6 files changed, 88 insertions, 29 deletions
diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go
index 4eb9bf32c..6989d3f5a 100644
--- a/weed/s3api/filer_multipart.go
+++ b/weed/s3api/filer_multipart.go
@@ -56,7 +56,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
- entries, err := s3a.list(uploadDirectory, "", "", false, 0)
+ entries, _, err := s3a.list(uploadDirectory, "", "", false, 0)
if err != nil || len(entries) == 0 {
glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries))
return nil, ErrNoSuchUpload
@@ -140,35 +140,50 @@ func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput
type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"`
- s3.ListMultipartUploadsOutput
+
+ // copied from s3.ListMultipartUploadsOutput, the Uploads is not converting to <Upload></Upload>
+ Bucket *string `type:"string"`
+ Delimiter *string `type:"string"`
+ EncodingType *string `type:"string" enum:"EncodingType"`
+ IsTruncated *bool `type:"boolean"`
+ KeyMarker *string `type:"string"`
+ MaxUploads *int64 `type:"integer"`
+ NextKeyMarker *string `type:"string"`
+ NextUploadIdMarker *string `type:"string"`
+ Prefix *string `type:"string"`
+ UploadIdMarker *string `type:"string"`
+ Upload []*s3.MultipartUpload `locationName:"Upload" type:"list" flattened:"true"`
}
func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
+ // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html
output = &ListMultipartUploadsResult{
- ListMultipartUploadsOutput: s3.ListMultipartUploadsOutput{
- Bucket: input.Bucket,
- Delimiter: input.Delimiter,
- EncodingType: input.EncodingType,
- KeyMarker: input.KeyMarker,
- MaxUploads: input.MaxUploads,
- Prefix: input.Prefix,
- },
+ Bucket: input.Bucket,
+ Delimiter: input.Delimiter,
+ EncodingType: input.EncodingType,
+ KeyMarker: input.KeyMarker,
+ MaxUploads: input.MaxUploads,
+ Prefix: input.Prefix,
}
- entries, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), *input.Prefix, *input.KeyMarker, true, uint32(*input.MaxUploads))
+ entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), *input.Prefix, *input.KeyMarker, true, uint32(*input.MaxUploads))
if err != nil {
glog.Errorf("listMultipartUploads %s error: %v", *input.Bucket, err)
return
}
+ output.IsTruncated = aws.Bool(!isLast)
for _, entry := range entries {
if entry.Extended != nil {
key := entry.Extended["key"]
- output.Uploads = append(output.Uploads, &s3.MultipartUpload{
+ output.Upload = append(output.Upload, &s3.MultipartUpload{
Key: objectKey(aws.String(string(key))),
UploadId: aws.String(entry.Name),
})
+ if !isLast {
+ output.NextUploadIdMarker = aws.String(entry.Name)
+ }
}
}
@@ -177,26 +192,39 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
type ListPartsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"`
- s3.ListPartsOutput
+
+ // copied from s3.ListPartsOutput, the Parts is not converting to <Part></Part>
+ Bucket *string `type:"string"`
+ IsTruncated *bool `type:"boolean"`
+ Key *string `min:"1" type:"string"`
+ MaxParts *int64 `type:"integer"`
+ NextPartNumberMarker *int64 `type:"integer"`
+ PartNumberMarker *int64 `type:"integer"`
+ Part []*s3.Part `locationName:"Part" type:"list" flattened:"true"`
+ StorageClass *string `type:"string" enum:"StorageClass"`
+ UploadId *string `type:"string"`
}
func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
+ // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html
+
output = &ListPartsResult{
- ListPartsOutput: s3.ListPartsOutput{
- Bucket: input.Bucket,
- Key: objectKey(input.Key),
- UploadId: input.UploadId,
- MaxParts: input.MaxParts, // the maximum number of parts to return.
- PartNumberMarker: input.PartNumberMarker, // the part number starts after this, exclusive
- },
+ Bucket: input.Bucket,
+ Key: objectKey(input.Key),
+ UploadId: input.UploadId,
+ MaxParts: input.MaxParts, // the maximum number of parts to return.
+ PartNumberMarker: input.PartNumberMarker, // the part number starts after this, exclusive
+ StorageClass: aws.String("STANDARD"),
}
- entries, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d.part", *input.PartNumberMarker), false, uint32(*input.MaxParts))
+ entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d.part", *input.PartNumberMarker), false, uint32(*input.MaxParts))
if err != nil {
glog.Errorf("listObjectParts %s %s error: %v", *input.Bucket, *input.UploadId, err)
return nil, ErrNoSuchUpload
}
+ output.IsTruncated = aws.Bool(!isLast)
+
for _, entry := range entries {
if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
partNumberString := entry.Name[:len(entry.Name)-len(".part")]
@@ -205,12 +233,15 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP
glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, err)
continue
}
- output.Parts = append(output.Parts, &s3.Part{
+ output.Part = append(output.Part, &s3.Part{
PartNumber: aws.Int64(int64(partNumber)),
LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0).UTC()),
Size: aws.Int64(int64(filer.FileSize(entry))),
ETag: aws.String("\"" + filer.ETag(entry) + "\""),
})
+ if !isLast {
+ output.NextPartNumberMarker = aws.Int64(int64(partNumber))
+ }
}
}
diff --git a/weed/s3api/filer_multipart_test.go b/weed/s3api/filer_multipart_test.go
index 835665dd6..f2568b6bc 100644
--- a/weed/s3api/filer_multipart_test.go
+++ b/weed/s3api/filer_multipart_test.go
@@ -4,6 +4,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"testing"
+ "time"
)
func TestInitiateMultipartUploadResult(t *testing.T) {
@@ -24,3 +25,25 @@ func TestInitiateMultipartUploadResult(t *testing.T) {
}
}
+
+func TestListPartsResult(t *testing.T) {
+
+ expected := `<?xml version="1.0" encoding="UTF-8"?>
+<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Part><ETag>&#34;12345678&#34;</ETag><LastModified>1970-01-01T00:00:00Z</LastModified><PartNumber>1</PartNumber><Size>123</Size></Part></ListPartsResult>`
+ response := &ListPartsResult{
+ Part: []*s3.Part{
+ {
+ PartNumber: aws.Int64(int64(1)),
+ LastModified: aws.Time(time.Unix(0, 0).UTC()),
+ Size: aws.Int64(int64(123)),
+ ETag: aws.String("\"12345678\""),
+ },
+ },
+ }
+
+ encoded := string(encodeResponse(response))
+ if encoded != expected {
+ t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
+ }
+
+}
diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go
index 7f49c320e..ebdbe8245 100644
--- a/weed/s3api/filer_util.go
+++ b/weed/s3api/filer_util.go
@@ -21,10 +21,13 @@ func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chun
}
-func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, inclusive bool, limit uint32) (entries []*filer_pb.Entry, err error) {
+func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, inclusive bool, limit uint32) (entries []*filer_pb.Entry, isLast bool, err error) {
- err = filer_pb.List(s3a, parentDirectoryPath, prefix, func(entry *filer_pb.Entry, isLast bool) error {
+ err = filer_pb.List(s3a, parentDirectoryPath, prefix, func(entry *filer_pb.Entry, isLastEntry bool) error {
entries = append(entries, entry)
+ if isLastEntry {
+ isLast = true
+ }
return nil
}, startFrom, inclusive, limit)
diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go
index 816db04f9..a014242c0 100644
--- a/weed/s3api/s3api_bucket_handlers.go
+++ b/weed/s3api/s3api_bucket_handlers.go
@@ -25,7 +25,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
var response ListAllMyBucketsResult
- entries, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)
+ entries, _, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)
if err != nil {
writeErrorResponse(w, ErrInternalError, r.URL)
diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go
index 80ca9afcb..6cbfe4e08 100644
--- a/weed/s3api/s3api_object_copy_handlers.go
+++ b/weed/s3api/s3api_object_copy_handlers.go
@@ -39,14 +39,14 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
srcUrl := fmt.Sprintf("http://%s%s/%s%s",
s3a.option.Filer, s3a.option.BucketsPath, srcBucket, srcObject)
- _, _, dataReader, err := util.DownloadFile(srcUrl)
+ _, _, resp, err := util.DownloadFile(srcUrl)
if err != nil {
writeErrorResponse(w, ErrInvalidCopySource, r.URL)
return
}
- defer dataReader.Close()
+ defer util.CloseResponse(resp)
- etag, errCode := s3a.putToFiler(r, dstUrl, dataReader)
+ etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body)
if errCode != ErrNone {
writeErrorResponse(w, errCode, r.URL)
diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go
index b6779dfb7..30d566f94 100644
--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -212,7 +212,9 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
InclusiveStartFrom: false,
}
- stream, listErr := client.ListEntries(context.Background(), request)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ stream, listErr := client.ListEntries(ctx, request)
if listErr != nil {
err = fmt.Errorf("list entires %+v: %v", request, listErr)
return
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-24 15:14:30 +0000