aboutsummaryrefslogtreecommitdiff
path: root/weed/s3api
diff options
context:
space:
mode:
Diffstat (limited to 'weed/s3api')
-rw-r--r--weed/s3api/filer_multipart.go15
-rw-r--r--weed/s3api/filer_util.go24
-rw-r--r--weed/s3api/s3api_objects_list_handlers.go25
-rw-r--r--weed/s3api/s3api_server.go6
-rw-r--r--weed/s3api/s3api_xsd_generated.go2
5 files changed, 52 insertions, 20 deletions
diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go
index 4de1dda36..d3bde66ee 100644
--- a/weed/s3api/filer_multipart.go
+++ b/weed/s3api/filer_multipart.go
@@ -14,7 +14,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
- "github.com/satori/go.uuid"
+ "github.com/google/uuid"
)
type InitiateMultipartUploadResult struct {
@@ -23,7 +23,7 @@ type InitiateMultipartUploadResult struct {
}
func (s3a *S3ApiServer) createMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code ErrorCode) {
- uploadId, _ := uuid.NewV4()
+ uploadId, _ := uuid.NewRandom()
uploadIdString := uploadId.String()
if err := s3a.mkdir(ctx, s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
@@ -91,6 +91,11 @@ func (s3a *S3ApiServer) completeMultipartUpload(ctx context.Context, input *s3.C
}
dirName = fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, *input.Bucket, dirName)
+ // remove suffix '/'
+ if strings.HasSuffix(dirName, "/") {
+ dirName = dirName[:len(dirName)-1]
+ }
+
err = s3a.mkFile(ctx, dirName, entryName, finalParts)
if err != nil {
@@ -101,9 +106,9 @@ func (s3a *S3ApiServer) completeMultipartUpload(ctx context.Context, input *s3.C
output = &CompleteMultipartUploadResult{
CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer, dirName, entryName)),
- Bucket: input.Bucket,
- ETag: aws.String("\"" + filer2.ETag(finalParts) + "\""),
- Key: objectKey(input.Key),
+ Bucket: input.Bucket,
+ ETag: aws.String("\"" + filer2.ETag(finalParts) + "\""),
+ Key: objectKey(input.Key),
},
}
diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go
index 84e3050cd..ed9612d35 100644
--- a/weed/s3api/filer_util.go
+++ b/weed/s3api/filer_util.go
@@ -3,6 +3,7 @@ package s3api
import (
"context"
"fmt"
+ "io"
"os"
"strings"
"time"
@@ -37,6 +38,7 @@ func (s3a *S3ApiServer) mkdir(ctx context.Context, parentDirectoryPath string, d
glog.V(1).Infof("mkdir: %v", request)
if _, err := client.CreateEntry(ctx, request); err != nil {
+ glog.V(0).Infof("mkdir %v: %v", request, err)
return fmt.Errorf("mkdir %s/%s: %v", parentDirectoryPath, dirName, err)
}
@@ -67,6 +69,7 @@ func (s3a *S3ApiServer) mkFile(ctx context.Context, parentDirectoryPath string,
glog.V(1).Infof("create file: %s/%s", parentDirectoryPath, fileName)
if _, err := client.CreateEntry(ctx, request); err != nil {
+ glog.V(0).Infof("create file %v:%v", request, err)
return fmt.Errorf("create file %s/%s: %v", parentDirectoryPath, fileName, err)
}
@@ -87,12 +90,25 @@ func (s3a *S3ApiServer) list(ctx context.Context, parentDirectoryPath, prefix, s
}
glog.V(4).Infof("read directory: %v", request)
- resp, err := client.ListEntries(ctx, request)
+ stream, err := client.ListEntries(ctx, request)
if err != nil {
+ glog.V(0).Infof("read directory %v: %v", request, err)
return fmt.Errorf("list dir %v: %v", parentDirectoryPath, err)
}
- entries = resp.Entries
+ for {
+ resp, recvErr := stream.Recv()
+ if recvErr != nil {
+ if recvErr == io.EOF {
+ break
+ } else {
+ return recvErr
+ }
+ }
+
+ entries = append(entries, resp.Entry)
+
+ }
return nil
})
@@ -114,6 +130,7 @@ func (s3a *S3ApiServer) rm(ctx context.Context, parentDirectoryPath string, entr
glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request)
if _, err := client.DeleteEntry(ctx, request); err != nil {
+ glog.V(0).Infof("delete entry %v: %v", request, err)
return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, err)
}
@@ -134,6 +151,7 @@ func (s3a *S3ApiServer) exists(ctx context.Context, parentDirectoryPath string,
glog.V(4).Infof("exists entry %v/%v: %v", parentDirectoryPath, entryName, request)
resp, err := client.LookupDirectoryEntry(ctx, request)
if err != nil {
+ glog.V(0).Infof("exists entry %v: %v", request, err)
return fmt.Errorf("exists entry %s/%s: %v", parentDirectoryPath, entryName, err)
}
@@ -151,4 +169,4 @@ func objectKey(key *string) *string {
return &t
}
return key
-} \ No newline at end of file
+}
diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go
index 4053913fb..aa6849cbd 100644
--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -3,6 +3,7 @@ package s3api
import (
"context"
"fmt"
+ "io"
"net/http"
"net/url"
"path/filepath"
@@ -107,7 +108,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
InclusiveStartFrom: false,
}
- resp, err := client.ListEntries(ctx, request)
+ stream, err := client.ListEntries(ctx, request)
if err != nil {
return fmt.Errorf("list buckets: %v", err)
}
@@ -117,7 +118,18 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
var counter int
var lastEntryName string
var isTruncated bool
- for _, entry := range resp.Entries {
+
+ for {
+ resp, recvErr := stream.Recv()
+ if recvErr != nil {
+ if recvErr == io.EOF {
+ break
+ } else {
+ return recvErr
+ }
+ }
+
+ entry := resp.Entry
counter++
if counter > maxKeys {
isTruncated = true
@@ -125,9 +137,11 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
}
lastEntryName = entry.Name
if entry.IsDirectory {
- commonPrefixes = append(commonPrefixes, PrefixEntry{
- Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
- })
+ if entry.Name != ".uploads" {
+ commonPrefixes = append(commonPrefixes, PrefixEntry{
+ Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
+ })
+ }
} else {
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s%s", dir, entry.Name),
@@ -141,6 +155,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
StorageClass: "STANDARD",
})
}
+
}
response = ListBucketResult{
diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go
index 24458592d..edf634444 100644
--- a/weed/s3api/s3api_server.go
+++ b/weed/s3api/s3api_server.go
@@ -1,12 +1,6 @@
package s3api
import (
- _ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra"
- _ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb"
- _ "github.com/chrislusf/seaweedfs/weed/filer2/memdb"
- _ "github.com/chrislusf/seaweedfs/weed/filer2/mysql"
- _ "github.com/chrislusf/seaweedfs/weed/filer2/postgres"
- _ "github.com/chrislusf/seaweedfs/weed/filer2/redis"
"github.com/gorilla/mux"
"google.golang.org/grpc"
"net/http"
diff --git a/weed/s3api/s3api_xsd_generated.go b/weed/s3api/s3api_xsd_generated.go
index 573c09ede..9d62afc4e 100644
--- a/weed/s3api/s3api_xsd_generated.go
+++ b/weed/s3api/s3api_xsd_generated.go
@@ -675,7 +675,7 @@ type PostResponse struct {
}
type PrefixEntry struct {
- Prefix string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix"`
+ Prefix string `xml:"Prefix"`
}
type PutObject struct {
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-24 14:05:52 +0000