Merge pull request #4 from chrislusf/master

Syncing  to the original repository

5 files changed, 52 insertions, 20 deletions

diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go
index 4de1dda36..d3bde66ee 100644
--- a/weed/s3api/filer_multipart.go
+++ b/weed/s3api/filer_multipart.go
@@ -14,7 +14,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/satori/go.uuid"
+	"github.com/google/uuid"
 )
 
 type InitiateMultipartUploadResult struct {
@@ -23,7 +23,7 @@ type InitiateMultipartUploadResult struct {
 }
 
 func (s3a *S3ApiServer) createMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code ErrorCode) {
-	uploadId, _ := uuid.NewV4()
+	uploadId, _ := uuid.NewRandom()
 	uploadIdString := uploadId.String()
 
 	if err := s3a.mkdir(ctx, s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
@@ -91,6 +91,11 @@ func (s3a *S3ApiServer) completeMultipartUpload(ctx context.Context, input *s3.C
 	}
 	dirName = fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, *input.Bucket, dirName)
 
+	// remove suffix '/'
+	if strings.HasSuffix(dirName, "/") {
+		dirName = dirName[:len(dirName)-1]
+	}
+
 	err = s3a.mkFile(ctx, dirName, entryName, finalParts)
 
 	if err != nil {
@@ -101,9 +106,9 @@ func (s3a *S3ApiServer) completeMultipartUpload(ctx context.Context, input *s3.C
 	output = &CompleteMultipartUploadResult{
 		CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
 			Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer, dirName, entryName)),
-			Bucket: input.Bucket,
-			ETag:   aws.String("\"" + filer2.ETag(finalParts) + "\""),
-			Key:    objectKey(input.Key),
+			Bucket:   input.Bucket,
+			ETag:     aws.String("\"" + filer2.ETag(finalParts) + "\""),
+			Key:      objectKey(input.Key),
 		},
 	}
 
diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go
index 84e3050cd..ed9612d35 100644
--- a/weed/s3api/filer_util.go
+++ b/weed/s3api/filer_util.go
@@ -3,6 +3,7 @@ package s3api
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 	"time"
@@ -37,6 +38,7 @@ func (s3a *S3ApiServer) mkdir(ctx context.Context, parentDirectoryPath string, d
 
 		glog.V(1).Infof("mkdir: %v", request)
 		if _, err := client.CreateEntry(ctx, request); err != nil {
+			glog.V(0).Infof("mkdir %v: %v", request, err)
 			return fmt.Errorf("mkdir %s/%s: %v", parentDirectoryPath, dirName, err)
 		}
 
@@ -67,6 +69,7 @@ func (s3a *S3ApiServer) mkFile(ctx context.Context, parentDirectoryPath string,
 
 		glog.V(1).Infof("create file: %s/%s", parentDirectoryPath, fileName)
 		if _, err := client.CreateEntry(ctx, request); err != nil {
+			glog.V(0).Infof("create file %v:%v", request, err)
 			return fmt.Errorf("create file %s/%s: %v", parentDirectoryPath, fileName, err)
 		}
 
@@ -87,12 +90,25 @@ func (s3a *S3ApiServer) list(ctx context.Context, parentDirectoryPath, prefix, s
 		}
 
 		glog.V(4).Infof("read directory: %v", request)
-		resp, err := client.ListEntries(ctx, request)
+		stream, err := client.ListEntries(ctx, request)
 		if err != nil {
+			glog.V(0).Infof("read directory %v: %v", request, err)
 			return fmt.Errorf("list dir %v: %v", parentDirectoryPath, err)
 		}
 
-		entries = resp.Entries
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				} else {
+					return recvErr
+				}
+			}
+
+			entries = append(entries, resp.Entry)
+
+		}
 
 		return nil
 	})
@@ -114,6 +130,7 @@ func (s3a *S3ApiServer) rm(ctx context.Context, parentDirectoryPath string, entr
 
 		glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request)
 		if _, err := client.DeleteEntry(ctx, request); err != nil {
+			glog.V(0).Infof("delete entry %v: %v", request, err)
 			return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, err)
 		}
 
@@ -134,6 +151,7 @@ func (s3a *S3ApiServer) exists(ctx context.Context, parentDirectoryPath string,
 		glog.V(4).Infof("exists entry %v/%v: %v", parentDirectoryPath, entryName, request)
 		resp, err := client.LookupDirectoryEntry(ctx, request)
 		if err != nil {
+			glog.V(0).Infof("exists entry %v: %v", request, err)
 			return fmt.Errorf("exists entry %s/%s: %v", parentDirectoryPath, entryName, err)
 		}
 
@@ -151,4 +169,4 @@ func objectKey(key *string) *string {
 		return &t
 	}
 	return key
-}
\ No newline at end of file
+}
diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go
index 4053913fb..aa6849cbd 100644
--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -3,6 +3,7 @@ package s3api
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"path/filepath"
@@ -107,7 +108,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
 			InclusiveStartFrom: false,
 		}
 
-		resp, err := client.ListEntries(ctx, request)
+		stream, err := client.ListEntries(ctx, request)
 		if err != nil {
 			return fmt.Errorf("list buckets: %v", err)
 		}
@@ -117,7 +118,18 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
 		var counter int
 		var lastEntryName string
 		var isTruncated bool
-		for _, entry := range resp.Entries {
+
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				} else {
+					return recvErr
+				}
+			}
+
+			entry := resp.Entry
 			counter++
 			if counter > maxKeys {
 				isTruncated = true
@@ -125,9 +137,11 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
 			}
 			lastEntryName = entry.Name
 			if entry.IsDirectory {
-				commonPrefixes = append(commonPrefixes, PrefixEntry{
-					Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
-				})
+				if entry.Name != ".uploads" {
+					commonPrefixes = append(commonPrefixes, PrefixEntry{
+						Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
+					})
+				}
 			} else {
 				contents = append(contents, ListEntry{
 					Key:          fmt.Sprintf("%s%s", dir, entry.Name),
@@ -141,6 +155,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
 					StorageClass: "STANDARD",
 				})
 			}
+
 		}
 
 		response = ListBucketResult{
diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go
index 24458592d..edf634444 100644
--- a/weed/s3api/s3api_server.go
+++ b/weed/s3api/s3api_server.go
@@ -1,12 +1,6 @@
 package s3api
 
 import (
-	_ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra"
-	_ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb"
-	_ "github.com/chrislusf/seaweedfs/weed/filer2/memdb"
-	_ "github.com/chrislusf/seaweedfs/weed/filer2/mysql"
-	_ "github.com/chrislusf/seaweedfs/weed/filer2/postgres"
-	_ "github.com/chrislusf/seaweedfs/weed/filer2/redis"
 	"github.com/gorilla/mux"
 	"google.golang.org/grpc"
 	"net/http"
diff --git a/weed/s3api/s3api_xsd_generated.go b/weed/s3api/s3api_xsd_generated.go
index 573c09ede..9d62afc4e 100644
--- a/weed/s3api/s3api_xsd_generated.go
+++ b/weed/s3api/s3api_xsd_generated.go
@@ -675,7 +675,7 @@ type PostResponse struct {
 }
 
 type PrefixEntry struct {
-	Prefix string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix"`
+	Prefix string `xml:"Prefix"`
 }
 
 type PutObject struct {