diff options
33 files changed, 281 insertions, 288 deletions
@@ -3,7 +3,7 @@ [](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY) [](https://twitter.com/intent/follow?screen_name=seaweedfs) -[](https://travis-ci.org/chrislusf/seaweedfs) +[](https://travis-ci.org/chrislusf/seaweedfs) [](https://godoc.org/github.com/chrislusf/seaweedfs/weed) [](https://github.com/chrislusf/seaweedfs/wiki) [](https://hub.docker.com/r/chrislusf/seaweedfs/) diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml index 851557e59..ef46c1bc4 100644 --- a/k8s/seaweedfs/Chart.yaml +++ b/k8s/seaweedfs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "2.52" -version: "2.52" +appVersion: "2.53" +version: "2.53" diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 48d36f1e9..e89e8c0f2 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - # imageTag: "2.52" - started using {.Chart.appVersion} + # imageTag: "2.53" - started using {.Chart.appVersion} imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index a5d29c451..9d21c40ef 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -213,11 +213,15 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi mode := fi.Mode() uid, gid := util.GetFileUidGid(fi) + fileSize := fi.Size() + if mode.IsDir() { + fileSize = 0 + } fileCopyTaskChan <- FileCopyTask{ sourceLocation: fileOrDir, destinationUrlPath: destPath, - fileSize: fi.Size(), + fileSize: fileSize, fileMode: fi.Mode(), uid: uid, gid: gid, diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index ab8f6bcbd..bb8ced81a 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -32,6 +32,9 @@ type AbstractSqlStore struct { dbsLock sync.Mutex } +func (store *AbstractSqlStore) CanDropWholeBucket() bool { + return store.SupportBucketTable +} func (store *AbstractSqlStore) OnBucketCreation(bucket string) { store.dbsLock.Lock() defer store.dbsLock.Unlock() diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index 3ef3cfff9..35187d034 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -71,7 +71,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry lastFileName := "" includeLastFile := false - if !isDeletingBucket { + if !isDeletingBucket || !f.Store.CanDropWholeBucket() { for { entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "") if err != nil { diff --git a/weed/filer/filerstore.go b/weed/filer/filerstore.go index a5b2f25de..38927d6fb 100644 --- a/weed/filer/filerstore.go +++ b/weed/filer/filerstore.go @@ -43,4 +43,5 @@ type FilerStore interface { type BucketAware interface { OnBucketCreation(bucket string) OnBucketDeletion(bucket string) + CanDropWholeBucket() bool } diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index 5175a87a1..2470f340c 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -23,6 +23,7 @@ type VirtualFilerStore interface { AddPathSpecificStore(path string, storeId string, store FilerStore) OnBucketCreation(bucket string) OnBucketDeletion(bucket string) + CanDropWholeBucket() bool } type FilerStoreWrapper struct { @@ -42,6 +43,13 @@ func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper { } } +func (fsw *FilerStoreWrapper) CanDropWholeBucket() bool { + if ba, ok := fsw.defaultStore.(BucketAware); ok { + return ba.CanDropWholeBucket() + } + return false +} + func (fsw *FilerStoreWrapper) OnBucketCreation(bucket string) { for _, store := range fsw.storeIdToStore { if ba, ok := store.(BucketAware); ok { diff --git a/weed/iamapi/iamapi_handlers.go b/weed/iamapi/iamapi_handlers.go index 2e5f709f3..7765d9e95 100644 --- a/weed/iamapi/iamapi_handlers.go +++ b/weed/iamapi/iamapi_handlers.go @@ -1,55 +1,13 @@ package iamapi import ( - "bytes" - "encoding/xml" "fmt" - "strconv" - - "net/http" - "net/url" - "time" - + "github.com/aws/aws-sdk-go/service/iam" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" - - "github.com/aws/aws-sdk-go/service/iam" -) - -type mimeType string - -const ( - mimeNone mimeType = "" - mimeXML mimeType = "application/xml" + "net/http" ) -func setCommonHeaders(w http.ResponseWriter) { - w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano())) - w.Header().Set("Accept-Ranges", "bytes") -} - -// Encodes the response headers into XML format. -func encodeResponse(response interface{}) []byte { - var bytesBuffer bytes.Buffer - bytesBuffer.WriteString(xml.Header) - e := xml.NewEncoder(&bytesBuffer) - e.Encode(response) - return bytesBuffer.Bytes() -} - -// If none of the http routes match respond with MethodNotAllowed -func notFoundHandler(w http.ResponseWriter, r *http.Request) { - glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI) - writeErrorResponse(w, s3err.ErrMethodNotAllowed, r.URL) -} - -func writeErrorResponse(w http.ResponseWriter, errorCode s3err.ErrorCode, reqURL *url.URL) { - apiError := s3err.GetAPIError(errorCode) - errorResponse := getRESTErrorResponse(apiError, reqURL.Path) - encodedErrorResponse := encodeResponse(errorResponse) - writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML) -} - func writeIamErrorResponse(w http.ResponseWriter, err error, object string, value string, msg error) { errCode := err.Error() errorResp := ErrorResponse{} @@ -64,42 +22,10 @@ func writeIamErrorResponse(w http.ResponseWriter, err error, object string, valu case iam.ErrCodeNoSuchEntityException: msg := fmt.Sprintf("The %s with name %s cannot be found.", object, value) errorResp.Error.Message = &msg - writeResponse(w, http.StatusNotFound, encodeResponse(errorResp), mimeXML) + s3err.WriteXMLResponse(w, http.StatusNotFound, errorResp) case iam.ErrCodeServiceFailureException: - writeResponse(w, http.StatusInternalServerError, encodeResponse(errorResp), mimeXML) + s3err.WriteXMLResponse(w, http.StatusInternalServerError, errorResp) default: - writeResponse(w, http.StatusInternalServerError, encodeResponse(errorResp), mimeXML) - } -} - -func getRESTErrorResponse(err s3err.APIError, resource string) s3err.RESTErrorResponse { - return s3err.RESTErrorResponse{ - Code: err.Code, - Message: err.Description, - Resource: resource, - RequestID: fmt.Sprintf("%d", time.Now().UnixNano()), + s3err.WriteXMLResponse(w, http.StatusInternalServerError, errorResp) } } - -func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { - setCommonHeaders(w) - if response != nil { - w.Header().Set("Content-Length", strconv.Itoa(len(response))) - } - if mType != mimeNone { - w.Header().Set("Content-Type", string(mType)) - } - w.WriteHeader(statusCode) - if response != nil { - glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response)) - _, err := w.Write(response) - if err != nil { - glog.V(0).Infof("write err: %v", err) - } - w.(http.Flusher).Flush() - } -} - -func writeSuccessResponseXML(w http.ResponseWriter, response []byte) { - writeResponse(w, http.StatusOK, response, mimeXML) -} diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index 89d283138..0826ce336 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -362,7 +362,7 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { - writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r) return } values := r.PostForm @@ -370,7 +370,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { s3cfgLock.RLock() s3cfg := &iam_pb.S3ApiConfiguration{} if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil { - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } s3cfgLock.RUnlock() @@ -411,14 +411,14 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { response, err = iama.CreatePolicy(s3cfg, values) if err != nil { glog.Errorf("CreatePolicy: %+v", err) - writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r) return } case "PutUserPolicy": response, err = iama.PutUserPolicy(s3cfg, values) if err != nil { glog.Errorf("PutUserPolicy: %+v", err) - writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r) return } case "GetUserPolicy": @@ -437,7 +437,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { errorResponse := ErrorResponse{} errorResponse.Error.Code = &errNotImplemented.Code errorResponse.Error.Message = &errNotImplemented.Description - writeResponse(w, errNotImplemented.HTTPStatusCode, encodeResponse(errorResponse), mimeXML) + s3err.WriteXMLResponse(w, errNotImplemented.HTTPStatusCode, errorResponse) return } if changed { @@ -449,5 +449,5 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { return } } - writeSuccessResponseXML(w, encodeResponse(response)) + s3err.WriteXMLResponse(w, http.StatusOK, response) } diff --git a/weed/iamapi/iamapi_server.go b/weed/iamapi/iamapi_server.go index 18af1a919..eb18e996d 100644 --- a/weed/iamapi/iamapi_server.go +++ b/weed/iamapi/iamapi_server.go @@ -12,6 +12,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" "github.com/chrislusf/seaweedfs/weed/s3api" . "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/gorilla/mux" "google.golang.org/grpc" @@ -71,7 +72,7 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) { apiRouter.Methods("POST").Path("/").HandlerFunc(iama.iam.Auth(iama.DoActions, ACTION_ADMIN)) // // NotFound - apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler) + apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler) } func (iam IamS3ApiConfigure) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) { diff --git a/weed/operation/delete_content.go b/weed/operation/delete_content.go index 8f87882b1..868cb5694 100644 --- a/weed/operation/delete_content.go +++ b/weed/operation/delete_content.go @@ -100,7 +100,7 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str go func(server string, fidList []string) { defer wg.Done() - if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, grpcDialOption, fidList, true); deleteErr != nil { + if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, grpcDialOption, fidList, false); deleteErr != nil { err = deleteErr } else if deleteResults != nil { resultChan <- deleteResults diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index d9d26756f..22df04dc0 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -150,7 +150,7 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt f(w, r) return } - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) } } diff --git a/weed/s3api/filer_multipart_test.go b/weed/s3api/filer_multipart_test.go index f2568b6bc..9e1d2307b 100644 --- a/weed/s3api/filer_multipart_test.go +++ b/weed/s3api/filer_multipart_test.go @@ -3,6 +3,7 @@ package s3api import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "testing" "time" ) @@ -19,7 +20,7 @@ func TestInitiateMultipartUploadResult(t *testing.T) { }, } - encoded := string(encodeResponse(response)) + encoded := string(s3err.EncodeXMLResponse(response)) if encoded != expected { t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected) } @@ -41,7 +42,7 @@ func TestListPartsResult(t *testing.T) { }, } - encoded := string(encodeResponse(response)) + encoded := string(s3err.EncodeXMLResponse(response)) if encoded != expected { t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected) } diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 48e8cb047..8beb954aa 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -32,7 +32,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques if s3a.iam.isEnabled() { identity, s3Err = s3a.iam.authUser(r) if s3Err != s3err.ErrNone { - writeErrorResponse(w, s3Err, r.URL) + s3err.WriteErrorResponse(w, s3Err, r) return } } @@ -42,7 +42,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques entries, _, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32) if err != nil { - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } @@ -69,7 +69,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques Buckets: buckets, } - writeSuccessResponseXML(w, encodeResponse(response)) + writeSuccessResponseXML(w, response) } func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) { @@ -95,14 +95,14 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) } return nil }); err != nil { - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } if exist, err := s3a.exists(s3a.option.BucketsPath, bucket, true); err == nil && exist { errCode = s3err.ErrBucketAlreadyExists } if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } @@ -118,7 +118,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) // create the folder for bucket, but lazily create actual collection if err := s3a.mkdir(s3a.option.BucketsPath, bucket, fn); err != nil { glog.Errorf("PutBucketHandler mkdir: %v", err) - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } @@ -130,7 +130,7 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque bucket, _ := getBucketAndObject(r) if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { - writeErrorResponse(w, err, r.URL) + s3err.WriteErrorResponse(w, err, r) return } @@ -152,11 +152,11 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque err = s3a.rm(s3a.option.BucketsPath, bucket, false, true) if err != nil { - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } - writeResponse(w, http.StatusNoContent, nil, mimeNone) + s3err.WriteEmptyResponse(w, http.StatusNoContent) } func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { @@ -164,7 +164,7 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request bucket, _ := getBucketAndObject(r) if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { - writeErrorResponse(w, err, r.URL) + s3err.WriteErrorResponse(w, err, r) return } diff --git a/weed/s3api/s3api_bucket_handlers_test.go b/weed/s3api/s3api_bucket_handlers_test.go index 7ab04830b..d5622c51c 100644 --- a/weed/s3api/s3api_bucket_handlers_test.go +++ b/weed/s3api/s3api_bucket_handlers_test.go @@ -1,6 +1,7 @@ package s3api import ( + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "testing" "time" @@ -32,7 +33,7 @@ func TestListBucketsHandler(t *testing.T) { Buckets: buckets, } - encoded := string(encodeResponse(response)) + encoded := string(s3err.EncodeXMLResponse(response)) if encoded != expected { t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected) } diff --git a/weed/s3api/s3api_handlers.go b/weed/s3api/s3api_handlers.go index 6935c75bd..b4d2c22e7 100644 --- a/weed/s3api/s3api_handlers.go +++ b/weed/s3api/s3api_handlers.go @@ -1,45 +1,16 @@ package s3api import ( - "bytes" "encoding/base64" - "encoding/xml" "fmt" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" - "net/http" - "net/url" - "strconv" - "time" - "google.golang.org/grpc" + "net/http" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) -type mimeType string - -const ( - mimeNone mimeType = "" - mimeJSON mimeType = "application/json" - mimeXML mimeType = "application/xml" -) - -func setCommonHeaders(w http.ResponseWriter) { - w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano())) - w.Header().Set("Accept-Ranges", "bytes") -} - -// Encodes the response headers into XML format. -func encodeResponse(response interface{}) []byte { - var bytesBuffer bytes.Buffer - bytesBuffer.WriteString(xml.Header) - e := xml.NewEncoder(&bytesBuffer) - e.Encode(response) - return bytesBuffer.Bytes() -} - var _ = filer_pb.FilerClient(&S3ApiServer{}) func (s3a *S3ApiServer) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { @@ -54,53 +25,12 @@ func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string { return location.Url } -// If none of the http routes match respond with MethodNotAllowed -func notFoundHandler(w http.ResponseWriter, r *http.Request) { - glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI) - writeErrorResponse(w, s3err.ErrMethodNotAllowed, r.URL) -} - -func writeErrorResponse(w http.ResponseWriter, errorCode s3err.ErrorCode, reqURL *url.URL) { - apiError := s3err.GetAPIError(errorCode) - errorResponse := getRESTErrorResponse(apiError, reqURL.Path) - encodedErrorResponse := encodeResponse(errorResponse) - writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML) -} - -func getRESTErrorResponse(err s3err.APIError, resource string) s3err.RESTErrorResponse { - return s3err.RESTErrorResponse{ - Code: err.Code, - Message: err.Description, - Resource: resource, - RequestID: fmt.Sprintf("%d", time.Now().UnixNano()), - } -} - -func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { - setCommonHeaders(w) - if response != nil { - w.Header().Set("Content-Length", strconv.Itoa(len(response))) - } - if mType != mimeNone { - w.Header().Set("Content-Type", string(mType)) - } - w.WriteHeader(statusCode) - if response != nil { - glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response)) - _, err := w.Write(response) - if err != nil { - glog.V(0).Infof("write err: %v", err) - } - w.(http.Flusher).Flush() - } -} - -func writeSuccessResponseXML(w http.ResponseWriter, response []byte) { - writeResponse(w, http.StatusOK, response, mimeXML) +func writeSuccessResponseXML(w http.ResponseWriter, response interface{}) { + s3err.WriteXMLResponse(w, http.StatusOK, response) } func writeSuccessResponseEmpty(w http.ResponseWriter) { - writeResponse(w, http.StatusOK, nil, mimeNone) + s3err.WriteEmptyResponse(w, http.StatusOK) } func validateContentMd5(h http.Header) ([]byte, error) { diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 84a85fd78..799483a18 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -32,28 +32,28 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request dir, name := fullPath.DirAndName() entry, err := s3a.getEntry(dir, name) if err != nil { - writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) } entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r)) err = s3a.touch(dir, name, entry) if err != nil { - writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) } - writeSuccessResponseXML(w, encodeResponse(CopyObjectResult{ + writeSuccessResponseXML(w, CopyObjectResult{ ETag: fmt.Sprintf("%x", entry.Attributes.Md5), LastModified: time.Now().UTC(), - })) + }) return } // If source object is empty or bucket is empty, reply back invalid copy source. if srcObject == "" || srcBucket == "" { - writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) return } if srcBucket == dstBucket && srcObject == dstObject { - writeErrorResponse(w, s3err.ErrInvalidCopyDest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidCopyDest, r) return } @@ -64,7 +64,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request _, _, resp, err := util.DownloadFile(srcUrl) if err != nil { - writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) return } defer util.CloseResponse(resp) @@ -73,7 +73,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } @@ -84,7 +84,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request LastModified: time.Now().UTC(), } - writeSuccessResponseXML(w, encodeResponse(response)) + writeSuccessResponseXML(w, response) } @@ -117,7 +117,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req srcBucket, srcObject := pathToBucketAndObject(cpSrcPath) // If source object is empty or bucket is empty, reply back invalid copy source. if srcObject == "" || srcBucket == "" { - writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) return } @@ -126,13 +126,13 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req partID, err := strconv.Atoi(partIDString) if err != nil { - writeErrorResponse(w, s3err.ErrInvalidPart, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r) return } // check partID with maximum part ID for multipart objects if partID > globalMaxPartID { - writeErrorResponse(w, s3err.ErrInvalidMaxParts, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r) return } @@ -145,7 +145,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, rangeHeader) if err != nil { - writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) return } defer dataReader.Close() @@ -154,7 +154,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req etag, errCode := s3a.putToFiler(r, dstUrl, dataReader) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } @@ -165,7 +165,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req LastModified: time.Now().UTC(), } - writeSuccessResponseXML(w, encodeResponse(response)) + writeSuccessResponseXML(w, response) } diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 17ead05a0..845c9a577 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -44,20 +44,20 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) _, err := validateContentMd5(r.Header) if err != nil { - writeErrorResponse(w, s3err.ErrInvalidDigest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r) return } if r.Header.Get("Cache-Control") != "" { if _, err = cacheobject.ParseRequestCacheControl(r.Header.Get("Cache-Control")); err != nil { - writeErrorResponse(w, s3err.ErrInvalidDigest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r) return } } if r.Header.Get("Expires") != "" { if _, err = time.Parse(http.TimeFormat, r.Header.Get("Expires")); err != nil { - writeErrorResponse(w, s3err.ErrInvalidDigest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r) return } } @@ -75,12 +75,12 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) _, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) } if s3ErrCode != s3err.ErrNone { - writeErrorResponse(w, s3ErrCode, r.URL) + s3err.WriteErrorResponse(w, s3ErrCode, r) return } } else { if authTypeStreamingSigned == rAuthType { - writeErrorResponse(w, s3err.ErrAuthNotSetup, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrAuthNotSetup, r) return } } @@ -88,7 +88,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) if strings.HasSuffix(object, "/") { if err := s3a.mkdir(s3a.option.BucketsPath, bucket+object, nil); err != nil { - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } } else { @@ -97,7 +97,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } @@ -120,7 +120,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) bucket, object := getBucketAndObject(r) if strings.HasSuffix(r.URL.Path, "/") { - writeErrorResponse(w, s3err.ErrNotImplemented, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r) return } @@ -195,13 +195,13 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h deleteXMLBytes, err := ioutil.ReadAll(r.Body) if err != nil { - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } deleteObjects := &DeleteObjectsRequest{} if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil { - writeErrorResponse(w, s3err.ErrMalformedXML, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrMalformedXML, r) return } @@ -253,7 +253,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h } deleteResp.Errors = deleteErrors - writeSuccessResponseXML(w, encodeResponse(deleteResp)) + writeSuccessResponseXML(w, deleteResp) } @@ -297,7 +297,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des if err != nil { glog.Errorf("NewRequest %s: %v", destUrl, err) - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } @@ -327,19 +327,19 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des if postErr != nil { glog.Errorf("post to filer: %v", postErr) - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } defer util.CloseResponse(resp) if resp.StatusCode == http.StatusPreconditionFailed { - writeErrorResponse(w, s3err.ErrPreconditionFailed, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrPreconditionFailed, r) return } if (resp.ContentLength == -1 || resp.StatusCode == 404) && resp.StatusCode != 304 { if r.Method != "DELETE" { - writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r) return } } diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go index 035302ae6..e1125689f 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy.go +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -26,23 +26,23 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R reader, err := r.MultipartReader() if err != nil { - writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r) return } form, err := reader.ReadForm(int64(5 * humanize.MiByte)) if err != nil { - writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r) return } defer form.RemoveAll() fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(form) if err != nil { - writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r) return } if fileBody == nil { - writeErrorResponse(w, s3err.ErrPOSTFileRequired, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrPOSTFileRequired, r) return } defer fileBody.Close() @@ -60,7 +60,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R if successRedirect != "" { redirectURL, err = url.Parse(successRedirect) if err != nil { - writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r) return } } @@ -68,13 +68,13 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R // Verify policy signature. errCode := s3a.iam.doesPolicySignatureMatch(formValues) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy")) if err != nil { - writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r) return } @@ -83,7 +83,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R postPolicyForm, err := policy.ParsePostPolicyForm(string(policyBytes)) if err != nil { - writeErrorResponse(w, s3err.ErrPostPolicyConditionInvalidFormat, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrPostPolicyConditionInvalidFormat, r) return } @@ -99,12 +99,12 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R lengthRange := postPolicyForm.Conditions.ContentLengthRange if lengthRange.Valid { if fileSize < lengthRange.Min { - writeErrorResponse(w, s3err.ErrEntityTooSmall, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrEntityTooSmall, r) return } if fileSize > lengthRange.Max { - writeErrorResponse(w, s3err.ErrEntityTooLarge, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrEntityTooLarge, r) return } } @@ -115,7 +115,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } @@ -123,7 +123,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R // Replace raw query params.. redirectURL.RawQuery = getRedirectPostRawQuery(bucket, object, etag) w.Header().Set("Location", redirectURL.String()) - writeResponse(w, http.StatusSeeOther, nil, mimeNone) + s3err.WriteEmptyResponse(w, http.StatusSeeOther) return } @@ -132,15 +132,15 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R // Decide what http response to send depending on success_action_status parameter switch successStatus { case "201": - resp := encodeResponse(PostResponse{ + resp := PostResponse{ Bucket: bucket, Key: object, ETag: `"` + etag + `"`, Location: w.Header().Get("Location"), - }) - writeResponse(w, http.StatusCreated, resp, mimeXML) + } + s3err.WriteXMLResponse(w, http.StatusCreated, resp) case "200": - writeResponse(w, http.StatusOK, nil, mimeNone) + s3err.WriteEmptyResponse(w, http.StatusOK) default: writeSuccessResponseEmpty(w) } diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 4ddb24e31..de3faaaaa 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -29,14 +29,14 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http Key: objectKey(aws.String(object)), }) - glog.V(2).Info("NewMultipartUploadHandler", string(encodeResponse(response)), errCode) + glog.V(2).Info("NewMultipartUploadHandler", s3err.EncodeXMLResponse(response), errCode) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } - writeSuccessResponseXML(w, encodeResponse(response)) + writeSuccessResponseXML(w, response) } @@ -53,14 +53,14 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r UploadId: aws.String(uploadID), }) - glog.V(2).Info("CompleteMultipartUploadHandler", string(encodeResponse(response)), errCode) + glog.V(2).Info("CompleteMultipartUploadHandler", s3err.EncodeXMLResponse(response), errCode) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } - writeSuccessResponseXML(w, encodeResponse(response)) + writeSuccessResponseXML(w, response) } @@ -78,13 +78,13 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht }) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } - glog.V(2).Info("AbortMultipartUploadHandler", string(encodeResponse(response))) + glog.V(2).Info("AbortMultipartUploadHandler", s3err.EncodeXMLResponse(response)) - writeSuccessResponseXML(w, encodeResponse(response)) + writeSuccessResponseXML(w, response) } @@ -94,13 +94,13 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query()) if maxUploads < 0 { - writeErrorResponse(w, s3err.ErrInvalidMaxUploads, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxUploads, r) return } if keyMarker != "" { // Marker not common with prefix is not implemented. if !strings.HasPrefix(keyMarker, prefix) { - writeErrorResponse(w, s3err.ErrNotImplemented, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r) return } } @@ -115,16 +115,16 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht UploadIdMarker: aws.String(uploadIDMarker), }) - glog.V(2).Info("ListMultipartUploadsHandler", string(encodeResponse(response)), errCode) + glog.V(2).Info("ListMultipartUploadsHandler", s3err.EncodeXMLResponse(response), errCode) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } // TODO handle encodingType - writeSuccessResponseXML(w, encodeResponse(response)) + writeSuccessResponseXML(w, response) } // ListObjectPartsHandler - Lists object parts in a multipart upload. @@ -133,11 +133,11 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query()) if partNumberMarker < 0 { - writeErrorResponse(w, s3err.ErrInvalidPartNumberMarker, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidPartNumberMarker, r) return } if maxParts < 0 { - writeErrorResponse(w, s3err.ErrInvalidMaxParts, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r) return } @@ -149,14 +149,14 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re UploadId: aws.String(uploadID), }) - glog.V(2).Info("ListObjectPartsHandler", string(encodeResponse(response)), errCode) + glog.V(2).Info("ListObjectPartsHandler", s3err.EncodeXMLResponse(response), errCode) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } - writeSuccessResponseXML(w, encodeResponse(response)) + writeSuccessResponseXML(w, response) } @@ -167,18 +167,18 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ uploadID := r.URL.Query().Get("uploadId") exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true) if !exists { - writeErrorResponse(w, s3err.ErrNoSuchUpload, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNoSuchUpload, r) return } partIDString := r.URL.Query().Get("partNumber") partID, err := strconv.Atoi(partIDString) if err != nil { - writeErrorResponse(w, s3err.ErrInvalidPart, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r) return } if partID > globalMaxPartID { - writeErrorResponse(w, s3err.ErrInvalidMaxParts, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r) return } @@ -195,7 +195,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ _, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) } if s3ErrCode != s3err.ErrNone { - writeErrorResponse(w, s3ErrCode, r.URL) + s3err.WriteErrorResponse(w, s3ErrCode, r) return } } @@ -207,7 +207,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader) if errCode != s3err.ErrNone { - writeErrorResponse(w, errCode, r.URL) + s3err.WriteErrorResponse(w, errCode, r) return } diff --git a/weed/s3api/s3api_object_tagging_handlers.go b/weed/s3api/s3api_object_tagging_handlers.go index 94719834c..fd3ec2ff7 100644 --- a/weed/s3api/s3api_object_tagging_handlers.go +++ b/weed/s3api/s3api_object_tagging_handlers.go @@ -25,15 +25,15 @@ func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.R if err != nil { if err == filer_pb.ErrNotFound { glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err) - writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r) } else { glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err) - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) } return } - writeSuccessResponseXML(w, encodeResponse(FromTags(tags))) + writeSuccessResponseXML(w, FromTags(tags)) } @@ -50,29 +50,29 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R input, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength)) if err != nil { glog.Errorf("PutObjectTaggingHandler read input %s: %v", r.URL, err) - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } if err = xml.Unmarshal(input, tagging); err != nil { glog.Errorf("PutObjectTaggingHandler Unmarshal %s: %v", r.URL, err) - writeErrorResponse(w, s3err.ErrMalformedXML, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrMalformedXML, r) return } tags := tagging.ToTags() if len(tags) > 10 { glog.Errorf("PutObjectTaggingHandler tags %s: %d tags more than 10", r.URL, len(tags)) - writeErrorResponse(w, s3err.ErrInvalidTag, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r) return } for k, v := range tags { if len(k) > 128 { glog.Errorf("PutObjectTaggingHandler tags %s: tag key %s longer than 128", r.URL, k) - writeErrorResponse(w, s3err.ErrInvalidTag, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r) return } if len(v) > 256 { glog.Errorf("PutObjectTaggingHandler tags %s: tag value %s longer than 256", r.URL, v) - writeErrorResponse(w, s3err.ErrInvalidTag, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r) return } } @@ -80,10 +80,10 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R if err = s3a.setTags(dir, name, tagging.ToTags()); err != nil { if err == filer_pb.ErrNotFound { glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err) - writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r) } else { glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err) - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) } return } @@ -105,10 +105,10 @@ func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *htt if err != nil { if err == filer_pb.ErrNotFound { glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err) - writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r) } else { glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err) - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) } return } diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 66c66d280..51a58af6a 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -44,11 +44,11 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query()) if maxKeys < 0 { - writeErrorResponse(w, s3err.ErrInvalidMaxKeys, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxKeys, r) return } if delimiter != "" && delimiter != "/" { - writeErrorResponse(w, s3err.ErrNotImplemented, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r) return } @@ -60,13 +60,13 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter) if err != nil { - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } if len(response.Contents) == 0 { if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists { - writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNoSuchBucket, r) return } } @@ -86,7 +86,7 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ StartAfter: startAfter, } - writeSuccessResponseXML(w, encodeResponse(responseV2)) + writeSuccessResponseXML(w, responseV2) } func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { @@ -99,29 +99,29 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query()) if maxKeys < 0 { - writeErrorResponse(w, s3err.ErrInvalidMaxKeys, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxKeys, r) return } if delimiter != "" && delimiter != "/" { - writeErrorResponse(w, s3err.ErrNotImplemented, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r) return } response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter) if err != nil { - writeErrorResponse(w, s3err.ErrInternalError, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) return } if len(response.Contents) == 0 { if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists { - writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL) + s3err.WriteErrorResponse(w, s3err.ErrNoSuchBucket, r) return } } - writeSuccessResponseXML(w, encodeResponse(response)) + writeSuccessResponseXML(w, response) } func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) { diff --git a/weed/s3api/s3api_objects_list_handlers_test.go b/weed/s3api/s3api_objects_list_handlers_test.go index 7b87b32fb..641f995b7 100644 --- a/weed/s3api/s3api_objects_list_handlers_test.go +++ b/weed/s3api/s3api_objects_list_handlers_test.go @@ -1,6 +1,7 @@ package s3api import ( + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "testing" "time" ) @@ -31,7 +32,7 @@ func TestListObjectsHandler(t *testing.T) { }}, } - encoded := string(encodeResponse(response)) + encoded := string(s3err.EncodeXMLResponse(response)) if encoded != expected { t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected) } diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 54df29492..57f4ba917 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" . "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "net/http" "strings" "time" @@ -132,6 +133,6 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.ListBucketsHandler, "LIST")) // NotFound - apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler) + apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler) } diff --git a/weed/s3api/s3api_test.go b/weed/s3api/s3api_test.go index 026766beb..6fcf8b165 100644 --- a/weed/s3api/s3api_test.go +++ b/weed/s3api/s3api_test.go @@ -1,6 +1,7 @@ package s3api import ( + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "testing" "time" ) @@ -14,7 +15,7 @@ func TestCopyObjectResponse(t *testing.T) { LastModified: time.Now(), } - println(string(encodeResponse(response))) + println(string(s3err.EncodeXMLResponse(response))) } @@ -27,6 +28,6 @@ func TestCopyPartResponse(t *testing.T) { LastModified: time.Now(), } - println(string(encodeResponse(response))) + println(string(s3err.EncodeXMLResponse(response))) } diff --git a/weed/s3api/s3err/error_handler.go b/weed/s3api/s3err/error_handler.go new file mode 100644 index 000000000..c1065fffc --- /dev/null +++ b/weed/s3api/s3err/error_handler.go @@ -0,0 +1,92 @@ +package s3err + +import ( + "bytes" + "encoding/xml" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/gorilla/mux" + "net/http" + "strconv" + "strings" + "time" +) + +type mimeType string + +const ( + mimeNone mimeType = "" + MimeXML mimeType = "application/xml" +) + +func WriteXMLResponse(w http.ResponseWriter, statusCode int, response interface{}) { + WriteResponse(w, statusCode, EncodeXMLResponse(response), MimeXML) +} + +func WriteEmptyResponse(w http.ResponseWriter, statusCode int) { + WriteResponse(w, statusCode, []byte{}, mimeNone) +} + +func WriteErrorResponse(w http.ResponseWriter, errorCode ErrorCode, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + object := vars["object"] + if strings.HasPrefix(object, "/") { + object = object[1:] + } + + apiError := GetAPIError(errorCode) + errorResponse := getRESTErrorResponse(apiError, r.URL.Path, bucket, object) + encodedErrorResponse := EncodeXMLResponse(errorResponse) + WriteResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, MimeXML) +} + +func getRESTErrorResponse(err APIError, resource string, bucket, object string) RESTErrorResponse { + return RESTErrorResponse{ + Code: err.Code, + BucketName: bucket, + Key: object, + Message: err.Description, + Resource: resource, + RequestID: fmt.Sprintf("%d", time.Now().UnixNano()), + } +} + +// Encodes the response headers into XML format. +func EncodeXMLResponse(response interface{}) []byte { + var bytesBuffer bytes.Buffer + bytesBuffer.WriteString(xml.Header) + e := xml.NewEncoder(&bytesBuffer) + e.Encode(response) + return bytesBuffer.Bytes() +} + +func setCommonHeaders(w http.ResponseWriter) { + w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano())) + w.Header().Set("Accept-Ranges", "bytes") +} + +func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { + setCommonHeaders(w) + if response != nil { + w.Header().Set("Content-Length", strconv.Itoa(len(response))) + } + if mType != mimeNone { + w.Header().Set("Content-Type", string(mType)) + } + w.WriteHeader(statusCode) + if response != nil { + glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response)) + _, err := w.Write(response) + if err != nil { + glog.V(0).Infof("write err: %v", err) + } + w.(http.Flusher).Flush() + } +} + +// If none of the http routes match respond with MethodNotAllowed +func NotFoundHandler(w http.ResponseWriter, r *http.Request) { + glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI) + WriteErrorResponse(w, ErrMethodNotAllowed, r) +} diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index 7f0ffdf86..a46bd0f04 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -15,11 +15,13 @@ type APIError struct { // RESTErrorResponse - error response format type RESTErrorResponse struct { - XMLName xml.Name `xml:"Error" json:"-"` - Code string `xml:"Code" json:"Code"` - Message string `xml:"Message" json:"Message"` - Resource string `xml:"Resource" json:"Resource"` - RequestID string `xml:"RequestId" json:"RequestId"` + XMLName xml.Name `xml:"Error" json:"-"` + Code string `xml:"Code" json:"Code"` + Message string `xml:"Message" json:"Message"` + Resource string `xml:"Resource" json:"Resource"` + RequestID string `xml:"RequestId" json:"RequestId"` + Key string `xml:"Key,omitempty" json:"Key,omitempty"` + BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"` // Underlying HTTP status code for the returned error StatusCode int `xml:"-" json:"-"` diff --git a/weed/s3api/tags_test.go b/weed/s3api/tags_test.go index 887843d6f..52adb36c1 100644 --- a/weed/s3api/tags_test.go +++ b/weed/s3api/tags_test.go @@ -2,6 +2,7 @@ package s3api import ( "encoding/xml" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "github.com/stretchr/testify/assert" "testing" ) @@ -41,7 +42,7 @@ func TestXMLMarshall(t *testing.T) { }, } - actual := string(encodeResponse(tags)) + actual := string(s3err.EncodeXMLResponse(tags)) expected := `<?xml version="1.0" encoding="UTF-8"?> <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><TagSet><Tag><Key>key1</Key><Value>value1</Value></Tag></TagSet></Tagging>` diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go index eadb970d5..ba9f15370 100644 --- a/weed/server/filer_grpc_server_rename.go +++ b/weed/server/filer_grpc_server_rename.go @@ -115,8 +115,7 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat Extended: entry.Extended, Content: entry.Content, } - createErr := fs.filer.CreateEntry(ctx, newEntry, false, false, nil) - if createErr != nil { + if createErr := fs.filer.CreateEntry(ctx, newEntry, false, false, nil); createErr != nil { return createErr } diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go index 3e6d9bb9e..50c9dbfdf 100644 --- a/weed/server/master_grpc_server.go +++ b/weed/server/master_grpc_server.go @@ -205,8 +205,8 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ _, err := stream.Recv() if err != nil { glog.V(2).Infof("- client %v: %v", clientName, err) - stopChan <- true - break + close(stopChan) + return } } }() diff --git a/weed/util/constants.go b/weed/util/constants.go index aadc09741..3dfe702ed 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 52) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 53) COMMIT = "" ) diff --git a/weed/util/grace/pprof.go b/weed/util/grace/pprof.go index 14686bfc8..0406b762c 100644 --- a/weed/util/grace/pprof.go +++ b/weed/util/grace/pprof.go @@ -14,9 +14,30 @@ func SetupProfiling(cpuProfile, memProfile string) { if err != nil { glog.Fatal(err) } + runtime.SetBlockProfileRate(1) + runtime.SetMutexProfileFraction(1) pprof.StartCPUProfile(f) OnInterrupt(func() { pprof.StopCPUProfile() + + // write block pprof + blockF, err := os.Create(cpuProfile+".block") + if err != nil { + return + } + p := pprof.Lookup("block") + p.WriteTo(blockF,0) + blockF.Close() + + // write mutex pprof + mutexF, err := os.Create(cpuProfile+".mutex") + if err != nil { + return + } + p = pprof.Lookup("mutex") + p.WriteTo(mutexF,0) + mutexF.Close() + }) } if memProfile != "" { |