diff options
| -rw-r--r-- | weed/s3api/auth_credentials.go | 2 | ||||
| -rw-r--r-- | weed/s3api/auth_signature_v2.go | 14 | ||||
| -rw-r--r-- | weed/s3api/auth_signature_v4.go | 32 | ||||
| -rw-r--r-- | weed/s3api/policy/post-policy.go | 321 | ||||
| -rw-r--r-- | weed/s3api/policy/post-policy_test.go | 380 | ||||
| -rw-r--r-- | weed/s3api/policy/postpolicyform.go | 276 | ||||
| -rw-r--r-- | weed/s3api/policy/postpolicyform_test.go | 106 | ||||
| -rw-r--r-- | weed/s3api/s3api_object_handlers_postpolicy.go | 242 | ||||
| -rw-r--r-- | weed/s3api/s3api_server.go | 3 | ||||
| -rw-r--r-- | weed/s3api/s3err/s3-error.go | 61 | ||||
| -rw-r--r-- | weed/s3api/s3err/s3api_errors.go | 48 |
11 files changed, 1482 insertions, 3 deletions
diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index bdd7f1f43..2b2e2858d 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -153,7 +153,7 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) identity, s3Err = iam.reqSignatureV4Verify(r) case authTypePostPolicy: glog.V(3).Infof("post policy auth type") - return s3err.ErrNotImplemented + identity, s3Err = iam.reqSignatureV4Verify(r) case authTypeJWT: glog.V(3).Infof("jwt auth type") return s3err.ErrNotImplemented diff --git a/weed/s3api/auth_signature_v2.go b/weed/s3api/auth_signature_v2.go index 3953e616a..5694a96ac 100644 --- a/weed/s3api/auth_signature_v2.go +++ b/weed/s3api/auth_signature_v2.go @@ -69,6 +69,20 @@ func (iam *IdentityAccessManagement) isReqAuthenticatedV2(r *http.Request) (*Ide return iam.doesPresignV2SignatureMatch(r) } +func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http.Header) s3err.ErrorCode { + accessKey := formValues.Get("AWSAccessKeyId") + _, cred, found := iam.lookupByAccessKey(accessKey) + if !found { + return s3err.ErrInvalidAccessKeyID + } + policy := formValues.Get("Policy") + signature := formValues.Get("Signature") + if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) { + return s3err.ErrSignatureDoesNotMatch + } + return s3err.ErrNone +} + // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature; // Signature = Base64( HMAC-SHA1( YourSecretKey, UTF-8-Encoding-Of( StringToSign ) ) ); // diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 121c793d8..5d428fdec 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -293,6 +293,38 @@ func parseSignature(signElement string) (string, s3err.ErrorCode) { return signature, s3err.ErrNone } + +// doesPolicySignatureMatch - Verify query headers with post policy +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html +// returns ErrNone if the signature matches. +func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.Header) s3err.ErrorCode { + + // Parse credential tag. + credHeader, err := parseCredentialHeader("Credential="+formValues.Get("X-Amz-Credential")) + if err != s3err.ErrNone { + return s3err.ErrMissingFields + } + + _, cred, found := iam.lookupByAccessKey(credHeader.accessKey) + if !found { + return s3err.ErrInvalidAccessKeyID + } + + // Get signing key. + signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region) + + // Get signature. + newSignature := getSignature(signingKey, formValues.Get("Policy")) + + // Verify signature. + if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) { + return s3err.ErrSignatureDoesNotMatch + } + + // Success. + return s3err.ErrNone +} + // check query headers with presigned signature // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) { diff --git a/weed/s3api/policy/post-policy.go b/weed/s3api/policy/post-policy.go new file mode 100644 index 000000000..5ef8d397d --- /dev/null +++ b/weed/s3api/policy/post-policy.go @@ -0,0 +1,321 @@ +package policy + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ( + "encoding/base64" + "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "net/http" + "strings" + "time" +) + +// expirationDateFormat date format for expiration key in json policy. +const expirationDateFormat = "2006-01-02T15:04:05.999Z" + +// policyCondition explanation: +// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html +// +// Example: +// +// policyCondition { +// matchType: "$eq", +// key: "$Content-Type", +// value: "image/png", +// } +// +type policyCondition struct { + matchType string + condition string + value string +} + +// PostPolicy - Provides strict static type conversion and validation +// for Amazon S3's POST policy JSON string. +type PostPolicy struct { + // Expiration date and time of the POST policy. + expiration time.Time + // Collection of different policy conditions. + conditions []policyCondition + // ContentLengthRange minimum and maximum allowable size for the + // uploaded content. + contentLengthRange struct { + min int64 + max int64 + } + + // Post form data. + formData map[string]string +} + +// NewPostPolicy - Instantiate new post policy. +func NewPostPolicy() *PostPolicy { + p := &PostPolicy{} + p.conditions = make([]policyCondition, 0) + p.formData = make(map[string]string) + return p +} + +// SetExpires - Sets expiration time for the new policy. +func (p *PostPolicy) SetExpires(t time.Time) error { + if t.IsZero() { + return errInvalidArgument("No expiry time set.") + } + p.expiration = t + return nil +} + +// SetKey - Sets an object name for the policy based upload. +func (p *PostPolicy) SetKey(key string) error { + if strings.TrimSpace(key) == "" || key == "" { + return errInvalidArgument("Object name is empty.") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$key", + value: key, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["key"] = key + return nil +} + +// SetKeyStartsWith - Sets an object name that an policy based upload +// can start with. +func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error { + if strings.TrimSpace(keyStartsWith) == "" || keyStartsWith == "" { + return errInvalidArgument("Object prefix is empty.") + } + policyCond := policyCondition{ + matchType: "starts-with", + condition: "$key", + value: keyStartsWith, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["key"] = keyStartsWith + return nil +} + +// SetBucket - Sets bucket at which objects will be uploaded to. +func (p *PostPolicy) SetBucket(bucketName string) error { + if strings.TrimSpace(bucketName) == "" || bucketName == "" { + return errInvalidArgument("Bucket name is empty.") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$bucket", + value: bucketName, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["bucket"] = bucketName + return nil +} + +// SetCondition - Sets condition for credentials, date and algorithm +func (p *PostPolicy) SetCondition(matchType, condition, value string) error { + if strings.TrimSpace(value) == "" || value == "" { + return errInvalidArgument("No value specified for condition") + } + + policyCond := policyCondition{ + matchType: matchType, + condition: "$" + condition, + value: value, + } + if condition == "X-Amz-Credential" || condition == "X-Amz-Date" || condition == "X-Amz-Algorithm" { + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData[condition] = value + return nil + } + return errInvalidArgument("Invalid condition in policy") +} + +// SetContentType - Sets content-type of the object for this policy +// based upload. +func (p *PostPolicy) SetContentType(contentType string) error { + if strings.TrimSpace(contentType) == "" || contentType == "" { + return errInvalidArgument("No content type specified.") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$Content-Type", + value: contentType, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["Content-Type"] = contentType + return nil +} + +// SetContentLengthRange - Set new min and max content length +// condition for all incoming uploads. +func (p *PostPolicy) SetContentLengthRange(min, max int64) error { + if min > max { + return errInvalidArgument("Minimum limit is larger than maximum limit.") + } + if min < 0 { + return errInvalidArgument("Minimum limit cannot be negative.") + } + if max < 0 { + return errInvalidArgument("Maximum limit cannot be negative.") + } + p.contentLengthRange.min = min + p.contentLengthRange.max = max + return nil +} + +// SetSuccessActionRedirect - Sets the redirect success url of the object for this policy +// based upload. +func (p *PostPolicy) SetSuccessActionRedirect(redirect string) error { + if strings.TrimSpace(redirect) == "" || redirect == "" { + return errInvalidArgument("Redirect is empty") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$success_action_redirect", + value: redirect, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["success_action_redirect"] = redirect + return nil +} + +// SetSuccessStatusAction - Sets the status success code of the object for this policy +// based upload. +func (p *PostPolicy) SetSuccessStatusAction(status string) error { + if strings.TrimSpace(status) == "" || status == "" { + return errInvalidArgument("Status is empty") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$success_action_status", + value: status, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["success_action_status"] = status + return nil +} + +// SetUserMetadata - Set user metadata as a key/value couple. +// Can be retrieved through a HEAD request or an event. +func (p *PostPolicy) SetUserMetadata(key string, value string) error { + if strings.TrimSpace(key) == "" || key == "" { + return errInvalidArgument("Key is empty") + } + if strings.TrimSpace(value) == "" || value == "" { + return errInvalidArgument("Value is empty") + } + headerName := fmt.Sprintf("x-amz-meta-%s", key) + policyCond := policyCondition{ + matchType: "eq", + condition: fmt.Sprintf("$%s", headerName), + value: value, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData[headerName] = value + return nil +} + +// SetUserData - Set user data as a key/value couple. +// Can be retrieved through a HEAD request or an event. +func (p *PostPolicy) SetUserData(key string, value string) error { + if key == "" { + return errInvalidArgument("Key is empty") + } + if value == "" { + return errInvalidArgument("Value is empty") + } + headerName := fmt.Sprintf("x-amz-%s", key) + policyCond := policyCondition{ + matchType: "eq", + condition: fmt.Sprintf("$%s", headerName), + value: value, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData[headerName] = value + return nil +} + +// addNewPolicy - internal helper to validate adding new policies. +func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { + if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" { + return errInvalidArgument("Policy fields are empty.") + } + p.conditions = append(p.conditions, policyCond) + return nil +} + +// String function for printing policy in json formatted string. +func (p PostPolicy) String() string { + return string(p.marshalJSON()) +} + +// marshalJSON - Provides Marshaled JSON in bytes. +func (p PostPolicy) marshalJSON() []byte { + expirationStr := `"expiration":"` + p.expiration.Format(expirationDateFormat) + `"` + var conditionsStr string + conditions := []string{} + for _, po := range p.conditions { + conditions = append(conditions, fmt.Sprintf("[\"%s\",\"%s\",\"%s\"]", po.matchType, po.condition, po.value)) + } + if p.contentLengthRange.min != 0 || p.contentLengthRange.max != 0 { + conditions = append(conditions, fmt.Sprintf("[\"content-length-range\", %d, %d]", + p.contentLengthRange.min, p.contentLengthRange.max)) + } + if len(conditions) > 0 { + conditionsStr = `"conditions":[` + strings.Join(conditions, ",") + "]" + } + retStr := "{" + retStr = retStr + expirationStr + "," + retStr = retStr + conditionsStr + retStr = retStr + "}" + return []byte(retStr) +} + +// base64 - Produces base64 of PostPolicy's Marshaled json. +func (p PostPolicy) base64() string { + return base64.StdEncoding.EncodeToString(p.marshalJSON()) +} + +// errInvalidArgument - Invalid argument response. +func errInvalidArgument(message string) error { + return s3err.RESTErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "InvalidArgument", + Message: message, + RequestID: "minio", + } +} diff --git a/weed/s3api/policy/post-policy_test.go b/weed/s3api/policy/post-policy_test.go new file mode 100644 index 000000000..1bd0985f0 --- /dev/null +++ b/weed/s3api/policy/post-policy_test.go @@ -0,0 +1,380 @@ +package policy + +/* + * MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "mime/multipart" + "net/http" + "net/url" + "regexp" + "strings" + "time" + "unicode/utf8" +) + +const ( + iso8601DateFormat = "20060102T150405Z" + iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision. +) + +func newPostPolicyBytesV4WithContentRange(credential, bucketName, objectKey string, expiration time.Time) []byte { + t := time.Now().UTC() + // Add the expiration date. + expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat)) + // Add the bucket condition, only accept buckets equal to the one passed. + bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) + // Add the key condition, only accept keys equal to the one passed. + keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s/upload.txt"]`, objectKey) + // Add content length condition, only accept content sizes of a given length. + contentLengthCondStr := `["content-length-range", 1024, 1048576]` + // Add the algorithm condition, only accept AWS SignV4 Sha256. + algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]` + // Add the date condition, only accept the current date. + dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat)) + // Add the credential string, only accept the credential passed. + credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential) + // Add the meta-uuid string, set to 1234 + uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234") + + // Combine all conditions into one string. + conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr, + keyConditionStr, contentLengthCondStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr) + retStr := "{" + retStr = retStr + expirationStr + "," + retStr = retStr + conditionStr + retStr = retStr + "}" + + return []byte(retStr) +} + +// newPostPolicyBytesV4 - creates a bare bones postpolicy string with key and bucket matches. +func newPostPolicyBytesV4(credential, bucketName, objectKey string, expiration time.Time) []byte { + t := time.Now().UTC() + // Add the expiration date. + expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat)) + // Add the bucket condition, only accept buckets equal to the one passed. + bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) + // Add the key condition, only accept keys equal to the one passed. + keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s/upload.txt"]`, objectKey) + // Add the algorithm condition, only accept AWS SignV4 Sha256. + algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]` + // Add the date condition, only accept the current date. + dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat)) + // Add the credential string, only accept the credential passed. + credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential) + // Add the meta-uuid string, set to 1234 + uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234") + + // Combine all conditions into one string. + conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr) + retStr := "{" + retStr = retStr + expirationStr + "," + retStr = retStr + conditionStr + retStr = retStr + "}" + + return []byte(retStr) +} + +// newPostPolicyBytesV2 - creates a bare bones postpolicy string with key and bucket matches. +func newPostPolicyBytesV2(bucketName, objectKey string, expiration time.Time) []byte { + // Add the expiration date. + expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat)) + // Add the bucket condition, only accept buckets equal to the one passed. + bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) + // Add the key condition, only accept keys equal to the one passed. + keyConditionStr := fmt.Sprintf(`["starts-with", "$key", "%s/upload.txt"]`, objectKey) + + // Combine all conditions into one string. + conditionStr := fmt.Sprintf(`"conditions":[%s, %s]`, bucketConditionStr, keyConditionStr) + retStr := "{" + retStr = retStr + expirationStr + "," + retStr = retStr + conditionStr + retStr = retStr + "}" + + return []byte(retStr) +} + +// Wrapper for calling TestPostPolicyBucketHandler tests for both Erasure multiple disks and single node setup. + +// testPostPolicyBucketHandler - Tests validate post policy handler uploading objects. + +// Wrapper for calling TestPostPolicyBucketHandlerRedirect tests for both Erasure multiple disks and single node setup. + +// testPostPolicyBucketHandlerRedirect tests POST Object when success_action_redirect is specified + +// postPresignSignatureV4 - presigned signature for PostPolicy requests. +func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string { + // Get signining key. + signingkey := getSigningKey(secretAccessKey, t, location) + // Calculate signature. + signature := getSignature(signingkey, policyBase64) + return signature +} + +// copied from auth_signature_v4.go to break import loop +// sumHMAC calculate hmac between two input byte array. +func sumHMAC(key []byte, data []byte) []byte { + hash := hmac.New(sha256.New, key) + hash.Write(data) + return hash.Sum(nil) +} + +// copied from auth_signature_v4.go to break import loop +// getSigningKey hmac seed to calculate final signature. +func getSigningKey(secretKey string, t time.Time, region string) []byte { + date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format("20060102"))) + regionBytes := sumHMAC(date, []byte(region)) + service := sumHMAC(regionBytes, []byte("s3")) + signingKey := sumHMAC(service, []byte("aws4_request")) + return signingKey +} + +// copied from auth_signature_v4.go to break import loop +// getSignature final signature in hexadecimal form. +func getSignature(signingKey []byte, stringToSign string) string { + return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) +} + +// copied from auth_signature_v4.go to break import loop +func calculateSignatureV2(stringToSign string, secret string) string { + hm := hmac.New(sha1.New, []byte(secret)) + hm.Write([]byte(stringToSign)) + return base64.StdEncoding.EncodeToString(hm.Sum(nil)) +} + +func newPostRequestV2(endPoint, bucketName, objectName string, accessKey, secretKey string) (*http.Request, error) { + // Expire the request five minutes from now. + expirationTime := time.Now().UTC().Add(time.Minute * 5) + // Create a new post policy. + policy := newPostPolicyBytesV2(bucketName, objectName, expirationTime) + // Only need the encoding. + encodedPolicy := base64.StdEncoding.EncodeToString(policy) + + // Presign with V4 signature based on the policy. + signature := calculateSignatureV2(encodedPolicy, secretKey) + + formData := map[string]string{ + "AWSAccessKeyId": accessKey, + "bucket": bucketName, + "key": objectName + "/${filename}", + "policy": encodedPolicy, + "signature": signature, + } + + // Create the multipart form. + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + // Set the normal formData + for k, v := range formData { + w.WriteField(k, v) + } + // Set the File formData + writer, err := w.CreateFormFile("file", "upload.txt") + if err != nil { + // return nil, err + return nil, err + } + writer.Write([]byte("hello world")) + // Close before creating the new request. + w.Close() + + // Set the body equal to the created policy. + reader := bytes.NewReader(buf.Bytes()) + + req, err := http.NewRequest(http.MethodPost, makeTestTargetURL(endPoint, bucketName, "", nil), reader) + if err != nil { + return nil, err + } + + // Set form content-type. + req.Header.Set("Content-Type", w.FormDataContentType()) + return req, nil +} + +func buildGenericPolicy(t time.Time, accessKey, region, bucketName, objectName string, contentLengthRange bool) []byte { + // Expire the request five minutes from now. + expirationTime := t.Add(time.Minute * 5) + + credStr := getCredentialString(accessKey, region, t) + // Create a new post policy. + policy := newPostPolicyBytesV4(credStr, bucketName, objectName, expirationTime) + if contentLengthRange { + policy = newPostPolicyBytesV4WithContentRange(credStr, bucketName, objectName, expirationTime) + } + return policy +} + +func newPostRequestV4Generic(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string, region string, + t time.Time, policy []byte, addFormData map[string]string, corruptedB64 bool, corruptedMultipart bool) (*http.Request, error) { + // Get the user credential. + credStr := getCredentialString(accessKey, region, t) + + // Only need the encoding. + encodedPolicy := base64.StdEncoding.EncodeToString(policy) + + if corruptedB64 { + encodedPolicy = "%!~&" + encodedPolicy + } + + // Presign with V4 signature based on the policy. + signature := postPresignSignatureV4(encodedPolicy, t, secretKey, region) + + formData := map[string]string{ + "bucket": bucketName, + "key": objectName + "/${filename}", + "x-amz-credential": credStr, + "policy": encodedPolicy, + "x-amz-signature": signature, + "x-amz-date": t.Format(iso8601DateFormat), + "x-amz-algorithm": "AWS4-HMAC-SHA256", + "x-amz-meta-uuid": "1234", + "Content-Encoding": "gzip", + } + + // Add form data + for k, v := range addFormData { + formData[k] = v + } + + // Create the multipart form. + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + // Set the normal formData + for k, v := range formData { + w.WriteField(k, v) + } + // Set the File formData but don't if we want send an incomplete multipart request + if !corruptedMultipart { + writer, err := w.CreateFormFile("file", "upload.txt") + if err != nil { + // return nil, err + return nil, err + } + writer.Write(objData) + // Close before creating the new request. + w.Close() + } + + // Set the body equal to the created policy. + reader := bytes.NewReader(buf.Bytes()) + + req, err := http.NewRequest(http.MethodPost, makeTestTargetURL(endPoint, bucketName, "", nil), reader) + if err != nil { + return nil, err + } + + // Set form content-type. + req.Header.Set("Content-Type", w.FormDataContentType()) + return req, nil +} + +func newPostRequestV4WithContentLength(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) { + t := time.Now().UTC() + region := "us-east-1" + policy := buildGenericPolicy(t, accessKey, region, bucketName, objectName, true) + return newPostRequestV4Generic(endPoint, bucketName, objectName, objData, accessKey, secretKey, region, t, policy, nil, false, false) +} + +func newPostRequestV4(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) { + t := time.Now().UTC() + region := "us-east-1" + policy := buildGenericPolicy(t, accessKey, region, bucketName, objectName, false) + return newPostRequestV4Generic(endPoint, bucketName, objectName, objData, accessKey, secretKey, region, t, policy, nil, false, false) +} + +// construct URL for http requests for bucket operations. +func makeTestTargetURL(endPoint, bucketName, objectName string, queryValues url.Values) string { + urlStr := endPoint + "/" + if bucketName != "" { + urlStr = urlStr + bucketName + "/" + } + if objectName != "" { + urlStr = urlStr + EncodePath(objectName) + } + if len(queryValues) > 0 { + urlStr = urlStr + "?" + queryValues.Encode() + } + return urlStr +} + + + +// if object matches reserved string, no need to encode them +var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") + +// EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences +// +// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 +// non english characters cannot be parsed due to the nature in which url.Encode() is written +// +// This function on the other hand is a direct replacement for url.Encode() technique to support +// pretty much every UTF-8 character. +func EncodePath(pathName string) string { + if reservedObjectNames.MatchString(pathName) { + return pathName + } + var encodedPathname string + for _, s := range pathName { + if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) + encodedPathname = encodedPathname + string(s) + continue + } + switch s { + case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) + encodedPathname = encodedPathname + string(s) + continue + default: + len := utf8.RuneLen(s) + if len < 0 { + // if utf8 cannot convert return the same string as is + return pathName + } + u := make([]byte, len) + utf8.EncodeRune(u, s) + for _, r := range u { + hex := hex.EncodeToString([]byte{r}) + encodedPathname = encodedPathname + "%" + strings.ToUpper(hex) + } + } + } + return encodedPathname +} + +// getCredentialString generate a credential string. +func getCredentialString(accessKeyID, location string, t time.Time) string { + return accessKeyID + "/" + getScope(t, location) +} + +// getScope generate a string of a specific date, an AWS region, and a service. +func getScope(t time.Time, region string) string { + scope := strings.Join([]string{ + t.Format("20060102"), + region, + string("s3"), + "aws4_request", + }, "/") + return scope +} diff --git a/weed/s3api/policy/postpolicyform.go b/weed/s3api/policy/postpolicyform.go new file mode 100644 index 000000000..798408aa4 --- /dev/null +++ b/weed/s3api/policy/postpolicyform.go @@ -0,0 +1,276 @@ +package policy + +/* + * MinIO Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "reflect" + "strconv" + "strings" + "time" +) + +// startWithConds - map which indicates if a given condition supports starts-with policy operator +var startsWithConds = map[string]bool{ + "$acl": true, + "$bucket": false, + "$cache-control": true, + "$content-type": true, + "$content-disposition": true, + "$content-encoding": true, + "$expires": true, + "$key": true, + "$success_action_redirect": true, + "$redirect": true, + "$success_action_status": false, + "$x-amz-algorithm": false, + "$x-amz-credential": false, + "$x-amz-date": false, +} + +// Add policy conditionals. +const ( + policyCondEqual = "eq" + policyCondStartsWith = "starts-with" + policyCondContentLength = "content-length-range" +) + +// toString - Safely convert interface to string without causing panic. +func toString(val interface{}) string { + switch v := val.(type) { + case string: + return v + default: + return "" + } +} + +// toLowerString - safely convert interface to lower string +func toLowerString(val interface{}) string { + return strings.ToLower(toString(val)) +} + +// toInteger _ Safely convert interface to integer without causing panic. +func toInteger(val interface{}) (int64, error) { + switch v := val.(type) { + case float64: + return int64(v), nil + case int64: + return v, nil + case int: + return int64(v), nil + case string: + i, err := strconv.Atoi(v) + return int64(i), err + default: + return 0, errors.New("Invalid number format") + } +} + +// isString - Safely check if val is of type string without causing panic. +func isString(val interface{}) bool { + _, ok := val.(string) + return ok +} + +// ContentLengthRange - policy content-length-range field. +type contentLengthRange struct { + Min int64 + Max int64 + Valid bool // If content-length-range was part of policy +} + +// PostPolicyForm provides strict static type conversion and validation for Amazon S3's POST policy JSON string. +type PostPolicyForm struct { + Expiration time.Time // Expiration date and time of the POST policy. + Conditions struct { // Conditional policy structure. + Policies []struct { + Operator string + Key string + Value string + } + ContentLengthRange contentLengthRange + } +} + +// ParsePostPolicyForm - Parse JSON policy string into typed PostPolicyForm structure. +func ParsePostPolicyForm(policy string) (ppf PostPolicyForm, e error) { + // Convert po into interfaces and + // perform strict type conversion using reflection. + var rawPolicy struct { + Expiration string `json:"expiration"` + Conditions []interface{} `json:"conditions"` + } + + err := json.Unmarshal([]byte(policy), &rawPolicy) + if err != nil { + return ppf, err + } + + parsedPolicy := PostPolicyForm{} + + // Parse expiry time. + parsedPolicy.Expiration, err = time.Parse(time.RFC3339Nano, rawPolicy.Expiration) + if err != nil { + return ppf, err + } + + // Parse conditions. + for _, val := range rawPolicy.Conditions { + switch condt := val.(type) { + case map[string]interface{}: // Handle key:value map types. + for k, v := range condt { + if !isString(v) { // Pre-check value type. + // All values must be of type string. + return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form", reflect.TypeOf(condt).String(), condt) + } + // {"acl": "public-read" } is an alternate way to indicate - [ "eq", "$acl", "public-read" ] + // In this case we will just collapse this into "eq" for all use cases. + parsedPolicy.Conditions.Policies = append(parsedPolicy.Conditions.Policies, struct { + Operator string + Key string + Value string + }{ + policyCondEqual, "$" + strings.ToLower(k), toString(v), + }) + } + case []interface{}: // Handle array types. + if len(condt) != 3 { // Return error if we have insufficient elements. + return parsedPolicy, fmt.Errorf("Malformed conditional fields %s of type %s found in POST policy form", condt, reflect.TypeOf(condt).String()) + } + switch toLowerString(condt[0]) { + case policyCondEqual, policyCondStartsWith: + for _, v := range condt { // Pre-check all values for type. + if !isString(v) { + // All values must be of type string. + return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form", reflect.TypeOf(condt).String(), condt) + } + } + operator, matchType, value := toLowerString(condt[0]), toLowerString(condt[1]), toString(condt[2]) + if !strings.HasPrefix(matchType, "$") { + return parsedPolicy, fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", operator, matchType, value) + } + parsedPolicy.Conditions.Policies = append(parsedPolicy.Conditions.Policies, struct { + Operator string + Key string + Value string + }{ + operator, matchType, value, + }) + case policyCondContentLength: + min, err := toInteger(condt[1]) + if err != nil { + return parsedPolicy, err + } + + max, err := toInteger(condt[2]) + if err != nil { + return parsedPolicy, err + } + + parsedPolicy.Conditions.ContentLengthRange = contentLengthRange{ + Min: min, + Max: max, + Valid: true, + } + default: + // Condition should be valid. + return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form", + reflect.TypeOf(condt).String(), condt) + } + default: + return parsedPolicy, fmt.Errorf("Unknown field %s of type %s found in POST policy form", + condt, reflect.TypeOf(condt).String()) + } + } + return parsedPolicy, nil +} + +// checkPolicyCond returns a boolean to indicate if a condition is satisified according +// to the passed operator +func checkPolicyCond(op string, input1, input2 string) bool { + switch op { + case policyCondEqual: + return input1 == input2 + case policyCondStartsWith: + return strings.HasPrefix(input1, input2) + } + return false +} + +// CheckPostPolicy - apply policy conditions and validate input values. +// (http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html) +func CheckPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) error { + // Check if policy document expiry date is still not reached + if !postPolicyForm.Expiration.After(time.Now().UTC()) { + return fmt.Errorf("Invalid according to Policy: Policy expired") + } + // map to store the metadata + metaMap := make(map[string]string) + for _, policy := range postPolicyForm.Conditions.Policies { + if strings.HasPrefix(policy.Key, "$x-amz-meta-") { + formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$")) + metaMap[formCanonicalName] = policy.Value + } + } + // Check if any extra metadata field is passed as input + for key := range formValues { + if strings.HasPrefix(key, "X-Amz-Meta-") { + if _, ok := metaMap[key]; !ok { + return fmt.Errorf("Invalid according to Policy: Extra input fields: %s", key) + } + } + } + + // Flag to indicate if all policies conditions are satisfied + var condPassed bool + + // Iterate over policy conditions and check them against received form fields + for _, policy := range postPolicyForm.Conditions.Policies { + // Form fields names are in canonical format, convert conditions names + // to canonical for simplification purpose, so `$key` will become `Key` + formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$")) + // Operator for the current policy condition + op := policy.Operator + // If the current policy condition is known + if startsWithSupported, condFound := startsWithConds[policy.Key]; condFound { + // Check if the current condition supports starts-with operator + if op == policyCondStartsWith && !startsWithSupported { + return fmt.Errorf("Invalid according to Policy: Policy Condition failed") + } + // Check if current policy condition is satisfied + condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value) + if !condPassed { + return fmt.Errorf("Invalid according to Policy: Policy Condition failed") + } + } else { + // This covers all conditions X-Amz-Meta-* and X-Amz-* + if strings.HasPrefix(policy.Key, "$x-amz-meta-") || strings.HasPrefix(policy.Key, "$x-amz-") { + // Check if policy condition is satisfied + condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value) + if !condPassed { + return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", op, policy.Key, policy.Value) + } + } + } + } + + return nil +} diff --git a/weed/s3api/policy/postpolicyform_test.go b/weed/s3api/policy/postpolicyform_test.go new file mode 100644 index 000000000..1a9d78b0e --- /dev/null +++ b/weed/s3api/policy/postpolicyform_test.go @@ -0,0 +1,106 @@ +package policy + +/* + * MinIO Cloud Storage, (C) 2016 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ( + "encoding/base64" + "fmt" + "net/http" + "testing" + "time" +) + +// Test Post Policy parsing and checking conditions +func TestPostPolicyForm(t *testing.T) { + pp := NewPostPolicy() + pp.SetBucket("testbucket") + pp.SetContentType("image/jpeg") + pp.SetUserMetadata("uuid", "14365123651274") + pp.SetKeyStartsWith("user/user1/filename") + pp.SetContentLengthRange(1048579, 10485760) + pp.SetSuccessStatusAction("201") + + type testCase struct { + Bucket string + Key string + XAmzDate string + XAmzAlgorithm string + XAmzCredential string + XAmzMetaUUID string + ContentType string + SuccessActionStatus string + Policy string + Expired bool + expectedErr error + } + + testCases := []testCase{ + // Everything is fine with this test + {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: nil}, + // Expired policy document + {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Expired: true, expectedErr: fmt.Errorf("Invalid according to Policy: Policy expired")}, + // Different AMZ date + {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "2017T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, + // Key which doesn't start with user/user1/filename + {Bucket: "testbucket", Key: "myfile.txt", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, + // Incorrect bucket name. + {Bucket: "incorrect", Key: "user/user1/filename/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, + // Incorrect key name + {Bucket: "testbucket", Key: "incorrect", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, + // Incorrect date + {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "incorrect", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, + // Incorrect ContentType + {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "incorrect", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, + // Incorrect Metadata + {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "151274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed: [eq, $x-amz-meta-uuid, 14365123651274]")}, + } + // Validate all the test cases. + for i, tt := range testCases { + formValues := make(http.Header) + formValues.Set("Bucket", tt.Bucket) + formValues.Set("Key", tt.Key) + formValues.Set("Content-Type", tt.ContentType) + formValues.Set("X-Amz-Date", tt.XAmzDate) + formValues.Set("X-Amz-Meta-Uuid", tt.XAmzMetaUUID) + formValues.Set("X-Amz-Algorithm", tt.XAmzAlgorithm) + formValues.Set("X-Amz-Credential", tt.XAmzCredential) + if tt.Expired { + // Expired already. + pp.SetExpires(time.Now().UTC().AddDate(0, 0, -10)) + } else { + // Expires in 10 days. + pp.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) + } + + formValues.Set("Policy", base64.StdEncoding.EncodeToString([]byte(pp.String()))) + formValues.Set("Success_action_status", tt.SuccessActionStatus) + policyBytes, err := base64.StdEncoding.DecodeString(base64.StdEncoding.EncodeToString([]byte(pp.String()))) + if err != nil { + t.Fatal(err) + } + + postPolicyForm, err := ParsePostPolicyForm(string(policyBytes)) + if err != nil { + t.Fatal(err) + } + + err = CheckPostPolicy(formValues, postPolicyForm) + if err != nil && tt.expectedErr != nil && err.Error() != tt.expectedErr.Error() { + t.Fatalf("Test %d:, Expected %s, got %s", i+1, tt.expectedErr.Error(), err.Error()) + } + } +} diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go new file mode 100644 index 000000000..f1605cc63 --- /dev/null +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -0,0 +1,242 @@ +package s3api + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/policy" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/dustin/go-humanize" + "github.com/gorilla/mux" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "net/url" + "strings" +) + +func (s3a *S3ApiServer) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + + // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html + // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-post-example.html + + bucket := mux.Vars(r)["bucket"] + + reader, err := r.MultipartReader() + if err != nil { + writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL) + return + } + form, err := reader.ReadForm(int64(5 * humanize.MiByte)) + if err != nil { + writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL) + return + } + defer form.RemoveAll() + + fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(form) + if err != nil { + writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL) + return + } + if fileBody == nil { + writeErrorResponse(w, s3err.ErrPOSTFileRequired, r.URL) + return + } + defer fileBody.Close() + + formValues.Set("Bucket", bucket) + + if fileName != "" && strings.Contains(formValues.Get("Key"), "${filename}") { + formValues.Set("Key", strings.Replace(formValues.Get("Key"), "${filename}", fileName, -1)) + } + object := formValues.Get("Key") + + successRedirect := formValues.Get("success_action_redirect") + successStatus := formValues.Get("success_action_status") + var redirectURL *url.URL + if successRedirect != "" { + redirectURL, err = url.Parse(successRedirect) + if err != nil { + writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL) + return + } + } + + // Verify policy signature. + errCode := s3a.iam.doesPolicySignatureMatch(formValues) + if errCode != s3err.ErrNone { + writeErrorResponse(w, errCode, r.URL) + return + } + + policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy")) + if err != nil { + writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL) + return + } + + // Handle policy if it is set. + if len(policyBytes) > 0 { + + postPolicyForm, err := policy.ParsePostPolicyForm(string(policyBytes)) + if err != nil { + writeErrorResponse(w, s3err.ErrPostPolicyConditionInvalidFormat, r.URL) + return + } + + // Make sure formValues adhere to policy restrictions. + if err = policy.CheckPostPolicy(formValues, postPolicyForm); err != nil { + w.Header().Set("Location", r.URL.Path) + w.WriteHeader(http.StatusTemporaryRedirect) + return + } + + // Ensure that the object size is within expected range, also the file size + // should not exceed the maximum single Put size (5 GiB) + lengthRange := postPolicyForm.Conditions.ContentLengthRange + if lengthRange.Valid { + if fileSize < lengthRange.Min { + writeErrorResponse(w, s3err.ErrEntityTooSmall, r.URL) + return + } + + if fileSize > lengthRange.Max { + writeErrorResponse(w, s3err.ErrEntityTooLarge, r.URL) + return + } + } + } + + uploadUrl := fmt.Sprintf("http://%s%s/%s/%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, object) + + etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody) + + if errCode != s3err.ErrNone { + writeErrorResponse(w, errCode, r.URL) + return + } + + if successRedirect != "" { + // Replace raw query params.. + redirectURL.RawQuery = getRedirectPostRawQuery(bucket, object, etag) + w.Header().Set("Location", redirectURL.String()) + writeResponse(w, http.StatusSeeOther, nil, mimeNone) + return + } + + setEtag(w, etag) + + // Decide what http response to send depending on success_action_status parameter + switch successStatus { + case "201": + resp := encodeResponse(PostResponse{ + Bucket: bucket, + Key: object, + ETag: `"` + etag + `"`, + Location: w.Header().Get("Location"), + }) + writeResponse(w, http.StatusCreated, resp, mimeXML) + case "200": + writeResponse(w, http.StatusOK, nil, mimeNone) + default: + writeSuccessResponseEmpty(w) + } + +} + + +// Extract form fields and file data from a HTTP POST Policy +func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, fileName string, fileSize int64, formValues http.Header, err error) { + /// HTML Form values + fileName = "" + + // Canonicalize the form values into http.Header. + formValues = make(http.Header) + for k, v := range form.Value { + formValues[http.CanonicalHeaderKey(k)] = v + } + + // Validate form values. + if err = validateFormFieldSize(formValues); err != nil { + return nil, "", 0, nil, err + } + + // this means that filename="" was not specified for file key and Go has + // an ugly way of handling this situation. Refer here + // https://golang.org/src/mime/multipart/formdata.go#L61 + if len(form.File) == 0 { + var b = &bytes.Buffer{} + for _, v := range formValues["File"] { + b.WriteString(v) + } + fileSize = int64(b.Len()) + filePart = ioutil.NopCloser(b) + return filePart, fileName, fileSize, formValues, nil + } + + // Iterator until we find a valid File field and break + for k, v := range form.File { + canonicalFormName := http.CanonicalHeaderKey(k) + if canonicalFormName == "File" { + if len(v) == 0 { + return nil, "", 0, nil, errors.New("Invalid arguments specified") + } + // Fetch fileHeader which has the uploaded file information + fileHeader := v[0] + // Set filename + fileName = fileHeader.Filename + // Open the uploaded part + filePart, err = fileHeader.Open() + if err != nil { + return nil, "", 0, nil, err + } + // Compute file size + fileSize, err = filePart.(io.Seeker).Seek(0, 2) + if err != nil { + return nil, "", 0, nil, err + } + // Reset Seek to the beginning + _, err = filePart.(io.Seeker).Seek(0, 0) + if err != nil { + return nil, "", 0, nil, err + } + // File found and ready for reading + break + } + } + return filePart, fileName, fileSize, formValues, nil +} + +// Validate form field size for s3 specification requirement. +func validateFormFieldSize(formValues http.Header) error { + // Iterate over form values + for k := range formValues { + // Check if value's field exceeds S3 limit + if int64(len(formValues.Get(k))) > int64(1 * humanize.MiByte) { + return errors.New("Data size larger than expected") + } + } + + // Success. + return nil +} + +func getRedirectPostRawQuery(bucket, key, etag string) string { + redirectValues := make(url.Values) + redirectValues.Set("bucket", bucket) + redirectValues.Set("key", key) + redirectValues.Set("etag", "\""+etag+"\"") + return redirectValues.Encode() +} + +// Check to see if Policy is signed correctly. +func (iam *IdentityAccessManagement) doesPolicySignatureMatch(formValues http.Header) s3err.ErrorCode { + // For SignV2 - Signature field will be valid + if _, ok := formValues["Signature"]; ok { + return iam.doesPolicySignatureV2Match(formValues) + } + return iam.doesPolicySignatureV4Match(formValues) +} diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index d76de1704..4eaacd66c 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -87,6 +87,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // ListObjectsV1 (Legacy) bucket.Methods("GET").HandlerFunc(stats(s3a.iam.Auth(s3a.ListObjectsV1Handler, ACTION_READ), "LIST")) + // PostPolicy + bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(stats(s3a.iam.Auth(s3a.PutBucketPolicyHandler, ACTION_WRITE), "POST")) + // DeleteMultipleObjects bucket.Methods("POST").HandlerFunc(stats(s3a.iam.Auth(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE), "DELETE")).Queries("delete", "") /* diff --git a/weed/s3api/s3err/s3-error.go b/weed/s3api/s3err/s3-error.go new file mode 100644 index 000000000..224378ec5 --- /dev/null +++ b/weed/s3api/s3err/s3-error.go @@ -0,0 +1,61 @@ +package s3err + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Non exhaustive list of AWS S3 standard error responses - +// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html +var s3ErrorResponseMap = map[string]string{ + "AccessDenied": "Access Denied.", + "BadDigest": "The Content-Md5 you specified did not match what we received.", + "EntityTooSmall": "Your proposed upload is smaller than the minimum allowed object size.", + "EntityTooLarge": "Your proposed upload exceeds the maximum allowed object size.", + "IncompleteBody": "You did not provide the number of bytes specified by the Content-Length HTTP header.", + "InternalError": "We encountered an internal error, please try again.", + "InvalidAccessKeyId": "The access key ID you provided does not exist in our records.", + "InvalidBucketName": "The specified bucket is not valid.", + "InvalidDigest": "The Content-Md5 you specified is not valid.", + "InvalidRange": "The requested range is not satisfiable", + "MalformedXML": "The XML you provided was not well-formed or did not validate against our published schema.", + "MissingContentLength": "You must provide the Content-Length HTTP header.", + "MissingContentMD5": "Missing required header for this request: Content-Md5.", + "MissingRequestBodyError": "Request body is empty.", + "NoSuchBucket": "The specified bucket does not exist.", + "NoSuchBucketPolicy": "The bucket policy does not exist", + "NoSuchKey": "The specified key does not exist.", + "NoSuchUpload": "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.", + "NotImplemented": "A header you provided implies functionality that is not implemented", + "PreconditionFailed": "At least one of the pre-conditions you specified did not hold", + "RequestTimeTooSkewed": "The difference between the request time and the server's time is too large.", + "SignatureDoesNotMatch": "The request signature we calculated does not match the signature you provided. Check your key and signing method.", + "MethodNotAllowed": "The specified method is not allowed against this resource.", + "InvalidPart": "One or more of the specified parts could not be found.", + "InvalidPartOrder": "The list of parts was not in ascending order. The parts list must be specified in order by part number.", + "InvalidObjectState": "The operation is not valid for the current state of the object.", + "AuthorizationHeaderMalformed": "The authorization header is malformed; the region is wrong.", + "MalformedPOSTRequest": "The body of your POST request is not well-formed multipart/form-data.", + "BucketNotEmpty": "The bucket you tried to delete is not empty", + "AllAccessDisabled": "All access to this bucket has been disabled.", + "MalformedPolicy": "Policy has invalid resource.", + "MissingFields": "Missing fields in request.", + "AuthorizationQueryParametersError": "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".", + "MalformedDate": "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.", + "BucketAlreadyOwnedByYou": "Your previous request to create the named bucket succeeded and you already own it.", + "InvalidDuration": "Duration provided in the request is invalid.", + "XAmzContentSHA256Mismatch": "The provided 'x-amz-content-sha256' header does not match what was computed.", + // Add new API errors here. +} diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index d5fdba0a0..cccef0227 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -2,6 +2,7 @@ package s3err import ( "encoding/xml" + "fmt" "net/http" ) @@ -19,6 +20,21 @@ type RESTErrorResponse struct { Message string `xml:"Message" json:"Message"` Resource string `xml:"Resource" json:"Resource"` RequestID string `xml:"RequestId" json:"RequestId"` + + // Underlying HTTP status code for the returned error + StatusCode int `xml:"-" json:"-"` +} + +// Error - Returns S3 error string. +func (e RESTErrorResponse) Error() string { + if e.Message == "" { + msg, ok := s3ErrorResponseMap[e.Code] + if !ok { + msg = fmt.Sprintf("Error response code %s.", e.Code) + } + return msg + } + return e.Message } // ErrorCode type of error status. @@ -47,6 +63,11 @@ const ( ErrInvalidCopySource ErrAuthHeaderEmpty ErrSignatureVersionNotSupported + ErrMalformedPOSTRequest + ErrPOSTFileRequired + ErrPostPolicyConditionInvalidFormat + ErrEntityTooSmall + ErrEntityTooLarge ErrMissingFields ErrMissingCredTag ErrCredMalformed @@ -167,13 +188,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.", HTTPStatusCode: http.StatusBadRequest, }, - ErrMalformedXML: { Code: "MalformedXML", Description: "The XML you provided was not well-formed or did not validate against our published schema.", HTTPStatusCode: http.StatusBadRequest, }, - ErrAuthHeaderEmpty: { Code: "InvalidArgument", Description: "Authorization header is invalid -- one and only one ' ' (space) required.", @@ -184,6 +203,31 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.", HTTPStatusCode: http.StatusBadRequest, }, + ErrMalformedPOSTRequest: { + Code: "MalformedPOSTRequest", + Description: "The body of your POST request is not well-formed multipart/form-data.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrPOSTFileRequired: { + Code: "InvalidArgument", + Description: "POST requires exactly one file upload per request.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrPostPolicyConditionInvalidFormat: { + Code: "PostPolicyInvalidKeyName", + Description: "Invalid according to Policy: Policy Condition failed", + HTTPStatusCode: http.StatusForbidden, + }, + ErrEntityTooSmall: { + Code: "EntityTooSmall", + Description: "Your proposed upload is smaller than the minimum allowed object size.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEntityTooLarge: { + Code: "EntityTooLarge", + Description: "Your proposed upload exceeds the maximum allowed object size.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrMissingFields: { Code: "MissingFields", Description: "Missing fields in request.", |