aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Lu <chrislusf@users.noreply.github.com>2025-07-15 10:11:49 -0700
committerGitHub <noreply@github.com>2025-07-15 10:11:49 -0700
commit74f4e9ba5ab691938e89b32017a89359ae49428f (patch)
tree30f306324c6e34c42452588fc8e4cb74c20e7ed7
parent4b040e8a8701199d4c680bb6f241c4751c8210a2 (diff)
downloadseaweedfs-74f4e9ba5ab691938e89b32017a89359ae49428f.tar.xz
seaweedfs-74f4e9ba5ab691938e89b32017a89359ae49428f.zip
rewrite, simplify, avoid unused functions (#6989)
* adding cors support * address some comments * optimize matchesWildcard * address comments * fix for tests * address comments * address comments * address comments * path building * refactor * Update weed/s3api/s3api_bucket_config.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * address comment Service-level responses need both Access-Control-Allow-Methods and Access-Control-Allow-Headers. After setting Access-Control-Allow-Origin and Access-Control-Expose-Headers, also set Access-Control-Allow-Methods: * and Access-Control-Allow-Headers: * so service endpoints satisfy CORS preflight requirements. * Update weed/s3api/s3api_bucket_config.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_object_handlers.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_object_handlers.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix * refactor * Update weed/s3api/s3api_bucket_config.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_object_handlers.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_server.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * simplify * add cors tests * fix tests * fix tests * remove unused functions * fix tests * simplify * address comments * fix * Update weed/s3api/auth_signature_v4.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * rename variable * Revert "Apply suggestion from @Copilot" This reverts commit fce2d4e57e6f712672e62e8c63468c6b89878c6c. --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
-rw-r--r--weed/s3api/auth_signature_v2.go341
-rw-r--r--weed/s3api/auth_signature_v4.go900
-rw-r--r--weed/s3api/auto_signature_v4_test.go112
-rw-r--r--weed/s3api/chunked_reader_v4.go71
4 files changed, 501 insertions, 923 deletions
diff --git a/weed/s3api/auth_signature_v2.go b/weed/s3api/auth_signature_v2.go
index 3fdc321b1..4cdc07df0 100644
--- a/weed/s3api/auth_signature_v2.go
+++ b/weed/s3api/auth_signature_v2.go
@@ -22,16 +22,14 @@ import (
"crypto/sha1"
"crypto/subtle"
"encoding/base64"
- "fmt"
- "github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
- "net"
"net/http"
- "net/url"
- "path"
"sort"
"strconv"
"strings"
"time"
+
+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
// Whitelist resource list that will be used in query string for signature-V2 calculation.
@@ -61,7 +59,7 @@ var resourceList = []string{
"website",
}
-// Verify if request has valid AWS Signature Version '2'.
+// Verify if request has AWS Signature Version '2'.
func (iam *IdentityAccessManagement) isReqAuthenticatedV2(r *http.Request) (*Identity, s3err.ErrorCode) {
if isRequestSignatureV2(r) {
return iam.doesSignV2Match(r)
@@ -70,277 +68,186 @@ func (iam *IdentityAccessManagement) isReqAuthenticatedV2(r *http.Request) (*Ide
}
func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http.Header) s3err.ErrorCode {
+
accessKey := formValues.Get("AWSAccessKeyId")
- _, cred, found := iam.lookupByAccessKey(accessKey)
+ if accessKey == "" {
+ return s3err.ErrMissingFields
+ }
+
+ identity, cred, found := iam.lookupByAccessKey(accessKey)
if !found {
return s3err.ErrInvalidAccessKeyID
}
- policy := formValues.Get("Policy")
- signature := formValues.Get("Signature")
- if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) {
- return s3err.ErrSignatureDoesNotMatch
- }
- return s3err.ErrNone
-}
-
-// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
-// Signature = Base64( HMAC-SHA1( YourSecretKey, UTF-8-Encoding-Of( StringToSign ) ) );
-//
-// StringToSign = HTTP-Verb + "\n" +
-// Content-Md5 + "\n" +
-// Content-Type + "\n" +
-// Date + "\n" +
-// CanonicalizedProtocolHeaders +
-// CanonicalizedResource;
-//
-// CanonicalizedResource = [ "/" + Bucket ] +
-// <HTTP-Request-URI, from the protocol name up to the query string> +
-// [ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
-//
-// CanonicalizedProtocolHeaders = <described below>
-// doesSignV2Match - Verify authorization header with calculated header in accordance with
-// - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
-// returns true if matches, false otherwise. if error is not nil then it is always false
-
-func validateV2AuthHeader(v2Auth string) (accessKey string, errCode s3err.ErrorCode) {
- if v2Auth == "" {
- return "", s3err.ErrAuthHeaderEmpty
- }
- // Verify if the header algorithm is supported or not.
- if !strings.HasPrefix(v2Auth, signV2Algorithm) {
- return "", s3err.ErrSignatureVersionNotSupported
+ bucket := formValues.Get("bucket")
+ if !identity.canDo(s3_constants.ACTION_WRITE, bucket, "") {
+ return s3err.ErrAccessDenied
}
- // below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
- // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature
- authFields := strings.Split(v2Auth, " ")
- if len(authFields) != 2 {
- return "", s3err.ErrMissingFields
+ policy := formValues.Get("Policy")
+ if policy == "" {
+ return s3err.ErrMissingFields
}
- // Then will be splitting on ":", this will separate `AWSAccessKeyId` and `Signature` string.
- keySignFields := strings.Split(strings.TrimSpace(authFields[1]), ":")
- if len(keySignFields) != 2 {
- return "", s3err.ErrMissingFields
+ signature := formValues.Get("Signature")
+ if signature == "" {
+ return s3err.ErrMissingFields
}
- return keySignFields[0], s3err.ErrNone
+ if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) {
+ return s3err.ErrSignatureDoesNotMatch
+ }
+ return s3err.ErrNone
}
+// doesSignV2Match - Verify authorization header with calculated header in accordance with
+// - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
+//
+// returns ErrNone if the signature matches.
func (iam *IdentityAccessManagement) doesSignV2Match(r *http.Request) (*Identity, s3err.ErrorCode) {
v2Auth := r.Header.Get("Authorization")
-
- accessKey, apiError := validateV2AuthHeader(v2Auth)
- if apiError != s3err.ErrNone {
- return nil, apiError
+ accessKey, errCode := validateV2AuthHeader(v2Auth)
+ if errCode != s3err.ErrNone {
+ return nil, errCode
}
- // Access credentials.
- // Validate if access key id same.
- ident, cred, found := iam.lookupByAccessKey(accessKey)
+ identity, cred, found := iam.lookupByAccessKey(accessKey)
if !found {
return nil, s3err.ErrInvalidAccessKeyID
}
- // r.RequestURI will have raw encoded URI as sent by the client.
- tokens := strings.SplitN(r.RequestURI, "?", 2)
- encodedResource := tokens[0]
- encodedQuery := ""
- if len(tokens) == 2 {
- encodedQuery = tokens[1]
+ bucket, object := s3_constants.GetBucketAndObject(r)
+ if !identity.canDo(s3_constants.ACTION_WRITE, bucket, object) {
+ return nil, s3err.ErrAccessDenied
}
- unescapedQueries, err := unescapeQueries(encodedQuery)
- if err != nil {
- return nil, s3err.ErrInvalidQueryParams
- }
-
- encodedResource, err = getResource(encodedResource, r.Host, iam.domain)
- if err != nil {
- return nil, s3err.ErrInvalidRequest
- }
-
- prefix := fmt.Sprintf("%s %s:", signV2Algorithm, cred.AccessKey)
- if !strings.HasPrefix(v2Auth, prefix) {
- return nil, s3err.ErrSignatureDoesNotMatch
- }
- v2Auth = v2Auth[len(prefix):]
- expectedAuth := signatureV2(cred, r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header)
+ expectedAuth := signatureV2(cred, r.Method, r.URL.Path, r.URL.Query().Encode(), r.Header)
if !compareSignatureV2(v2Auth, expectedAuth) {
return nil, s3err.ErrSignatureDoesNotMatch
}
- return ident, s3err.ErrNone
+ return identity, s3err.ErrNone
}
-// doesPresignV2SignatureMatch - Verify query headers with presigned signature
-// - http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationQueryStringAuth
+// doesPresignV2SignatureMatch - Verify query headers with calculated header in accordance with
+// - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
//
-// returns ErrNone if matches. S3 errors otherwise.
+// returns ErrNone if the signature matches.
func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request) (*Identity, s3err.ErrorCode) {
-
- // r.RequestURI will have raw encoded URI as sent by the client.
- tokens := strings.SplitN(r.RequestURI, "?", 2)
- encodedResource := tokens[0]
- encodedQuery := ""
- if len(tokens) == 2 {
- encodedQuery = tokens[1]
+ query := r.URL.Query()
+ expires := query.Get("Expires")
+ if expires == "" {
+ return nil, s3err.ErrMissingFields
}
- var (
- filteredQueries []string
- gotSignature string
- expires string
- accessKey string
- err error
- )
-
- var unescapedQueries []string
- unescapedQueries, err = unescapeQueries(encodedQuery)
+ expireTimestamp, err := strconv.ParseInt(expires, 10, 64)
if err != nil {
- return nil, s3err.ErrInvalidQueryParams
- }
-
- // Extract the necessary values from presigned query, construct a list of new filtered queries.
- for _, query := range unescapedQueries {
- keyval := strings.SplitN(query, "=", 2)
- if len(keyval) != 2 {
- return nil, s3err.ErrInvalidQueryParams
- }
- switch keyval[0] {
- case "AWSAccessKeyId":
- accessKey = keyval[1]
- case "Signature":
- gotSignature = keyval[1]
- case "Expires":
- expires = keyval[1]
- default:
- filteredQueries = append(filteredQueries, query)
- }
+ return nil, s3err.ErrMalformedExpires
}
- // Invalid values returns error.
- if accessKey == "" || gotSignature == "" || expires == "" {
- return nil, s3err.ErrInvalidQueryParams
+ if time.Unix(expireTimestamp, 0).Before(time.Now().UTC()) {
+ return nil, s3err.ErrExpiredPresignRequest
}
- // Validate if access key id same.
- ident, cred, found := iam.lookupByAccessKey(accessKey)
- if !found {
+ accessKey := query.Get("AWSAccessKeyId")
+ if accessKey == "" {
return nil, s3err.ErrInvalidAccessKeyID
}
- // Make sure the request has not expired.
- expiresInt, err := strconv.ParseInt(expires, 10, 64)
- if err != nil {
- return nil, s3err.ErrMalformedExpires
+ signature := query.Get("Signature")
+ if signature == "" {
+ return nil, s3err.ErrMissingFields
}
- // Check if the presigned URL has expired.
- if expiresInt < time.Now().UTC().Unix() {
- return nil, s3err.ErrExpiredPresignRequest
+ identity, cred, found := iam.lookupByAccessKey(accessKey)
+ if !found {
+ return nil, s3err.ErrInvalidAccessKeyID
}
- encodedResource, err = getResource(encodedResource, r.Host, iam.domain)
- if err != nil {
- return nil, s3err.ErrInvalidRequest
+ bucket, object := s3_constants.GetBucketAndObject(r)
+ if !identity.canDo(s3_constants.ACTION_READ, bucket, object) {
+ return nil, s3err.ErrAccessDenied
}
- expectedSignature := preSignatureV2(cred, r.Method, encodedResource, strings.Join(filteredQueries, "&"), r.Header, expires)
- if !compareSignatureV2(gotSignature, expectedSignature) {
+ expectedSignature := preSignatureV2(cred, r.Method, r.URL.Path, r.URL.Query().Encode(), r.Header, expires)
+ if !compareSignatureV2(signature, expectedSignature) {
return nil, s3err.ErrSignatureDoesNotMatch
}
-
- return ident, s3err.ErrNone
+ return identity, s3err.ErrNone
}
-// Escape encodedQuery string into unescaped list of query params, returns error
-// if any while unescaping the values.
-func unescapeQueries(encodedQuery string) (unescapedQueries []string, err error) {
- for _, query := range strings.Split(encodedQuery, "&") {
- var unescapedQuery string
- unescapedQuery, err = url.QueryUnescape(query)
- if err != nil {
- return nil, err
- }
- unescapedQueries = append(unescapedQueries, unescapedQuery)
+// validateV2AuthHeader validates AWS Signature Version '2' authentication header.
+func validateV2AuthHeader(v2Auth string) (accessKey string, errCode s3err.ErrorCode) {
+ if v2Auth == "" {
+ return "", s3err.ErrAuthHeaderEmpty
}
- return unescapedQueries, nil
-}
-// Returns "/bucketName/objectName" for path-style or virtual-host-style requests.
-func getResource(path string, host string, domain string) (string, error) {
- if domain == "" {
- return path, nil
- }
- // If virtual-host-style is enabled construct the "resource" properly.
- if strings.Contains(host, ":") {
- // In bucket.mydomain.com:9000, strip out :9000
- var err error
- if host, _, err = net.SplitHostPort(host); err != nil {
- return "", err
- }
+ // Signature V2 authorization header format:
+ // Authorization: AWS AKIAIOSFODNN7EXAMPLE:frJIUN8DYpKDtOLCwo//yllqDzg=
+ if !strings.HasPrefix(v2Auth, signV2Algorithm) {
+ return "", s3err.ErrSignatureVersionNotSupported
}
- if !strings.HasSuffix(host, "."+domain) {
- return path, nil
+
+ // Strip off the Algorithm prefix.
+ v2Auth = v2Auth[len(signV2Algorithm):]
+ authFields := strings.Split(v2Auth, ":")
+ if len(authFields) != 2 {
+ return "", s3err.ErrMissingFields
}
- bucket := strings.TrimSuffix(host, "."+domain)
- return "/" + pathJoin(bucket, path), nil
-}
-// pathJoin - like path.Join() but retains trailing "/" of the last element
-func pathJoin(elem ...string) string {
- trailingSlash := ""
- if len(elem) > 0 {
- if strings.HasSuffix(elem[len(elem)-1], "/") {
- trailingSlash = "/"
- }
+ // The first field is Access Key ID.
+ if authFields[0] == "" {
+ return "", s3err.ErrInvalidAccessKeyID
+ }
+
+ // The second field is signature.
+ if authFields[1] == "" {
+ return "", s3err.ErrMissingFields
}
- return path.Join(elem...) + trailingSlash
+
+ return authFields[0], s3err.ErrNone
}
-// Return the signature v2 of a given request.
+// signatureV2 - calculates signature version 2 for request.
func signatureV2(cred *Credential, method string, encodedResource string, encodedQuery string, headers http.Header) string {
stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, "")
signature := calculateSignatureV2(stringToSign, cred.SecretKey)
- return signature
+ return signV2Algorithm + cred.AccessKey + ":" + signature
}
-// Return string to sign under two different conditions.
-// - if expires string is set then string to sign includes date instead of the Date header.
-// - if expires string is empty then string to sign includes date header instead.
+// getStringToSignV2 - string to sign in accordance with
+// - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
func getStringToSignV2(method string, encodedResource, encodedQuery string, headers http.Header, expires string) string {
canonicalHeaders := canonicalizedAmzHeadersV2(headers)
if len(canonicalHeaders) > 0 {
canonicalHeaders += "\n"
}
- date := expires // Date is set to expires date for presign operations.
- if date == "" {
- // If expires date is empty then request header Date is used.
- date = headers.Get("Date")
- }
-
// From the Amazon docs:
//
// StringToSign = HTTP-Verb + "\n" +
- // Content-Md5 + "\n" +
+ // Content-MD5 + "\n" +
// Content-Type + "\n" +
- // Date/Expires + "\n" +
- // CanonicalizedProtocolHeaders +
+ // Date + "\n" +
+ // CanonicalizedAmzHeaders +
// CanonicalizedResource;
- stringToSign := strings.Join([]string{
- method,
- headers.Get("Content-MD5"),
- headers.Get("Content-Type"),
- date,
- canonicalHeaders,
- }, "\n")
-
- return stringToSign + canonicalizedResourceV2(encodedResource, encodedQuery)
+ stringToSign := method + "\n"
+ stringToSign += headers.Get("Content-Md5") + "\n"
+ stringToSign += headers.Get("Content-Type") + "\n"
+
+ if expires != "" {
+ stringToSign += expires + "\n"
+ } else {
+ stringToSign += headers.Get("Date") + "\n"
+ if v := headers.Get("x-amz-date"); v != "" {
+ stringToSign = strings.Replace(stringToSign, headers.Get("Date")+"\n", "\n", -1)
+ }
+ }
+ stringToSign += canonicalHeaders
+ stringToSign += canonicalizedResourceV2(encodedResource, encodedQuery)
+ return stringToSign
}
-// Return canonical resource string.
+// canonicalizedResourceV2 - canonicalize the resource string for signature V2.
func canonicalizedResourceV2(encodedResource, encodedQuery string) string {
queries := strings.Split(encodedQuery, "&")
keyval := make(map[string]string)
@@ -356,28 +263,26 @@ func canonicalizedResourceV2(encodedResource, encodedQuery string) string {
}
var canonicalQueries []string
- for _, key := range resourceList {
- val, ok := keyval[key]
- if !ok {
- continue
- }
- if val == "" {
- canonicalQueries = append(canonicalQueries, key)
- continue
+ for _, resource := range resourceList {
+ if val, ok := keyval[resource]; ok {
+ if val == "" {
+ canonicalQueries = append(canonicalQueries, resource)
+ continue
+ }
+ canonicalQueries = append(canonicalQueries, resource+"="+val)
}
- canonicalQueries = append(canonicalQueries, key+"="+val)
}
- // The queries will be already sorted as resourceList is sorted, if canonicalQueries
- // is empty strings.Join returns empty.
- canonicalQuery := strings.Join(canonicalQueries, "&")
- if canonicalQuery != "" {
- return encodedResource + "?" + canonicalQuery
+ // The queries will be already sorted as resourceList is sorted.
+ if len(canonicalQueries) == 0 {
+ return encodedResource
}
- return encodedResource
+
+ // If queries are present then the canonicalized resource is set to encodedResource + "?" + strings.Join(canonicalQueries, "&")
+ return encodedResource + "?" + strings.Join(canonicalQueries, "&")
}
-// Return canonical headers.
+// canonicalizedAmzHeadersV2 - canonicalize the x-amz-* headers for signature V2.
func canonicalizedAmzHeadersV2(headers http.Header) string {
var keys []string
keyval := make(map[string]string)
@@ -390,6 +295,7 @@ func canonicalizedAmzHeadersV2(headers http.Header) string {
keyval[lkey] = strings.Join(headers[key], ",")
}
sort.Strings(keys)
+
var canonicalHeaders []string
for _, key := range keys {
canonicalHeaders = append(canonicalHeaders, key+":"+keyval[key])
@@ -397,6 +303,7 @@ func canonicalizedAmzHeadersV2(headers http.Header) string {
return strings.Join(canonicalHeaders, "\n")
}
+// calculateSignatureV2 - calculates signature version 2.
func calculateSignatureV2(stringToSign string, secret string) string {
hm := hmac.New(sha1.New, []byte(secret))
hm.Write([]byte(stringToSign))
diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go
index 0ddbaa917..8d9011f0d 100644
--- a/weed/s3api/auth_signature_v4.go
+++ b/weed/s3api/auth_signature_v4.go
@@ -23,21 +23,15 @@ import (
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
- "hash"
- "io"
- "net"
"net/http"
- "net/url"
"regexp"
"sort"
"strconv"
"strings"
- "sync"
- "sync/atomic"
"time"
"unicode/utf8"
- "github.com/seaweedfs/seaweedfs/weed/glog"
+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
@@ -52,77 +46,99 @@ func (iam *IdentityAccessManagement) reqSignatureV4Verify(r *http.Request) (*Ide
return nil, s3err.ErrAccessDenied
}
-// Streaming AWS Signature Version '4' constants.
+// Constants specific to this file
const (
- emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
- streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
- signV4ChunkedAlgorithm = "AWS4-HMAC-SHA256-PAYLOAD"
-
- // http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" or "STREAMING-UNSIGNED-PAYLOAD-TRAILER" indicates that the
- // client did not calculate sha256 of the payload.
- unsignedPayload = "UNSIGNED-PAYLOAD"
+ emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+ streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
streamingUnsignedPayload = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
+ unsignedPayload = "UNSIGNED-PAYLOAD"
)
-// AWS S3 authentication headers that should be skipped when signing the request
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
-var awsS3AuthHeaders = map[string]struct{}{
- "x-amz-content-sha256": {},
- "x-amz-security-token": {},
- "x-amz-algorithm": {},
- "x-amz-date": {},
- "x-amz-expires": {},
- "x-amz-signedheaders": {},
- "x-amz-credential": {},
- "x-amz-signature": {},
-}
-
-// Returns SHA256 for calculating canonical-request.
+// getContentSha256Cksum retrieves the "x-amz-content-sha256" header value.
func getContentSha256Cksum(r *http.Request) string {
- var (
- defaultSha256Cksum string
- v []string
- ok bool
- )
+ // If the client sends a SHA256 checksum of the object in this header, use it.
+ if v := r.Header.Get("X-Amz-Content-Sha256"); v != "" {
+ return v
+ }
// For a presigned request we look at the query param for sha256.
if isRequestPresignedSignatureV4(r) {
- // X-Amz-Content-Sha256, if not set in presigned requests, checksum
- // will default to 'UNSIGNED-PAYLOAD'.
- defaultSha256Cksum = unsignedPayload
- v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
- if !ok {
- v, ok = r.Header["X-Amz-Content-Sha256"]
- }
- } else {
- // X-Amz-Content-Sha256, if not set in signed requests, checksum
- // will default to sha256([]byte("")).
- defaultSha256Cksum = emptySHA256
- v, ok = r.Header["X-Amz-Content-Sha256"]
+ // X-Amz-Content-Sha256 header value is optional for presigned requests.
+ return unsignedPayload
}
- // We found 'X-Amz-Content-Sha256' return the captured value.
- if ok {
- return v[0]
+ // X-Amz-Content-Sha256 header value is required for all non-presigned requests.
+ return emptySHA256
+}
+
+// signValues data type represents structured form of AWS Signature V4 header.
+type signValues struct {
+ Credential credentialHeader
+ SignedHeaders []string
+ Signature string
+}
+
+// parseSignV4 parses the authorization header for signature v4.
+func parseSignV4(v4Auth string) (sv signValues, aec s3err.ErrorCode) {
+ // Replace all spaced strings, some clients can send spaced
+ // parameters and some won't. So we pro-actively remove any spaces
+ // to make parsing easier.
+ v4Auth = strings.Replace(v4Auth, " ", "", -1)
+ if v4Auth == "" {
+ return sv, s3err.ErrAuthHeaderEmpty
}
- // We couldn't find 'X-Amz-Content-Sha256'.
- return defaultSha256Cksum
+ // Verify if the header algorithm is supported or not.
+ if !strings.HasPrefix(v4Auth, signV4Algorithm) {
+ return sv, s3err.ErrSignatureVersionNotSupported
+ }
+
+ // Strip off the Algorithm prefix.
+ v4Auth = strings.TrimPrefix(v4Auth, signV4Algorithm)
+ authFields := strings.Split(strings.TrimSpace(v4Auth), ",")
+ if len(authFields) != 3 {
+ return sv, s3err.ErrMissingFields
+ }
+
+ // Initialize signature version '4' structured header.
+ signV4Values := signValues{}
+
+ var err s3err.ErrorCode
+ // Save credential values.
+ signV4Values.Credential, err = parseCredentialHeader(authFields[0])
+ if err != s3err.ErrNone {
+ return sv, err
+ }
+
+ // Save signed headers.
+ signV4Values.SignedHeaders, err = parseSignedHeader(authFields[1])
+ if err != s3err.ErrNone {
+ return sv, err
+ }
+
+ // Save signature.
+ signV4Values.Signature, err = parseSignature(authFields[2])
+ if err != s3err.ErrNone {
+ return sv, err
+ }
+
+ // Return the structure here.
+ return signV4Values, s3err.ErrNone
}
-// Verify authorization header - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
+// Wrapper to verify if request came with a valid signature.
func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) {
- // Copy request.
+ // Copy request
req := *r
// Save authorization header.
v4Auth := req.Header.Get("Authorization")
// Parse signature version '4' header.
- signV4Values, err := parseSignV4(v4Auth)
- if err != s3err.ErrNone {
- return nil, err
+ signV4Values, errCode := parseSignV4(v4Auth)
+ if errCode != s3err.ErrNone {
+ return nil, errCode
}
// Extract all the signed headers along with its values.
@@ -131,21 +147,26 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r
return nil, errCode
}
- // Verify if the access key id matches.
- identity, cred, found := iam.lookupByAccessKey(signV4Values.Credential.accessKey)
+ cred := signV4Values.Credential
+ identity, foundCred, found := iam.lookupByAccessKey(cred.accessKey)
if !found {
return nil, s3err.ErrInvalidAccessKeyID
}
+ bucket, object := s3_constants.GetBucketAndObject(r)
+ if !identity.canDo(s3_constants.ACTION_WRITE, bucket, object) {
+ return nil, s3err.ErrAccessDenied
+ }
+
// Extract date, if not present throw error.
- var date string
- if date = req.Header.Get(http.CanonicalHeaderKey("X-Amz-Date")); date == "" {
- if date = r.Header.Get("Date"); date == "" {
+ var dateStr string
+ if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
+ if dateStr = r.Header.Get("Date"); dateStr == "" {
return nil, s3err.ErrMissingDateHeader
}
}
// Parse date header.
- t, e := time.Parse(iso8601Format, date)
+ t, e := time.Parse(iso8601Format, dateStr)
if e != nil {
return nil, s3err.ErrMalformedDate
}
@@ -153,62 +174,134 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r
// Query string.
queryStr := req.URL.Query().Encode()
- // Get hashed Payload
- if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil {
- buf, _ := io.ReadAll(r.Body)
- r.Body = io.NopCloser(bytes.NewBuffer(buf))
- b, _ := io.ReadAll(bytes.NewBuffer(buf))
- if len(b) != 0 {
- bodyHash := sha256.Sum256(b)
- hashedPayload = hex.EncodeToString(bodyHash[:])
- }
+ // Get canonical request.
+ canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method)
+
+ // Get string to sign from canonical request.
+ stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope())
+
+ // Get hmac signing key.
+ signingKey := getSigningKey(foundCred.SecretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), signV4Values.Credential.scope.region, "s3")
+
+ // Calculate signature.
+ newSignature := getSignature(signingKey, stringToSign)
+
+ // Verify if signature match.
+ if !compareSignatureV4(newSignature, signV4Values.Signature) {
+ return nil, s3err.ErrSignatureDoesNotMatch
}
- if forwardedPrefix := r.Header.Get("X-Forwarded-Prefix"); forwardedPrefix != "" {
- // Handling usage of reverse proxy at prefix.
- // Trying with prefix before main path.
+ // Return error none.
+ return identity, s3err.ErrNone
+}
- // Get canonical request.
- glog.V(4).Infof("Forwarded Prefix: %s", forwardedPrefix)
+// Simple implementation for presigned signature verification
+func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) {
+ // Parse presigned signature values from query parameters
+ query := r.URL.Query()
- canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, forwardedPrefix+req.URL.Path, req.Method)
- errCode = iam.genAndCompareSignatureV4(canonicalRequest, cred.SecretKey, t, signV4Values)
- if errCode == s3err.ErrNone {
- return identity, errCode
- }
+ // Check required parameters
+ algorithm := query.Get("X-Amz-Algorithm")
+ if algorithm != signV4Algorithm {
+ return nil, s3err.ErrSignatureVersionNotSupported
}
- // Get canonical request.
- canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method)
+ credential := query.Get("X-Amz-Credential")
+ if credential == "" {
+ return nil, s3err.ErrMissingFields
+ }
- errCode = iam.genAndCompareSignatureV4(canonicalRequest, cred.SecretKey, t, signV4Values)
+ signature := query.Get("X-Amz-Signature")
+ if signature == "" {
+ return nil, s3err.ErrMissingFields
+ }
- if errCode == s3err.ErrNone {
- return identity, errCode
+ signedHeadersStr := query.Get("X-Amz-SignedHeaders")
+ if signedHeadersStr == "" {
+ return nil, s3err.ErrMissingFields
}
- return nil, errCode
-}
-// Generate and compare signature for request.
-func (iam *IdentityAccessManagement) genAndCompareSignatureV4(canonicalRequest, secretKey string, t time.Time, signV4Values signValues) s3err.ErrorCode {
- // Get string to sign from canonical request.
- stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope())
- glog.V(4).Infof("String to Sign:\n%s", stringToSign)
- // Calculate signature.
- newSignature := iam.getSignature(
- secretKey,
- signV4Values.Credential.scope.date,
- signV4Values.Credential.scope.region,
- signV4Values.Credential.scope.service,
- stringToSign,
- )
- glog.V(4).Infof("Signature:\n%s", newSignature)
+ dateStr := query.Get("X-Amz-Date")
+ if dateStr == "" {
+ return nil, s3err.ErrMissingDateHeader
+ }
- // Verify if signature match.
- if !compareSignatureV4(newSignature, signV4Values.Signature) {
- return s3err.ErrSignatureDoesNotMatch
+ // Parse credential
+ credHeader, err := parseCredentialHeader("Credential=" + credential)
+ if err != s3err.ErrNone {
+ return nil, err
}
- return s3err.ErrNone
+
+ // Look up identity by access key
+ identity, foundCred, found := iam.lookupByAccessKey(credHeader.accessKey)
+ if !found {
+ return nil, s3err.ErrInvalidAccessKeyID
+ }
+
+ // Check permissions
+ bucket, object := s3_constants.GetBucketAndObject(r)
+ if !identity.canDo(s3_constants.ACTION_READ, bucket, object) {
+ return nil, s3err.ErrAccessDenied
+ }
+
+ // Parse date
+ t, e := time.Parse(iso8601Format, dateStr)
+ if e != nil {
+ return nil, s3err.ErrMalformedDate
+ }
+
+ // Check expiration
+ expiresStr := query.Get("X-Amz-Expires")
+ if expiresStr != "" {
+ expires, parseErr := strconv.ParseInt(expiresStr, 10, 64)
+ if parseErr != nil {
+ return nil, s3err.ErrMalformedDate
+ }
+ // Check if current time is after the expiration time
+ expirationTime := t.Add(time.Duration(expires) * time.Second)
+ if time.Now().UTC().After(expirationTime) {
+ return nil, s3err.ErrExpiredPresignRequest
+ }
+ }
+
+ // Parse signed headers
+ signedHeaders := strings.Split(signedHeadersStr, ";")
+
+ // Extract signed headers from request
+ extractedSignedHeaders := make(http.Header)
+ for _, header := range signedHeaders {
+ if header == "host" {
+ extractedSignedHeaders[header] = []string{r.Host}
+ continue
+ }
+ if values := r.Header[http.CanonicalHeaderKey(header)]; len(values) > 0 {
+ extractedSignedHeaders[http.CanonicalHeaderKey(header)] = values
+ }
+ }
+
+ // Remove signature from query for canonical request calculation
+ queryForCanonical := r.URL.Query()
+ queryForCanonical.Del("X-Amz-Signature")
+ queryStr := strings.Replace(queryForCanonical.Encode(), "+", "%20", -1)
+
+ // Get canonical request
+ canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, r.URL.Path, r.Method)
+
+ // Get string to sign
+ stringToSign := getStringToSign(canonicalRequest, t, credHeader.getScope())
+
+ // Get signing key
+ signingKey := getSigningKey(foundCred.SecretKey, credHeader.scope.date.Format(yyyymmdd), credHeader.scope.region, "s3")
+
+ // Calculate expected signature
+ expectedSignature := getSignature(signingKey, stringToSign)
+
+ // Verify signature
+ if !compareSignatureV4(expectedSignature, signature) {
+ return nil, s3err.ErrSignatureDoesNotMatch
+ }
+
+ return identity, s3err.ErrNone
}
// credentialHeader data type represents structured form of Credential
@@ -223,14 +316,6 @@ type credentialHeader struct {
}
}
-// signValues data type represents structured form of AWS Signature V4 header.
-type signValues struct {
- Credential credentialHeader
- SignedHeaders []string
- Signature string
-}
-
-// Return scope string.
func (c credentialHeader) getScope() string {
return strings.Join([]string{
c.scope.date.Format(yyyymmdd),
@@ -240,58 +325,9 @@ func (c credentialHeader) getScope() string {
}, "/")
}
-// Authorization: algorithm Credential=accessKeyID/credScope, \
-// SignedHeaders=signedHeaders, Signature=signature
-func parseSignV4(v4Auth string) (sv signValues, aec s3err.ErrorCode) {
- // Replace all spaced strings, some clients can send spaced
- // parameters and some won't. So we pro-actively remove any spaces
- // to make parsing easier.
- v4Auth = strings.Replace(v4Auth, " ", "", -1)
- if v4Auth == "" {
- return sv, s3err.ErrAuthHeaderEmpty
- }
-
- // Verify if the header algorithm is supported or not.
- if !strings.HasPrefix(v4Auth, signV4Algorithm) {
- return sv, s3err.ErrSignatureVersionNotSupported
- }
-
- // Strip off the Algorithm prefix.
- v4Auth = strings.TrimPrefix(v4Auth, signV4Algorithm)
- authFields := strings.Split(strings.TrimSpace(v4Auth), ",")
- if len(authFields) != 3 {
- return sv, s3err.ErrMissingFields
- }
-
- // Initialize signature version '4' structured header.
- signV4Values := signValues{}
-
- var err s3err.ErrorCode
- // Save credential values.
- signV4Values.Credential, err = parseCredentialHeader(authFields[0])
- if err != s3err.ErrNone {
- return sv, err
- }
-
- // Save signed headers.
- signV4Values.SignedHeaders, err = parseSignedHeader(authFields[1])
- if err != s3err.ErrNone {
- return sv, err
- }
-
- // Save signature.
- signV4Values.Signature, err = parseSignature(authFields[2])
- if err != s3err.ErrNone {
- return sv, err
- }
-
- // Return the structure here.
- return signV4Values, s3err.ErrNone
-}
-
// parse credentialHeader string into its structured form.
func parseCredentialHeader(credElement string) (ch credentialHeader, aec s3err.ErrorCode) {
- creds := strings.Split(strings.TrimSpace(credElement), "=")
+ creds := strings.SplitN(strings.TrimSpace(credElement), "=", 2)
if len(creds) != 2 {
return ch, s3err.ErrMissingFields
}
@@ -318,22 +354,6 @@ func parseCredentialHeader(credElement string) (ch credentialHeader, aec s3err.E
return cred, s3err.ErrNone
}
-// Parse slice of signed headers from signed headers tag.
-func parseSignedHeader(signedHdrElement string) ([]string, s3err.ErrorCode) {
- signedHdrFields := strings.Split(strings.TrimSpace(signedHdrElement), "=")
- if len(signedHdrFields) != 2 {
- return nil, s3err.ErrMissingFields
- }
- if signedHdrFields[0] != "SignedHeaders" {
- return nil, s3err.ErrMissingSignHeadersTag
- }
- if signedHdrFields[1] == "" {
- return nil, s3err.ErrMissingFields
- }
- signedHeaders := strings.Split(signedHdrFields[1], ";")
- return signedHeaders, s3err.ErrNone
-}
-
// Parse signature from signature tag.
func parseSignature(signElement string) (string, s3err.ErrorCode) {
signFields := strings.Split(strings.TrimSpace(signElement), "=")
@@ -350,437 +370,90 @@ func parseSignature(signElement string) (string, s3err.ErrorCode) {
return signature, s3err.ErrNone
}
-// doesPolicySignatureV4Match - Verify query headers with post policy
-// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
-//
-// returns ErrNone if the signature matches.
-func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.Header) s3err.ErrorCode {
-
- // Parse credential tag.
- credHeader, err := parseCredentialHeader("Credential=" + formValues.Get("X-Amz-Credential"))
- if err != s3err.ErrNone {
- return s3err.ErrMissingFields
+// Parse slice of signed headers from signed headers tag.
+func parseSignedHeader(signedHdrElement string) ([]string, s3err.ErrorCode) {
+ signedHdrFields := strings.Split(strings.TrimSpace(signedHdrElement), "=")
+ if len(signedHdrFields) != 2 {
+ return nil, s3err.ErrMissingFields
}
-
- _, cred, found := iam.lookupByAccessKey(credHeader.accessKey)
- if !found {
- return s3err.ErrInvalidAccessKeyID
+ if signedHdrFields[0] != "SignedHeaders" {
+ return nil, s3err.ErrMissingSignHeadersTag
}
-
- // Get signature.
- newSignature := iam.getSignature(
- cred.SecretKey,
- credHeader.scope.date,
- credHeader.scope.region,
- credHeader.scope.service,
- formValues.Get("Policy"),
- )
-
- // Verify signature.
- if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) {
- return s3err.ErrSignatureDoesNotMatch
+ if signedHdrFields[1] == "" {
+ return nil, s3err.ErrMissingFields
}
-
- // Success.
- return s3err.ErrNone
+ signedHeaders := strings.Split(signedHdrFields[1], ";")
+ return signedHeaders, s3err.ErrNone
}
-// check query headers with presigned signature
-// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
-func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) {
-
- // Copy request
- req := *r
+func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.Header) s3err.ErrorCode {
- // Parse request query string.
- pSignValues, err := parsePreSignV4(req.URL.Query())
+ // Parse credential tag.
+ credHeader, err := parseCredentialHeader("Credential=" + formValues.Get("X-Amz-Credential"))
if err != s3err.ErrNone {
- return nil, err
+ return err
}
- // Verify if the access key id matches.
- identity, cred, found := iam.lookupByAccessKey(pSignValues.Credential.accessKey)
+ identity, cred, found := iam.lookupByAccessKey(credHeader.accessKey)
if !found {
- return nil, s3err.ErrInvalidAccessKeyID
- }
-
- // Extract all the signed headers along with its values.
- extractedSignedHeaders, errCode := extractSignedHeaders(pSignValues.SignedHeaders, r)
- if errCode != s3err.ErrNone {
- return nil, errCode
- }
- // Construct new query.
- query := make(url.Values)
- if req.URL.Query().Get("X-Amz-Content-Sha256") != "" {
- query.Set("X-Amz-Content-Sha256", hashedPayload)
- }
-
- query.Set("X-Amz-Algorithm", signV4Algorithm)
-
- now := time.Now().UTC()
-
- // If the host which signed the request is slightly ahead in time (by less than globalMaxSkewTime) the
- // request should still be allowed.
- if pSignValues.Date.After(now.Add(15 * time.Minute)) {
- return nil, s3err.ErrRequestNotReadyYet
- }
-
- if now.Sub(pSignValues.Date) > pSignValues.Expires {
- return nil, s3err.ErrExpiredPresignRequest
- }
-
- // Save the date and expires.
- t := pSignValues.Date
- expireSeconds := int(pSignValues.Expires / time.Second)
-
- // Construct the query.
- query.Set("X-Amz-Date", t.Format(iso8601Format))
- query.Set("X-Amz-Expires", strconv.Itoa(expireSeconds))
- query.Set("X-Amz-SignedHeaders", getSignedHeaders(extractedSignedHeaders))
- query.Set("X-Amz-Credential", cred.AccessKey+"/"+getScope(t, pSignValues.Credential.scope.region))
-
- // Save other headers available in the request parameters.
- for k, v := range req.URL.Query() {
- // Skip AWS S3 authentication headers
- if _, ok := awsS3AuthHeaders[strings.ToLower(k)]; ok {
- continue
- }
-
- query[k] = v
+ return s3err.ErrInvalidAccessKeyID
}
- // Get the encoded query.
- encodedQuery := query.Encode()
-
- // Verify if date query is same.
- if req.URL.Query().Get("X-Amz-Date") != query.Get("X-Amz-Date") {
- return nil, s3err.ErrSignatureDoesNotMatch
- }
- // Verify if expires query is same.
- if req.URL.Query().Get("X-Amz-Expires") != query.Get("X-Amz-Expires") {
- return nil, s3err.ErrSignatureDoesNotMatch
- }
- // Verify if signed headers query is same.
- if req.URL.Query().Get("X-Amz-SignedHeaders") != query.Get("X-Amz-SignedHeaders") {
- return nil, s3err.ErrSignatureDoesNotMatch
- }
- // Verify if credential query is same.
- if req.URL.Query().Get("X-Amz-Credential") != query.Get("X-Amz-Credential") {
- return nil, s3err.ErrSignatureDoesNotMatch
- }
- // Verify if sha256 payload query is same.
- if req.URL.Query().Get("X-Amz-Content-Sha256") != "" {
- if req.URL.Query().Get("X-Amz-Content-Sha256") != query.Get("X-Amz-Content-Sha256") {
- return nil, s3err.ErrContentSHA256Mismatch
- }
+ bucket := formValues.Get("bucket")
+ if !identity.canDo(s3_constants.ACTION_WRITE, bucket, "") {
+ return s3err.ErrAccessDenied
}
- // / Verify finally if signature is same.
-
- // Get canonical request.
- presignedCanonicalReq := getCanonicalRequest(extractedSignedHeaders, hashedPayload, encodedQuery, req.URL.Path, req.Method)
-
- // Get string to sign from canonical request.
- presignedStringToSign := getStringToSign(presignedCanonicalReq, t, pSignValues.Credential.getScope())
+ // Get signing key.
+ signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date.Format(yyyymmdd), credHeader.scope.region, "s3")
- // Get new signature.
- newSignature := iam.getSignature(
- cred.SecretKey,
- pSignValues.Credential.scope.date,
- pSignValues.Credential.scope.region,
- pSignValues.Credential.scope.service,
- presignedStringToSign,
- )
+ // Get signature.
+ newSignature := getSignature(signingKey, formValues.Get("Policy"))
// Verify signature.
- if !compareSignatureV4(req.URL.Query().Get("X-Amz-Signature"), newSignature) {
- return nil, s3err.ErrSignatureDoesNotMatch
- }
- return identity, s3err.ErrNone
-}
-
-func (iam *IdentityAccessManagement) getSignature(secretKey string, t time.Time, region string, service string, stringToSign string) string {
- pool := iam.getSignatureHashPool(secretKey, t, region, service)
- h := pool.Get().(hash.Hash)
- defer pool.Put(h)
-
- h.Reset()
- h.Write([]byte(stringToSign))
- sig := hex.EncodeToString(h.Sum(nil))
-
- return sig
-}
-
-func (iam *IdentityAccessManagement) getSignatureHashPool(secretKey string, t time.Time, region string, service string) *sync.Pool {
- // Build a caching key for the pool.
- date := t.Format(yyyymmdd)
- hashID := "AWS4" + secretKey + "/" + date + "/" + region + "/" + service + "/" + "aws4_request"
-
- // Try to find an existing pool and return it.
- iam.hashMu.RLock()
- pool, ok := iam.hashes[hashID]
- iam.hashMu.RUnlock()
-
- if !ok {
- iam.hashMu.Lock()
- defer iam.hashMu.Unlock()
- pool, ok = iam.hashes[hashID]
- }
-
- if ok {
- atomic.StoreInt32(iam.hashCounters[hashID], 1)
- return pool
- }
-
- // Create a pool that returns HMAC hashers for the requested parameters to avoid expensive re-initializing
- // of new instances on every request.
- iam.hashes[hashID] = &sync.Pool{
- New: func() any {
- signingKey := getSigningKey(secretKey, date, region, service)
- return hmac.New(sha256.New, signingKey)
- },
- }
- iam.hashCounters[hashID] = new(int32)
-
- // Clean up unused pools automatically after one hour of inactivity
- ticker := time.NewTicker(time.Hour)
- go func() {
- for range ticker.C {
- old := atomic.SwapInt32(iam.hashCounters[hashID], 0)
- if old == 0 {
- break
- }
- }
-
- ticker.Stop()
- iam.hashMu.Lock()
- delete(iam.hashes, hashID)
- delete(iam.hashCounters, hashID)
- iam.hashMu.Unlock()
- }()
-
- return iam.hashes[hashID]
-}
-
-func contains(list []string, elem string) bool {
- for _, t := range list {
- if t == elem {
- return true
- }
- }
- return false
-}
-
-// preSignValues data type represents structured form of AWS Signature V4 query string.
-type preSignValues struct {
- signValues
- Date time.Time
- Expires time.Duration
-}
-
-// Parses signature version '4' query string of the following form.
-//
-// querystring = X-Amz-Algorithm=algorithm
-// querystring += &X-Amz-Credential= urlencode(accessKey + '/' + credential_scope)
-// querystring += &X-Amz-Date=date
-// querystring += &X-Amz-Expires=timeout interval
-// querystring += &X-Amz-SignedHeaders=signed_headers
-// querystring += &X-Amz-Signature=signature
-//
-// verifies if any of the necessary query params are missing in the presigned request.
-func doesV4PresignParamsExist(query url.Values) s3err.ErrorCode {
- v4PresignQueryParams := []string{"X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Signature", "X-Amz-Date", "X-Amz-SignedHeaders", "X-Amz-Expires"}
- for _, v4PresignQueryParam := range v4PresignQueryParams {
- if _, ok := query[v4PresignQueryParam]; !ok {
- return s3err.ErrInvalidQueryParams
- }
+ if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) {
+ return s3err.ErrSignatureDoesNotMatch
}
return s3err.ErrNone
}
-// Parses all the presigned signature values into separate elements.
-func parsePreSignV4(query url.Values) (psv preSignValues, aec s3err.ErrorCode) {
- var err s3err.ErrorCode
- // verify whether the required query params exist.
- err = doesV4PresignParamsExist(query)
- if err != s3err.ErrNone {
- return psv, err
- }
-
- // Verify if the query algorithm is supported or not.
- if query.Get("X-Amz-Algorithm") != signV4Algorithm {
- return psv, s3err.ErrInvalidQuerySignatureAlgo
- }
-
- // Initialize signature version '4' structured header.
- preSignV4Values := preSignValues{}
-
- // Save credential.
- preSignV4Values.Credential, err = parseCredentialHeader("Credential=" + query.Get("X-Amz-Credential"))
- if err != s3err.ErrNone {
- return psv, err
- }
-
- var e error
- // Save date in native time.Time.
- preSignV4Values.Date, e = time.Parse(iso8601Format, query.Get("X-Amz-Date"))
- if e != nil {
- return psv, s3err.ErrMalformedPresignedDate
- }
-
- // Save expires in native time.Duration.
- preSignV4Values.Expires, e = time.ParseDuration(query.Get("X-Amz-Expires") + "s")
- if e != nil {
- return psv, s3err.ErrMalformedExpires
- }
-
- if preSignV4Values.Expires < 0 {
- return psv, s3err.ErrNegativeExpires
- }
-
- // Check if Expiry time is less than 7 days (value in seconds).
- if preSignV4Values.Expires.Seconds() > 604800 {
- return psv, s3err.ErrMaximumExpires
- }
-
- // Save signed headers.
- preSignV4Values.SignedHeaders, err = parseSignedHeader("SignedHeaders=" + query.Get("X-Amz-SignedHeaders"))
- if err != s3err.ErrNone {
- return psv, err
- }
-
- // Save signature.
- preSignV4Values.Signature, err = parseSignature("Signature=" + query.Get("X-Amz-Signature"))
- if err != s3err.ErrNone {
- return psv, err
- }
-
- // Return structured form of signature query string.
- return preSignV4Values, s3err.ErrNone
-}
-
-// extractSignedHeaders extract signed headers from Authorization header
+// Verify if extracted signed headers are not properly signed.
func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header, s3err.ErrorCode) {
reqHeaders := r.Header
- // find whether "host" is part of list of signed headers.
- // if not return ErrUnsignedHeaders. "host" is mandatory.
- if !contains(signedHeaders, "host") {
- return nil, s3err.ErrUnsignedHeaders
+ // If no signed headers are provided, then return an error.
+ if len(signedHeaders) == 0 {
+ return nil, s3err.ErrMissingFields
}
extractedSignedHeaders := make(http.Header)
for _, header := range signedHeaders {
- // `host` will not be found in the headers, can be found in r.Host.
- // but its alway necessary that the list of signed headers containing host in it.
- val, ok := reqHeaders[http.CanonicalHeaderKey(header)]
- if ok {
- for _, enc := range val {
- extractedSignedHeaders.Add(header, enc)
- }
+ // `host` is not a case-sensitive header, unlike other headers such as `x-amz-date`.
+ if header == "host" {
+ // Get host value.
+ hostHeaderValue := extractHostHeader(r)
+ extractedSignedHeaders[header] = []string{hostHeaderValue}
continue
}
- switch header {
- case "expect":
- // Set the default value of the Expect header for compatibility.
- //
- // In NGINX v1.1, the Expect header is removed when handling 100-continue requests.
- //
- // `aws-cli` sets this as part of signed headers.
- //
- // According to
- // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.20
- // Expect header is always of form:
- //
- // Expect = "Expect" ":" 1#expectation
- // expectation = "100-continue" | expectation-extension
- //
- // So it safe to assume that '100-continue' is what would
- // be sent, for the time being keep this work around.
- extractedSignedHeaders.Set(header, "100-continue")
- case "host":
- extractedHost := extractHostHeader(r)
- extractedSignedHeaders.Set(header, extractedHost)
- case "transfer-encoding":
- for _, enc := range r.TransferEncoding {
- extractedSignedHeaders.Add(header, enc)
- }
- case "content-length":
- // Signature-V4 spec excludes Content-Length from signed headers list for signature calculation.
- // But some clients deviate from this rule. Hence we consider Content-Length for signature
- // calculation to be compatible with such clients.
- extractedSignedHeaders.Set(header, strconv.FormatInt(r.ContentLength, 10))
- default:
- return nil, s3err.ErrUnsignedHeaders
+ // For all other headers we need to find them in the HTTP headers and copy them over.
+ // We skip non-existent headers to be compatible with AWS signatures.
+ if values, ok := reqHeaders[http.CanonicalHeaderKey(header)]; ok {
+ extractedSignedHeaders[header] = values
}
}
return extractedSignedHeaders, s3err.ErrNone
}
+// extractHostHeader returns the value of host header if available.
func extractHostHeader(r *http.Request) string {
-
- forwardedHost := r.Header.Get("X-Forwarded-Host")
- forwardedPort := r.Header.Get("X-Forwarded-Port")
- forwardedProto := r.Header.Get("X-Forwarded-Proto")
-
- // If X-Forwarded-Host is set, use that as the host.
- // If X-Forwarded-Port is set, use that too to form the host.
- // If X-Forwarded-Proto is set, check if is it default to omit the port.
- if forwardedHost != "" {
- extractedHost := forwardedHost
- host, port, err := net.SplitHostPort(extractedHost)
- if err == nil {
- extractedHost = host
- if forwardedPort == "" {
- forwardedPort = port
- }
- }
- scheme := r.URL.Scheme
- if forwardedProto != "" {
- scheme = forwardedProto
- }
- if !isDefaultPort(scheme, forwardedPort) {
- extractedHost = net.JoinHostPort(extractedHost, forwardedPort)
- }
- return extractedHost
- } else {
- // Go http server removes "host" from Request.Header
- host := r.Host
- if host == "" {
- host = r.URL.Host
- }
- h, port, err := net.SplitHostPort(host)
- if err != nil {
- return host
- }
- if isDefaultPort(r.URL.Scheme, port) {
- return h
- }
- return host
+ hostHeaderValue := r.Host
+ // For standard requests, this should be fine.
+ if r.Host != "" {
+ return hostHeaderValue
}
-}
-
-func isDefaultPort(scheme, port string) bool {
- if port == "" {
- return true
- }
-
- switch port {
- case "80":
- return strings.EqualFold(scheme, "http")
- case "443":
- return strings.EqualFold(scheme, "https")
- default:
- return false
- }
-}
-
-// getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names
-func getSignedHeaders(signedHeaders http.Header) string {
- var headers []string
- for k := range signedHeaders {
- headers = append(headers, strings.ToLower(k))
+ // If no host header is found, then check for host URL value.
+ if r.URL.Host != "" {
+ hostHeaderValue = r.URL.Host
}
- sort.Strings(headers)
- return strings.Join(headers, ";")
+ return hostHeaderValue
}
// getScope generate a string of a specific date, an AWS region, and a service.
@@ -815,8 +488,6 @@ func getCanonicalRequest(extractedSignedHeaders http.Header, payload, queryStr,
getSignedHeaders(extractedSignedHeaders),
payload,
}, "\n")
-
- glog.V(4).Infof("Canonical Request:\n%s", canonicalRequest)
return canonicalRequest
}
@@ -824,11 +495,16 @@ func getCanonicalRequest(extractedSignedHeaders http.Header, payload, queryStr,
func getStringToSign(canonicalRequest string, t time.Time, scope string) string {
stringToSign := signV4Algorithm + "\n" + t.Format(iso8601Format) + "\n"
stringToSign = stringToSign + scope + "\n"
- canonicalRequestBytes := sha256.Sum256([]byte(canonicalRequest))
- stringToSign = stringToSign + hex.EncodeToString(canonicalRequestBytes[:])
+ stringToSign = stringToSign + getSHA256Hash([]byte(canonicalRequest))
return stringToSign
}
+// getSHA256Hash returns hex-encoded SHA256 hash of the input data.
+func getSHA256Hash(data []byte) string {
+ hash := sha256.Sum256(data)
+ return hex.EncodeToString(hash[:])
+}
+
// sumHMAC calculate hmac between two input byte array.
func sumHMAC(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
@@ -850,9 +526,11 @@ func getCanonicalHeaders(signedHeaders http.Header) string {
var headers []string
vals := make(http.Header)
for k, vv := range signedHeaders {
- headers = append(headers, strings.ToLower(k))
vals[strings.ToLower(k)] = vv
}
+ for k := range vals {
+ headers = append(headers, k)
+ }
sort.Strings(headers)
var buf bytes.Buffer
@@ -870,18 +548,28 @@ func getCanonicalHeaders(signedHeaders http.Header) string {
return buf.String()
}
-// Trim leading and trailing spaces and replace sequential spaces with one space, following Trimall()
-// in http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+// signV4TrimAll trims leading and trailing spaces from each string in the slice, and trims sequential spaces.
func signV4TrimAll(input string) string {
// Compress adjacent spaces (a space is determined by
- // unicode.IsSpace() internally here) to one space and return
+ // unicode.IsSpace() internally here) to a single space and trim
+ // leading and trailing spaces.
return strings.Join(strings.Fields(input), " ")
}
+// getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names
+func getSignedHeaders(signedHeaders http.Header) string {
+ var headers []string
+ for k := range signedHeaders {
+ headers = append(headers, strings.ToLower(k))
+ }
+ sort.Strings(headers)
+ return strings.Join(headers, ";")
+}
+
// if object matches reserved string, no need to encode them
var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
-// EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
+// encodePath encodes the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
// non english characters cannot be parsed due to the nature in which url.Encode() is written
@@ -896,34 +584,38 @@ func encodePath(pathName string) string {
for _, s := range pathName {
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
encodedPathname = encodedPathname + string(s)
- continue
- }
- switch s {
- case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
- encodedPathname = encodedPathname + string(s)
- continue
- default:
- len := utf8.RuneLen(s)
- if len < 0 {
- // if utf8 cannot convert return the same string as is
- return pathName
- }
- u := make([]byte, len)
- utf8.EncodeRune(u, s)
- for _, r := range u {
- hex := hex.EncodeToString([]byte{r})
- encodedPathname = encodedPathname + "%" + strings.ToUpper(hex)
+ } else {
+ switch s {
+ case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
+ encodedPathname = encodedPathname + string(s)
+ default:
+ runeLen := utf8.RuneLen(s)
+ if runeLen < 0 {
+ return pathName
+ }
+ u := make([]byte, runeLen)
+ utf8.EncodeRune(u, s)
+ for _, r := range u {
+ hex := hex.EncodeToString([]byte{r})
+ encodedPathname = encodedPathname + "%" + strings.ToUpper(hex)
+ }
}
}
}
return encodedPathname
}
+// getSignature final signature in hexadecimal form.
+func getSignature(signingKey []byte, stringToSign string) string {
+ return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
+}
+
// compareSignatureV4 returns true if and only if both signatures
-// are equal. The signatures are expected to be HEX encoded strings
+// are equal. The signatures are expected to be hex-encoded strings
// according to the AWS S3 signature V4 spec.
func compareSignatureV4(sig1, sig2 string) bool {
- // The CTC using []byte(str) works because the hex encoding
- // is unique for a sequence of bytes. See also compareSignatureV2.
+ // The CTC using []byte(str) works because the hex encoding doesn't use
+ // non-ASCII characters. Otherwise, we'd need to convert the strings to
+ // a []rune of UTF-8 characters.
return subtle.ConstantTimeCompare([]byte(sig1), []byte(sig2)) == 1
}
diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go
index 86fbbd19e..b8b817ab8 100644
--- a/weed/s3api/auto_signature_v4_test.go
+++ b/weed/s3api/auto_signature_v4_test.go
@@ -6,13 +6,10 @@ import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
- "errors"
"fmt"
"io"
"net/http"
- "net/url"
"sort"
- "strconv"
"strings"
"sync"
"testing"
@@ -76,7 +73,7 @@ func TestIsReqAuthenticated(t *testing.T) {
SecretKey: "secret_key_1",
},
},
- Actions: []string{},
+ Actions: []string{"Read", "Write"},
},
},
})
@@ -149,7 +146,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
SecretKey: "secret_key_1",
},
},
- Actions: []string{},
+ Actions: []string{"Admin", "Read", "Write"},
},
},
})
@@ -170,15 +167,12 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
func BenchmarkGetSignature(b *testing.B) {
t := time.Now()
- iam := IdentityAccessManagement{
- hashes: make(map[string]*sync.Pool),
- hashCounters: make(map[string]*int32),
- }
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
- iam.getSignature("secret-key", t, "us-east-1", "s3", "random data")
+ signingKey := getSigningKey("secret-key", t.Format(yyyymmdd), "us-east-1", "s3")
+ getSignature(signingKey, "random data")
}
}
@@ -213,6 +207,53 @@ func mustNewPresignedRequest(iam *IdentityAccessManagement, method string, urlSt
return req
}
+// preSignV4 adds presigned URL parameters to the request
+func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKey, secretKey string, expires int64) error {
+ // Create credential scope
+ now := time.Now().UTC()
+ dateStr := now.Format(iso8601Format)
+
+ // Create credential header
+ scope := fmt.Sprintf("%s/%s/%s/%s", now.Format(yyyymmdd), "us-east-1", "s3", "aws4_request")
+ credential := fmt.Sprintf("%s/%s", accessKey, scope)
+
+ // Get the query parameters
+ query := req.URL.Query()
+ query.Set("X-Amz-Algorithm", signV4Algorithm)
+ query.Set("X-Amz-Credential", credential)
+ query.Set("X-Amz-Date", dateStr)
+ query.Set("X-Amz-Expires", fmt.Sprintf("%d", expires))
+ query.Set("X-Amz-SignedHeaders", "host")
+
+ // Set the query on the URL (without signature yet)
+ req.URL.RawQuery = query.Encode()
+
+ // Get the payload hash
+ hashedPayload := getContentSha256Cksum(req)
+
+ // Extract signed headers
+ extractedSignedHeaders := make(http.Header)
+ extractedSignedHeaders["host"] = []string{req.Host}
+
+ // Get canonical request
+ canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, req.URL.RawQuery, req.URL.Path, req.Method)
+
+ // Get string to sign
+ stringToSign := getStringToSign(canonicalRequest, now, scope)
+
+ // Get signing key
+ signingKey := getSigningKey(secretKey, now.Format(yyyymmdd), "us-east-1", "s3")
+
+ // Calculate signature
+ signature := getSignature(signingKey, stringToSign)
+
+ // Add signature to query
+ query.Set("X-Amz-Signature", signature)
+ req.URL.RawQuery = query.Encode()
+
+ return nil
+}
+
// Returns new HTTP request object.
func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) {
if method == "" {
@@ -254,11 +295,6 @@ func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeek
return req, nil
}
-// getSHA256Hash returns SHA-256 hash in hex encoding of given data.
-func getSHA256Hash(data []byte) string {
- return hex.EncodeToString(getSHA256Sum(data))
-}
-
// getMD5HashBase64 returns MD5 hash in base64 encoding of given data.
func getMD5HashBase64(data []byte) string {
return base64.StdEncoding.EncodeToString(getMD5Sum(data))
@@ -467,46 +503,6 @@ func signRequestV4(req *http.Request, accessKey, secretKey string) error {
return nil
}
-// preSignV4 presign the request, in accordance with
-// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html.
-func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKeyID, secretAccessKey string, expires int64) error {
- // Presign is not needed for anonymous credentials.
- if accessKeyID == "" || secretAccessKey == "" {
- return errors.New("Presign cannot be generated without access and secret keys")
- }
-
- region := "us-east-1"
- date := time.Now().UTC()
- scope := getScope(date, region)
- credential := fmt.Sprintf("%s/%s", accessKeyID, scope)
-
- // Set URL query.
- query := req.URL.Query()
- query.Set("X-Amz-Algorithm", signV4Algorithm)
- query.Set("X-Amz-Date", date.Format(iso8601Format))
- query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
- query.Set("X-Amz-SignedHeaders", "host")
- query.Set("X-Amz-Credential", credential)
- query.Set("X-Amz-Content-Sha256", unsignedPayload)
-
- // "host" is the only header required to be signed for Presigned URLs.
- extractedSignedHeaders := make(http.Header)
- extractedSignedHeaders.Set("host", req.Host)
-
- queryStr := strings.Replace(query.Encode(), "+", "%20", -1)
- canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method)
- stringToSign := getStringToSign(canonicalRequest, date, scope)
- signature := iam.getSignature(secretAccessKey, date, region, "s3", stringToSign)
-
- req.URL.RawQuery = query.Encode()
-
- // Add signature header to RawQuery.
- req.URL.RawQuery += "&X-Amz-Signature=" + url.QueryEscape(signature)
-
- // Construct the final presigned URL.
- return nil
-}
-
// EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
@@ -529,12 +525,12 @@ func EncodePath(pathName string) string {
encodedPathname = encodedPathname + string(s)
continue
default:
- len := utf8.RuneLen(s)
- if len < 0 {
+ runeLen := utf8.RuneLen(s)
+ if runeLen < 0 {
// if utf8 cannot convert return the same string as is
return pathName
}
- u := make([]byte, len)
+ u := make([]byte, runeLen)
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go
index 6ee46c82b..53ea8e768 100644
--- a/weed/s3api/chunked_reader_v4.go
+++ b/weed/s3api/chunked_reader_v4.go
@@ -102,13 +102,12 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
return nil, "", "", time.Time{}, s3err.ErrMissingDateHeader
}
}
+
// Parse date header.
- var err error
- date, err = time.Parse(iso8601Format, dateStr)
+ date, err := time.Parse(iso8601Format, dateStr)
if err != nil {
return nil, "", "", time.Time{}, s3err.ErrMalformedDate
}
-
// Query string.
queryStr := req.URL.Query().Encode()
@@ -118,14 +117,11 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
// Get string to sign from canonical request.
stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())
+ // Get hmac signing key.
+ signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), region, "s3")
+
// Calculate signature.
- newSignature := iam.getSignature(
- cred.SecretKey,
- signV4Values.Credential.scope.date,
- region,
- "s3",
- stringToSign,
- )
+ newSignature := getSignature(signingKey, stringToSign)
// Verify if signature match.
if !compareSignatureV4(newSignature, signV4Values.Signature) {
@@ -469,58 +465,47 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
// getChunkSignature - get chunk signature.
func (cr *s3ChunkedReader) getChunkSignature(hashedChunk string) string {
// Calculate string to sign.
- stringToSign := signV4ChunkedAlgorithm + "\n" +
+ stringToSign := signV4Algorithm + "-PAYLOAD" + "\n" +
cr.seedDate.Format(iso8601Format) + "\n" +
getScope(cr.seedDate, cr.region) + "\n" +
cr.seedSignature + "\n" +
emptySHA256 + "\n" +
hashedChunk
- // Calculate signature.
- return cr.iam.getSignature(
- cr.cred.SecretKey,
- cr.seedDate,
- cr.region,
- "s3",
- stringToSign,
- )
+ // Get hmac signing key.
+ signingKey := getSigningKey(cr.cred.SecretKey, cr.seedDate.Format(yyyymmdd), cr.region, "s3")
+
+ // Calculate and return signature.
+ return getSignature(signingKey, stringToSign)
}
-// readCRLF - check if reader only has '\r\n' CRLF character.
-// returns malformed encoding if it doesn't.
func readCRLF(reader *bufio.Reader) error {
buf := make([]byte, 2)
- _, err := reader.Read(buf)
+ _, err := io.ReadFull(reader, buf)
if err != nil {
return err
}
return checkCRLF(buf)
}
-// peekCRLF - peeks at the next two bytes to check for CRLF without consuming them.
func peekCRLF(reader *bufio.Reader) error {
- peeked, err := reader.Peek(2)
+ buf, err := reader.Peek(2)
if err != nil {
return err
}
- if err := checkCRLF(peeked); err != nil {
+ if err := checkCRLF(buf); err != nil {
return err
}
return nil
}
-// checkCRLF - checks if the buffer contains '\r\n' CRLF character.
func checkCRLF(buf []byte) error {
- if buf[0] != '\r' || buf[1] != '\n' {
+ if len(buf) != 2 || buf[0] != '\r' || buf[1] != '\n' {
return errMalformedEncoding
}
return nil
}
-// Read a line of bytes (up to \n) from b.
-// Give up if the line exceeds maxLineLength.
-// The returned bytes are owned by the bufio.Reader
-// so they are only valid until the next bufio read.
func readChunkLine(b *bufio.Reader) ([]byte, error) {
buf, err := b.ReadSlice('\n')
if err != nil {
@@ -536,8 +521,7 @@ func readChunkLine(b *bufio.Reader) ([]byte, error) {
if len(buf) >= maxLineLength {
return nil, errLineTooLong
}
-
- return buf, nil
+ return trimTrailingWhitespace(buf), nil
}
// trimTrailingWhitespace - trim trailing white space.
@@ -608,13 +592,11 @@ func parseChunkChecksum(b *bufio.Reader) (ChecksumAlgorithm, []byte) {
return extractedAlgorithm, checksumValue
}
-// parseChunkSignature - parse chunk signature.
func parseChunkSignature(chunk []byte) []byte {
- chunkSplits := bytes.SplitN(chunk, []byte(s3ChunkSignatureStr), 2)
- return chunkSplits[1]
+ chunkSplits := bytes.SplitN(chunk, []byte("="), 2)
+ return chunkSplits[1] // Keep only the signature.
}
-// parse hex to uint64.
func parseHexUint(v []byte) (n uint64, err error) {
for i, b := range v {
switch {
@@ -636,6 +618,7 @@ func parseHexUint(v []byte) (n uint64, err error) {
return
}
+// Checksum Algorithm represents the various checksum algorithms supported.
type ChecksumAlgorithm int
const (
@@ -649,18 +632,18 @@ const (
func (ca ChecksumAlgorithm) String() string {
switch ca {
+ case ChecksumAlgorithmNone:
+ return ""
case ChecksumAlgorithmCRC32:
- return "CRC32"
+ return "x-amz-checksum-crc32"
case ChecksumAlgorithmCRC32C:
- return "CRC32C"
+ return "x-amz-checksum-crc32c"
case ChecksumAlgorithmCRC64NVMe:
- return "CRC64NVMe"
+ return "x-amz-checksum-crc64nvme"
case ChecksumAlgorithmSHA1:
- return "SHA1"
+ return "x-amz-checksum-sha1"
case ChecksumAlgorithmSHA256:
- return "SHA256"
- case ChecksumAlgorithmNone:
- return ""
+ return "x-amz-checksum-sha256"
}
return ""
}