aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTom Crasset <25140344+tcrasset@users.noreply.github.com>2025-10-25 10:11:45 +0200
committerGitHub <noreply@github.com>2025-10-25 01:11:45 -0700
commit824dcac3bf5b75fd4b74bf83d7b08895422d4374 (patch)
tree6abc89b9b6fa88ba474b9e96e8636a50da85ad30
parent6a8c53bc44beb057f64d5ba1f7ac026f8410fe04 (diff)
downloadseaweedfs-824dcac3bf5b75fd4b74bf83d7b08895422d4374.tar.xz
seaweedfs-824dcac3bf5b75fd4b74bf83d7b08895422d4374.zip
s3: combine all signature verification checks into a single function (#7330)
-rw-r--r--weed/s3api/auth_signature_v4.go408
-rw-r--r--weed/s3api/auth_signature_v4_test.go91
-rw-r--r--weed/s3api/auto_signature_v4_test.go108
-rw-r--r--weed/s3api/chunked_reader_v4.go82
-rw-r--r--weed/s3api/chunked_reader_v4_test.go228
-rw-r--r--weed/s3api/s3err/s3api_errors.go7
6 files changed, 592 insertions, 332 deletions
diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go
index 05e5c7b5f..3a8e59392 100644
--- a/weed/s3api/auth_signature_v4.go
+++ b/weed/s3api/auth_signature_v4.go
@@ -25,7 +25,6 @@ import (
"encoding/hex"
"io"
"net/http"
- "path"
"regexp"
"sort"
"strconv"
@@ -33,17 +32,20 @@ import (
"time"
"unicode/utf8"
+ "github.com/seaweedfs/seaweedfs/weed/glog"
+
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
func (iam *IdentityAccessManagement) reqSignatureV4Verify(r *http.Request) (*Identity, s3err.ErrorCode) {
- sha256sum := getContentSha256Cksum(r)
switch {
case isRequestSignatureV4(r):
- return iam.doesSignatureMatch(sha256sum, r)
+ identity, _, errCode := iam.doesSignatureMatch(r)
+ return identity, errCode
case isRequestPresignedSignatureV4(r):
- return iam.doesPresignedSignatureMatch(sha256sum, r)
+ identity, _, errCode := iam.doesPresignedSignatureMatch(r)
+ return identity, errCode
}
return nil, s3err.ErrAccessDenied
}
@@ -154,236 +156,298 @@ func parseSignV4(v4Auth string) (sv signValues, aec s3err.ErrorCode) {
return signV4Values, s3err.ErrNone
}
-// doesSignatureMatch verifies the request signature.
-func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) {
-
- // Copy request
- req := *r
+// buildPathWithForwardedPrefix combines forwarded prefix with URL path while preserving S3 key semantics.
+// This function avoids path.Clean which would collapse "//" and dot segments, breaking S3 signatures.
+// It only normalizes the join boundary to avoid double slashes between prefix and path.
+func buildPathWithForwardedPrefix(forwardedPrefix, urlPath string) string {
+ if forwardedPrefix == "" {
+ return urlPath
+ }
+ // Ensure single leading slash on prefix
+ if !strings.HasPrefix(forwardedPrefix, "/") {
+ forwardedPrefix = "/" + forwardedPrefix
+ }
+ // Join without collapsing interior segments; only fix a double slash at the boundary
+ var joined string
+ if strings.HasSuffix(forwardedPrefix, "/") && strings.HasPrefix(urlPath, "/") {
+ joined = forwardedPrefix + urlPath[1:]
+ } else if !strings.HasSuffix(forwardedPrefix, "/") && !strings.HasPrefix(urlPath, "/") {
+ joined = forwardedPrefix + "/" + urlPath
+ } else {
+ joined = forwardedPrefix + urlPath
+ }
+ // Trailing slash semantics inherited from urlPath (already present if needed)
+ return joined
+}
- // Save authorization header.
- v4Auth := req.Header.Get("Authorization")
+// v4AuthInfo holds the parsed authentication data from a request,
+// whether it's from the Authorization header or presigned URL query parameters.
+type v4AuthInfo struct {
+ Signature string
+ AccessKey string
+ SignedHeaders []string
+ Date time.Time
+ Region string
+ Service string
+ Scope string
+ HashedPayload string
+ IsPresigned bool
+}
- // Parse signature version '4' header.
- signV4Values, errCode := parseSignV4(v4Auth)
+// verifyV4Signature is the single entry point for verifying any AWS Signature V4 request.
+// It handles standard requests, presigned URLs, and the seed signature for streaming uploads.
+func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCheckPermissions bool) (identity *Identity, credential *Credential, calculatedSignature string, authInfo *v4AuthInfo, errCode s3err.ErrorCode) {
+ // 1. Extract authentication information from header or query parameters
+ authInfo, errCode = extractV4AuthInfo(r)
if errCode != s3err.ErrNone {
- return nil, errCode
- }
-
- // Compute payload hash for non-S3 services
- if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil {
- var err error
- hashedPayload, err = streamHashRequestBody(r, iamRequestBodyLimit)
- if err != nil {
- return nil, s3err.ErrInternalError
- }
+ return nil, nil, "", nil, errCode
}
- // Extract all the signed headers along with its values.
- extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
- if errCode != s3err.ErrNone {
- return nil, errCode
+ // 2. Lookup user and credentials
+ identity, cred, found := iam.lookupByAccessKey(authInfo.AccessKey)
+ if !found {
+ return nil, nil, "", nil, s3err.ErrInvalidAccessKeyID
}
- cred := signV4Values.Credential
- identity, foundCred, found := iam.lookupByAccessKey(cred.accessKey)
- if !found {
- return nil, s3err.ErrInvalidAccessKeyID
+ // 3. Perform permission check
+ if shouldCheckPermissions {
+ bucket, object := s3_constants.GetBucketAndObject(r)
+ action := s3_constants.ACTION_READ
+ if r.Method != http.MethodGet && r.Method != http.MethodHead {
+ action = s3_constants.ACTION_WRITE
+ }
+ if !identity.canDo(Action(action), bucket, object) {
+ return nil, nil, "", nil, s3err.ErrAccessDenied
+ }
}
- // Extract date, if not present throw error.
- var dateStr string
- if dateStr = req.Header.Get("x-amz-date"); dateStr == "" {
- if dateStr = r.Header.Get("Date"); dateStr == "" {
- return nil, s3err.ErrMissingDateHeader
+ // 4. Handle presigned request expiration
+ if authInfo.IsPresigned {
+ if errCode = checkPresignedRequestExpiry(r, authInfo.Date); errCode != s3err.ErrNone {
+ return nil, nil, "", nil, errCode
}
}
- // Parse date header.
- t, e := time.Parse(iso8601Format, dateStr)
- if e != nil {
- return nil, s3err.ErrMalformedDate
+
+ // 5. Extract headers that were part of the signature
+ extractedSignedHeaders, errCode := extractSignedHeaders(authInfo.SignedHeaders, r)
+ if errCode != s3err.ErrNone {
+ return nil, nil, "", nil, errCode
}
- // Query string.
- queryStr := req.URL.Query().Encode()
+ // 6. Get the query string for the canonical request
+ queryStr := getCanonicalQueryString(r, authInfo.IsPresigned)
+
+ // 7. Define a closure for the core verification logic to avoid repetition
+ verify := func(urlPath string) (string, s3err.ErrorCode) {
+ return calculateAndVerifySignature(
+ cred.SecretKey,
+ r.Method,
+ urlPath,
+ queryStr,
+ extractedSignedHeaders,
+ authInfo,
+ )
+ }
- // Check if reverse proxy is forwarding with prefix
+ // 8. Verify the signature, trying with X-Forwarded-Prefix first
if forwardedPrefix := r.Header.Get("X-Forwarded-Prefix"); forwardedPrefix != "" {
- // Try signature verification with the forwarded prefix first.
- // This handles cases where reverse proxies strip URL prefixes and add the X-Forwarded-Prefix header.
- cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, req.URL.Path)
- errCode = iam.verifySignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, cleanedPath, req.Method, foundCred.SecretKey, t, signV4Values)
+ cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, r.URL.Path)
+ calculatedSignature, errCode = verify(cleanedPath)
if errCode == s3err.ErrNone {
- return identity, errCode
+ return identity, cred, calculatedSignature, authInfo, s3err.ErrNone
}
}
- // Try normal signature verification (without prefix)
- errCode = iam.verifySignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method, foundCred.SecretKey, t, signV4Values)
- if errCode == s3err.ErrNone {
- return identity, errCode
+ // 9. Verify with the original path
+ calculatedSignature, errCode = verify(r.URL.Path)
+ if errCode != s3err.ErrNone {
+ return nil, nil, "", nil, errCode
}
- return nil, errCode
+ return identity, cred, calculatedSignature, authInfo, s3err.ErrNone
}
-// buildPathWithForwardedPrefix combines forwarded prefix with URL path while preserving trailing slashes.
-// This ensures compatibility with S3 SDK signatures that include trailing slashes for directory operations.
-func buildPathWithForwardedPrefix(forwardedPrefix, urlPath string) string {
- fullPath := forwardedPrefix + urlPath
- hasTrailingSlash := strings.HasSuffix(urlPath, "/") && urlPath != "/"
- cleanedPath := path.Clean(fullPath)
- if hasTrailingSlash && !strings.HasSuffix(cleanedPath, "/") {
- cleanedPath += "/"
- }
- return cleanedPath
-}
-
-// verifySignatureWithPath verifies signature with a given path (used for both normal and prefixed paths).
-func (iam *IdentityAccessManagement) verifySignatureWithPath(extractedSignedHeaders http.Header, hashedPayload, queryStr, urlPath, method, secretKey string, t time.Time, signV4Values signValues) s3err.ErrorCode {
- // Get canonical request.
- canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, urlPath, method)
-
- // Get string to sign from canonical request.
- stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope())
-
- // Get hmac signing key.
- signingKey := getSigningKey(secretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), signV4Values.Credential.scope.region, signV4Values.Credential.scope.service)
-
- // Calculate signature.
+// calculateAndVerifySignature contains the core logic for creating the canonical request,
+// string-to-sign, and comparing the final signature.
+func calculateAndVerifySignature(secretKey, method, urlPath, queryStr string, extractedSignedHeaders http.Header, authInfo *v4AuthInfo) (string, s3err.ErrorCode) {
+ canonicalRequest := getCanonicalRequest(extractedSignedHeaders, authInfo.HashedPayload, queryStr, urlPath, method)
+ stringToSign := getStringToSign(canonicalRequest, authInfo.Date, authInfo.Scope)
+ signingKey := getSigningKey(secretKey, authInfo.Date.Format(yyyymmdd), authInfo.Region, authInfo.Service)
newSignature := getSignature(signingKey, stringToSign)
- // Verify if signature match.
- if !compareSignatureV4(newSignature, signV4Values.Signature) {
- return s3err.ErrSignatureDoesNotMatch
+ if !compareSignatureV4(newSignature, authInfo.Signature) {
+ glog.V(4).Infof("Signature mismatch. Details:\n- CanonicalRequest: %q\n- StringToSign: %q\n- Calculated: %s, Provided: %s",
+ canonicalRequest, stringToSign, newSignature, authInfo.Signature)
+ return "", s3err.ErrSignatureDoesNotMatch
}
- return s3err.ErrNone
+ return newSignature, s3err.ErrNone
}
-// verifyPresignedSignatureWithPath verifies presigned signature with a given path (used for both normal and prefixed paths).
-func (iam *IdentityAccessManagement) verifyPresignedSignatureWithPath(extractedSignedHeaders http.Header, hashedPayload, queryStr, urlPath, method, secretKey string, t time.Time, credHeader credentialHeader, signature string) s3err.ErrorCode {
- // Get canonical request.
- canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, urlPath, method)
+func extractV4AuthInfo(r *http.Request) (*v4AuthInfo, s3err.ErrorCode) {
+ if isRequestPresignedSignatureV4(r) {
+ return extractV4AuthInfoFromQuery(r)
+ }
+ return extractV4AuthInfoFromHeader(r)
+}
- // Get string to sign from canonical request.
- stringToSign := getStringToSign(canonicalRequest, t, credHeader.getScope())
+func extractV4AuthInfoFromHeader(r *http.Request) (*v4AuthInfo, s3err.ErrorCode) {
+ authHeader := r.Header.Get("Authorization")
+ signV4Values, errCode := parseSignV4(authHeader)
+ if errCode != s3err.ErrNone {
+ return nil, errCode
+ }
- // Get hmac signing key.
- signingKey := getSigningKey(secretKey, credHeader.scope.date.Format(yyyymmdd), credHeader.scope.region, credHeader.scope.service)
+ var t time.Time
+ if xamz := r.Header.Get("x-amz-date"); xamz != "" {
+ parsed, err := time.Parse(iso8601Format, xamz)
+ if err != nil {
+ return nil, s3err.ErrMalformedDate
+ }
+ t = parsed
+ } else {
+ ds := r.Header.Get("Date")
+ if ds == "" {
+ return nil, s3err.ErrMissingDateHeader
+ }
+ parsed, err := http.ParseTime(ds)
+ if err != nil {
+ return nil, s3err.ErrMalformedDate
+ }
+ t = parsed.UTC()
+ }
- // Calculate expected signature.
- expectedSignature := getSignature(signingKey, stringToSign)
+ // Validate clock skew: requests cannot be older than 15 minutes from server time to prevent replay attacks
+ const maxSkew = 15 * time.Minute
+ now := time.Now().UTC()
+ if now.Sub(t) > maxSkew || t.Sub(now) > maxSkew {
+ return nil, s3err.ErrRequestTimeTooSkewed
+ }
- // Verify if signature match.
- if !compareSignatureV4(expectedSignature, signature) {
- return s3err.ErrSignatureDoesNotMatch
+ hashedPayload := getContentSha256Cksum(r)
+ if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil {
+ var hashErr error
+ hashedPayload, hashErr = streamHashRequestBody(r, iamRequestBodyLimit)
+ if hashErr != nil {
+ return nil, s3err.ErrInternalError
+ }
}
- return s3err.ErrNone
+ return &v4AuthInfo{
+ Signature: signV4Values.Signature,
+ AccessKey: signV4Values.Credential.accessKey,
+ SignedHeaders: signV4Values.SignedHeaders,
+ Date: t,
+ Region: signV4Values.Credential.scope.region,
+ Service: signV4Values.Credential.scope.service,
+ Scope: signV4Values.Credential.getScope(),
+ HashedPayload: hashedPayload,
+ IsPresigned: false,
+ }, s3err.ErrNone
}
-// Simple implementation for presigned signature verification
-func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) {
- // Parse presigned signature values from query parameters
+func extractV4AuthInfoFromQuery(r *http.Request) (*v4AuthInfo, s3err.ErrorCode) {
query := r.URL.Query()
- // Check required parameters
- algorithm := query.Get("X-Amz-Algorithm")
- if algorithm != signV4Algorithm {
+ // Validate all required query parameters upfront for fail-fast behavior
+ if query.Get("X-Amz-Algorithm") != signV4Algorithm {
return nil, s3err.ErrSignatureVersionNotSupported
}
-
- credential := query.Get("X-Amz-Credential")
- if credential == "" {
- return nil, s3err.ErrMissingFields
+ if query.Get("X-Amz-Date") == "" {
+ return nil, s3err.ErrMissingDateHeader
}
-
- signature := query.Get("X-Amz-Signature")
- if signature == "" {
+ if query.Get("X-Amz-Credential") == "" {
return nil, s3err.ErrMissingFields
}
-
- signedHeadersStr := query.Get("X-Amz-SignedHeaders")
- if signedHeadersStr == "" {
+ if query.Get("X-Amz-Signature") == "" {
return nil, s3err.ErrMissingFields
}
-
- dateStr := query.Get("X-Amz-Date")
- if dateStr == "" {
- return nil, s3err.ErrMissingDateHeader
- }
-
- // Parse credential
- credHeader, err := parseCredentialHeader("Credential=" + credential)
- if err != s3err.ErrNone {
- return nil, err
+ if query.Get("X-Amz-SignedHeaders") == "" {
+ return nil, s3err.ErrMissingFields
}
-
- // Look up identity by access key
- identity, foundCred, found := iam.lookupByAccessKey(credHeader.accessKey)
- if !found {
- return nil, s3err.ErrInvalidAccessKeyID
+ if query.Get("X-Amz-Expires") == "" {
+ return nil, s3err.ErrInvalidQueryParams
}
// Parse date
- t, e := time.Parse(iso8601Format, dateStr)
- if e != nil {
+ dateStr := query.Get("X-Amz-Date")
+ t, err := time.Parse(iso8601Format, dateStr)
+ if err != nil {
return nil, s3err.ErrMalformedDate
}
- // Check expiration
- expiresStr := query.Get("X-Amz-Expires")
- if expiresStr != "" {
- expires, parseErr := strconv.ParseInt(expiresStr, 10, 64)
- if parseErr != nil {
- return nil, s3err.ErrMalformedDate
- }
- // Check if current time is after the expiration time
- expirationTime := t.Add(time.Duration(expires) * time.Second)
- if time.Now().UTC().After(expirationTime) {
- return nil, s3err.ErrExpiredPresignRequest
- }
+ // Parse credential header
+ credHeader, errCode := parseCredentialHeader("Credential=" + query.Get("X-Amz-Credential"))
+ if errCode != s3err.ErrNone {
+ return nil, errCode
}
- // Parse signed headers
- signedHeaders := strings.Split(signedHeadersStr, ";")
+ // For presigned URLs, X-Amz-Content-Sha256 must come from the query parameter
+ // (or default to UNSIGNED-PAYLOAD) because that's what was used for signing.
+ // We must NOT check the request header as it wasn't part of the signature calculation.
+ hashedPayload := query.Get("X-Amz-Content-Sha256")
+ if hashedPayload == "" {
+ hashedPayload = unsignedPayload
+ }
+
+ return &v4AuthInfo{
+ Signature: query.Get("X-Amz-Signature"),
+ AccessKey: credHeader.accessKey,
+ SignedHeaders: strings.Split(query.Get("X-Amz-SignedHeaders"), ";"),
+ Date: t,
+ Region: credHeader.scope.region,
+ Service: credHeader.scope.service,
+ Scope: credHeader.getScope(),
+ HashedPayload: hashedPayload,
+ IsPresigned: true,
+ }, s3err.ErrNone
+}
- // Extract signed headers from request
- extractedSignedHeaders := make(http.Header)
- for _, header := range signedHeaders {
- if header == "host" {
- extractedSignedHeaders[header] = []string{extractHostHeader(r)}
- continue
- }
- if values := r.Header[http.CanonicalHeaderKey(header)]; len(values) > 0 {
- extractedSignedHeaders[http.CanonicalHeaderKey(header)] = values
- }
+func getCanonicalQueryString(r *http.Request, isPresigned bool) string {
+ var queryToEncode string
+ if !isPresigned {
+ queryToEncode = r.URL.Query().Encode()
+ } else {
+ queryForCanonical := r.URL.Query()
+ queryForCanonical.Del("X-Amz-Signature")
+ queryToEncode = queryForCanonical.Encode()
}
+ return queryToEncode
+}
- // Remove signature from query for canonical request calculation
- queryForCanonical := r.URL.Query()
- queryForCanonical.Del("X-Amz-Signature")
- queryStr := strings.Replace(queryForCanonical.Encode(), "+", "%20", -1)
+func checkPresignedRequestExpiry(r *http.Request, t time.Time) s3err.ErrorCode {
+ expiresStr := r.URL.Query().Get("X-Amz-Expires")
+ // X-Amz-Expires is validated as required in extractV4AuthInfoFromQuery,
+ // so it should never be empty here
+ expires, err := strconv.ParseInt(expiresStr, 10, 64)
+ if err != nil {
+ return s3err.ErrMalformedDate
+ }
- var errCode s3err.ErrorCode
- // Check if reverse proxy is forwarding with prefix for presigned URLs
- if forwardedPrefix := r.Header.Get("X-Forwarded-Prefix"); forwardedPrefix != "" {
- // Try signature verification with the forwarded prefix first.
- // This handles cases where reverse proxies strip URL prefixes and add the X-Forwarded-Prefix header.
- cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, r.URL.Path)
- errCode = iam.verifyPresignedSignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, cleanedPath, r.Method, foundCred.SecretKey, t, credHeader, signature)
- if errCode == s3err.ErrNone {
- return identity, errCode
- }
+ // The maximum value for X-Amz-Expires is 604800 seconds (7 days)
+ // Allow 0 but it will immediately fail expiration check
+ if expires < 0 {
+ return s3err.ErrNegativeExpires
+ }
+ if expires > 604800 {
+ return s3err.ErrMaximumExpires
}
- // Try normal signature verification (without prefix)
- errCode = iam.verifyPresignedSignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, r.URL.Path, r.Method, foundCred.SecretKey, t, credHeader, signature)
- if errCode == s3err.ErrNone {
- return identity, errCode
+ expirationTime := t.Add(time.Duration(expires) * time.Second)
+ if time.Now().UTC().After(expirationTime) {
+ return s3err.ErrExpiredPresignRequest
}
+ return s3err.ErrNone
+}
+
+func (iam *IdentityAccessManagement) doesSignatureMatch(r *http.Request) (*Identity, string, s3err.ErrorCode) {
+ identity, _, calculatedSignature, _, errCode := iam.verifyV4Signature(r, false)
+ return identity, calculatedSignature, errCode
+}
- return nil, errCode
+func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(r *http.Request) (*Identity, string, s3err.ErrorCode) {
+ identity, _, calculatedSignature, _, errCode := iam.verifyV4Signature(r, false)
+ return identity, calculatedSignature, errCode
}
// credentialHeader data type represents structured form of Credential
@@ -531,7 +595,7 @@ func extractHostHeader(r *http.Request) string {
// Check if reverse proxy also forwarded the port
if forwardedPort := r.Header.Get("X-Forwarded-Port"); forwardedPort != "" {
// Determine the protocol to check for standard ports
- proto := r.Header.Get("X-Forwarded-Proto")
+ proto := strings.ToLower(r.Header.Get("X-Forwarded-Proto"))
// Only add port if it's not the standard port for the protocol
if (proto == "https" && forwardedPort != "443") || (proto != "https" && forwardedPort != "80") {
return forwardedHost + ":" + forwardedPort
diff --git a/weed/s3api/auth_signature_v4_test.go b/weed/s3api/auth_signature_v4_test.go
new file mode 100644
index 000000000..312e88767
--- /dev/null
+++ b/weed/s3api/auth_signature_v4_test.go
@@ -0,0 +1,91 @@
+package s3api
+
+import (
+ "testing"
+)
+
+func TestBuildPathWithForwardedPrefix(t *testing.T) {
+ tests := []struct {
+ name string
+ forwardedPrefix string
+ urlPath string
+ expected string
+ }{
+ {
+ name: "empty prefix returns urlPath",
+ forwardedPrefix: "",
+ urlPath: "/bucket/obj",
+ expected: "/bucket/obj",
+ },
+ {
+ name: "prefix without trailing slash",
+ forwardedPrefix: "/storage",
+ urlPath: "/bucket/obj",
+ expected: "/storage/bucket/obj",
+ },
+ {
+ name: "prefix with trailing slash",
+ forwardedPrefix: "/storage/",
+ urlPath: "/bucket/obj",
+ expected: "/storage/bucket/obj",
+ },
+ {
+ name: "prefix without leading slash",
+ forwardedPrefix: "storage",
+ urlPath: "/bucket/obj",
+ expected: "/storage/bucket/obj",
+ },
+ {
+ name: "prefix without leading slash and with trailing slash",
+ forwardedPrefix: "storage/",
+ urlPath: "/bucket/obj",
+ expected: "/storage/bucket/obj",
+ },
+ {
+ name: "preserve double slashes in key",
+ forwardedPrefix: "/storage",
+ urlPath: "/bucket//obj",
+ expected: "/storage/bucket//obj",
+ },
+ {
+ name: "preserve trailing slash in urlPath",
+ forwardedPrefix: "/storage",
+ urlPath: "/bucket/folder/",
+ expected: "/storage/bucket/folder/",
+ },
+ {
+ name: "preserve trailing slash with prefix having trailing slash",
+ forwardedPrefix: "/storage/",
+ urlPath: "/bucket/folder/",
+ expected: "/storage/bucket/folder/",
+ },
+ {
+ name: "root path",
+ forwardedPrefix: "/storage",
+ urlPath: "/",
+ expected: "/storage/",
+ },
+ {
+ name: "complex key with multiple slashes",
+ forwardedPrefix: "/api/v1",
+ urlPath: "/bucket/path//with///slashes",
+ expected: "/api/v1/bucket/path//with///slashes",
+ },
+ {
+ name: "urlPath without leading slash",
+ forwardedPrefix: "/storage",
+ urlPath: "bucket/obj",
+ expected: "/storage/bucket/obj",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := buildPathWithForwardedPrefix(tt.forwardedPrefix, tt.urlPath)
+ if result != tt.expected {
+ t.Errorf("buildPathWithForwardedPrefix(%q, %q) = %q, want %q",
+ tt.forwardedPrefix, tt.urlPath, result, tt.expected)
+ }
+ })
+ }
+}
diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go
index bf11a0906..71cae3546 100644
--- a/weed/s3api/auto_signature_v4_test.go
+++ b/weed/s3api/auto_signature_v4_test.go
@@ -229,8 +229,12 @@ func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKey, secr
// Set the query on the URL (without signature yet)
req.URL.RawQuery = query.Encode()
- // Get the payload hash
- hashedPayload := getContentSha256Cksum(req)
+ // For presigned URLs, the payload hash must be UNSIGNED-PAYLOAD (or from query param if explicitly set)
+ // We should NOT use request headers as they're not part of the presigned URL
+ hashedPayload := query.Get("X-Amz-Content-Sha256")
+ if hashedPayload == "" {
+ hashedPayload = unsignedPayload
+ }
// Extract signed headers
extractedSignedHeaders := make(http.Header)
@@ -314,7 +318,7 @@ func TestSignatureV4WithForwardedPrefix(t *testing.T) {
signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", tt.expectedPath)
// Test signature verification
- _, errCode := iam.doesSignatureMatch(getContentSha256Cksum(r), r)
+ _, _, errCode := iam.doesSignatureMatch(r)
if errCode != s3err.ErrNone {
t.Errorf("Expected successful signature validation with X-Forwarded-Prefix %q, got error: %v (code: %d)", tt.forwardedPrefix, errCode, int(errCode))
}
@@ -380,7 +384,7 @@ func TestSignatureV4WithForwardedPrefixTrailingSlash(t *testing.T) {
signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", tt.expectedPath)
// Test signature verification - this should succeed even with trailing slashes
- _, errCode := iam.doesSignatureMatch(getContentSha256Cksum(r), r)
+ _, _, errCode := iam.doesSignatureMatch(r)
if errCode != s3err.ErrNone {
t.Errorf("Expected successful signature validation with trailing slash in path %q, got error: %v (code: %d)", tt.urlPath, errCode, int(errCode))
}
@@ -475,7 +479,7 @@ func TestSignatureV4WithForwardedPort(t *testing.T) {
signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", r.URL.Path)
// Test signature verification
- _, errCode := iam.doesSignatureMatch(getContentSha256Cksum(r), r)
+ _, _, errCode := iam.doesSignatureMatch(r)
if errCode != s3err.ErrNone {
t.Errorf("Expected successful signature validation with forwarded port, got error: %v (code: %d)", errCode, int(errCode))
}
@@ -508,12 +512,50 @@ func TestPresignedSignatureV4Basic(t *testing.T) {
}
// Test presigned signature verification
- _, errCode := iam.doesPresignedSignatureMatch(getContentSha256Cksum(r), r)
+ _, _, errCode := iam.doesPresignedSignatureMatch(r)
if errCode != s3err.ErrNone {
t.Errorf("Expected successful presigned signature validation, got error: %v (code: %d)", errCode, int(errCode))
}
}
+// TestPresignedSignatureV4MissingExpires verifies that X-Amz-Expires is required for presigned URLs
+func TestPresignedSignatureV4MissingExpires(t *testing.T) {
+ iam := newTestIAM()
+
+ // Create a presigned request
+ r, err := newTestRequest("GET", "https://example.com/test-bucket/test-object", 0, nil)
+ if err != nil {
+ t.Fatalf("Failed to create test request: %v", err)
+ }
+
+ r = mux.SetURLVars(r, map[string]string{
+ "bucket": "test-bucket",
+ "object": "test-object",
+ })
+ r.Header.Set("Host", "example.com")
+
+ // Manually construct presigned URL query parameters WITHOUT X-Amz-Expires
+ now := time.Now().UTC()
+ dateStr := now.Format(iso8601Format)
+ scope := fmt.Sprintf("%s/%s/%s/%s", now.Format(yyyymmdd), "us-east-1", "s3", "aws4_request")
+ credential := fmt.Sprintf("%s/%s", "AKIAIOSFODNN7EXAMPLE", scope)
+
+ query := r.URL.Query()
+ query.Set("X-Amz-Algorithm", signV4Algorithm)
+ query.Set("X-Amz-Credential", credential)
+ query.Set("X-Amz-Date", dateStr)
+ // Intentionally NOT setting X-Amz-Expires
+ query.Set("X-Amz-SignedHeaders", "host")
+ query.Set("X-Amz-Signature", "dummy-signature") // Signature doesn't matter, should fail earlier
+ r.URL.RawQuery = query.Encode()
+
+ // Test presigned signature verification - should fail with ErrInvalidQueryParams
+ _, _, errCode := iam.doesPresignedSignatureMatch(r)
+ if errCode != s3err.ErrInvalidQueryParams {
+ t.Errorf("Expected ErrInvalidQueryParams for missing X-Amz-Expires, got: %v (code: %d)", errCode, int(errCode))
+ }
+}
+
// Test X-Forwarded-Prefix support for presigned URLs
func TestPresignedSignatureV4WithForwardedPrefix(t *testing.T) {
tests := []struct {
@@ -573,7 +615,8 @@ func TestPresignedSignatureV4WithForwardedPrefix(t *testing.T) {
r.Header.Set("X-Forwarded-Host", "example.com")
// Test presigned signature verification
- _, errCode := iam.doesPresignedSignatureMatch(getContentSha256Cksum(r), r)
+ _, _, errCode := iam.doesPresignedSignatureMatch(r)
+
if errCode != s3err.ErrNone {
t.Errorf("Expected successful presigned signature validation with X-Forwarded-Prefix %q, got error: %v (code: %d)", tt.forwardedPrefix, errCode, int(errCode))
}
@@ -640,7 +683,8 @@ func TestPresignedSignatureV4WithForwardedPrefixTrailingSlash(t *testing.T) {
r.Header.Set("X-Forwarded-Host", "example.com")
// Test presigned signature verification - this should succeed with trailing slashes
- _, errCode := iam.doesPresignedSignatureMatch(getContentSha256Cksum(r), r)
+ _, _, errCode := iam.doesPresignedSignatureMatch(r)
+
if errCode != s3err.ErrNone {
t.Errorf("Expected successful presigned signature validation with trailing slash in path %q, got error: %v (code: %d)", tt.strippedPath, errCode, int(errCode))
}
@@ -669,8 +713,12 @@ func preSignV4WithPath(iam *IdentityAccessManagement, req *http.Request, accessK
// Set the query on the URL (without signature yet)
req.URL.RawQuery = query.Encode()
- // Get the payload hash
- hashedPayload := getContentSha256Cksum(req)
+ // For presigned URLs, the payload hash must be UNSIGNED-PAYLOAD (or from query param if explicitly set)
+ // We should NOT use request headers as they're not part of the presigned URL
+ hashedPayload := query.Get("X-Amz-Content-Sha256")
+ if hashedPayload == "" {
+ hashedPayload = unsignedPayload
+ }
// Extract signed headers
extractedSignedHeaders := make(http.Header)
@@ -884,7 +932,7 @@ func signRequestV4(req *http.Request, accessKey, secretKey string) error {
return fmt.Errorf("Invalid hashed payload")
}
- currTime := time.Now()
+ currTime := time.Now().UTC()
// Set x-amz-date.
req.Header.Set("x-amz-date", currTime.Format(iso8601Format))
@@ -1061,10 +1109,6 @@ func TestIAMPayloadHashComputation(t *testing.T) {
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
req.Header.Set("Host", "localhost:8111")
- // Compute expected payload hash
- expectedHash := sha256.Sum256([]byte(testPayload))
- expectedHashStr := hex.EncodeToString(expectedHash[:])
-
// Create an IAM-style authorization header with "iam" service instead of "s3"
now := time.Now().UTC()
dateStr := now.Format("20060102T150405Z")
@@ -1079,7 +1123,7 @@ func TestIAMPayloadHashComputation(t *testing.T) {
// Test the doesSignatureMatch function directly
// This should now compute the correct payload hash for IAM requests
- identity, errCode := iam.doesSignatureMatch(expectedHashStr, req)
+ identity, _, errCode := iam.doesSignatureMatch(req)
// Even though the signature will fail (dummy signature),
// the fact that we get past the credential parsing means the payload hash was computed correctly
@@ -1141,7 +1185,7 @@ func TestS3PayloadHashNoRegression(t *testing.T) {
req.Header.Set("Authorization", authHeader)
// This should use the emptySHA256 hash and not try to read the body
- identity, errCode := iam.doesSignatureMatch(emptySHA256, req)
+ identity, _, errCode := iam.doesSignatureMatch(req)
// Should get signature mismatch (because of dummy signature) but not other errors
assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
@@ -1192,7 +1236,7 @@ func TestIAMEmptyBodyPayloadHash(t *testing.T) {
req.Header.Set("Authorization", authHeader)
// Even with an IAM request, empty body should result in emptySHA256
- identity, errCode := iam.doesSignatureMatch(emptySHA256, req)
+ identity, _, errCode := iam.doesSignatureMatch(req)
// Should get signature mismatch (because of dummy signature) but not other errors
assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
@@ -1235,10 +1279,6 @@ func TestSTSPayloadHashComputation(t *testing.T) {
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
req.Header.Set("Host", "localhost:8112")
- // Compute expected payload hash
- expectedHash := sha256.Sum256([]byte(testPayload))
- expectedHashStr := hex.EncodeToString(expectedHash[:])
-
// Create an STS-style authorization header with "sts" service
now := time.Now().UTC()
dateStr := now.Format("20060102T150405Z")
@@ -1252,7 +1292,7 @@ func TestSTSPayloadHashComputation(t *testing.T) {
// Test the doesSignatureMatch function
// This should compute the correct payload hash for STS requests (non-S3 service)
- identity, errCode := iam.doesSignatureMatch(expectedHashStr, req)
+ identity, _, errCode := iam.doesSignatureMatch(req)
// Should get signature mismatch (dummy signature) but payload hash should be computed correctly
assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
@@ -1317,7 +1357,7 @@ func TestGitHubIssue7080Scenario(t *testing.T) {
// Since we're using a dummy signature, we expect signature mismatch, but the important
// thing is that it doesn't fail earlier due to payload hash computation issues
- identity, errCode := iam.doesSignatureMatch(emptySHA256, req)
+ identity, _, errCode := iam.doesSignatureMatch(req)
// The error should be signature mismatch, not payload related
assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
@@ -1357,32 +1397,37 @@ func TestIAMSignatureServiceMatching(t *testing.T) {
// Use the exact payload and headers from the failing logs
testPayload := "Action=CreateAccessKey&UserName=admin&Version=2010-05-08"
+ // Use current time to avoid clock skew validation failures
+ now := time.Now().UTC()
+ amzDate := now.Format(iso8601Format)
+ dateStamp := now.Format(yyyymmdd)
+
// Create request exactly as shown in logs
req, err := http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(testPayload))
assert.NoError(t, err)
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
req.Header.Set("Host", "localhost:8111")
- req.Header.Set("X-Amz-Date", "20250805T082934Z")
+ req.Header.Set("X-Amz-Date", amzDate)
// Calculate the expected signature using the correct IAM service
// This simulates what botocore/AWS SDK would calculate
- credentialScope := "20250805/us-east-1/iam/aws4_request"
+ credentialScope := dateStamp + "/us-east-1/iam/aws4_request"
// Calculate the actual payload hash for our test payload
actualPayloadHash := getSHA256Hash([]byte(testPayload))
// Build the canonical request with the actual payload hash
- canonicalRequest := "POST\n/\n\ncontent-type:application/x-www-form-urlencoded; charset=utf-8\nhost:localhost:8111\nx-amz-date:20250805T082934Z\n\ncontent-type;host;x-amz-date\n" + actualPayloadHash
+ canonicalRequest := "POST\n/\n\ncontent-type:application/x-www-form-urlencoded; charset=utf-8\nhost:localhost:8111\nx-amz-date:" + amzDate + "\n\ncontent-type;host;x-amz-date\n" + actualPayloadHash
// Calculate the canonical request hash
canonicalRequestHash := getSHA256Hash([]byte(canonicalRequest))
// Build the string to sign
- stringToSign := "AWS4-HMAC-SHA256\n20250805T082934Z\n" + credentialScope + "\n" + canonicalRequestHash
+ stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + credentialScope + "\n" + canonicalRequestHash
// Calculate expected signature using IAM service (what client sends)
- expectedSigningKey := getSigningKey("power_user_secret", "20250805", "us-east-1", "iam")
+ expectedSigningKey := getSigningKey("power_user_secret", dateStamp, "us-east-1", "iam")
expectedSignature := getSignature(expectedSigningKey, stringToSign)
// Create authorization header with the correct signature
@@ -1391,7 +1436,8 @@ func TestIAMSignatureServiceMatching(t *testing.T) {
req.Header.Set("Authorization", authHeader)
// Now test that SeaweedFS computes the same signature with our fix
- identity, errCode := iam.doesSignatureMatch(actualPayloadHash, req)
+ identity, computedSignature, errCode := iam.doesSignatureMatch(req)
+ assert.Equal(t, expectedSignature, computedSignature)
// With the fix, the signatures should match and we should get a successful authentication
assert.Equal(t, s3err.ErrNone, errCode)
@@ -1481,7 +1527,7 @@ func TestIAMLargeBodySecurityLimit(t *testing.T) {
req.Header.Set("Authorization", authHeader)
// The function should complete successfully but limit the body to 10 MiB
- identity, errCode := iam.doesSignatureMatch(emptySHA256, req)
+ identity, _, errCode := iam.doesSignatureMatch(req)
// Should get signature mismatch (dummy signature) but not internal error
assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go
index ca35fe3cd..39d8336f0 100644
--- a/weed/s3api/chunked_reader_v4.go
+++ b/weed/s3api/chunked_reader_v4.go
@@ -34,7 +34,6 @@ import (
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
- "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/dustin/go-humanize"
@@ -47,23 +46,13 @@ import (
// returns signature, error otherwise if the signature mismatches or any other
// error while parsing and validating.
func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cred *Credential, signature string, region string, service string, date time.Time, errCode s3err.ErrorCode) {
-
- // Copy request.
- req := *r
-
- // Save authorization header.
- v4Auth := req.Header.Get("Authorization")
-
- // Parse signature version '4' header.
- signV4Values, errCode := parseSignV4(v4Auth)
+ _, credential, calculatedSignature, authInfo, errCode := iam.verifyV4Signature(r, true)
if errCode != s3err.ErrNone {
return nil, "", "", "", time.Time{}, errCode
}
- contentSha256Header := req.Header.Get("X-Amz-Content-Sha256")
-
- switch contentSha256Header {
- // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
+ // This check ensures we only proceed for streaming uploads.
+ switch authInfo.HashedPayload {
case streamingContentSHA256:
glog.V(3).Infof("streaming content sha256")
case streamingUnsignedPayload:
@@ -72,64 +61,7 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
return nil, "", "", "", time.Time{}, s3err.ErrContentSHA256Mismatch
}
- // Payload streaming.
- payload := contentSha256Header
-
- // Extract all the signed headers along with its values.
- extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
- if errCode != s3err.ErrNone {
- return nil, "", "", "", time.Time{}, errCode
- }
- // Verify if the access key id matches.
- identity, cred, found := iam.lookupByAccessKey(signV4Values.Credential.accessKey)
- if !found {
- return nil, "", "", "", time.Time{}, s3err.ErrInvalidAccessKeyID
- }
-
- bucket, object := s3_constants.GetBucketAndObject(r)
- if !identity.canDo(s3_constants.ACTION_WRITE, bucket, object) {
- errCode = s3err.ErrAccessDenied
- return
- }
-
- // Verify if region is valid.
- region = signV4Values.Credential.scope.region
-
- // Extract date, if not present throw error.
- var dateStr string
- if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
- if dateStr = r.Header.Get("Date"); dateStr == "" {
- return nil, "", "", "", time.Time{}, s3err.ErrMissingDateHeader
- }
- }
-
- // Parse date header.
- date, err := time.Parse(iso8601Format, dateStr)
- if err != nil {
- return nil, "", "", "", time.Time{}, s3err.ErrMalformedDate
- }
- // Query string.
- queryStr := req.URL.Query().Encode()
-
- // Get canonical request.
- canonicalRequest := getCanonicalRequest(extractedSignedHeaders, payload, queryStr, req.URL.Path, req.Method)
-
- // Get string to sign from canonical request.
- stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())
-
- // Get hmac signing key.
- signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), region, signV4Values.Credential.scope.service)
-
- // Calculate signature.
- newSignature := getSignature(signingKey, stringToSign)
-
- // Verify if signature match.
- if !compareSignatureV4(newSignature, signV4Values.Signature) {
- return nil, "", "", "", time.Time{}, s3err.ErrSignatureDoesNotMatch
- }
-
- // Return calculated signature.
- return cred, newSignature, region, signV4Values.Credential.scope.service, date, s3err.ErrNone
+ return credential, calculatedSignature, authInfo.Region, authInfo.Service, authInfo.Date, s3err.ErrNone
}
const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB
@@ -149,7 +81,7 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
contentSha256Header := req.Header.Get("X-Amz-Content-Sha256")
authorizationHeader := req.Header.Get("Authorization")
- var ident *Credential
+ var credential *Credential
var seedSignature, region, service string
var seedDate time.Time
var errCode s3err.ErrorCode
@@ -158,7 +90,7 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
case streamingContentSHA256:
glog.V(3).Infof("streaming content sha256")
- ident, seedSignature, region, service, seedDate, errCode = iam.calculateSeedSignature(req)
+ credential, seedSignature, region, service, seedDate, errCode = iam.calculateSeedSignature(req)
if errCode != s3err.ErrNone {
return nil, errCode
}
@@ -186,7 +118,7 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
checkSumWriter := getCheckSumWriter(checksumAlgorithm)
return &s3ChunkedReader{
- cred: ident,
+ cred: credential,
reader: bufio.NewReader(req.Body),
seedSignature: seedSignature,
seedDate: seedDate,
diff --git a/weed/s3api/chunked_reader_v4_test.go b/weed/s3api/chunked_reader_v4_test.go
index 786df3465..c9bad1d8a 100644
--- a/weed/s3api/chunked_reader_v4_test.go
+++ b/weed/s3api/chunked_reader_v4_test.go
@@ -9,6 +9,7 @@ import (
"strings"
"sync"
"testing"
+ "time"
"hash/crc32"
@@ -16,66 +17,19 @@ import (
"github.com/stretchr/testify/assert"
)
+// getDefaultTimestamp returns a current timestamp for tests
+func getDefaultTimestamp() string {
+ return time.Now().UTC().Format(iso8601Format)
+}
+
const (
- defaultTimestamp = "20130524T000000Z"
+ defaultTimestamp = "20130524T000000Z" // Legacy constant for reference
defaultBucketName = "examplebucket"
defaultAccessKeyId = "AKIAIOSFODNN7EXAMPLE"
defaultSecretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
defaultRegion = "us-east-1"
)
-func generatestreamingAws4HmacSha256Payload() string {
- // This test will implement the following scenario:
- // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
-
- chunk1 := "10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n" +
- strings.Repeat("a", 65536) + "\r\n"
- chunk2 := "400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\r\n" +
- strings.Repeat("a", 1024) + "\r\n"
- chunk3 := "0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n" +
- "\r\n" // The last chunk is empty
-
- payload := chunk1 + chunk2 + chunk3
- return payload
-}
-
-func NewRequeststreamingAws4HmacSha256Payload() (*http.Request, error) {
- // This test will implement the following scenario:
- // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
-
- payload := generatestreamingAws4HmacSha256Payload()
- req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/examplebucket/chunkObject.txt", bytes.NewReader([]byte(payload)))
- if err != nil {
- return nil, err
- }
-
- req.Header.Set("Host", "s3.amazonaws.com")
- req.Header.Set("x-amz-date", defaultTimestamp)
- req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
- req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9")
- req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
- req.Header.Set("Content-Encoding", "aws-chunked")
- req.Header.Set("x-amz-decoded-content-length", "66560")
- req.Header.Set("Content-Length", "66824")
-
- return req, nil
-}
-
-func TestNewSignV4ChunkedReaderstreamingAws4HmacSha256Payload(t *testing.T) {
- // This test will implement the following scenario:
- // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
- req, err := NewRequeststreamingAws4HmacSha256Payload()
- if err != nil {
- t.Fatalf("Failed to create request: %v", err)
- }
- iam := setupIam()
-
- // The expected payload a long string of 'a's
- expectedPayload := strings.Repeat("a", 66560)
-
- runWithRequest(iam, req, t, expectedPayload)
-}
-
func generateStreamingUnsignedPayloadTrailerPayload(includeFinalCRLF bool) string {
// This test will implement the following scenario:
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
@@ -117,7 +71,7 @@ func NewRequestStreamingUnsignedPayloadTrailer(includeFinalCRLF bool) (*http.Req
}
req.Header.Set("Host", "amzn-s3-demo-bucket")
- req.Header.Set("x-amz-date", defaultTimestamp)
+ req.Header.Set("x-amz-date", getDefaultTimestamp())
req.Header.Set("Content-Encoding", "aws-chunked")
req.Header.Set("x-amz-decoded-content-length", "17408")
req.Header.Set("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER")
@@ -194,3 +148,169 @@ func setupIam() IdentityAccessManagement {
iam.accessKeyIdent[defaultAccessKeyId] = iam.identities[0]
return iam
}
+
+// TestSignedStreamingUpload tests streaming uploads with signed chunks
+// This replaces the removed AWS example test with a dynamic signature generation approach
+func TestSignedStreamingUpload(t *testing.T) {
+ iam := setupIam()
+
+ // Create a simple streaming upload with 2 chunks
+ chunk1Data := strings.Repeat("a", 1024)
+ chunk2Data := strings.Repeat("b", 512)
+
+ // Use current time for signatures
+ now := time.Now().UTC()
+ amzDate := now.Format(iso8601Format)
+ dateStamp := now.Format(yyyymmdd)
+
+ // Calculate seed signature
+ scope := dateStamp + "/" + defaultRegion + "/s3/aws4_request"
+
+ // Build canonical request for seed signature
+ hashedPayload := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+ canonicalHeaders := "content-encoding:aws-chunked\n" +
+ "host:s3.amazonaws.com\n" +
+ "x-amz-content-sha256:" + hashedPayload + "\n" +
+ "x-amz-date:" + amzDate + "\n" +
+ "x-amz-decoded-content-length:1536\n"
+ signedHeaders := "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length"
+
+ canonicalRequest := "PUT\n" +
+ "/test-bucket/test-object\n" +
+ "\n" +
+ canonicalHeaders + "\n" +
+ signedHeaders + "\n" +
+ hashedPayload
+
+ canonicalRequestHash := getSHA256Hash([]byte(canonicalRequest))
+ stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + canonicalRequestHash
+
+ signingKey := getSigningKey(defaultSecretAccessKey, dateStamp, defaultRegion, "s3")
+ seedSignature := getSignature(signingKey, stringToSign)
+
+ // Calculate chunk signatures
+ chunk1Hash := getSHA256Hash([]byte(chunk1Data))
+ chunk1StringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+ seedSignature + "\n" + emptySHA256 + "\n" + chunk1Hash
+ chunk1Signature := getSignature(signingKey, chunk1StringToSign)
+
+ chunk2Hash := getSHA256Hash([]byte(chunk2Data))
+ chunk2StringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+ chunk1Signature + "\n" + emptySHA256 + "\n" + chunk2Hash
+ chunk2Signature := getSignature(signingKey, chunk2StringToSign)
+
+ finalStringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+ chunk2Signature + "\n" + emptySHA256 + "\n" + emptySHA256
+ finalSignature := getSignature(signingKey, finalStringToSign)
+
+ // Build the chunked payload
+ payload := fmt.Sprintf("400;chunk-signature=%s\r\n%s\r\n", chunk1Signature, chunk1Data) +
+ fmt.Sprintf("200;chunk-signature=%s\r\n%s\r\n", chunk2Signature, chunk2Data) +
+ fmt.Sprintf("0;chunk-signature=%s\r\n\r\n", finalSignature)
+
+ // Create the request
+ req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/test-bucket/test-object",
+ bytes.NewReader([]byte(payload)))
+ assert.NoError(t, err)
+
+ req.Header.Set("Host", "s3.amazonaws.com")
+ req.Header.Set("x-amz-date", amzDate)
+ req.Header.Set("x-amz-content-sha256", hashedPayload)
+ req.Header.Set("Content-Encoding", "aws-chunked")
+ req.Header.Set("x-amz-decoded-content-length", "1536")
+
+ authHeader := fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s",
+ defaultAccessKeyId, scope, signedHeaders, seedSignature)
+ req.Header.Set("Authorization", authHeader)
+
+ // Test the chunked reader
+ reader, errCode := iam.newChunkedReader(req)
+ assert.Equal(t, s3err.ErrNone, errCode)
+ assert.NotNil(t, reader)
+
+ // Read and verify the payload
+ data, err := io.ReadAll(reader)
+ assert.NoError(t, err)
+ assert.Equal(t, chunk1Data+chunk2Data, string(data))
+}
+
+// TestSignedStreamingUploadInvalidSignature tests that invalid chunk signatures are rejected
+// This is a negative test case to ensure signature validation is actually working
+func TestSignedStreamingUploadInvalidSignature(t *testing.T) {
+ iam := setupIam()
+
+ // Create a simple streaming upload with 1 chunk
+ chunk1Data := strings.Repeat("a", 1024)
+
+ // Use current time for signatures
+ now := time.Now().UTC()
+ amzDate := now.Format(iso8601Format)
+ dateStamp := now.Format(yyyymmdd)
+
+ // Calculate seed signature
+ scope := dateStamp + "/" + defaultRegion + "/s3/aws4_request"
+
+ // Build canonical request for seed signature
+ hashedPayload := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+ canonicalHeaders := "content-encoding:aws-chunked\n" +
+ "host:s3.amazonaws.com\n" +
+ "x-amz-content-sha256:" + hashedPayload + "\n" +
+ "x-amz-date:" + amzDate + "\n" +
+ "x-amz-decoded-content-length:1024\n"
+ signedHeaders := "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length"
+
+ canonicalRequest := "PUT\n" +
+ "/test-bucket/test-object\n" +
+ "\n" +
+ canonicalHeaders + "\n" +
+ signedHeaders + "\n" +
+ hashedPayload
+
+ canonicalRequestHash := getSHA256Hash([]byte(canonicalRequest))
+ stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + canonicalRequestHash
+
+ signingKey := getSigningKey(defaultSecretAccessKey, dateStamp, defaultRegion, "s3")
+ seedSignature := getSignature(signingKey, stringToSign)
+
+ // Calculate chunk signature (correct)
+ chunk1Hash := getSHA256Hash([]byte(chunk1Data))
+ chunk1StringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+ seedSignature + "\n" + emptySHA256 + "\n" + chunk1Hash
+ chunk1Signature := getSignature(signingKey, chunk1StringToSign)
+
+ // Calculate final signature (correct)
+ finalStringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+ chunk1Signature + "\n" + emptySHA256 + "\n" + emptySHA256
+ finalSignature := getSignature(signingKey, finalStringToSign)
+
+ // Build the chunked payload with INTENTIONALLY WRONG chunk signature
+ // We'll use a modified signature to simulate a tampered request
+ wrongChunkSignature := strings.Replace(chunk1Signature, "a", "b", 1)
+ payload := fmt.Sprintf("400;chunk-signature=%s\r\n%s\r\n", wrongChunkSignature, chunk1Data) +
+ fmt.Sprintf("0;chunk-signature=%s\r\n\r\n", finalSignature)
+
+ // Create the request
+ req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/test-bucket/test-object",
+ bytes.NewReader([]byte(payload)))
+ assert.NoError(t, err)
+
+ req.Header.Set("Host", "s3.amazonaws.com")
+ req.Header.Set("x-amz-date", amzDate)
+ req.Header.Set("x-amz-content-sha256", hashedPayload)
+ req.Header.Set("Content-Encoding", "aws-chunked")
+ req.Header.Set("x-amz-decoded-content-length", "1024")
+
+ authHeader := fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s",
+ defaultAccessKeyId, scope, signedHeaders, seedSignature)
+ req.Header.Set("Authorization", authHeader)
+
+ // Test the chunked reader - it should be created successfully
+ reader, errCode := iam.newChunkedReader(req)
+ assert.Equal(t, s3err.ErrNone, errCode)
+ assert.NotNil(t, reader)
+
+ // Try to read the payload - this should fail with signature validation error
+ _, err = io.ReadAll(reader)
+ assert.Error(t, err, "Expected error when reading chunk with invalid signature")
+ assert.Contains(t, err.Error(), "chunk signature does not match", "Error should indicate chunk signature mismatch")
+}
diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go
index 0d354ee8c..762289bce 100644
--- a/weed/s3api/s3err/s3api_errors.go
+++ b/weed/s3api/s3err/s3api_errors.go
@@ -102,6 +102,7 @@ const (
ErrContentSHA256Mismatch
ErrInvalidAccessKeyID
ErrRequestNotReadyYet
+ ErrRequestTimeTooSkewed
ErrMissingDateHeader
ErrInvalidRequest
ErrAuthNotSetup
@@ -432,6 +433,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
HTTPStatusCode: http.StatusForbidden,
},
+ ErrRequestTimeTooSkewed: {
+ Code: "RequestTimeTooSkewed",
+ Description: "The difference between the request time and the server's time is too large.",
+ HTTPStatusCode: http.StatusForbidden,
+ },
+
ErrSignatureDoesNotMatch: {
Code: "SignatureDoesNotMatch",
Description: "The request signature we calculated does not match the signature you provided. Check your key and signing method.",