aboutsummaryrefslogtreecommitdiff
path: root/weed/s3api
diff options
context:
space:
mode:
Diffstat (limited to 'weed/s3api')
-rw-r--r--weed/s3api/auth_signature_v4.go30
-rw-r--r--weed/s3api/auto_signature_v4_test.go2
-rw-r--r--weed/s3api/chunked_reader_v4.go4
-rw-r--r--weed/s3api/filer_util.go10
-rw-r--r--weed/s3api/s3api_bucket_handlers.go2
-rw-r--r--weed/s3api/s3api_object_copy_handlers.go27
-rw-r--r--weed/s3api/s3api_object_handlers.go28
-rw-r--r--weed/s3api/s3api_object_handlers_postpolicy.go2
-rw-r--r--weed/s3api/s3api_objects_list_handlers.go1
9 files changed, 87 insertions, 19 deletions
diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go
index 5ef7439c8..0df26e6fc 100644
--- a/weed/s3api/auth_signature_v4.go
+++ b/weed/s3api/auth_signature_v4.go
@@ -24,6 +24,7 @@ import (
"crypto/subtle"
"encoding/hex"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+ "io/ioutil"
"net/http"
"net/url"
"regexp"
@@ -132,6 +133,17 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r
// Query string.
queryStr := req.URL.Query().Encode()
+ // Get hashed Payload
+ if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil {
+ buf, _ := ioutil.ReadAll(r.Body)
+ r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
+ b, _ := ioutil.ReadAll(bytes.NewBuffer(buf))
+ if len(b) != 0 {
+ bodyHash := sha256.Sum256(b)
+ hashedPayload = hex.EncodeToString(bodyHash[:])
+ }
+ }
+
// Get canonical request.
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method)
@@ -139,7 +151,10 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r
stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope())
// Get hmac signing key.
- signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, signV4Values.Credential.scope.region)
+ signingKey := getSigningKey(cred.SecretKey,
+ signV4Values.Credential.scope.date,
+ signV4Values.Credential.scope.region,
+ signV4Values.Credential.scope.service)
// Calculate signature.
newSignature := getSignature(signingKey, stringToSign)
@@ -310,7 +325,7 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.
}
// Get signing key.
- signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region)
+ signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, credHeader.scope.service)
// Get signature.
newSignature := getSignature(signingKey, formValues.Get("Policy"))
@@ -427,7 +442,10 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s
presignedStringToSign := getStringToSign(presignedCanonicalReq, t, pSignValues.Credential.getScope())
// Get hmac presigned signing key.
- presignedSigningKey := getSigningKey(cred.SecretKey, pSignValues.Credential.scope.date, pSignValues.Credential.scope.region)
+ presignedSigningKey := getSigningKey(cred.SecretKey,
+ pSignValues.Credential.scope.date,
+ pSignValues.Credential.scope.region,
+ pSignValues.Credential.scope.service)
// Get new signature.
newSignature := getSignature(presignedSigningKey, presignedStringToSign)
@@ -655,11 +673,11 @@ func sumHMAC(key []byte, data []byte) []byte {
}
// getSigningKey hmac seed to calculate final signature.
-func getSigningKey(secretKey string, t time.Time, region string) []byte {
+func getSigningKey(secretKey string, t time.Time, region string, service string) []byte {
date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format(yyyymmdd)))
regionBytes := sumHMAC(date, []byte(region))
- service := sumHMAC(regionBytes, []byte("s3"))
- signingKey := sumHMAC(service, []byte("aws4_request"))
+ serviceBytes := sumHMAC(regionBytes, []byte(service))
+ signingKey := sumHMAC(serviceBytes, []byte("aws4_request"))
return signingKey
}
diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go
index 4c8255768..b47cd5f2d 100644
--- a/weed/s3api/auto_signature_v4_test.go
+++ b/weed/s3api/auto_signature_v4_test.go
@@ -370,7 +370,7 @@ func preSignV4(req *http.Request, accessKeyID, secretAccessKey string, expires i
queryStr := strings.Replace(query.Encode(), "+", "%20", -1)
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method)
stringToSign := getStringToSign(canonicalRequest, date, scope)
- signingKey := getSigningKey(secretAccessKey, date, region)
+ signingKey := getSigningKey(secretAccessKey, date, region, "s3")
signature := getSignature(signingKey, stringToSign)
req.URL.RawQuery = query.Encode()
diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go
index 734c9faee..b163ec2f6 100644
--- a/weed/s3api/chunked_reader_v4.go
+++ b/weed/s3api/chunked_reader_v4.go
@@ -45,7 +45,7 @@ func getChunkSignature(secretKey string, seedSignature string, region string, da
hashedChunk
// Get hmac signing key.
- signingKey := getSigningKey(secretKey, date, region)
+ signingKey := getSigningKey(secretKey, date, region, "s3")
// Calculate signature.
newSignature := getSignature(signingKey, stringToSign)
@@ -117,7 +117,7 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())
// Get hmac signing key.
- signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region)
+ signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, "s3")
// Calculate signature.
newSignature := getSignature(signingKey, stringToSign)
diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go
index 3626ece98..1803332a3 100644
--- a/weed/s3api/filer_util.go
+++ b/weed/s3api/filer_util.go
@@ -31,6 +31,10 @@ func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, incl
return nil
}, startFrom, inclusive, limit)
+ if len(entries) == 0 {
+ isLast = true
+ }
+
return
}
@@ -75,6 +79,12 @@ func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isD
}
+func (s3a *S3ApiServer) touch(parentDirectoryPath string, entryName string, entry *filer_pb.Entry) (err error) {
+
+ return filer_pb.Touch(s3a, parentDirectoryPath, entryName, entry)
+
+}
+
func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *filer_pb.Entry, err error) {
fullPath := util.NewFullPath(parentDirectoryPath, entryName)
return filer_pb.GetEntry(s3a, fullPath)
diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go
index 338f82668..48e8cb047 100644
--- a/weed/s3api/s3api_bucket_handlers.go
+++ b/weed/s3api/s3api_bucket_handlers.go
@@ -51,7 +51,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
var buckets []*s3.Bucket
for _, entry := range entries {
if entry.IsDirectory {
- if identity != nil && !identity.canDo(s3_constants.ACTION_ADMIN, entry.Name) {
+ if identity != nil && !identity.canDo(s3_constants.ACTION_LIST, entry.Name) {
continue
}
buckets = append(buckets, &s3.Bucket{
diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go
index ca578e7e5..84a85fd78 100644
--- a/weed/s3api/s3api_object_copy_handlers.go
+++ b/weed/s3api/s3api_object_copy_handlers.go
@@ -4,6 +4,7 @@ import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+ weed_server "github.com/chrislusf/seaweedfs/weed/server"
"net/http"
"net/url"
"strconv"
@@ -25,6 +26,26 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
}
srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
+
+ if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && isReplace(r) {
+ fullPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
+ dir, name := fullPath.DirAndName()
+ entry, err := s3a.getEntry(dir, name)
+ if err != nil {
+ writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+ }
+ entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r))
+ err = s3a.touch(dir, name, entry)
+ if err != nil {
+ writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+ }
+ writeSuccessResponseXML(w, encodeResponse(CopyObjectResult{
+ ETag: fmt.Sprintf("%x", entry.Attributes.Md5),
+ LastModified: time.Now().UTC(),
+ }))
+ return
+ }
+
// If source object is empty or bucket is empty, reply back invalid copy source.
if srcObject == "" || srcBucket == "" {
writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
@@ -32,7 +53,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
}
if srcBucket == dstBucket && srcObject == dstObject {
- writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+ writeErrorResponse(w, s3err.ErrInvalidCopyDest, r.URL)
return
}
@@ -147,3 +168,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
writeSuccessResponseXML(w, encodeResponse(response))
}
+
+func isReplace(r *http.Request) bool {
+ return r.Header.Get("X-Amz-Metadata-Directive") == "REPLACE"
+}
diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go
index 19d85c495..f1a539ac5 100644
--- a/weed/s3api/s3api_object_handlers.go
+++ b/weed/s3api/s3api_object_handlers.go
@@ -5,9 +5,11 @@ import (
"encoding/json"
"encoding/xml"
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/filer"
"io"
"io/ioutil"
"net/http"
+ "net/url"
"sort"
"strings"
@@ -69,7 +71,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
return
}
} else {
- uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
+ uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, urlPathEscape(object))
etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
@@ -84,6 +86,14 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
writeSuccessResponseEmpty(w)
}
+func urlPathEscape(object string) string {
+ var escapedParts []string
+ for _, part := range strings.Split(object, "/") {
+ escapedParts = append(escapedParts, url.PathEscape(part))
+ }
+ return strings.Join(escapedParts, "/")
+}
+
func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
bucket, object := getBucketAndObject(r)
@@ -94,7 +104,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
}
destUrl := fmt.Sprintf("http://%s%s/%s%s",
- s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
+ s3a.option.Filer, s3a.option.BucketsPath, bucket, urlPathEscape(object))
s3a.proxyToFiler(w, r, destUrl, passThroughResponse)
@@ -105,7 +115,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
bucket, object := getBucketAndObject(r)
destUrl := fmt.Sprintf("http://%s%s/%s%s",
- s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
+ s3a.option.Filer, s3a.option.BucketsPath, bucket, urlPathEscape(object))
s3a.proxyToFiler(w, r, destUrl, passThroughResponse)
@@ -116,7 +126,7 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
bucket, object := getBucketAndObject(r)
destUrl := fmt.Sprintf("http://%s%s/%s%s?recursive=true",
- s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
+ s3a.option.Filer, s3a.option.BucketsPath, bucket, urlPathEscape(object))
s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) {
for k, v := range proxyResponse.Header {
@@ -196,6 +206,8 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
if err == nil {
directoriesWithDeletion[parentDirectoryPath]++
deletedObjects = append(deletedObjects, object)
+ } else if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
+ deletedObjects = append(deletedObjects, object)
} else {
delete(directoriesWithDeletion, parentDirectoryPath)
deleteErrors = append(deleteErrors, DeleteError{
@@ -299,7 +311,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
}
defer util.CloseResponse(resp)
- if (resp.ContentLength == -1 || resp.StatusCode == 404) && !strings.HasSuffix(destUrl, "/") {
+ if (resp.ContentLength == -1 || resp.StatusCode == 404) && resp.StatusCode != 304 {
if r.Method != "DELETE" {
writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
return
@@ -314,7 +326,11 @@ func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
for k, v := range proxyResponse.Header {
w.Header()[k] = v
}
- w.WriteHeader(proxyResponse.StatusCode)
+ if proxyResponse.Header.Get("Content-Range") != "" && proxyResponse.StatusCode == 200 {
+ w.WriteHeader(http.StatusPartialContent)
+ } else {
+ w.WriteHeader(proxyResponse.StatusCode)
+ }
io.Copy(w, proxyResponse.Body)
}
diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go
index 044e732db..035302ae6 100644
--- a/weed/s3api/s3api_object_handlers_postpolicy.go
+++ b/weed/s3api/s3api_object_handlers_postpolicy.go
@@ -110,7 +110,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
}
}
- uploadUrl := fmt.Sprintf("http://%s%s/%s/%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
+ uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, urlPathEscape(object))
etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody)
diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go
index 2d36c6ec9..739cdd8f9 100644
--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -206,7 +206,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
isTruncated = isTruncated || subIsTruncated
maxKeys -= subCounter
nextMarker = subDir + "/" + subNextMarker
- counter += subCounter
// finished processing this sub directory
marker = subDir
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-22 10:01:28 +0000