aboutsummaryrefslogtreecommitdiff
path: root/weed/s3api
diff options
context:
space:
mode:
Diffstat (limited to 'weed/s3api')
-rw-r--r--weed/s3api/auth_credentials.go25
-rw-r--r--weed/s3api/chunked_reader_v4.go3
-rw-r--r--weed/s3api/filer_multipart.go2
-rw-r--r--weed/s3api/http/header.go27
-rw-r--r--weed/s3api/s3api_bucket_handlers.go11
-rw-r--r--weed/s3api/s3api_handlers.go1
-rw-r--r--weed/s3api/s3api_object_copy_handlers.go11
-rw-r--r--weed/s3api/s3api_object_handlers.go86
-rw-r--r--weed/s3api/s3api_object_handlers_postpolicy.go1
-rw-r--r--weed/s3api/s3api_object_multipart_handlers.go13
-rw-r--r--weed/s3api/s3api_object_tagging_handlers.go10
-rw-r--r--weed/s3api/s3api_objects_list_handlers.go4
-rw-r--r--weed/s3api/s3api_server.go2
-rw-r--r--weed/s3api/s3api_status_handlers.go7
-rw-r--r--weed/s3api/s3err/audit_fluent.go183
-rw-r--r--weed/s3api/s3err/error_handler.go2
16 files changed, 304 insertions, 84 deletions
diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go
index d29e8692f..0d46ad7ca 100644
--- a/weed/s3api/auth_credentials.go
+++ b/weed/s3api/auth_credentials.go
@@ -203,40 +203,51 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
var identity *Identity
var s3Err s3err.ErrorCode
var found bool
+ var authType string
switch getRequestAuthType(r) {
case authTypeStreamingSigned:
return identity, s3err.ErrNone
case authTypeUnknown:
glog.V(3).Infof("unknown auth type")
+ r.Header.Set(xhttp.AmzAuthType, "Unknown")
return identity, s3err.ErrAccessDenied
case authTypePresignedV2, authTypeSignedV2:
glog.V(3).Infof("v2 auth type")
identity, s3Err = iam.isReqAuthenticatedV2(r)
+ authType = "SigV2"
case authTypeSigned, authTypePresigned:
glog.V(3).Infof("v4 auth type")
identity, s3Err = iam.reqSignatureV4Verify(r)
+ authType = "SigV4"
case authTypePostPolicy:
glog.V(3).Infof("post policy auth type")
+ r.Header.Set(xhttp.AmzAuthType, "PostPolicy")
return identity, s3err.ErrNone
case authTypeJWT:
glog.V(3).Infof("jwt auth type")
+ r.Header.Set(xhttp.AmzAuthType, "Jwt")
return identity, s3err.ErrNotImplemented
case authTypeAnonymous:
+ authType = "Anonymous"
identity, found = iam.lookupAnonymous()
if !found {
+ r.Header.Set(xhttp.AmzAuthType, authType)
return identity, s3err.ErrAccessDenied
}
default:
return identity, s3err.ErrNotImplemented
}
+ if len(authType) > 0 {
+ r.Header.Set(xhttp.AmzAuthType, authType)
+ }
if s3Err != s3err.ErrNone {
return identity, s3Err
}
glog.V(3).Infof("user name: %v actions: %v, action: %v", identity.Name, identity.Actions, action)
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
if !identity.canDo(action, bucket) {
return identity, s3err.ErrAccessDenied
@@ -250,33 +261,45 @@ func (iam *IdentityAccessManagement) authUser(r *http.Request) (*Identity, s3err
var identity *Identity
var s3Err s3err.ErrorCode
var found bool
+ var authType string
switch getRequestAuthType(r) {
case authTypeStreamingSigned:
return identity, s3err.ErrNone
case authTypeUnknown:
glog.V(3).Infof("unknown auth type")
+ r.Header.Set(xhttp.AmzAuthType, "Unknown")
return identity, s3err.ErrAccessDenied
case authTypePresignedV2, authTypeSignedV2:
glog.V(3).Infof("v2 auth type")
identity, s3Err = iam.isReqAuthenticatedV2(r)
+ authType = "SigV2"
case authTypeSigned, authTypePresigned:
glog.V(3).Infof("v4 auth type")
identity, s3Err = iam.reqSignatureV4Verify(r)
+ authType = "SigV4"
case authTypePostPolicy:
glog.V(3).Infof("post policy auth type")
+ r.Header.Set(xhttp.AmzAuthType, "PostPolicy")
return identity, s3err.ErrNone
case authTypeJWT:
glog.V(3).Infof("jwt auth type")
+ r.Header.Set(xhttp.AmzAuthType, "Jwt")
return identity, s3err.ErrNotImplemented
case authTypeAnonymous:
+ authType = "Anonymous"
identity, found = iam.lookupAnonymous()
if !found {
+ r.Header.Set(xhttp.AmzAuthType, authType)
return identity, s3err.ErrAccessDenied
}
default:
return identity, s3err.ErrNotImplemented
}
+ if len(authType) > 0 {
+ r.Header.Set(xhttp.AmzAuthType, authType)
+ }
+
glog.V(3).Infof("auth error: %v", s3Err)
if s3Err != s3err.ErrNone {
return identity, s3Err
diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go
index ec26f693a..5dd0648c6 100644
--- a/weed/s3api/chunked_reader_v4.go
+++ b/weed/s3api/chunked_reader_v4.go
@@ -24,6 +24,7 @@ import (
"crypto/sha256"
"encoding/hex"
"errors"
+ xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"hash"
"io"
@@ -90,7 +91,7 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
return nil, "", "", time.Time{}, s3err.ErrInvalidAccessKeyID
}
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
if !identity.canDo("Write", bucket) {
errCode = s3err.ErrAccessDenied
return
diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go
index d93ac63ea..1795ade93 100644
--- a/weed/s3api/filer_multipart.go
+++ b/weed/s3api/filer_multipart.go
@@ -142,7 +142,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
output = &CompleteMultipartUploadResult{
CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
- Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), dirName, entryName)),
+ Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlPathEscape(dirName), urlPathEscape(entryName))),
Bucket: input.Bucket,
ETag: aws.String("\"" + filer.ETagChunks(finalParts) + "\""),
Key: objectKey(input.Key),
diff --git a/weed/s3api/http/header.go b/weed/s3api/http/header.go
index 6614b0af0..d63d50443 100644
--- a/weed/s3api/http/header.go
+++ b/weed/s3api/http/header.go
@@ -16,6 +16,12 @@
package http
+import (
+ "github.com/gorilla/mux"
+ "net/http"
+ "strings"
+)
+
// Standard S3 HTTP request constants
const (
// S3 storage class
@@ -32,5 +38,26 @@ const (
// Non-Standard S3 HTTP request constants
const (
AmzIdentityId = "s3-identity-id"
+ AmzAuthType = "s3-auth-type"
AmzIsAdmin = "s3-is-admin" // only set to http request header as a context
)
+
+func GetBucketAndObject(r *http.Request) (bucket, object string) {
+ vars := mux.Vars(r)
+ bucket = vars["bucket"]
+ object = vars["object"]
+ if !strings.HasPrefix(object, "/") {
+ object = "/" + object
+ }
+
+ return
+}
+
+var PassThroughHeaders = map[string]string{
+ "response-cache-control": "Cache-Control",
+ "response-content-disposition": "Content-Disposition",
+ "response-content-encoding": "Content-Encoding",
+ "response-content-language": "Content-Language",
+ "response-content-type": "Content-Type",
+ "response-expires": "Expires",
+}
diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go
index b932edbac..247e33104 100644
--- a/weed/s3api/s3api_bucket_handlers.go
+++ b/weed/s3api/s3api_bucket_handlers.go
@@ -78,7 +78,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("PutBucketHandler %s", bucket)
// avoid duplicated buckets
@@ -133,13 +133,12 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
-
writeSuccessResponseEmpty(w, r)
}
func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("DeleteBucketHandler %s", bucket)
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
@@ -174,7 +173,7 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("HeadBucketHandler %s", bucket)
if entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket); entry == nil || err == filer_pb.ErrNotFound {
@@ -219,7 +218,7 @@ func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool {
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html
func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Request) {
// collect parameters
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("GetBucketAclHandler %s", bucket)
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
@@ -259,7 +258,7 @@ func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Reque
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html
func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
// collect parameters
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("GetBucketAclHandler %s", bucket)
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
diff --git a/weed/s3api/s3api_handlers.go b/weed/s3api/s3api_handlers.go
index 5bc94bb04..e42fb6c44 100644
--- a/weed/s3api/s3api_handlers.go
+++ b/weed/s3api/s3api_handlers.go
@@ -28,6 +28,7 @@ func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string {
func writeSuccessResponseXML(w http.ResponseWriter, r *http.Request, response interface{}) {
s3err.WriteXMLResponse(w, r, http.StatusOK, response)
+ s3err.PostLog(r, http.StatusOK, s3err.ErrNone)
}
func writeSuccessResponseEmpty(w http.ResponseWriter, r *http.Request) {
diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go
index e2b191435..7756e1348 100644
--- a/weed/s3api/s3api_object_copy_handlers.go
+++ b/weed/s3api/s3api_object_copy_handlers.go
@@ -3,6 +3,7 @@ package s3api
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
+ xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
"net/http"
@@ -16,7 +17,7 @@ import (
func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
- dstBucket, dstObject := getBucketAndObject(r)
+ dstBucket, dstObject := xhttp.GetBucketAndObject(r)
// Copy source path.
cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source"))
@@ -69,9 +70,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
}
dstUrl := fmt.Sprintf("http://%s%s/%s%s?collection=%s",
- s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, dstBucket, dstObject, dstBucket)
+ s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, dstBucket, urlPathEscape(dstObject), dstBucket)
srcUrl := fmt.Sprintf("http://%s%s/%s%s",
- s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, srcObject)
+ s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, urlPathEscape(srcObject))
_, _, resp, err := util.DownloadFile(srcUrl, "")
if err != nil {
@@ -116,7 +117,7 @@ type CopyPartResult struct {
func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
// https://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsUsingRESTMPUapi.html
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
- dstBucket, _ := getBucketAndObject(r)
+ dstBucket, _ := xhttp.GetBucketAndObject(r)
// Copy source path.
cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source"))
@@ -154,7 +155,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
dstUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s",
s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(dstBucket), uploadID, partID, dstBucket)
srcUrl := fmt.Sprintf("http://%s%s/%s%s",
- s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, srcObject)
+ s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, urlPathEscape(srcObject))
dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, rangeHeader)
if err != nil {
diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go
index 4defe28da..2ac9c8102 100644
--- a/weed/s3api/s3api_object_handlers.go
+++ b/weed/s3api/s3api_object_handlers.go
@@ -16,10 +16,9 @@ import (
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/pquerna/cachecontrol/cacheobject"
+ xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
- "github.com/gorilla/mux"
-
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
@@ -51,7 +50,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("PutObjectHandler %s %s", bucket, object)
_, err := validateContentMd5(r.Header)
@@ -133,7 +132,7 @@ func urlPathEscape(object string) string {
func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("GetObjectHandler %s %s", bucket, object)
if strings.HasSuffix(r.URL.Path, "/") {
@@ -145,34 +144,34 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object))
s3a.proxyToFiler(w, r, destUrl, passThroughResponse)
-
}
func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("HeadObjectHandler %s %s", bucket, object)
destUrl := fmt.Sprintf("http://%s%s/%s%s",
s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object))
s3a.proxyToFiler(w, r, destUrl, passThroughResponse)
-
}
func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("DeleteObjectHandler %s %s", bucket, object)
destUrl := fmt.Sprintf("http://%s%s/%s%s?recursive=true",
s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object))
- s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) {
+ s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int) {
+ statusCode = http.StatusNoContent
for k, v := range proxyResponse.Header {
w.Header()[k] = v
}
- w.WriteHeader(http.StatusNoContent)
+ w.WriteHeader(statusCode)
+ return statusCode
})
}
@@ -210,7 +209,7 @@ type DeleteObjectsResponse struct {
// DeleteMultipleObjectsHandler - Delete multiple objects
func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("DeleteMultipleObjectsHandler %s", bucket)
deleteXMLBytes, err := io.ReadAll(r.Body)
@@ -227,14 +226,17 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
var deletedObjects []ObjectIdentifier
var deleteErrors []DeleteError
+ var auditLog *s3err.AccessLog
directoriesWithDeletion := make(map[string]int)
+ if s3err.Logger != nil {
+ auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
+ }
s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// delete file entries
for _, object := range deleteObjects.Objects {
-
lastSeparator := strings.LastIndex(object.ObjectName, "/")
parentDirectoryPath, entryName, isDeleteData, isRecursive := "", object.ObjectName, true, false
if lastSeparator > 0 && lastSeparator+1 < len(object.ObjectName) {
@@ -257,6 +259,10 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
Key: object.ObjectName,
})
}
+ if auditLog != nil {
+ auditLog.Key = entryName
+ s3err.PostAccessLog(*auditLog)
+ }
}
// purge empty folders, only checking folders with deletions
@@ -300,16 +306,7 @@ func (s3a *S3ApiServer) doDeleteEmptyDirectories(client filer_pb.SeaweedFilerCli
return
}
-var passThroughHeaders = []string{
- "response-cache-control",
- "response-content-disposition",
- "response-content-encoding",
- "response-content-language",
- "response-content-type",
- "response-expires",
-}
-
-func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResponse *http.Response, w http.ResponseWriter)) {
+func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int)) {
glog.V(3).Infof("s3 proxying %s to %s", r.Method, destUrl)
@@ -322,25 +319,14 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
}
proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)
-
- for header, values := range r.Header {
- // handle s3 related headers
- passed := false
- for _, h := range passThroughHeaders {
- if strings.ToLower(header) == h && len(values) > 0 {
- proxyReq.Header.Add(header[len("response-"):], values[0])
- passed = true
- break
- }
- }
- if passed {
- continue
- }
- // handle other headers
- for _, value := range values {
- proxyReq.Header.Add(header, value)
+ for k, v := range r.URL.Query() {
+ if _, ok := xhttp.PassThroughHeaders[strings.ToLower(k)]; ok {
+ proxyReq.Header[k] = v
}
}
+ for header, values := range r.Header {
+ proxyReq.Header[header] = values
+ }
resp, postErr := client.Do(proxyReq)
@@ -363,20 +349,23 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
}
}
- responseFn(resp, w)
-
+ responseStatusCode := responseFn(resp, w)
+ s3err.PostLog(r, responseStatusCode, s3err.ErrNone)
}
-func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
+func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int) {
for k, v := range proxyResponse.Header {
w.Header()[k] = v
}
if proxyResponse.Header.Get("Content-Range") != "" && proxyResponse.StatusCode == 200 {
w.WriteHeader(http.StatusPartialContent)
+ statusCode = http.StatusPartialContent
} else {
- w.WriteHeader(proxyResponse.StatusCode)
+ statusCode = proxyResponse.StatusCode
}
+ w.WriteHeader(statusCode)
io.Copy(w, proxyResponse.Body)
+ return statusCode
}
func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader) (etag string, code s3err.ErrorCode) {
@@ -438,17 +427,6 @@ func setEtag(w http.ResponseWriter, etag string) {
}
}
-func getBucketAndObject(r *http.Request) (bucket, object string) {
- vars := mux.Vars(r)
- bucket = vars["bucket"]
- object = vars["object"]
- if !strings.HasPrefix(object, "/") {
- object = "/" + object
- }
-
- return
-}
-
func filerErrorToS3Error(errString string) s3err.ErrorCode {
if strings.HasPrefix(errString, "existing ") && strings.HasSuffix(errString, "is a directory") {
return s3err.ErrExistingObjectIsDirectory
diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go
index 23027253e..b0b71b1de 100644
--- a/weed/s3api/s3api_object_handlers_postpolicy.go
+++ b/weed/s3api/s3api_object_handlers_postpolicy.go
@@ -142,6 +142,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
Location: w.Header().Get("Location"),
}
s3err.WriteXMLResponse(w, r, http.StatusCreated, resp)
+ s3err.PostLog(r, http.StatusCreated, s3err.ErrNone)
case "200":
s3err.WriteEmptyResponse(w, r, http.StatusOK)
default:
diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go
index 926e048a8..8cbaf9393 100644
--- a/weed/s3api/s3api_object_multipart_handlers.go
+++ b/weed/s3api/s3api_object_multipart_handlers.go
@@ -3,6 +3,7 @@ package s3api
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
+ xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
"net/http"
@@ -23,7 +24,7 @@ const (
// NewMultipartUploadHandler - New multipart upload.
func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
createMultipartUploadInput := &s3.CreateMultipartUploadInput{
Bucket: aws.String(bucket),
@@ -55,7 +56,7 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
// CompleteMultipartUploadHandler - Completes multipart upload.
func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
// Get upload id.
uploadID, _, _, _ := getObjectResources(r.URL.Query())
@@ -79,7 +80,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
// AbortMultipartUploadHandler - Aborts multipart upload.
func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
// Get upload id.
uploadID, _, _, _ := getObjectResources(r.URL.Query())
@@ -103,7 +104,7 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht
// ListMultipartUploadsHandler - Lists multipart uploads.
func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query())
if maxUploads < 0 {
@@ -142,7 +143,7 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht
// ListObjectPartsHandler - Lists object parts in a multipart upload.
func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query())
if partNumberMarker < 0 {
@@ -175,7 +176,7 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
// PutObjectPartHandler - Put an object part in a multipart upload.
func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
uploadID := r.URL.Query().Get("uploadId")
exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
diff --git a/weed/s3api/s3api_object_tagging_handlers.go b/weed/s3api/s3api_object_tagging_handlers.go
index f97f32f0b..5c66fb961 100644
--- a/weed/s3api/s3api_object_tagging_handlers.go
+++ b/weed/s3api/s3api_object_tagging_handlers.go
@@ -3,6 +3,7 @@ package s3api
import (
"encoding/xml"
"fmt"
+ xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
"io"
"net/http"
@@ -16,7 +17,7 @@ import (
// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html
func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("GetObjectTaggingHandler %s %s", bucket, object)
target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object))
@@ -42,7 +43,7 @@ func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.R
// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html
func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("PutObjectTaggingHandler %s %s", bucket, object)
target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object))
@@ -91,14 +92,14 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R
}
w.WriteHeader(http.StatusOK)
-
+ s3err.PostLog(r, http.StatusOK, s3err.ErrNone)
}
// DeleteObjectTaggingHandler Delete object tagging
// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjectTagging.html
func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
- bucket, object := getBucketAndObject(r)
+ bucket, object := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("DeleteObjectTaggingHandler %s %s", bucket, object)
target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object))
@@ -117,4 +118,5 @@ func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *htt
}
w.WriteHeader(http.StatusNoContent)
+ s3err.PostLog(r, http.StatusNoContent, s3err.ErrNone)
}
diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go
index 20ab1d4d6..4decb5eac 100644
--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -39,7 +39,7 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
// collect parameters
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("ListObjectsV2Handler %s", bucket)
originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())
@@ -95,7 +95,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
// collect parameters
- bucket, _ := getBucketAndObject(r)
+ bucket, _ := xhttp.GetBucketAndObject(r)
glog.V(3).Infof("ListObjectsV1Handler %s", bucket)
originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())
diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go
index 6f7767d66..1abf9259d 100644
--- a/weed/s3api/s3api_server.go
+++ b/weed/s3api/s3api_server.go
@@ -38,7 +38,6 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
s3ApiServer.registerRouter(router)
go s3ApiServer.subscribeMetaEvents("s3", filer.IamConfigDirecotry+"/"+filer.IamIdentityFile, time.Now().UnixNano())
-
return s3ApiServer, nil
}
@@ -132,7 +131,6 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
// DeleteBucketLifecycleConfiguration
bucket.Methods("DELETE").HandlerFunc(s3a.iam.Auth(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)).Queries("lifecycle", "")
-
// ListObjectsV1 (Legacy)
bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV1Handler, ACTION_LIST), "LIST"))
diff --git a/weed/s3api/s3api_status_handlers.go b/weed/s3api/s3api_status_handlers.go
index 2ee6c37b2..fafb6ac2f 100644
--- a/weed/s3api/s3api_status_handlers.go
+++ b/weed/s3api/s3api_status_handlers.go
@@ -1,8 +1,11 @@
package s3api
-import "net/http"
+import (
+ "github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+ "net/http"
+)
func (s3a *S3ApiServer) StatusHandler(w http.ResponseWriter, r *http.Request) {
// write out the response code and content type header
- writeSuccessResponseEmpty(w, r)
+ s3err.WriteResponse(w, r, http.StatusOK, []byte{}, "")
}
diff --git a/weed/s3api/s3err/audit_fluent.go b/weed/s3api/s3err/audit_fluent.go
new file mode 100644
index 000000000..fcc5f9a0f
--- /dev/null
+++ b/weed/s3api/s3err/audit_fluent.go
@@ -0,0 +1,183 @@
+package s3err
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/chrislusf/seaweedfs/weed/glog"
+ xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
+ "github.com/fluent/fluent-logger-golang/fluent"
+ "net/http"
+ "os"
+ "time"
+)
+
+type AccessLogExtend struct {
+ AccessLog
+ AccessLogHTTP
+}
+
+type AccessLog struct {
+ Bucket string `msg:"bucket" json:"bucket"` // awsexamplebucket1
+ Time int64 `msg:"time" json:"time"` // [06/Feb/2019:00:00:38 +0000]
+ RemoteIP string `msg:"remote_ip" json:"remote_ip,omitempty"` // 192.0.2.3
+ Requester string `msg:"requester" json:"requester,omitempty"` // IAM user id
+ RequestID string `msg:"request_id" json:"request_id,omitempty"` // 3E57427F33A59F07
+ Operation string `msg:"operation" json:"operation,omitempty"` // REST.HTTP_method.resource_type REST.PUT.OBJECT
+ Key string `msg:"key" json:"key,omitempty"` // /photos/2019/08/puppy.jpg
+ ErrorCode string `msg:"error_code" json:"error_code,omitempty"`
+ HostId string `msg:"host_id" json:"host_id,omitempty"`
+ HostHeader string `msg:"host_header" json:"host_header,omitempty"` // s3.us-west-2.amazonaws.com
+ UserAgent string `msg:"user_agent" json:"user_agent,omitempty"`
+ HTTPStatus int `msg:"status" json:"status,omitempty"`
+ SignatureVersion string `msg:"signature_version" json:"signature_version,omitempty"`
+}
+
+type AccessLogHTTP struct {
+ RequestURI string `json:"request_uri,omitempty"` // "GET /awsexamplebucket1/photos/2019/08/puppy.jpg?x-foo=bar HTTP/1.1"
+ BytesSent string `json:"bytes_sent,omitempty"`
+ ObjectSize string `json:"object_size,omitempty"`
+ TotalTime int `json:"total_time,omitempty"`
+ TurnAroundTime int `json:"turn_around_time,omitempty"`
+ Referer string `json:"Referer,omitempty"`
+ VersionId string `json:"version_id,omitempty"`
+ CipherSuite string `json:"cipher_suite,omitempty"`
+ AuthenticationType string `json:"auth_type,omitempty"`
+ TLSVersion string `json:"TLS_version,omitempty"`
+}
+
+const tag = "s3.access"
+
+var (
+ Logger *fluent.Fluent
+ hostname = os.Getenv("HOSTNAME")
+ environment = os.Getenv("ENVIRONMENT")
+)
+
+func InitAuditLog(config string) {
+ configContent, readErr := os.ReadFile(config)
+ if readErr != nil {
+ glog.Errorf("fail to read fluent config %s : %v", config, readErr)
+ return
+ }
+ fluentConfig := &fluent.Config{}
+ if err := json.Unmarshal(configContent, fluentConfig); err != nil {
+ glog.Errorf("fail to parse fluent config %s : %v", string(configContent), err)
+ return
+ }
+ if len(fluentConfig.TagPrefix) == 0 && len(environment) > 0 {
+ fluentConfig.TagPrefix = environment
+ }
+ fluentConfig.Async = true
+ fluentConfig.AsyncResultCallback = func(data []byte, err error) {
+ if err != nil {
+ glog.Warning("Error while posting log: ", err)
+ }
+ }
+ var err error
+ Logger, err = fluent.New(*fluentConfig)
+ if err != nil {
+ glog.Errorf("fail to load fluent config: %v", err)
+ }
+}
+
+func getREST(httpMetod string, resourceType string) string {
+ return fmt.Sprintf("REST.%s.%s", httpMetod, resourceType)
+}
+
+func getResourceType(object string, query_key string, metod string) (string, bool) {
+ if object == "/" {
+ switch query_key {
+ case "delete":
+ return "BATCH.DELETE.OBJECT", true
+ case "tagging":
+ return getREST(metod, "OBJECTTAGGING"), true
+ case "lifecycle":
+ return getREST(metod, "LIFECYCLECONFIGURATION"), true
+ case "acl":
+ return getREST(metod, "ACCESSCONTROLPOLICY"), true
+ case "policy":
+ return getREST(metod, "BUCKETPOLICY"), true
+ default:
+ return getREST(metod, "BUCKET"), false
+ }
+ } else {
+ switch query_key {
+ case "tagging":
+ return getREST(metod, "OBJECTTAGGING"), true
+ default:
+ return getREST(metod, "OBJECT"), false
+ }
+ }
+}
+
+func getOperation(object string, r *http.Request) string {
+ queries := r.URL.Query()
+ var operation string
+ var queryFound bool
+ for key, _ := range queries {
+ operation, queryFound = getResourceType(object, key, r.Method)
+ if queryFound {
+ return operation
+ }
+ }
+ if len(queries) == 0 {
+ operation, _ = getResourceType(object, "", r.Method)
+ }
+ return operation
+}
+
+func GetAccessHttpLog(r *http.Request, statusCode int, s3errCode ErrorCode) AccessLogHTTP {
+ return AccessLogHTTP{
+ RequestURI: r.RequestURI,
+ Referer: r.Header.Get("Referer"),
+ }
+}
+
+func GetAccessLog(r *http.Request, HTTPStatusCode int, s3errCode ErrorCode) *AccessLog {
+ bucket, key := xhttp.GetBucketAndObject(r)
+ var errorCode string
+ if s3errCode != ErrNone {
+ errorCode = GetAPIError(s3errCode).Code
+ }
+ remoteIP := r.Header.Get("X-Real-IP")
+ if len(remoteIP) == 0 {
+ remoteIP = r.RemoteAddr
+ }
+ hostHeader := r.Header.Get("X-Forwarded-Host")
+ if len(hostHeader) == 0 {
+ hostHeader = r.Host
+ }
+ return &AccessLog{
+ HostHeader: hostHeader,
+ RequestID: r.Header.Get("X-Request-ID"),
+ RemoteIP: remoteIP,
+ Requester: r.Header.Get(xhttp.AmzIdentityId),
+ SignatureVersion: r.Header.Get(xhttp.AmzAuthType),
+ UserAgent: r.Header.Get("user-agent"),
+ HostId: hostname,
+ Bucket: bucket,
+ HTTPStatus: HTTPStatusCode,
+ Time: time.Now().Unix(),
+ Key: key,
+ Operation: getOperation(key, r),
+ ErrorCode: errorCode,
+ }
+}
+
+func PostLog(r *http.Request, HTTPStatusCode int, errorCode ErrorCode) {
+ if Logger == nil {
+ return
+ }
+ if err := Logger.Post(tag, *GetAccessLog(r, HTTPStatusCode, errorCode)); err != nil {
+ glog.Warning("Error while posting log: ", err)
+ }
+}
+
+func PostAccessLog(log AccessLog) {
+ if Logger == nil || len(log.Key) == 0 {
+ return
+ }
+ if err := Logger.Post(tag, log); err != nil {
+ glog.Warning("Error while posting log: ", err)
+ }
+}
diff --git a/weed/s3api/s3err/error_handler.go b/weed/s3api/s3err/error_handler.go
index 3cfdaafef..6753a1641 100644
--- a/weed/s3api/s3err/error_handler.go
+++ b/weed/s3api/s3err/error_handler.go
@@ -25,6 +25,7 @@ func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, re
func WriteEmptyResponse(w http.ResponseWriter, r *http.Request, statusCode int) {
WriteResponse(w, r, statusCode, []byte{}, mimeNone)
+ PostLog(r, statusCode, ErrNone)
}
func WriteErrorResponse(w http.ResponseWriter, r *http.Request, errorCode ErrorCode) {
@@ -39,6 +40,7 @@ func WriteErrorResponse(w http.ResponseWriter, r *http.Request, errorCode ErrorC
errorResponse := getRESTErrorResponse(apiError, r.URL.Path, bucket, object)
encodedErrorResponse := EncodeXMLResponse(errorResponse)
WriteResponse(w, r, apiError.HTTPStatusCode, encodedErrorResponse, MimeXML)
+ PostLog(r, apiError.HTTPStatusCode, errorCode)
}
func getRESTErrorResponse(err APIError, resource string, bucket, object string) RESTErrorResponse {
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-23 20:59:43 +0000