aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--docker/compose/local-dev-compose.yml3
-rw-r--r--go.mod1
-rw-r--r--go.sum2
-rw-r--r--weed/filer/filer_conf.go1
-rw-r--r--weed/filesys/dir.go11
-rw-r--r--weed/filesys/file.go57
-rw-r--r--weed/filesys/filehandle.go25
-rw-r--r--weed/filesys/wfs.go2
-rw-r--r--weed/glog/glog.go2
-rw-r--r--weed/iamapi/iamapi_handlers.go9
-rw-r--r--weed/iamapi/iamapi_management_handlers.go171
-rw-r--r--weed/iamapi/iamapi_response.go10
-rw-r--r--weed/iamapi/iamapi_server.go51
-rw-r--r--weed/iamapi/iamapi_test.go60
-rw-r--r--weed/operation/assign_file_id.go1
-rw-r--r--weed/s3api/auth_signature_v4.go30
-rw-r--r--weed/s3api/auto_signature_v4_test.go2
-rw-r--r--weed/s3api/chunked_reader_v4.go4
-rw-r--r--weed/s3api/s3api_object_handlers.go2
-rw-r--r--weed/server/filer_server_handlers_read.go2
-rw-r--r--weed/server/filer_server_handlers_write_autochunk.go8
-rw-r--r--weed/shell/shell_liner.go4
22 files changed, 331 insertions, 127 deletions
diff --git a/docker/compose/local-dev-compose.yml b/docker/compose/local-dev-compose.yml
index 05103a7fc..01d0594a6 100644
--- a/docker/compose/local-dev-compose.yml
+++ b/docker/compose/local-dev-compose.yml
@@ -26,9 +26,10 @@ services:
filer:
image: chrislusf/seaweedfs:local
ports:
+ - 8111:8111
- 8888:8888
- 18888:18888
- command: '-v=1 filer -master="master:9333"'
+ command: '-v=1 filer -master="master:9333" -iam'
depends_on:
- master
- volume
diff --git a/go.mod b/go.mod
index 310f5585e..bd07b3312 100644
--- a/go.mod
+++ b/go.mod
@@ -42,6 +42,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
+ github.com/jinzhu/copier v0.2.8
github.com/json-iterator/go v1.1.10
github.com/karlseguin/ccache v2.0.3+incompatible // indirect
github.com/karlseguin/ccache/v2 v2.0.7
diff --git a/go.sum b/go.sum
index 1ea4ee761..4899bd7b8 100644
--- a/go.sum
+++ b/go.sum
@@ -439,6 +439,8 @@ github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjL
github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE=
+github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go
index 8e549f5ad..ab5afc5cc 100644
--- a/weed/filer/filer_conf.go
+++ b/weed/filer/filer_conf.go
@@ -18,6 +18,7 @@ const (
FilerConfName = "filer.conf"
IamConfigDirecotry = "/etc/iam"
IamIdentityFile = "identity.json"
+ IamPoliciesFile = "policies.json"
)
type FilerConf struct {
diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go
index 7b918e769..63631b857 100644
--- a/weed/filesys/dir.go
+++ b/weed/filesys/dir.go
@@ -105,11 +105,10 @@ func (dir *Dir) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
func (dir *Dir) newFile(name string, entry *filer_pb.Entry) fs.Node {
f := dir.wfs.fsNodeCache.EnsureFsNode(util.NewFullPath(dir.FullPath(), name), func() fs.Node {
return &File{
- Name: name,
- dir: dir,
- wfs: dir.wfs,
- entry: entry,
- entryViewCache: nil,
+ Name: name,
+ dir: dir,
+ wfs: dir.wfs,
+ entry: entry,
}
})
f.(*File).dir = dir // in case dir node was created later
@@ -408,7 +407,7 @@ func (dir *Dir) removeOneFile(req *fuse.RemoveRequest) error {
dir.wfs.fsNodeCache.DeleteFsNode(filePath)
if fsNode != nil {
if file, ok := fsNode.(*File); ok {
- file.clearEntry()
+ file.entry = nil
}
}
diff --git a/weed/filesys/file.go b/weed/filesys/file.go
index 2433be590..2d1c9a86e 100644
--- a/weed/filesys/file.go
+++ b/weed/filesys/file.go
@@ -2,10 +2,8 @@ package filesys
import (
"context"
- "io"
"os"
"sort"
- "sync"
"time"
"github.com/seaweedfs/fuse"
@@ -30,15 +28,12 @@ var _ = fs.NodeListxattrer(&File{})
var _ = fs.NodeForgetter(&File{})
type File struct {
- Name string
- dir *Dir
- wfs *WFS
- entry *filer_pb.Entry
- entryLock sync.RWMutex
- entryViewCache []filer.VisibleInterval
- isOpen int
- reader io.ReaderAt
- dirtyMetadata bool
+ Name string
+ dir *Dir
+ wfs *WFS
+ entry *filer_pb.Entry
+ isOpen int
+ dirtyMetadata bool
}
func (file *File) fullpath() util.FullPath {
@@ -154,8 +149,6 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
}
}
entry.Chunks = chunks
- file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), chunks)
- file.setReader(nil)
}
entry.Attributes.FileSize = req.Size
file.dirtyMetadata = true
@@ -274,7 +267,6 @@ func (file *File) Forget() {
glog.V(4).Infof("Forget file %s", t)
file.wfs.fsNodeCache.DeleteFsNode(t)
file.wfs.ReleaseHandle(t, 0)
- file.setReader(nil)
}
func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, err error) {
@@ -294,7 +286,7 @@ func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, er
return entry, err
}
if entry != nil {
- file.setEntry(entry)
+ file.entry = entry
} else {
glog.Warningf("maybeLoadEntry not found entry %s/%s: %v", file.dir.FullPath(), file.Name, err)
}
@@ -336,44 +328,11 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
return lessThan(chunks[i], chunks[j])
})
- // add to entry view cache
- for _, chunk := range chunks {
- file.entryViewCache = filer.MergeIntoVisibles(file.entryViewCache, chunk)
- }
-
- file.setReader(nil)
-
glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(entry.Chunks), len(chunks))
entry.Chunks = append(entry.Chunks, newChunks...)
}
-func (file *File) setReader(reader io.ReaderAt) {
- r := file.reader
- if r != nil {
- if closer, ok := r.(io.Closer); ok {
- closer.Close()
- }
- }
- file.reader = reader
-}
-
-func (file *File) setEntry(entry *filer_pb.Entry) {
- file.entryLock.Lock()
- defer file.entryLock.Unlock()
- file.entry = entry
- file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), entry.Chunks)
- file.setReader(nil)
-}
-
-func (file *File) clearEntry() {
- file.entryLock.Lock()
- defer file.entryLock.Unlock()
- file.entry = nil
- file.entryViewCache = nil
- file.setReader(nil)
-}
-
func (file *File) saveEntry(entry *filer_pb.Entry) error {
return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
@@ -400,7 +359,5 @@ func (file *File) saveEntry(entry *filer_pb.Entry) error {
}
func (file *File) getEntry() *filer_pb.Entry {
- file.entryLock.RLock()
- defer file.entryLock.RUnlock()
return file.entry
}
diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go
index f04952e96..0236fd1cd 100644
--- a/weed/filesys/filehandle.go
+++ b/weed/filesys/filehandle.go
@@ -20,9 +20,11 @@ import (
type FileHandle struct {
// cache file has been written to
- dirtyPages *ContinuousDirtyPages
- contentType string
- handle uint64
+ dirtyPages *ContinuousDirtyPages
+ entryViewCache []filer.VisibleInterval
+ reader io.ReaderAt
+ contentType string
+ handle uint64
sync.Mutex
f *File
@@ -125,20 +127,20 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
}
var chunkResolveErr error
- if fh.f.entryViewCache == nil {
- fh.f.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.f.wfs.LookupFn(), entry.Chunks)
+ if fh.entryViewCache == nil {
+ fh.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.f.wfs.LookupFn(), entry.Chunks)
if chunkResolveErr != nil {
return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr)
}
- fh.f.setReader(nil)
+ fh.reader = nil
}
- reader := fh.f.reader
+ reader := fh.reader
if reader == nil {
- chunkViews := filer.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64)
+ chunkViews := filer.ViewFromVisibleIntervals(fh.entryViewCache, 0, math.MaxInt64)
reader = filer.NewChunkReaderAtFromClient(fh.f.wfs.LookupFn(), chunkViews, fh.f.wfs.chunkCache, fileSize)
}
- fh.f.setReader(reader)
+ fh.reader = reader
totalRead, err := reader.ReadAt(buff, offset)
@@ -200,8 +202,6 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
fh.Lock()
defer fh.Unlock()
- fh.f.entryViewCache = nil
-
if fh.f.isOpen <= 0 {
glog.V(0).Infof("Release reset %s open count %d => %d", fh.f.Name, fh.f.isOpen, 0)
fh.f.isOpen = 0
@@ -211,9 +211,10 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
if fh.f.isOpen == 1 {
fh.f.isOpen--
+ fh.entryViewCache = nil
+ fh.reader = nil
fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
- fh.f.setReader(nil)
}
return nil
diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go
index ba5eb4b6b..aaff1377b 100644
--- a/weed/filesys/wfs.go
+++ b/weed/filesys/wfs.go
@@ -111,7 +111,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
if err := wfs.Server.InvalidateNodeData(file); err != nil {
glog.V(4).Infof("InvalidateNodeData %s : %v", filePath, err)
}
- file.clearEntry()
+ file.entry = nil
}
}
dir, name := filePath.DirAndName()
diff --git a/weed/glog/glog.go b/weed/glog/glog.go
index adb6ab5aa..352a7e185 100644
--- a/weed/glog/glog.go
+++ b/weed/glog/glog.go
@@ -398,7 +398,7 @@ type flushSyncWriter interface {
func init() {
flag.BoolVar(&logging.toStderr, "logtostderr", false, "log to standard error instead of files")
flag.BoolVar(&logging.alsoToStderr, "alsologtostderr", true, "log to standard error as well as files")
- flag.Var(&logging.verbosity, "v", "log level for V logs")
+ flag.Var(&logging.verbosity, "v", "log levels [0|1|2|3|4], default to 0")
flag.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr")
flag.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging")
flag.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace")
diff --git a/weed/iamapi/iamapi_handlers.go b/weed/iamapi/iamapi_handlers.go
index fdaf4dd69..2e5f709f3 100644
--- a/weed/iamapi/iamapi_handlers.go
+++ b/weed/iamapi/iamapi_handlers.go
@@ -50,20 +50,25 @@ func writeErrorResponse(w http.ResponseWriter, errorCode s3err.ErrorCode, reqURL
writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
}
-func writeIamErrorResponse(w http.ResponseWriter, err error, object string, value string) {
+func writeIamErrorResponse(w http.ResponseWriter, err error, object string, value string, msg error) {
errCode := err.Error()
errorResp := ErrorResponse{}
errorResp.Error.Type = "Sender"
errorResp.Error.Code = &errCode
+ if msg != nil {
+ errMsg := msg.Error()
+ errorResp.Error.Message = &errMsg
+ }
glog.Errorf("Response %+v", err)
switch errCode {
case iam.ErrCodeNoSuchEntityException:
msg := fmt.Sprintf("The %s with name %s cannot be found.", object, value)
errorResp.Error.Message = &msg
writeResponse(w, http.StatusNotFound, encodeResponse(errorResp), mimeXML)
+ case iam.ErrCodeServiceFailureException:
+ writeResponse(w, http.StatusInternalServerError, encodeResponse(errorResp), mimeXML)
default:
writeResponse(w, http.StatusInternalServerError, encodeResponse(errorResp), mimeXML)
-
}
}
diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go
index 470731064..b00ada234 100644
--- a/weed/iamapi/iamapi_management_handlers.go
+++ b/weed/iamapi/iamapi_management_handlers.go
@@ -11,6 +11,7 @@ import (
"math/rand"
"net/http"
"net/url"
+ "reflect"
"strings"
"sync"
"time"
@@ -19,27 +20,77 @@ import (
)
const (
- charsetUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
- charset = charsetUpper + "abcdefghijklmnopqrstuvwxyz/"
+ charsetUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+ charset = charsetUpper + "abcdefghijklmnopqrstuvwxyz/"
+ policyDocumentVersion = "2012-10-17"
+ StatementActionAdmin = "*"
+ StatementActionWrite = "Put*"
+ StatementActionRead = "Get*"
+ StatementActionList = "List*"
+ StatementActionTagging = "Tagging*"
)
var (
seededRand *rand.Rand = rand.New(
rand.NewSource(time.Now().UnixNano()))
policyDocuments = map[string]*PolicyDocument{}
+ policyLock = sync.RWMutex{}
)
+func MapToStatementAction(action string) string {
+ switch action {
+ case StatementActionAdmin:
+ return s3_constants.ACTION_ADMIN
+ case StatementActionWrite:
+ return s3_constants.ACTION_WRITE
+ case StatementActionRead:
+ return s3_constants.ACTION_READ
+ case StatementActionList:
+ return s3_constants.ACTION_LIST
+ case StatementActionTagging:
+ return s3_constants.ACTION_TAGGING
+ default:
+ return ""
+ }
+}
+
+func MapToIdentitiesAction(action string) string {
+ switch action {
+ case s3_constants.ACTION_ADMIN:
+ return StatementActionAdmin
+ case s3_constants.ACTION_WRITE:
+ return StatementActionWrite
+ case s3_constants.ACTION_READ:
+ return StatementActionRead
+ case s3_constants.ACTION_LIST:
+ return StatementActionList
+ case s3_constants.ACTION_TAGGING:
+ return StatementActionTagging
+ default:
+ return ""
+ }
+}
+
type Statement struct {
Effect string `json:"Effect"`
Action []string `json:"Action"`
Resource []string `json:"Resource"`
}
+type Policies struct {
+ Policies map[string]PolicyDocument `json:"policies"`
+}
+
type PolicyDocument struct {
Version string `json:"Version"`
Statement []*Statement `json:"Statement"`
}
+func (p PolicyDocument) String() string {
+ b, _ := json.Marshal(p)
+ return string(b)
+}
+
func Hash(s *string) string {
h := sha1.New()
h.Write([]byte(*s))
@@ -83,7 +134,7 @@ func (iama *IamApiServer) CreateUser(s3cfg *iam_pb.S3ApiConfiguration, values ur
func (iama *IamApiServer) DeleteUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (resp DeleteUserResponse, err error) {
for i, ident := range s3cfg.Identities {
if userName == ident.Name {
- ident.Credentials = append(ident.Credentials[:i], ident.Credentials[i+1:]...)
+ s3cfg.Identities = append(s3cfg.Identities[:i], s3cfg.Identities[i+1:]...)
return resp, nil
}
}
@@ -119,7 +170,16 @@ func (iama *IamApiServer) CreatePolicy(s3cfg *iam_pb.S3ApiConfiguration, values
resp.CreatePolicyResult.Policy.PolicyName = &policyName
resp.CreatePolicyResult.Policy.Arn = &arn
resp.CreatePolicyResult.Policy.PolicyId = &policyId
- policyDocuments[policyName] = &policyDocument
+ policies := Policies{}
+ policyLock.Lock()
+ defer policyLock.Unlock()
+ if err = iama.s3ApiConfig.GetPolicies(&policies); err != nil {
+ return resp, err
+ }
+ policies.Policies[policyName] = policyDocument
+ if err = iama.s3ApiConfig.PutPolicies(&policies); err != nil {
+ return resp, err
+ }
return resp, nil
}
@@ -144,19 +204,69 @@ func (iama *IamApiServer) PutUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values
return resp, nil
}
-func MapAction(action string) string {
- switch action {
- case "*":
- return s3_constants.ACTION_ADMIN
- case "Put*":
- return s3_constants.ACTION_WRITE
- case "Get*":
- return s3_constants.ACTION_READ
- case "List*":
- return s3_constants.ACTION_LIST
- default:
- return s3_constants.ACTION_TAGGING
+func (iama *IamApiServer) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp GetUserPolicyResponse, err error) {
+ userName := values.Get("UserName")
+ policyName := values.Get("PolicyName")
+ for _, ident := range s3cfg.Identities {
+ if userName != ident.Name {
+ continue
+ }
+
+ resp.GetUserPolicyResult.UserName = userName
+ resp.GetUserPolicyResult.PolicyName = policyName
+ if len(ident.Actions) == 0 {
+ return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException)
+ }
+
+ policyDocument := PolicyDocument{Version: policyDocumentVersion}
+ statements := make(map[string][]string)
+ for _, action := range ident.Actions {
+ // parse "Read:EXAMPLE-BUCKET"
+ act := strings.Split(action, ":")
+
+ resource := "*"
+ if len(act) == 2 {
+ resource = fmt.Sprintf("arn:aws:s3:::%s/*", act[1])
+ }
+ statements[resource] = append(statements[resource],
+ fmt.Sprintf("s3:%s", MapToIdentitiesAction(act[0])),
+ )
+ }
+ for resource, actions := range statements {
+ isEqAction := false
+ for i, statement := range policyDocument.Statement {
+ if reflect.DeepEqual(statement.Action, actions) {
+ policyDocument.Statement[i].Resource = append(
+ policyDocument.Statement[i].Resource, resource)
+ isEqAction = true
+ break
+ }
+ }
+ if isEqAction {
+ continue
+ }
+ policyDocumentStatement := Statement{
+ Effect: "Allow",
+ Action: actions,
+ }
+ policyDocumentStatement.Resource = append(policyDocumentStatement.Resource, resource)
+ policyDocument.Statement = append(policyDocument.Statement, &policyDocumentStatement)
+ }
+ resp.GetUserPolicyResult.PolicyDocument = policyDocument.String()
+ return resp, nil
+ }
+ return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException)
+}
+
+func (iama *IamApiServer) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp PutUserPolicyResponse, err error) {
+ userName := values.Get("UserName")
+ for i, ident := range s3cfg.Identities {
+ if ident.Name == userName {
+ s3cfg.Identities = append(s3cfg.Identities[:i], s3cfg.Identities[i+1:]...)
+ return resp, nil
+ }
}
+ return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException)
}
func GetActions(policy *PolicyDocument) (actions []string) {
@@ -178,8 +288,9 @@ func GetActions(policy *PolicyDocument) (actions []string) {
glog.Infof("not match action: %s", act)
continue
}
+ statementAction := MapToStatementAction(act[1])
if res[5] == "*" {
- actions = append(actions, MapAction(act[1]))
+ actions = append(actions, statementAction)
continue
}
// Parse my-bucket/shared/*
@@ -188,7 +299,7 @@ func GetActions(policy *PolicyDocument) (actions []string) {
glog.Infof("not match bucket: %s", path)
continue
}
- actions = append(actions, fmt.Sprintf("%s:%s", MapAction(act[1]), path[0]))
+ actions = append(actions, fmt.Sprintf("%s:%s", statementAction, path[0]))
}
}
}
@@ -277,14 +388,15 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
userName := values.Get("UserName")
response, err = iama.GetUser(s3cfg, userName)
if err != nil {
- writeIamErrorResponse(w, err, "user", userName)
+ writeIamErrorResponse(w, err, "user", userName, nil)
return
}
+ changed = false
case "DeleteUser":
userName := values.Get("UserName")
response, err = iama.DeleteUser(s3cfg, userName)
if err != nil {
- writeIamErrorResponse(w, err, "user", userName)
+ writeIamErrorResponse(w, err, "user", userName, nil)
return
}
case "CreateAccessKey":
@@ -305,8 +417,23 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL)
return
}
+ case "GetUserPolicy":
+ response, err = iama.GetUserPolicy(s3cfg, values)
+ if err != nil {
+ writeIamErrorResponse(w, err, "user", values.Get("UserName"), nil)
+ return
+ }
+ changed = false
+ case "DeleteUserPolicy":
+ if response, err = iama.DeleteUserPolicy(s3cfg, values); err != nil {
+ writeIamErrorResponse(w, err, "user", values.Get("UserName"), nil)
+ }
default:
- writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
+ errNotImplemented := s3err.GetAPIError(s3err.ErrNotImplemented)
+ errorResponse := ErrorResponse{}
+ errorResponse.Error.Code = &errNotImplemented.Code
+ errorResponse.Error.Message = &errNotImplemented.Description
+ writeResponse(w, errNotImplemented.HTTPStatusCode, encodeResponse(errorResponse), mimeXML)
return
}
if changed {
@@ -314,7 +441,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
err := iama.s3ApiConfig.PutS3ApiConfiguration(s3cfg)
s3cfgLock.Unlock()
if err != nil {
- writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+ writeIamErrorResponse(w, fmt.Errorf(iam.ErrCodeServiceFailureException), "", "", err)
return
}
}
diff --git a/weed/iamapi/iamapi_response.go b/weed/iamapi/iamapi_response.go
index 26dd0f263..77328b608 100644
--- a/weed/iamapi/iamapi_response.go
+++ b/weed/iamapi/iamapi_response.go
@@ -79,6 +79,16 @@ type PutUserPolicyResponse struct {
XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ PutUserPolicyResponse"`
}
+type GetUserPolicyResponse struct {
+ CommonResponse
+ XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetUserPolicyResponse"`
+ GetUserPolicyResult struct {
+ UserName string `xml:"UserName"`
+ PolicyName string `xml:"PolicyName"`
+ PolicyDocument string `xml:"PolicyDocument"`
+ } `xml:"GetUserPolicyResult"`
+}
+
type ErrorResponse struct {
CommonResponse
XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ErrorResponse"`
diff --git a/weed/iamapi/iamapi_server.go b/weed/iamapi/iamapi_server.go
index 7698fab71..18af1a919 100644
--- a/weed/iamapi/iamapi_server.go
+++ b/weed/iamapi/iamapi_server.go
@@ -4,11 +4,14 @@ package iamapi
import (
"bytes"
+ "encoding/json"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
+ "github.com/chrislusf/seaweedfs/weed/s3api"
+ . "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/gorilla/mux"
"google.golang.org/grpc"
@@ -19,6 +22,8 @@ import (
type IamS3ApiConfig interface {
GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error)
PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error)
+ GetPolicies(policies *Policies) (err error)
+ PutPolicies(policies *Policies) (err error)
}
type IamS3ApiConfigure struct {
@@ -36,7 +41,7 @@ type IamServerOption struct {
type IamApiServer struct {
s3ApiConfig IamS3ApiConfig
- filerclient *filer_pb.SeaweedFilerClient
+ iam *s3api.IdentityAccessManagement
}
var s3ApiConfigure IamS3ApiConfig
@@ -46,9 +51,10 @@ func NewIamApiServer(router *mux.Router, option *IamServerOption) (iamApiServer
option: option,
masterClient: wdclient.NewMasterClient(option.GrpcDialOption, pb.AdminShellClient, "", 0, "", strings.Split(option.Masters, ",")),
}
-
+ s3Option := s3api.S3ApiServerOption{Filer: option.Filer}
iamApiServer = &IamApiServer{
s3ApiConfig: s3ApiConfigure,
+ iam: s3api.NewIdentityAccessManagement(&s3Option),
}
iamApiServer.registerRouter(router)
@@ -62,7 +68,8 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) {
// ListBuckets
// apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST"))
- apiRouter.Path("/").Methods("POST").HandlerFunc(iama.DoActions)
+ apiRouter.Methods("POST").Path("/").HandlerFunc(iama.iam.Auth(iama.DoActions, ACTION_ADMIN))
+ //
// NotFound
apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler)
}
@@ -102,3 +109,41 @@ func (iam IamS3ApiConfigure) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfigurat
},
)
}
+
+func (iam IamS3ApiConfigure) GetPolicies(policies *Policies) (err error) {
+ var buf bytes.Buffer
+ err = pb.WithGrpcFilerClient(iam.option.FilerGrpcAddress, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+ if err = filer.ReadEntry(iam.masterClient, client, filer.IamConfigDirecotry, filer.IamPoliciesFile, &buf); err != nil {
+ return err
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ if buf.Len() == 0 {
+ policies.Policies = make(map[string]PolicyDocument)
+ return nil
+ }
+ if err := json.Unmarshal(buf.Bytes(), policies); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (iam IamS3ApiConfigure) PutPolicies(policies *Policies) (err error) {
+ var b []byte
+ if b, err = json.Marshal(policies); err != nil {
+ return err
+ }
+ return pb.WithGrpcFilerClient(
+ iam.option.FilerGrpcAddress,
+ iam.option.GrpcDialOption,
+ func(client filer_pb.SeaweedFilerClient) error {
+ if err := filer.SaveInsideFiler(client, filer.IamConfigDirecotry, filer.IamPoliciesFile, b); err != nil {
+ return err
+ }
+ return nil
+ },
+ )
+}
diff --git a/weed/iamapi/iamapi_test.go b/weed/iamapi/iamapi_test.go
index f989626e6..09aaf0ac8 100644
--- a/weed/iamapi/iamapi_test.go
+++ b/weed/iamapi/iamapi_test.go
@@ -7,29 +7,43 @@ import (
"github.com/aws/aws-sdk-go/service/iam"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
"github.com/gorilla/mux"
+ "github.com/jinzhu/copier"
"github.com/stretchr/testify/assert"
"net/http"
"net/http/httptest"
"testing"
)
-var S3config iam_pb.S3ApiConfiguration
var GetS3ApiConfiguration func(s3cfg *iam_pb.S3ApiConfiguration) (err error)
var PutS3ApiConfiguration func(s3cfg *iam_pb.S3ApiConfiguration) (err error)
+var GetPolicies func(policies *Policies) (err error)
+var PutPolicies func(policies *Policies) (err error)
+
+var s3config = iam_pb.S3ApiConfiguration{}
+var policiesFile = Policies{Policies: make(map[string]PolicyDocument)}
+var ias = IamApiServer{s3ApiConfig: iamS3ApiConfigureMock{}}
type iamS3ApiConfigureMock struct{}
func (iam iamS3ApiConfigureMock) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) {
- s3cfg = &S3config
+ _ = copier.Copy(&s3cfg.Identities, &s3config.Identities)
return nil
}
func (iam iamS3ApiConfigureMock) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) {
- S3config = *s3cfg
+ _ = copier.Copy(&s3config.Identities, &s3cfg.Identities)
return nil
}
-var a = IamApiServer{}
+func (iam iamS3ApiConfigureMock) GetPolicies(policies *Policies) (err error) {
+ _ = copier.Copy(&policies, &policiesFile)
+ return nil
+}
+
+func (iam iamS3ApiConfigureMock) PutPolicies(policies *Policies) (err error) {
+ _ = copier.Copy(&policiesFile, &policies)
+ return nil
+}
func TestCreateUser(t *testing.T) {
userName := aws.String("Test")
@@ -64,17 +78,6 @@ func TestListAccessKeys(t *testing.T) {
assert.Equal(t, http.StatusOK, response.Code)
}
-func TestDeleteUser(t *testing.T) {
- userName := aws.String("Test")
- params := &iam.DeleteUserInput{UserName: userName}
- req, _ := iam.New(session.New()).DeleteUserRequest(params)
- _ = req.Build()
- out := DeleteUserResponse{}
- response, err := executeRequest(req.HTTPRequest, out)
- assert.Equal(t, nil, err)
- assert.Equal(t, http.StatusNotFound, response.Code)
-}
-
func TestGetUser(t *testing.T) {
userName := aws.String("Test")
params := &iam.GetUserInput{UserName: userName}
@@ -83,7 +86,7 @@ func TestGetUser(t *testing.T) {
out := GetUserResponse{}
response, err := executeRequest(req.HTTPRequest, out)
assert.Equal(t, nil, err)
- assert.Equal(t, http.StatusNotFound, response.Code)
+ assert.Equal(t, http.StatusOK, response.Code)
}
// Todo flat statement
@@ -147,11 +150,32 @@ func TestPutUserPolicy(t *testing.T) {
assert.Equal(t, http.StatusOK, response.Code)
}
+func TestGetUserPolicy(t *testing.T) {
+ userName := aws.String("Test")
+ params := &iam.GetUserPolicyInput{UserName: userName, PolicyName: aws.String("S3-read-only-example-bucket")}
+ req, _ := iam.New(session.New()).GetUserPolicyRequest(params)
+ _ = req.Build()
+ out := GetUserPolicyResponse{}
+ response, err := executeRequest(req.HTTPRequest, out)
+ assert.Equal(t, nil, err)
+ assert.Equal(t, http.StatusOK, response.Code)
+}
+
+func TestDeleteUser(t *testing.T) {
+ userName := aws.String("Test")
+ params := &iam.DeleteUserInput{UserName: userName}
+ req, _ := iam.New(session.New()).DeleteUserRequest(params)
+ _ = req.Build()
+ out := DeleteUserResponse{}
+ response, err := executeRequest(req.HTTPRequest, out)
+ assert.Equal(t, nil, err)
+ assert.Equal(t, http.StatusOK, response.Code)
+}
+
func executeRequest(req *http.Request, v interface{}) (*httptest.ResponseRecorder, error) {
rr := httptest.NewRecorder()
apiRouter := mux.NewRouter().SkipClean(true)
- a.s3ApiConfig = iamS3ApiConfigureMock{}
- apiRouter.Path("/").Methods("POST").HandlerFunc(a.DoActions)
+ apiRouter.Path("/").Methods("POST").HandlerFunc(ias.DoActions)
apiRouter.ServeHTTP(rr, req)
return rr, xml.Unmarshal(rr.Body.Bytes(), &v)
}
diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go
index cc1359961..ffd3e4938 100644
--- a/weed/operation/assign_file_id.go
+++ b/weed/operation/assign_file_id.go
@@ -86,6 +86,7 @@ func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest
continue
}
+ break
}
return ret, lastError
diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go
index 5ef7439c8..0df26e6fc 100644
--- a/weed/s3api/auth_signature_v4.go
+++ b/weed/s3api/auth_signature_v4.go
@@ -24,6 +24,7 @@ import (
"crypto/subtle"
"encoding/hex"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+ "io/ioutil"
"net/http"
"net/url"
"regexp"
@@ -132,6 +133,17 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r
// Query string.
queryStr := req.URL.Query().Encode()
+ // Get hashed Payload
+ if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil {
+ buf, _ := ioutil.ReadAll(r.Body)
+ r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
+ b, _ := ioutil.ReadAll(bytes.NewBuffer(buf))
+ if len(b) != 0 {
+ bodyHash := sha256.Sum256(b)
+ hashedPayload = hex.EncodeToString(bodyHash[:])
+ }
+ }
+
// Get canonical request.
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method)
@@ -139,7 +151,10 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r
stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope())
// Get hmac signing key.
- signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, signV4Values.Credential.scope.region)
+ signingKey := getSigningKey(cred.SecretKey,
+ signV4Values.Credential.scope.date,
+ signV4Values.Credential.scope.region,
+ signV4Values.Credential.scope.service)
// Calculate signature.
newSignature := getSignature(signingKey, stringToSign)
@@ -310,7 +325,7 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.
}
// Get signing key.
- signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region)
+ signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, credHeader.scope.service)
// Get signature.
newSignature := getSignature(signingKey, formValues.Get("Policy"))
@@ -427,7 +442,10 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s
presignedStringToSign := getStringToSign(presignedCanonicalReq, t, pSignValues.Credential.getScope())
// Get hmac presigned signing key.
- presignedSigningKey := getSigningKey(cred.SecretKey, pSignValues.Credential.scope.date, pSignValues.Credential.scope.region)
+ presignedSigningKey := getSigningKey(cred.SecretKey,
+ pSignValues.Credential.scope.date,
+ pSignValues.Credential.scope.region,
+ pSignValues.Credential.scope.service)
// Get new signature.
newSignature := getSignature(presignedSigningKey, presignedStringToSign)
@@ -655,11 +673,11 @@ func sumHMAC(key []byte, data []byte) []byte {
}
// getSigningKey hmac seed to calculate final signature.
-func getSigningKey(secretKey string, t time.Time, region string) []byte {
+func getSigningKey(secretKey string, t time.Time, region string, service string) []byte {
date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format(yyyymmdd)))
regionBytes := sumHMAC(date, []byte(region))
- service := sumHMAC(regionBytes, []byte("s3"))
- signingKey := sumHMAC(service, []byte("aws4_request"))
+ serviceBytes := sumHMAC(regionBytes, []byte(service))
+ signingKey := sumHMAC(serviceBytes, []byte("aws4_request"))
return signingKey
}
diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go
index 4c8255768..b47cd5f2d 100644
--- a/weed/s3api/auto_signature_v4_test.go
+++ b/weed/s3api/auto_signature_v4_test.go
@@ -370,7 +370,7 @@ func preSignV4(req *http.Request, accessKeyID, secretAccessKey string, expires i
queryStr := strings.Replace(query.Encode(), "+", "%20", -1)
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method)
stringToSign := getStringToSign(canonicalRequest, date, scope)
- signingKey := getSigningKey(secretAccessKey, date, region)
+ signingKey := getSigningKey(secretAccessKey, date, region, "s3")
signature := getSignature(signingKey, stringToSign)
req.URL.RawQuery = query.Encode()
diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go
index 734c9faee..b163ec2f6 100644
--- a/weed/s3api/chunked_reader_v4.go
+++ b/weed/s3api/chunked_reader_v4.go
@@ -45,7 +45,7 @@ func getChunkSignature(secretKey string, seedSignature string, region string, da
hashedChunk
// Get hmac signing key.
- signingKey := getSigningKey(secretKey, date, region)
+ signingKey := getSigningKey(secretKey, date, region, "s3")
// Calculate signature.
newSignature := getSignature(signingKey, stringToSign)
@@ -117,7 +117,7 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())
// Get hmac signing key.
- signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region)
+ signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, "s3")
// Calculate signature.
newSignature := getSignature(signingKey, stringToSign)
diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go
index b3cfd9ec7..f1a539ac5 100644
--- a/weed/s3api/s3api_object_handlers.go
+++ b/weed/s3api/s3api_object_handlers.go
@@ -311,7 +311,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
}
defer util.CloseResponse(resp)
- if resp.ContentLength == -1 || resp.StatusCode == 404 {
+ if (resp.ContentLength == -1 || resp.StatusCode == 404) && resp.StatusCode != 304 {
if r.Method != "DELETE" {
writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
return
diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go
index f90b070a2..6bc09e953 100644
--- a/weed/server/filer_server_handlers_read.go
+++ b/weed/server/filer_server_handlers_read.go
@@ -79,7 +79,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat))
if r.Header.Get("If-Modified-Since") != "" {
if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil {
- if t.After(entry.Attr.Mtime) {
+ if !t.Before(entry.Attr.Mtime) {
w.WriteHeader(http.StatusNotModified)
return
}
diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go
index 2808042c7..c4f10d94e 100644
--- a/weed/server/filer_server_handlers_write_autochunk.go
+++ b/weed/server/filer_server_handlers_write_autochunk.go
@@ -142,6 +142,14 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
if fileName != "" {
path += fileName
}
+ } else {
+ if fileName != "" {
+ if possibleDirEntry, findDirErr := fs.filer.FindEntry(ctx, util.FullPath(path)); findDirErr == nil {
+ if possibleDirEntry.IsDirectory() {
+ path += "/" + fileName
+ }
+ }
+ }
}
var entry *filer.Entry
diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go
index d79f67032..1dd611ca5 100644
--- a/weed/shell/shell_liner.go
+++ b/weed/shell/shell_liner.go
@@ -2,6 +2,7 @@ package shell
import (
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/util/grace"
"io"
"os"
"path"
@@ -25,6 +26,9 @@ func RunShell(options ShellOptions) {
line = liner.NewLiner()
defer line.Close()
+ grace.OnInterrupt(func() {
+ line.Close()
+ })
line.SetCtrlCAborts(true)
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 21:09:30 +0000