aboutsummaryrefslogtreecommitdiff
path: root/weed/command
diff options
context:
space:
mode:
Diffstat (limited to 'weed/command')
-rw-r--r--weed/command/filer.go2
-rw-r--r--weed/command/filer_remote_gateway_buckets.go37
-rw-r--r--weed/command/filer_remote_sync.go7
-rw-r--r--weed/command/s3.go14
-rw-r--r--weed/command/scaffold/filer.toml8
-rw-r--r--weed/command/server.go2
6 files changed, 53 insertions, 17 deletions
diff --git a/weed/command/filer.go b/weed/command/filer.go
index 633e02d79..876b1bbf0 100644
--- a/weed/command/filer.go
+++ b/weed/command/filer.go
@@ -84,6 +84,7 @@ func init() {
filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
+ filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
// start webdav on filer
@@ -137,6 +138,7 @@ func runFiler(cmd *Command, args []string) bool {
startDelay := time.Duration(2)
if *filerStartS3 {
filerS3Options.filer = &filerAddress
+ filerS3Options.bindIp = f.bindIp
go func() {
time.Sleep(startDelay * time.Second)
filerS3Options.startS3Server()
diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go
index d70e96904..fc11cdbc5 100644
--- a/weed/command/filer_remote_gateway_buckets.go
+++ b/weed/command/filer_remote_gateway_buckets.go
@@ -86,12 +86,26 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
return nil
}
}
- if *option.createBucketRandomSuffix {
- // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
- if len(bucketName)+5 > 63 {
- bucketName = bucketName[:58]
+
+ bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name)
+ remoteLocation, found := option.mappings.Mappings[string(bucketPath)]
+ if !found {
+ if *option.createBucketRandomSuffix {
+ // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
+ if len(bucketName)+5 > 63 {
+ bucketName = bucketName[:58]
+ }
+ bucketName = fmt.Sprintf("%s-%04d", bucketName, rand.Uint32()%10000)
+ }
+ remoteLocation = &remote_pb.RemoteStorageLocation{
+ Name: *option.createBucketAt,
+ Bucket: bucketName,
+ Path: "/",
}
- bucketName = fmt.Sprintf("%s-%4d", bucketName, rand.Uint32()%10000)
+ // need to add new mapping here before getting updates from metadata tailing
+ option.mappings.Mappings[string(bucketPath)] = remoteLocation
+ } else {
+ bucketName = remoteLocation.Bucket
}
glog.V(0).Infof("create bucket %s", bucketName)
@@ -99,16 +113,6 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
return fmt.Errorf("create bucket %s in %s: %v", bucketName, remoteConf.Name, err)
}
- bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name)
- remoteLocation := &remote_pb.RemoteStorageLocation{
- Name: *option.createBucketAt,
- Bucket: bucketName,
- Path: "/",
- }
-
- // need to add new mapping here before getting upates from metadata tailing
- option.mappings.Mappings[string(bucketPath)] = remoteLocation
-
return filer.InsertMountMapping(option, string(bucketPath), remoteLocation)
}
@@ -177,6 +181,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
if message.NewParentPath == option.bucketsDir {
return handleCreateBucket(message.NewEntry)
}
+ if strings.HasPrefix(message.NewParentPath, option.bucketsDir) && strings.Contains(message.NewParentPath, "/.uploads/") {
+ return nil
+ }
if !filer.HasData(message.NewEntry) {
return nil
}
diff --git a/weed/command/filer_remote_sync.go b/weed/command/filer_remote_sync.go
index c55544925..bceeb097e 100644
--- a/weed/command/filer_remote_sync.go
+++ b/weed/command/filer_remote_sync.go
@@ -40,7 +40,7 @@ func init() {
remoteSyncOptions.filerAddress = cmdFilerRemoteSynchronize.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "", "a mounted directory on filer")
remoteSyncOptions.readChunkFromFiler = cmdFilerRemoteSynchronize.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
- remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
+ remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now, skipping previous metadata changes. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
}
var cmdFilerRemoteSynchronize = &Command{
@@ -54,6 +54,11 @@ var cmdFilerRemoteSynchronize = &Command{
weed filer.remote.sync -dir=/mount/s3_on_cloud
+ The metadata sync starting time is determined with the following priority order:
+ 1. specified by timeAgo
+ 2. last sync timestamp for this directory
+ 3. directory creation time
+
`,
}
diff --git a/weed/command/s3.go b/weed/command/s3.go
index e9f4ea885..d7cd7818d 100644
--- a/weed/command/s3.go
+++ b/weed/command/s3.go
@@ -3,6 +3,7 @@ package command
import (
"context"
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"net/http"
"time"
@@ -24,6 +25,7 @@ var (
type S3Options struct {
filer *string
+ bindIp *string
port *int
config *string
domainName *string
@@ -31,14 +33,17 @@ type S3Options struct {
tlsCertificate *string
metricsHttpPort *int
allowEmptyFolder *bool
+ auditLogConfig *string
}
func init() {
cmdS3.Run = runS3 // break init cycle
s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
+ s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to")
s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
+ s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file")
s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
@@ -186,12 +191,19 @@ func (s3opt *S3Options) startS3Server() bool {
httpS := &http.Server{Handler: router}
- listenAddress := fmt.Sprintf(":%d", *s3opt.port)
+ listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port)
s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
if err != nil {
glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
}
+ if len(*s3opt.auditLogConfig) > 0 {
+ s3err.InitAuditLog(*s3opt.auditLogConfig)
+ if s3err.Logger != nil {
+ defer s3err.Logger.Close()
+ }
+ }
+
if *s3opt.tlsPrivateKey != "" {
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml
index bf895f484..a6a45923e 100644
--- a/weed/command/scaffold/filer.toml
+++ b/weed/command/scaffold/filer.toml
@@ -199,6 +199,14 @@ address = "localhost:6379"
password = ""
database = 0
+[redis3_sentinel]
+enabled = false
+addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"]
+masterName = "master"
+username = ""
+password = ""
+database = 0
+
[redis_cluster3] # beta
enabled = false
addresses = [
diff --git a/weed/command/server.go b/weed/command/server.go
index 5c6c4b1cf..0cb748381 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -131,6 +131,7 @@ func init() {
s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
+ s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")
@@ -179,6 +180,7 @@ func runServer(cmd *Command, args []string) bool {
filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
filerOptions.ip = serverIp
filerOptions.bindIp = serverBindIp
+ s3Options.bindIp = serverBindIp
serverOptions.v.ip = serverIp
serverOptions.v.bindIp = serverBindIp
serverOptions.v.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-23 17:06:28 +0000