From e7b63d50b16e435dab31eda172896976c30d252c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 9 Feb 2020 14:31:51 -0800 Subject: add s3.config to server.go --- weed/command/server.go | 1 + 1 file changed, 1 insertion(+) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index 6aa68b6d2..d7d768df1 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -97,6 +97,7 @@ func init() { s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}") s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file") s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file") + s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file") } -- cgit v1.2.3 From d8dec2323bde1a5ab787b719a240969852004456 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 24 Feb 2020 14:34:14 -0800 Subject: s3: move buckets folder configuration to filer --- weed/command/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index d7d768df1..aa693618c 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -82,6 +82,7 @@ func init() { filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") + filerOptions.dirBucketsPath = cmdServer.Flag.String("filer.dir.buckets", "/buckets", "folder to store all buckets") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") @@ -92,7 +93,6 @@ func init() { serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory") serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address") - s3Options.filerBucketsPath = cmdServer.Flag.String("s3.filer.dir.buckets", "/buckets", "folder on filer to store all buckets") s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port") s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}") s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file") -- cgit v1.2.3 From 5bcb44eda9b1dba57abf8cd9ce3b2d18518bd100 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 24 Feb 2020 14:42:57 -0800 Subject: filer: move buckets folder to filer.toml since it should not be changed often --- weed/command/server.go | 1 - 1 file changed, 1 deletion(-) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index aa693618c..c9d27555c 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -82,7 +82,6 @@ func init() { filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") - filerOptions.dirBucketsPath = cmdServer.Flag.String("filer.dir.buckets", "/buckets", "folder to store all buckets") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") -- cgit v1.2.3 From 13e215ee5cb5f4c2873f89c263d8c970e9978b19 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 6 Mar 2020 00:49:47 -0800 Subject: filer: option to encrypt data on volume server --- weed/command/server.go | 1 + 1 file changed, 1 insertion(+) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index c9d27555c..f45429193 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -82,6 +82,7 @@ func init() { filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") + filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") -- cgit v1.2.3 From 9b3109a5d88dc4bc3353dae0dff95f3a7890c691 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 8 Mar 2020 15:42:44 -0700 Subject: filer: processing all response headers, no pass through to volume server * filer calculate MD5 etag * filer handle response headers, instread of pass it to volume servers --- weed/command/server.go | 5 ----- 1 file changed, 5 deletions(-) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index f45429193..560b90037 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -78,7 +78,6 @@ func init() { filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") - filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") @@ -115,10 +114,6 @@ func runServer(cmd *Command, args []string) bool { defer pprof.StopCPUProfile() } - if *filerOptions.redirectOnRead { - *isStartingFiler = true - } - if *isStartingS3 { *isStartingFiler = true } -- cgit v1.2.3 From 1dc30214cbaca4a93285270a29ff77cd3d1f728f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 8 Mar 2020 17:02:47 -0700 Subject: mark encryptVolumeData as not ready --- weed/command/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index 560b90037..ea2844db9 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -81,7 +81,7 @@ func init() { filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") - filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers") + filerOptions.cipher = cmdServer.Flag.Bool("filer.work_in_progress_encryptVolumeData", false, " encrypt data on volume servers") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") -- cgit v1.2.3 From e73b0c24578f15900d88427c5c770056651c369d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 9 Mar 2020 22:31:14 -0700 Subject: remove not ready status --- weed/command/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index ea2844db9..560b90037 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -81,7 +81,7 @@ func init() { filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") - filerOptions.cipher = cmdServer.Flag.Bool("filer.work_in_progress_encryptVolumeData", false, " encrypt data on volume servers") + filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") -- cgit v1.2.3 From e32999108abbbc3973d7f423498899435313d58b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 22 Mar 2020 16:27:15 -0700 Subject: add auto configured volume max count help messge --- weed/command/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index 560b90037..8e0d41277 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -53,7 +53,7 @@ var ( serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") serverDisableHttp = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.") volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...") - volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...") + volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]... If set to zero on non-windows OS, the limit will be auto configured.") pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway") -- cgit v1.2.3 From 076c8bd3bcb6f76c84a8df50aff923d493a6bb9d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 18 Apr 2020 15:17:27 -0700 Subject: filer master start up with default ip address instead of just localhost --- weed/command/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index 8e0d41277..2f3bbe94d 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -45,7 +45,7 @@ var cmdServer = &Command{ } var ( - serverIp = cmdServer.Flag.String("ip", "localhost", "ip or server name") + serverIp = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name") serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds") serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name") -- cgit v1.2.3 From cf54d703114256f439edf3b16a69d9ff8bd85aec Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 Apr 2020 14:21:06 -0700 Subject: filer: add -ip.bind option --- weed/command/server.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index 2f3bbe94d..0d54004f8 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -125,7 +125,8 @@ func runServer(cmd *Command, args []string) bool { masterOptions.ip = serverIp masterOptions.ipBind = serverBindIp filerOptions.masters = &peers - filerOptions.ip = serverBindIp + filerOptions.ip = serverIp + filerOptions.bindIp = serverBindIp serverOptions.v.ip = serverIp serverOptions.v.bindIp = serverBindIp serverOptions.v.masters = &peers -- cgit v1.2.3 From 2bfd810912b5c0967f30abf6ebacd8f8afa314aa Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 May 2020 22:44:20 -0700 Subject: combine msgBroker into weed server command --- weed/command/server.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'weed/command/server.go') diff --git a/weed/command/server.go b/weed/command/server.go index 0d54004f8..c006f00eb 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -18,10 +18,11 @@ type ServerOptions struct { } var ( - serverOptions ServerOptions - masterOptions MasterOptions - filerOptions FilerOptions - s3Options S3Options + serverOptions ServerOptions + masterOptions MasterOptions + filerOptions FilerOptions + s3Options S3Options + msgBrokerOptions MessageBrokerOptions ) func init() { @@ -57,6 +58,7 @@ var ( pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway") + isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker") serverWhiteList []string ) @@ -98,6 +100,8 @@ func init() { s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file") s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file") + msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port") + } func runServer(cmd *Command, args []string) bool { @@ -117,6 +121,9 @@ func runServer(cmd *Command, args []string) bool { if *isStartingS3 { *isStartingFiler = true } + if *isStartingMsgBroker { + *isStartingFiler = true + } _, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers) peers := strings.Join(peerList, ",") @@ -133,6 +140,7 @@ func runServer(cmd *Command, args []string) bool { serverOptions.v.idleConnectionTimeout = serverTimeout serverOptions.v.dataCenter = serverDataCenter serverOptions.v.rack = serverRack + msgBrokerOptions.ip = serverIp serverOptions.v.pulseSeconds = pulseSeconds masterOptions.pulseSeconds = pulseSeconds @@ -145,6 +153,7 @@ func runServer(cmd *Command, args []string) bool { filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port) s3Options.filer = &filerAddress + msgBrokerOptions.filer = &filerAddress if *filerOptions.defaultReplicaPlacement == "" { *filerOptions.defaultReplicaPlacement = *masterOptions.defaultReplication @@ -188,6 +197,13 @@ func runServer(cmd *Command, args []string) bool { }() } + if *isStartingMsgBroker { + go func() { + time.Sleep(2 * time.Second) + msgBrokerOptions.startQueueServer() + }() + } + // start volume server { go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption) -- cgit v1.2.3