aboutsummaryrefslogtreecommitdiff
path: root/weed/command
diff options
context:
space:
mode:
Diffstat (limited to 'weed/command')
-rw-r--r--weed/command/backup.go11
-rw-r--r--weed/command/benchmark.go28
-rw-r--r--weed/command/filer.go11
-rw-r--r--weed/command/filer_copy.go59
-rw-r--r--weed/command/filer_replication.go3
-rw-r--r--weed/command/master.go17
-rw-r--r--weed/command/mount_std.go7
-rw-r--r--weed/command/s3.go6
-rw-r--r--weed/command/scaffold.go44
-rw-r--r--weed/command/server.go12
-rw-r--r--weed/command/upload.go19
-rw-r--r--weed/command/volume.go7
12 files changed, 160 insertions, 64 deletions
diff --git a/weed/command/backup.go b/weed/command/backup.go
index 0641f2e5d..86391f9c4 100644
--- a/weed/command/backup.go
+++ b/weed/command/backup.go
@@ -2,6 +2,9 @@ package command
import (
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/security"
+ "github.com/chrislusf/seaweedfs/weed/server"
+ "github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/storage"
@@ -46,6 +49,10 @@ var cmdBackup = &Command{
}
func runBackup(cmd *Command, args []string) bool {
+
+ weed_server.LoadConfiguration("security", false)
+ grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
if *s.volumeId == -1 {
return false
}
@@ -59,7 +66,7 @@ func runBackup(cmd *Command, args []string) bool {
}
volumeServer := lookup.Locations[0].Url
- stats, err := operation.GetVolumeSyncStatus(volumeServer, uint32(vid))
+ stats, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
if err != nil {
fmt.Printf("Error get volume %d status: %v\n", vid, err)
return true
@@ -81,7 +88,7 @@ func runBackup(cmd *Command, args []string) bool {
return true
}
- if err := v.Synchronize(volumeServer); err != nil {
+ if err := v.Synchronize(volumeServer, grpcDialOption); err != nil {
fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
return true
}
diff --git a/weed/command/benchmark.go b/weed/command/benchmark.go
index 60fd88ccd..44601e567 100644
--- a/weed/command/benchmark.go
+++ b/weed/command/benchmark.go
@@ -4,6 +4,9 @@ import (
"bufio"
"context"
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/server"
+ "github.com/spf13/viper"
+ "google.golang.org/grpc"
"io"
"math"
"math/rand"
@@ -35,13 +38,14 @@ type BenchmarkOptions struct {
collection *string
cpuprofile *string
maxCpu *int
- secretKey *string
+ grpcDialOption grpc.DialOption
}
var (
b BenchmarkOptions
sharedBytes []byte
masterClient *wdclient.MasterClient
+ isSecure bool
)
func init() {
@@ -59,7 +63,6 @@ func init() {
b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
- b.secretKey = cmdBenchmark.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
sharedBytes = make([]byte, 1024)
}
@@ -102,6 +105,10 @@ var (
)
func runBenchmark(cmd *Command, args []string) bool {
+
+ weed_server.LoadConfiguration("security", false)
+ b.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+
fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
if *b.maxCpu < 1 {
*b.maxCpu = runtime.NumCPU()
@@ -116,7 +123,7 @@ func runBenchmark(cmd *Command, args []string) bool {
defer pprof.StopCPUProfile()
}
- masterClient = wdclient.NewMasterClient(context.Background(), "benchmark", strings.Split(*b.masters, ","))
+ masterClient = wdclient.NewMasterClient(context.Background(), b.grpcDialOption, "client", strings.Split(*b.masters, ","))
go masterClient.KeepConnectedToMaster()
masterClient.WaitUntilConnected()
@@ -188,7 +195,6 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
defer wait.Done()
delayedDeleteChan := make(chan *delayedFile, 100)
var waitForDeletions sync.WaitGroup
- secret := security.Secret(*b.secretKey)
for i := 0; i < 7; i++ {
waitForDeletions.Add(1)
@@ -198,8 +204,11 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
if df.enterTime.After(time.Now()) {
time.Sleep(df.enterTime.Sub(time.Now()))
}
- if e := util.Delete("http://"+df.fp.Server+"/"+df.fp.Fid,
- security.GenJwt(secret, df.fp.Fid)); e == nil {
+ var jwtAuthorization security.EncodedJwt
+ if isSecure {
+ jwtAuthorization = operation.LookupJwt(masterClient.GetMaster(), df.fp.Fid)
+ }
+ if e := util.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil {
s.completed++
} else {
s.failed++
@@ -222,9 +231,12 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
Count: 1,
Collection: *b.collection,
}
- if assignResult, err := operation.Assign(masterClient.GetMaster(), ar); err == nil {
+ if assignResult, err := operation.Assign(masterClient.GetMaster(), b.grpcDialOption, ar); err == nil {
fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection
- if _, err := fp.Upload(0, masterClient.GetMaster(), secret); err == nil {
+ if !isSecure && assignResult.Auth != "" {
+ isSecure = true
+ }
+ if _, err := fp.Upload(0, masterClient.GetMaster(), assignResult.Auth, b.grpcDialOption); err == nil {
if random.Intn(100) < *b.deletePercentage {
s.total++
delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
diff --git a/weed/command/filer.go b/weed/command/filer.go
index 0c1950f96..478b7d6bf 100644
--- a/weed/command/filer.go
+++ b/weed/command/filer.go
@@ -1,6 +1,8 @@
package command
import (
+ "github.com/chrislusf/seaweedfs/weed/security"
+ "github.com/spf13/viper"
"net/http"
"strconv"
"strings"
@@ -28,7 +30,6 @@ type FilerOptions struct {
redirectOnRead *bool
disableDirListing *bool
maxMB *int
- secretKey *string
dirListingLimit *int
dataCenter *string
enableNotification *bool
@@ -49,7 +50,6 @@ func init() {
f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
- f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
}
@@ -70,13 +70,15 @@ var cmdFiler = &Command{
The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
- The example filer.toml configuration file can be generated by "weed scaffold filer"
+ The example filer.toml configuration file can be generated by "weed scaffold -config=filer"
`,
}
func runFiler(cmd *Command, args []string) bool {
+ weed_server.LoadConfiguration("security", false)
+
f.startFiler()
return true
@@ -103,7 +105,6 @@ func (fo *FilerOptions) startFiler() {
RedirectOnRead: *fo.redirectOnRead,
DisableDirListing: *fo.disableDirListing,
MaxMB: *fo.maxMB,
- SecretKey: *fo.secretKey,
DirListingLimit: *fo.dirListingLimit,
DataCenter: *fo.dataCenter,
DefaultLevelDbDir: defaultLevelDbDirectory,
@@ -144,7 +145,7 @@ func (fo *FilerOptions) startFiler() {
if err != nil {
glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
}
- grpcS := util.NewGrpcServer()
+ grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "filer"))
filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
reflection.Register(grpcS)
go grpcS.Serve(grpcL)
diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go
index 3638bcb27..650757442 100644
--- a/weed/command/filer_copy.go
+++ b/weed/command/filer_copy.go
@@ -2,6 +2,10 @@ package command
import (
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/security"
+ "github.com/chrislusf/seaweedfs/weed/server"
+ "github.com/spf13/viper"
+ "google.golang.org/grpc"
"io/ioutil"
"net/url"
"os"
@@ -11,7 +15,6 @@ import (
"context"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
- "github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"io"
"net/http"
@@ -24,16 +27,14 @@ var (
)
type CopyOptions struct {
- filerGrpcPort *int
- master *string
- include *string
- replication *string
- collection *string
- ttl *string
- maxMB *int
- secretKey *string
-
- secret security.Secret
+ filerGrpcPort *int
+ master *string
+ include *string
+ replication *string
+ collection *string
+ ttl *string
+ maxMB *int
+ grpcDialOption grpc.DialOption
}
func init() {
@@ -46,7 +47,6 @@ func init() {
copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
- copy.secretKey = cmdCopy.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
}
var cmdCopy = &Command{
@@ -66,7 +66,9 @@ var cmdCopy = &Command{
}
func runCopy(cmd *Command, args []string) bool {
- copy.secret = security.Secret(*copy.secretKey)
+
+ weed_server.LoadConfiguration("security", false)
+
if len(args) <= 1 {
return false
}
@@ -101,16 +103,17 @@ func runCopy(cmd *Command, args []string) bool {
}
filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
+ copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
for _, fileOrDir := range fileOrDirs {
- if !doEachCopy(fileOrDir, filerUrl.Host, filerGrpcAddress, urlPath) {
+ if !doEachCopy(fileOrDir, filerUrl.Host, filerGrpcAddress, copy.grpcDialOption, urlPath) {
return false
}
}
return true
}
-func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, path string) bool {
+func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, path string) bool {
f, err := os.Open(fileOrDir)
if err != nil {
fmt.Printf("Failed to open file %s: %v\n", fileOrDir, err)
@@ -128,7 +131,7 @@ func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, path st
if mode.IsDir() {
files, _ := ioutil.ReadDir(fileOrDir)
for _, subFileOrDir := range files {
- if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), filerAddress, filerGrpcAddress, path+fi.Name()+"/") {
+ if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), filerAddress, filerGrpcAddress, grpcDialOption, path+fi.Name()+"/") {
return false
}
}
@@ -150,13 +153,13 @@ func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, path st
}
if chunkCount == 1 {
- return uploadFileAsOne(filerAddress, filerGrpcAddress, path, f, fi)
+ return uploadFileAsOne(filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi)
}
- return uploadFileInChunks(filerAddress, filerGrpcAddress, path, f, fi, chunkCount, chunkSize)
+ return uploadFileInChunks(filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi, chunkCount, chunkSize)
}
-func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f *os.File, fi os.FileInfo) bool {
+func uploadFileAsOne(filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo) bool {
// upload the file content
fileName := filepath.Base(f.Name())
@@ -167,7 +170,7 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
if fi.Size() > 0 {
// assign a volume
- assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{
+ assignResult, err := operation.Assign(*copy.master, grpcDialOption, &operation.VolumeAssignRequest{
Count: 1,
Replication: *copy.replication,
Collection: *copy.collection,
@@ -179,7 +182,7 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
- uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, "")
+ uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth)
if err != nil {
fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
return false
@@ -201,7 +204,7 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
}
- if err := withFilerClient(filerGrpcAddress, func(client filer_pb.SeaweedFilerClient) error {
+ if err := withFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.CreateEntryRequest{
Directory: urlFolder,
Entry: &filer_pb.Entry{
@@ -234,7 +237,7 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
return true
}
-func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool {
+func uploadFileInChunks(filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool {
fileName := filepath.Base(f.Name())
mimeType := detectMimeType(f)
@@ -244,7 +247,7 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
for i := int64(0); i < int64(chunkCount); i++ {
// assign a volume
- assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{
+ assignResult, err := operation.Assign(*copy.master, grpcDialOption, &operation.VolumeAssignRequest{
Count: 1,
Replication: *copy.replication,
Collection: *copy.collection,
@@ -259,7 +262,7 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
uploadResult, err := operation.Upload(targetUrl,
fileName+"-"+strconv.FormatInt(i+1, 10),
io.LimitReader(f, chunkSize),
- false, "application/octet-stream", nil, "")
+ false, "application/octet-stream", nil, assignResult.Auth)
if err != nil {
fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
return false
@@ -278,7 +281,7 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
}
- if err := withFilerClient(filerGrpcAddress, func(client filer_pb.SeaweedFilerClient) error {
+ if err := withFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.CreateEntryRequest{
Directory: urlFolder,
Entry: &filer_pb.Entry{
@@ -329,9 +332,9 @@ func detectMimeType(f *os.File) string {
return mimeType
}
-func withFilerClient(filerAddress string, fn func(filer_pb.SeaweedFilerClient) error) error {
+func withFilerClient(filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
- grpcConnection, err := util.GrpcDial(filerAddress)
+ grpcConnection, err := util.GrpcDial(filerAddress, grpcDialOption)
if err != nil {
return fmt.Errorf("fail to dial %s: %v", filerAddress, err)
}
diff --git a/weed/command/filer_replication.go b/weed/command/filer_replication.go
index 3384e4023..c9afbdc8a 100644
--- a/weed/command/filer_replication.go
+++ b/weed/command/filer_replication.go
@@ -28,13 +28,14 @@ var cmdFilerReplicate = &Command{
filer.replicate listens on filer notifications. If any file is updated, it will fetch the updated content,
and write to the other destination.
- Run "weed scaffold -config replication" to generate a replication.toml file and customize the parameters.
+ Run "weed scaffold -config=replication" to generate a replication.toml file and customize the parameters.
`,
}
func runFilerReplicate(cmd *Command, args []string) bool {
+ weed_server.LoadConfiguration("security", false)
weed_server.LoadConfiguration("replication", true)
weed_server.LoadConfiguration("notification", true)
config := viper.GetViper()
diff --git a/weed/command/master.go b/weed/command/master.go
index bd2267b9e..5b45c9627 100644
--- a/weed/command/master.go
+++ b/weed/command/master.go
@@ -1,6 +1,8 @@
package command
import (
+ "github.com/chrislusf/seaweedfs/weed/security"
+ "github.com/spf13/viper"
"net/http"
"os"
"runtime"
@@ -23,8 +25,11 @@ func init() {
var cmdMaster = &Command{
UsageLine: "master -port=9333",
Short: "start a master server",
- Long: `start a master server to provide volume=>location mapping service
- and sequence number of file ids
+ Long: `start a master server to provide volume=>location mapping service and sequence number of file ids
+
+ The configuration file "security.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
+
+ The example security.toml configuration file can be generated by "weed scaffold -config=security"
`,
}
@@ -44,7 +49,6 @@ var (
mMaxCpu = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
- masterSecureKey = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
masterCpuProfile = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
masterMemProfile = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
@@ -52,6 +56,9 @@ var (
)
func runMaster(cmd *Command, args []string) bool {
+
+ weed_server.LoadConfiguration("security", false)
+
if *mMaxCpu < 1 {
*mMaxCpu = runtime.NumCPU()
}
@@ -72,7 +79,7 @@ func runMaster(cmd *Command, args []string) bool {
ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
*volumeSizeLimitMB, *volumePreallocate,
*mpulse, *defaultReplicaPlacement, *garbageThreshold,
- masterWhiteList, *masterSecureKey,
+ masterWhiteList,
)
listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)
@@ -102,7 +109,7 @@ func runMaster(cmd *Command, args []string) bool {
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
}
// Create your protocol servers.
- grpcS := util.NewGrpcServer()
+ grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
master_pb.RegisterSeaweedServer(grpcS, ms)
reflection.Register(grpcS)
diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go
index 2937b9ef1..3e4249bfc 100644
--- a/weed/command/mount_std.go
+++ b/weed/command/mount_std.go
@@ -4,6 +4,9 @@ package command
import (
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/security"
+ "github.com/chrislusf/seaweedfs/weed/server"
+ "github.com/spf13/viper"
"os"
"os/user"
"runtime"
@@ -19,6 +22,9 @@ import (
)
func runMount(cmd *Command, args []string) bool {
+
+ weed_server.LoadConfiguration("security", false)
+
fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
if *mountOptions.dir == "" {
fmt.Printf("Please specify the mount directory via \"-dir\"")
@@ -91,6 +97,7 @@ func runMount(cmd *Command, args []string) bool {
err = fs.Serve(c, filesys.NewSeaweedFileSystem(&filesys.Option{
FilerGrpcAddress: filerGrpcAddress,
+ GrpcDialOption: security.LoadClientTLS(viper.Sub("grpc"), "client"),
FilerMountRootPath: mountRoot,
Collection: *mountOptions.collection,
Replication: *mountOptions.replication,
diff --git a/weed/command/s3.go b/weed/command/s3.go
index 16a9490ff..a54ddd2f7 100644
--- a/weed/command/s3.go
+++ b/weed/command/s3.go
@@ -1,6 +1,9 @@
package command
import (
+ "github.com/chrislusf/seaweedfs/weed/security"
+ "github.com/chrislusf/seaweedfs/weed/server"
+ "github.com/spf13/viper"
"net/http"
"time"
@@ -46,6 +49,8 @@ var cmdS3 = &Command{
func runS3(cmd *Command, args []string) bool {
+ weed_server.LoadConfiguration("security", false)
+
filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer, *s3options.filerGrpcPort)
if err != nil {
glog.Fatal(err)
@@ -59,6 +64,7 @@ func runS3(cmd *Command, args []string) bool {
FilerGrpcAddress: filerGrpcAddress,
DomainName: *s3options.domainName,
BucketsPath: *s3options.filerBucketsPath,
+ GrpcDialOption: security.LoadClientTLS(viper.Sub("grpc"), "client"),
})
if s3ApiServer_err != nil {
glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index ec0723859..e8608e9dd 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -10,7 +10,7 @@ func init() {
}
var cmdScaffold = &Command{
- UsageLine: "scaffold [filer]",
+ UsageLine: "scaffold -config=[filer|notification|replication|security]",
Short: "generate basic configuration files",
Long: `Generate filer.toml with all possible configurations for you to customize.
@@ -19,7 +19,7 @@ var cmdScaffold = &Command{
var (
outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
- config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication] the configuration file to generate")
+ config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security] the configuration file to generate")
)
func runScaffold(cmd *Command, args []string) bool {
@@ -32,6 +32,8 @@ func runScaffold(cmd *Command, args []string) bool {
content = NOTIFICATION_TOML_EXAMPLE
case "replication":
content = REPLICATION_TOML_EXAMPLE
+ case "security":
+ content = SECURITY_TOML_EXAMPLE
}
if content == "" {
println("need a valid -config option")
@@ -240,4 +242,42 @@ bucket = "mybucket" # an existing bucket
directory = "/" # destination directory
`
+
+ SECURITY_TOML_EXAMPLE = `
+# Put this file to one of the location, with descending priority
+# ./security.toml
+# $HOME/.seaweedfs/security.toml
+# /etc/seaweedfs/security.toml
+# this file is read by master, volume server, and filer
+
+# the jwt signing key is read by master and volume server
+# a jwt expires in 10 seconds
+[jwt.signing]
+key = ""
+
+# volume server also uses grpc that should be secured.
+
+# all grpc tls authentications are mutual
+[grpc]
+ca = ""
+
+[grpc.volume]
+cert = ""
+key = ""
+
+[grpc.master]
+cert = ""
+key = ""
+
+[grpc.filer]
+cert = ""
+key = ""
+
+# use this for any place needs a grpc client
+# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
+[grpc.client]
+cert = ""
+key = ""
+
+`
)
diff --git a/weed/command/server.go b/weed/command/server.go
index ba5305a97..a9415d068 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -1,6 +1,8 @@
package command
import (
+ "github.com/chrislusf/seaweedfs/weed/security"
+ "github.com/spf13/viper"
"net/http"
"os"
"runtime"
@@ -58,7 +60,6 @@ var (
serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
serverPeers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
- serverSecureKey = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
serverGarbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
masterGrpcPort = cmdServer.Flag.Int("master.port.grpc", 0, "master grpc server listen port, default to http port + 10000")
@@ -96,7 +97,9 @@ func init() {
}
func runServer(cmd *Command, args []string) bool {
- filerOptions.secretKey = serverSecureKey
+
+ weed_server.LoadConfiguration("security", false)
+
if *serverOptions.cpuprofile != "" {
f, err := os.Create(*serverOptions.cpuprofile)
if err != nil {
@@ -170,7 +173,7 @@ func runServer(cmd *Command, args []string) bool {
ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
*masterVolumeSizeLimitMB, *masterVolumePreallocate,
*pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
- serverWhiteList, *serverSecureKey,
+ serverWhiteList,
)
glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
@@ -190,7 +193,8 @@ func runServer(cmd *Command, args []string) bool {
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
}
// Create your protocol servers.
- grpcS := util.NewGrpcServer()
+ glog.V(0).Infof("grpc config %+v", viper.Sub("grpc"))
+ grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
master_pb.RegisterSeaweedServer(grpcS, ms)
reflection.Register(grpcS)
diff --git a/weed/command/upload.go b/weed/command/upload.go
index 244caaa4c..80fc635c1 100644
--- a/weed/command/upload.go
+++ b/weed/command/upload.go
@@ -3,11 +3,13 @@ package command
import (
"encoding/json"
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/security"
+ "github.com/chrislusf/seaweedfs/weed/server"
+ "github.com/spf13/viper"
"os"
"path/filepath"
"github.com/chrislusf/seaweedfs/weed/operation"
- "github.com/chrislusf/seaweedfs/weed/security"
)
var (
@@ -23,7 +25,6 @@ type UploadOptions struct {
dataCenter *string
ttl *string
maxMB *int
- secretKey *string
}
func init() {
@@ -37,7 +38,6 @@ func init() {
upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
upload.maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit")
- upload.secretKey = cmdUpload.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
}
var cmdUpload = &Command{
@@ -60,7 +60,10 @@ var cmdUpload = &Command{
}
func runUpload(cmd *Command, args []string) bool {
- secret := security.Secret(*upload.secretKey)
+
+ weed_server.LoadConfiguration("security", false)
+ grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
if len(args) == 0 {
if *upload.dir == "" {
return false
@@ -77,9 +80,9 @@ func runUpload(cmd *Command, args []string) bool {
if e != nil {
return e
}
- results, e := operation.SubmitFiles(*upload.master, parts,
+ results, e := operation.SubmitFiles(*upload.master, grpcDialOption, parts,
*upload.replication, *upload.collection, *upload.dataCenter,
- *upload.ttl, *upload.maxMB, secret)
+ *upload.ttl, *upload.maxMB)
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
if e != nil {
@@ -96,9 +99,9 @@ func runUpload(cmd *Command, args []string) bool {
if e != nil {
fmt.Println(e.Error())
}
- results, _ := operation.SubmitFiles(*upload.master, parts,
+ results, _ := operation.SubmitFiles(*upload.master, grpcDialOption, parts,
*upload.replication, *upload.collection, *upload.dataCenter,
- *upload.ttl, *upload.maxMB, secret)
+ *upload.ttl, *upload.maxMB)
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
}
diff --git a/weed/command/volume.go b/weed/command/volume.go
index 27a075b5b..32ec7819b 100644
--- a/weed/command/volume.go
+++ b/weed/command/volume.go
@@ -1,6 +1,8 @@
package command
import (
+ "github.com/chrislusf/seaweedfs/weed/security"
+ "github.com/spf13/viper"
"net/http"
"os"
"runtime"
@@ -78,6 +80,9 @@ var (
)
func runVolume(cmd *Command, args []string) bool {
+
+ weed_server.LoadConfiguration("security", false)
+
if *v.maxCpu < 1 {
*v.maxCpu = runtime.NumCPU()
}
@@ -185,7 +190,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
if err != nil {
glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
}
- grpcS := util.NewGrpcServer()
+ grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "volume"))
volume_server_pb.RegisterVolumeServerServer(grpcS, volumeServer)
reflection.Register(grpcS)
go grpcS.Serve(grpcL)
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-22 15:47:46 +0000