Merge branch 'master' into add_remote_storage

18 files changed, 152 insertions, 23 deletions

diff --git a/go.mod b/go.mod
index 53188faf2..8fdbf5f60 100644
--- a/go.mod
+++ b/go.mod
@@ -90,6 +90,7 @@ require (
 	gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
 	modernc.org/sqlite v1.10.7
+	github.com/posener/complete v1.2.3
 )
 
 // replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
diff --git a/go.sum b/go.sum
index c96845979..f23ca0020 100644
--- a/go.sum
+++ b/go.sum
@@ -364,10 +364,12 @@ github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
 github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
@@ -571,6 +573,8 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo=
+github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
 github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc=
 github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml
index cc8e4bdca..83b0148e7 100644
--- a/k8s/seaweedfs/Chart.yaml
+++ b/k8s/seaweedfs/Chart.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.60"
-version: "2.60"
+appVersion: "2.61"
+version: "2.61"
diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml
index 3f1c6da1c..adc0f1d2c 100644
--- a/k8s/seaweedfs/values.yaml
+++ b/k8s/seaweedfs/values.yaml
@@ -4,7 +4,6 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  # imageTag: "2.60" - started using {.Chart.appVersion}
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always
diff --git a/weed/command/autocomplete.go b/weed/command/autocomplete.go
new file mode 100644
index 000000000..9a545a183
--- /dev/null
+++ b/weed/command/autocomplete.go
@@ -0,0 +1,109 @@
+package command
+
+import (
+	"fmt"
+	flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
+	"github.com/posener/complete"
+	completeinstall "github.com/posener/complete/cmd/install"
+	"runtime"
+)
+
+func AutocompleteMain(commands []*Command) bool {
+	subCommands := make(map[string]complete.Command)
+	helpSubCommands := make(map[string]complete.Command)
+	for _, cmd := range commands {
+		flags := make(map[string]complete.Predictor)
+		cmd.Flag.VisitAll(func(flag *flag.Flag) {
+			flags["-"+flag.Name] = complete.PredictAnything
+		})
+
+		subCommands[cmd.Name()] = complete.Command{
+			Flags: flags,
+		}
+		helpSubCommands[cmd.Name()] = complete.Command{}
+	}
+	subCommands["help"] = complete.Command{Sub: helpSubCommands}
+
+	globalFlags := make(map[string]complete.Predictor)
+	flag.VisitAll(func(flag *flag.Flag) {
+		globalFlags["-"+flag.Name] = complete.PredictAnything
+	})
+
+	weedCmd := complete.Command{
+		Sub:         subCommands,
+		Flags:       globalFlags,
+		GlobalFlags: complete.Flags{"-h": complete.PredictNothing},
+	}
+	cmp := complete.New("weed", weedCmd)
+
+	return cmp.Complete()
+}
+
+func installAutoCompletion() bool {
+	if runtime.GOOS == "windows" {
+		fmt.Println("windows is not supported")
+		return false
+	}
+
+	err := completeinstall.Install("weed")
+	if err != nil {
+		fmt.Printf("install failed! %s\n", err)
+		return false
+	}
+	fmt.Printf("autocompletion is enabled. Please restart your shell.\n")
+	return true
+}
+
+func uninstallAutoCompletion() bool {
+	if runtime.GOOS == "windows" {
+		fmt.Println("windows is not supported")
+		return false
+	}
+
+	err := completeinstall.Uninstall("weed")
+	if err != nil {
+		fmt.Printf("uninstall failed! %s\n", err)
+		return false
+	}
+	fmt.Printf("autocompletion is disable. Please restart your shell.\n")
+	return true
+}
+
+var cmdAutocomplete = &Command{
+	Run:       runAutocomplete,
+	UsageLine: "autocomplete",
+	Short:     "install autocomplete",
+	Long: `weed autocomplete is installed in the shell.
+
+    Supported shells are bash, zsh, and fish.
+    Windows is not supported.
+
+`,
+}
+
+func runAutocomplete(cmd *Command, args []string) bool {
+	if len(args) != 0 {
+		cmd.Usage()
+	}
+
+	return installAutoCompletion()
+}
+
+var cmdUnautocomplete = &Command{
+	Run:       runUnautocomplete,
+	UsageLine: "autocomplete.uninstall",
+	Short:     "uninstall autocomplete",
+	Long: `weed autocomplete is uninstalled in the shell.
+
+    Windows is not supported.
+
+`,
+}
+
+func runUnautocomplete(cmd *Command, args []string) bool {
+	if len(args) != 0 {
+		cmd.Usage()
+	}
+
+	return uninstallAutoCompletion()
+}
diff --git a/weed/command/command.go b/weed/command/command.go
index 0bac56442..9ae93fe61 100644
--- a/weed/command/command.go
+++ b/weed/command/command.go
@@ -8,6 +8,8 @@ import (
 )
 
 var Commands = []*Command{
+	cmdAutocomplete,
+	cmdUnautocomplete,
 	cmdBackup,
 	cmdBenchmark,
 	cmdCompact,
diff --git a/weed/command/filer_backup.go b/weed/command/filer_backup.go
index 888b46fe7..fc4dd8298 100644
--- a/weed/command/filer_backup.go
+++ b/weed/command/filer_backup.go
@@ -52,11 +52,11 @@ var cmdFilerBackup = &Command{
 
 func runFilerBackup(cmd *Command, args []string) bool {
 
-	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
-
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("replication", true)
 
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
 	for {
 		err := doFilerBackup(grpcDialOption, &filerBackupOptions)
 		if err != nil {
diff --git a/weed/command/filer_meta_backup.go b/weed/command/filer_meta_backup.go
index ba0b44659..28bd367e7 100644
--- a/weed/command/filer_meta_backup.go
+++ b/weed/command/filer_meta_backup.go
@@ -53,6 +53,7 @@ The backup writes to another filer store specified in a backup_filer.toml.
 
 func runFilerMetaBackup(cmd *Command, args []string) bool {
 
+	util.LoadConfiguration("security", false)
 	metaBackup.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	// load backup_filer.toml
diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go
index 8451ffd78..76699bb5e 100644
--- a/weed/command/filer_meta_tail.go
+++ b/weed/command/filer_meta_tail.go
@@ -45,6 +45,7 @@ var (
 
 func runFilerMetaTail(cmd *Command, args []string) bool {
 
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	var filterFunc func(dir, fname string) bool
diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go
index 211c34aea..7cfc8a7fe 100644
--- a/weed/command/filer_sync.go
+++ b/weed/command/filer_sync.go
@@ -89,6 +89,7 @@ var cmdFilerSynchronize = &Command{
 
 func runFilerSynchronize(cmd *Command, args []string) bool {
 
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	grace.SetupProfiling(*syncCpuProfile, *syncMemProfile)
diff --git a/weed/command/iam.go b/weed/command/iam.go
index 17d0832cb..ed4eea543 100644
--- a/weed/command/iam.go
+++ b/weed/command/iam.go
@@ -49,6 +49,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 		return false
 	}
 
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	for {
 		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
diff --git a/weed/command/scaffold/master.toml b/weed/command/scaffold/master.toml
index cdd92016c..020f48e36 100644
--- a/weed/command/scaffold/master.toml
+++ b/weed/command/scaffold/master.toml
@@ -11,7 +11,7 @@ scripts = """
   ec.encode -fullPercent=95 -quietFor=1h
   ec.rebuild -force
   ec.balance -force
-  volume.deleteEmpty -quietFor=24h
+  volume.deleteEmpty -quietFor=24h -force
   volume.balance -force
   volume.fix.replication
   unlock
diff --git a/weed/command/upload.go b/weed/command/upload.go
index ccdec561f..9ae1befab 100644
--- a/weed/command/upload.go
+++ b/weed/command/upload.go
@@ -71,13 +71,13 @@ func runUpload(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
-	defaultCollection, err := readMasterConfiguration(grpcDialOption, *upload.master)
+	defaultReplication, err := readMasterConfiguration(grpcDialOption, *upload.master)
 	if err != nil {
 		fmt.Printf("upload: %v", err)
 		return false
 	}
 	if *upload.replication == "" {
-		*upload.replication = defaultCollection
+		*upload.replication = defaultReplication
 	}
 
 	if len(args) == 0 {
diff --git a/weed/shell/command_volume_balance.go b/weed/shell/command_volume_balance.go
index ad7da0e44..6da128c68 100644
--- a/weed/shell/command_volume_balance.go
+++ b/weed/shell/command_volume_balance.go
@@ -120,7 +120,7 @@ func balanceVolumeServers(commandEnv *CommandEnv, diskTypes []types.DiskType, vo
 
 func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, volumeSizeLimit uint64, collection string, applyBalancing bool) error {
 
-	// balance writable volumes
+	// balance read only volumes
 	for _, n := range nodes {
 		n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
 			if collection != "ALL_COLLECTIONS" {
@@ -128,14 +128,14 @@ func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskT
 					return false
 				}
 			}
-			return v.DiskType == string(diskType) && (!v.ReadOnly && v.Size < volumeSizeLimit)
+			return v.DiskType == string(diskType) && (v.ReadOnly || v.Size >= volumeSizeLimit)
 		})
 	}
-	if err := balanceSelectedVolume(commandEnv, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortWritableVolumes, applyBalancing); err != nil {
+	if err := balanceSelectedVolume(commandEnv, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortReadOnlyVolumes, applyBalancing); err != nil {
 		return err
 	}
 
-	// balance readable volumes
+	// balance writable volumes
 	for _, n := range nodes {
 		n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
 			if collection != "ALL_COLLECTIONS" {
@@ -143,10 +143,10 @@ func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskT
 					return false
 				}
 			}
-			return v.DiskType == string(diskType) && (v.ReadOnly || v.Size >= volumeSizeLimit)
+			return v.DiskType == string(diskType) && (!v.ReadOnly && v.Size < volumeSizeLimit)
 		})
 	}
-	if err := balanceSelectedVolume(commandEnv, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortReadOnlyVolumes, applyBalancing); err != nil {
+	if err := balanceSelectedVolume(commandEnv, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortWritableVolumes, applyBalancing); err != nil {
 		return err
 	}
 
diff --git a/weed/shell/command_volume_delete_empty.go b/weed/shell/command_volume_delete_empty.go
index 07c6d5b2c..079915f66 100644
--- a/weed/shell/command_volume_delete_empty.go
+++ b/weed/shell/command_volume_delete_empty.go
@@ -3,11 +3,10 @@ package shell
 import (
 	"flag"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"io"
 	"log"
 	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 func init() {
@@ -24,7 +23,7 @@ func (c *commandVolumeDeleteEmpty) Name() string {
 func (c *commandVolumeDeleteEmpty) Help() string {
 	return `delete empty volumes from all volume servers
 
-	volume.deleteEmpty -quietFor=24h
+	volume.deleteEmpty -quietFor=24h -force
 
 	This command deletes all empty volumes from one volume server.
 
@@ -39,6 +38,7 @@ func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, wri
 
 	volDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
 	quietPeriod := volDeleteCommand.Duration("quietFor", 24*time.Hour, "select empty volumes with no recent writes, avoid newly created ones")
+	applyBalancing := volDeleteCommand.Bool("force", false, "apply to delete empty volumes")
 	if err = volDeleteCommand.Parse(args); err != nil {
 		return nil
 	}
@@ -55,10 +55,15 @@ func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, wri
 	eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
 		for _, diskInfo := range dn.DiskInfos {
 			for _, v := range diskInfo.VolumeInfos {
-				if v.Size <= 8 && v.ModifiedAtSecond + quietSeconds < nowUnixSeconds {
-					log.Printf("deleting empty volume %d from %s", v.Id, dn.Id)
-					if deleteErr := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), dn.Id); deleteErr != nil {
-						err = deleteErr
+				if v.Size <= 8 && v.ModifiedAtSecond+quietSeconds < nowUnixSeconds {
+					if *applyBalancing {
+						log.Printf("deleting empty volume %d from %s", v.Id, dn.Id)
+						if deleteErr := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), dn.Id); deleteErr != nil {
+							err = deleteErr
+						}
+						continue
+					} else {
+						log.Printf("empty volume %d from %s", v.Id, dn.Id)
 					}
 				}
 			}
diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go
index 9e6280788..7e7c0a93d 100644
--- a/weed/shell/command_volume_fix_replication.go
+++ b/weed/shell/command_volume_fix_replication.go
@@ -160,7 +160,7 @@ func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *Comm
 	for _, vid := range underReplicatedVolumeIds {
 		for i := 0; i < retryCount+1; i++ {
 			if err = c.fixOneUnderReplicatedVolume(commandEnv, writer, takeAction, volumeReplicas, vid, allLocations); err == nil {
-				continue
+				break
 			}
 		}
 	}
diff --git a/weed/util/constants.go b/weed/util/constants.go
index 2636adf45..108dfb2e4 100644
--- a/weed/util/constants.go
+++ b/weed/util/constants.go
@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 60)
+	VERSION = fmt.Sprintf("%s %.02f", sizeLimit, 2.61)
 	COMMIT  = ""
 )
 
diff --git a/weed/weed.go b/weed/weed.go
index 91c17d9ff..068d2077c 100644
--- a/weed/weed.go
+++ b/weed/weed.go
@@ -46,6 +46,11 @@ func main() {
 	glog.MaxSize = 1024 * 1024 * 32
 	rand.Seed(time.Now().UnixNano())
 	flag.Usage = usage
+
+	if command.AutocompleteMain(commands) {
+		return
+	}
+
 	flag.Parse()
 
 	args := flag.Args()