Merge pull request #87 from chrislusf/master

sync

9 files changed, 116 insertions, 10 deletions

diff --git a/weed/shell/command_cluster_ps.go b/weed/shell/command_cluster_ps.go
new file mode 100644
index 000000000..5ed1677c8
--- /dev/null
+++ b/weed/shell/command_cluster_ps.go
@@ -0,0 +1,55 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/cluster"
+	"io"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandClusterPs{})
+}
+
+type commandClusterPs struct {
+}
+
+func (c *commandClusterPs) Name() string {
+	return "cluster.ps"
+}
+
+func (c *commandClusterPs) Help() string {
+	return `check current cluster process status
+
+	cluster.ps
+
+`
+}
+
+func (c *commandClusterPs) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	clusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	if err = clusterPsCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
+		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
+			ClientType: cluster.FilerType,
+		})
+
+		fmt.Fprintf(writer, "the cluster has %d filers\n", len(resp.ClusterNodes))
+		for _, node := range resp.ClusterNodes {
+			fmt.Fprintf(writer, "  * %s (%v)\n", node.Address, node.Version)
+		}
+		return err
+	})
+	if err != nil {
+		return
+	}
+
+	return nil
+}
diff --git a/weed/shell/command_ec_encode.go b/weed/shell/command_ec_encode.go
index 33eef9ca7..3606f42e3 100644
--- a/weed/shell/command_ec_encode.go
+++ b/weed/shell/command_ec_encode.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"io"
+	"math/rand"
 	"sync"
 	"time"
 
@@ -33,7 +34,7 @@ func (c *commandEcEncode) Name() string {
 func (c *commandEcEncode) Help() string {
 	return `apply erasure coding to a volume
 
-	ec.encode [-collection=""] [-fullPercent=95] [-quietFor=1h]
+	ec.encode [-collection=""] [-fullPercent=95 -quietFor=1h]
 	ec.encode [-collection=""] [-volumeId=<volume_id>]
 
 	This command will:
@@ -248,7 +249,7 @@ func parallelCopyEcShardsFromSource(grpcDialOption grpc.DialOption, targetServer
 func balancedEcDistribution(servers []*EcNode) (allocated [][]uint32) {
 	allocated = make([][]uint32, len(servers))
 	allocatedShardIdIndex := uint32(0)
-	serverIndex := 0
+	serverIndex := rand.Intn(len(servers))
 	for allocatedShardIdIndex < erasure_coding.TotalShardsCount {
 		if servers[serverIndex].freeEcSlot > 0 {
 			allocated[serverIndex] = append(allocated[serverIndex], allocatedShardIdIndex)
diff --git a/weed/shell/command_remote_cache.go b/weed/shell/command_remote_cache.go
index 2ae20d143..2bd8d4eda 100644
--- a/weed/shell/command_remote_cache.go
+++ b/weed/shell/command_remote_cache.go
@@ -26,7 +26,7 @@ func (c *commandRemoteCache) Help() string {
 	return `cache the file content for mounted directories or files
 
 	# assume a remote storage is configured to name "cloud1"
-	remote.configure -name=cloud1 -type=s3 -access_key=xxx -secret_key=yyy
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
 	# mount and pull one bucket
 	remote.mount -dir=/xxx -remote=cloud1/bucket
 
@@ -163,10 +163,10 @@ func (c *commandRemoteCache) cacheContentData(commandEnv *CommandEnv, writer io.
 
 			remoteLocation := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dir.Child(entry.Name))
 
-			if err := filer.DownloadToLocal(commandEnv, remoteConf, remoteLocation, dir, entry); err != nil {
-				fmt.Fprintf(writer, "DownloadToLocal %+v: %v\n", remoteLocation, err)
+			if err := filer.CacheRemoteObjectToLocalCluster(commandEnv, remoteConf, remoteLocation, dir, entry); err != nil {
+				fmt.Fprintf(writer, "CacheRemoteObjectToLocalCluster %+v: %v\n", remoteLocation, err)
 				if executionErr == nil {
-					executionErr = fmt.Errorf("DownloadToLocal %+v: %v\n", remoteLocation, err)
+					executionErr = fmt.Errorf("CacheRemoteObjectToLocalCluster %+v: %v\n", remoteLocation, err)
 				}
 				return
 			}
diff --git a/weed/shell/command_remote_meta_sync.go b/weed/shell/command_remote_meta_sync.go
index 5dbf55987..277c4c2be 100644
--- a/weed/shell/command_remote_meta_sync.go
+++ b/weed/shell/command_remote_meta_sync.go
@@ -27,7 +27,7 @@ func (c *commandRemoteMetaSync) Help() string {
 	return `synchronize the local file meta data with the remote file metadata
 
 	# assume a remote storage is configured to name "cloud1"
-	remote.configure -name=cloud1 -type=s3 -access_key=xxx -secret_key=yyy
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
 	# mount and pull one bucket
 	remote.mount -dir=/xxx -remote=cloud1/bucket
 
diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go
index c2d9ec6ba..2b57db707 100644
--- a/weed/shell/command_remote_mount.go
+++ b/weed/shell/command_remote_mount.go
@@ -32,7 +32,7 @@ func (c *commandRemoteMount) Help() string {
 	return `mount remote storage and pull its metadata
 
 	# assume a remote storage is configured to name "cloud1"
-	remote.configure -name=cloud1 -type=s3 -access_key=xxx -secret_key=yyy
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
 
 	# mount and pull one bucket
 	remote.mount -dir=/xxx -remote=cloud1/bucket
diff --git a/weed/shell/command_remote_mount_buckets.go b/weed/shell/command_remote_mount_buckets.go
index f76629193..c4411e639 100644
--- a/weed/shell/command_remote_mount_buckets.go
+++ b/weed/shell/command_remote_mount_buckets.go
@@ -27,7 +27,7 @@ func (c *commandRemoteMountBuckets) Help() string {
 	return `mount all buckets in remote storage and pull its metadata
 
 	# assume a remote storage is configured to name "cloud1"
-	remote.configure -name=cloud1 -type=s3 -access_key=xxx -secret_key=yyy
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
 
 	# mount all buckets
 	remote.mount.buckets -remote=cloud1
diff --git a/weed/shell/command_remote_unmount.go b/weed/shell/command_remote_unmount.go
index d030143a3..c947a19e6 100644
--- a/weed/shell/command_remote_unmount.go
+++ b/weed/shell/command_remote_unmount.go
@@ -27,7 +27,7 @@ func (c *commandRemoteUnmount) Help() string {
 	return `unmount remote storage
 
 	# assume a remote storage is configured to name "s3_1"
-	remote.configure -name=s3_1 -type=s3 -access_key=xxx -secret_key=yyy
+	remote.configure -name=s3_1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
 	# mount and pull one bucket
 	remote.mount -dir=/xxx -remote=s3_1/bucket
 
diff --git a/weed/shell/command_volume_fix_replication_test.go b/weed/shell/command_volume_fix_replication_test.go
index bb61be1ef..4d9cd8188 100644
--- a/weed/shell/command_volume_fix_replication_test.go
+++ b/weed/shell/command_volume_fix_replication_test.go
@@ -261,6 +261,29 @@ func TestSatisfyReplicaPlacement00x(t *testing.T) {
 
 }
 
+func TestSatisfyReplicaPlacement100(t *testing.T) {
+
+	var tests = []testcase{
+		{
+			name:        "test 100",
+			replication: "100",
+			replicas: []*VolumeReplica{
+				{
+					location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
+				},
+				{
+					location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
+				},
+			},
+			possibleLocation: location{"dc2", "r3", &master_pb.DataNodeInfo{Id: "dn3"}},
+			expected:         true,
+		},
+	}
+
+	runTests(tests, t)
+
+}
+
 func runTests(tests []testcase, t *testing.T) {
 	for _, tt := range tests {
 		replicaPlacement, _ := super_block.NewReplicaPlacementFromString(tt.replication)
diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go
index db9e815ff..caf8da859 100644
--- a/weed/shell/shell_liner.go
+++ b/weed/shell/shell_liner.go
@@ -3,9 +3,13 @@ package shell
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/cluster"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/util/grace"
 	"io"
+	"math/rand"
 	"os"
 	"path"
 	"regexp"
@@ -47,6 +51,29 @@ func RunShell(options ShellOptions) {
 	go commandEnv.MasterClient.KeepConnectedToMaster()
 	commandEnv.MasterClient.WaitUntilConnected()
 
+	if commandEnv.option.FilerAddress == "" {
+		var filers []pb.ServerAddress
+		commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
+			resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
+				ClientType: cluster.FilerType,
+			})
+			if err != nil {
+				return err
+			}
+
+			for _, clusterNode := range resp.ClusterNodes {
+				filers = append(filers, pb.ServerAddress(clusterNode.Address))
+			}
+			return nil
+		})
+		fmt.Printf("master: %s ", *options.Masters)
+		if len(filers) > 0 {
+			fmt.Printf("filers: %v", filers)
+			commandEnv.option.FilerAddress = filers[rand.Intn(len(filers))]
+		}
+		fmt.Println()
+	}
+
 	if commandEnv.option.FilerAddress != "" {
 		commandEnv.WithFilerClient(func(filerClient filer_pb.SeaweedFilerClient) error {
 			resp, err := filerClient.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})