Merge pull request #2868 from kmlebedev/hashicorp_raft

hashicorp raft

1 files changed, 127 insertions, 35 deletions

diff --git a/weed/server/master_server.go b/weed/server/master_server.go
index b63e3a418..9f29d4ba7 100644
--- a/weed/server/master_server.go
+++ b/weed/server/master_server.go
@@ -1,6 +1,7 @@
 package weed_server
 
 import (
+	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/stats"
 	"net/http"
@@ -17,6 +18,7 @@ import (
 
 	"github.com/chrislusf/raft"
 	"github.com/gorilla/mux"
+	hashicorpRaft "github.com/hashicorp/raft"
 	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -30,8 +32,9 @@ import (
 )
 
 const (
-	SequencerType        = "master.sequencer.type"
-	SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
+	SequencerType         = "master.sequencer.type"
+	SequencerSnowflakeId  = "master.sequencer.sequencer_snowflake_id"
+	RaftServerRemovalTime = 72 * time.Minute
 )
 
 type MasterOption struct {
@@ -62,6 +65,9 @@ type MasterServer struct {
 
 	boundedLeaderChan chan int
 
+	onPeerUpdatDoneCn      chan string
+	onPeerUpdatDoneCnExist bool
+
 	// notifying clients
 	clientChansLock sync.RWMutex
 	clientChans     map[string]chan *master_pb.KeepConnectedResponse
@@ -112,6 +118,9 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
 		Cluster:         cluster.NewCluster(),
 	}
 	ms.boundedLeaderChan = make(chan int, 16)
+	ms.onPeerUpdatDoneCn = make(chan string)
+
+	ms.MasterClient.OnPeerUpdate = ms.OnPeerUpdate
 
 	seq := ms.createSequencer(option)
 	if nil == seq {
@@ -160,19 +169,41 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
 }
 
 func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
-	ms.Topo.RaftServer = raftServer.raftServer
-	ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
-		glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
-		stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
-		if ms.Topo.RaftServer.Leader() != "" {
-			glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
-		}
-	})
+	var raftServerName string
+	if raftServer.raftServer != nil {
+		ms.Topo.RaftServer = raftServer.raftServer
+		ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
+			glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
+			stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
+			if ms.Topo.RaftServer.Leader() != "" {
+				glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
+			}
+		})
+		raftServerName = ms.Topo.RaftServer.Name()
+	} else if raftServer.RaftHashicorp != nil {
+		ms.Topo.HashicorpRaft = raftServer.RaftHashicorp
+		leaderCh := raftServer.RaftHashicorp.LeaderCh()
+		prevLeader := ms.Topo.HashicorpRaft.Leader()
+		go func() {
+			for {
+				select {
+				case isLeader := <-leaderCh:
+					leader := ms.Topo.HashicorpRaft.Leader()
+					glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader)
+					stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc()
+					prevLeader = leader
+				}
+			}
+		}()
+		raftServerName = ms.Topo.HashicorpRaft.String()
+	}
 	if ms.Topo.IsLeader() {
-		glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", "I am the leader!")
+		glog.V(0).Infoln("[", raftServerName, "]", "I am the leader!")
 	} else {
-		if ms.Topo.RaftServer.Leader() != "" {
+		if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
 			glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
+		} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
+			glog.V(0).Infoln("[", ms.Topo.HashicorpRaft.String(), "]", ms.Topo.HashicorpRaft.Leader(), "is the leader.")
 		}
 	}
 }
@@ -181,31 +212,38 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if ms.Topo.IsLeader() {
 			f(w, r)
-		} else if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
-			ms.boundedLeaderChan <- 1
-			defer func() { <-ms.boundedLeaderChan }()
-			targetUrl, err := url.Parse("http://" + ms.Topo.RaftServer.Leader())
-			if err != nil {
-				writeJsonError(w, r, http.StatusInternalServerError,
-					fmt.Errorf("Leader URL http://%s Parse Error: %v", ms.Topo.RaftServer.Leader(), err))
-				return
-			}
-			glog.V(4).Infoln("proxying to leader", ms.Topo.RaftServer.Leader())
-			proxy := httputil.NewSingleHostReverseProxy(targetUrl)
-			director := proxy.Director
-			proxy.Director = func(req *http.Request) {
-				actualHost, err := security.GetActualRemoteHost(req)
-				if err == nil {
-					req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost)
-				}
-				director(req)
-			}
-			proxy.Transport = util.Transport
-			proxy.ServeHTTP(w, r)
-		} else {
-			// handle requests locally
+			return
+		}
+		var raftServerLeader string
+		if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
+			raftServerLeader = ms.Topo.RaftServer.Leader()
+		} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
+			raftServerLeader = string(ms.Topo.HashicorpRaft.Leader())
+		}
+		if raftServerLeader == "" {
 			f(w, r)
+			return
+		}
+		ms.boundedLeaderChan <- 1
+		defer func() { <-ms.boundedLeaderChan }()
+		targetUrl, err := url.Parse("http://" + raftServerLeader)
+		if err != nil {
+			writeJsonError(w, r, http.StatusInternalServerError,
+				fmt.Errorf("Leader URL http://%s Parse Error: %v", raftServerLeader, err))
+			return
+		}
+		glog.V(4).Infoln("proxying to leader", raftServerLeader)
+		proxy := httputil.NewSingleHostReverseProxy(targetUrl)
+		director := proxy.Director
+		proxy.Director = func(req *http.Request) {
+			actualHost, err := security.GetActualRemoteHost(req)
+			if err == nil {
+				req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost)
+			}
+			director(req)
 		}
+		proxy.Transport = util.Transport
+		proxy.ServeHTTP(w, r)
 	}
 }
 
@@ -301,3 +339,57 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer
 	}
 	return seq
 }
+
+func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate) {
+	if update.NodeType != cluster.MasterType || ms.Topo.HashicorpRaft == nil {
+		return
+	}
+	glog.V(4).Infof("OnPeerUpdate: %+v", update)
+
+	peerAddress := pb.ServerAddress(update.Address)
+	peerName := string(peerAddress)
+	isLeader := ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader
+	if update.IsAdd {
+		if isLeader {
+			raftServerFound := false
+			for _, server := range ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers {
+				if string(server.ID) == peerName {
+					raftServerFound = true
+				}
+			}
+			if !raftServerFound {
+				glog.V(0).Infof("adding new raft server: %s", peerName)
+				ms.Topo.HashicorpRaft.AddVoter(
+					hashicorpRaft.ServerID(peerName),
+					hashicorpRaft.ServerAddress(peerAddress.ToGrpcAddress()), 0, 0)
+			}
+		}
+		if ms.onPeerUpdatDoneCnExist {
+			ms.onPeerUpdatDoneCn <- peerName
+		}
+	} else if isLeader {
+		go func(peerName string) {
+			for {
+				select {
+				case <-time.After(RaftServerRemovalTime):
+					err := ms.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
+						_, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{
+							Id:    peerName,
+							Force: false,
+						})
+						return err
+					})
+					if err != nil {
+						glog.Warningf("failed to removing old raft server %s: %v", peerName, err)
+					}
+					return
+				case peerDone := <-ms.onPeerUpdatDoneCn:
+					if peerName == peerDone {
+						return
+					}
+				}
+			}
+		}(peerName)
+		ms.onPeerUpdatDoneCnExist = true
+	}
+}