Merge accumulated changes related to message queue (#5098)

* balance partitions on brokers

* prepare topic partition first and then publish, move partition

* purge unused APIs

* clean up

* adjust logs

* add BalanceTopics() grpc API

* configure topic

* configure topic command

* refactor

* repair missing partitions

* sequence of operations to ensure ordering

* proto to close publishers and consumers

* rename file

* topic partition versioned by unixTimeNs

* create local topic partition

* close publishers

* randomize the client name

* wait until no publishers

* logs

* close stop publisher channel

* send last ack

* comments

* comment

* comments

* support list of brokers

* add cli options

* Update .gitignore

* logs

* return io.eof directly

* refactor

* optionally create topic

* refactoring

* detect consumer disconnection

* sub client wait for more messages

* subscribe by time stamp

* rename

* rename to sub_balancer

* rename

* adjust comments

* rename

* fix compilation

* rename

* rename

* SubscriberToSubCoordinator

* sticky rebalance

* go fmt

* add tests

* balance partitions on brokers

* prepare topic partition first and then publish, move partition

* purge unused APIs

* clean up

* adjust logs

* add BalanceTopics() grpc API

* configure topic

* configure topic command

* refactor

* repair missing partitions

* sequence of operations to ensure ordering

* proto to close publishers and consumers

* rename file

* topic partition versioned by unixTimeNs

* create local topic partition

* close publishers

* randomize the client name

* wait until no publishers

* logs

* close stop publisher channel

* send last ack

* comments

* comment

* comments

* support list of brokers

* add cli options

* Update .gitignore

* logs

* return io.eof directly

* refactor

* optionally create topic

* refactoring

* detect consumer disconnection

* sub client wait for more messages

* subscribe by time stamp

* rename

* rename to sub_balancer

* rename

* adjust comments

* rename

* fix compilation

* rename

* rename

* SubscriberToSubCoordinator

* sticky rebalance

* go fmt

* add tests

* tracking topic=>broker

* merge

* comment

1 files changed, 36 insertions, 23 deletions

diff --git a/weed/mq/topic/local_partition.go b/weed/mq/topic/local_partition.go
index 49b639dfa..aa1274ff5 100644
--- a/weed/mq/topic/local_partition.go
+++ b/weed/mq/topic/local_partition.go
@@ -11,19 +11,23 @@ import (
 
 type LocalPartition struct {
 	Partition
-	isLeader        bool
-	FollowerBrokers []pb.ServerAddress
-	logBuffer       *log_buffer.LogBuffer
-	ConsumerCount   int32
+	isLeader          bool
+	FollowerBrokers   []pb.ServerAddress
+	logBuffer         *log_buffer.LogBuffer
+	ConsumerCount     int32
+	StopPublishersCh  chan struct{}
+	Publishers        *LocalPartitionPublishers
+	StopSubscribersCh chan struct{}
+	Subscribers       *LocalPartitionSubscribers
 }
 
-func NewLocalPartition(topic Topic, partition Partition, isLeader bool, followerBrokers []pb.ServerAddress) *LocalPartition {
+func NewLocalPartition(partition Partition, isLeader bool, followerBrokers []pb.ServerAddress) *LocalPartition {
 	return &LocalPartition{
 		Partition:       partition,
 		isLeader:        isLeader,
 		FollowerBrokers: followerBrokers,
 		logBuffer: log_buffer.NewLogBuffer(
-			fmt.Sprintf("%s/%s/%4d-%4d", topic.Namespace, topic.Name, partition.RangeStart, partition.RangeStop),
+			fmt.Sprintf("%d/%4d-%4d", partition.UnixTimeNs, partition.RangeStart, partition.RangeStop),
 			2*time.Minute,
 			func(startTime, stopTime time.Time, buf []byte) {
 
@@ -32,34 +36,43 @@ func NewLocalPartition(topic Topic, partition Partition, isLeader bool, follower
 
 			},
 		),
+		Publishers:  NewLocalPartitionPublishers(),
+		Subscribers: NewLocalPartitionSubscribers(),
 	}
 }
 
 type OnEachMessageFn func(logEntry *filer_pb.LogEntry) error
 
-func (p LocalPartition) Publish(message *mq_pb.DataMessage) {
+func (p *LocalPartition) Publish(message *mq_pb.DataMessage) {
 	p.logBuffer.AddToBuffer(message.Key, message.Value, time.Now().UnixNano())
 }
 
-func (p LocalPartition) Subscribe(clientName string, startReadTime time.Time, eachMessageFn OnEachMessageFn) {
-	p.logBuffer.LoopProcessLogData(clientName, startReadTime, 0, func() bool {
-		return true
-	}, eachMessageFn)
+func (p *LocalPartition) Subscribe(clientName string, startReadTime time.Time, onNoMessageFn func() bool, eachMessageFn OnEachMessageFn) {
+	p.logBuffer.LoopProcessLogData(clientName, startReadTime, 0, onNoMessageFn, eachMessageFn)
 }
 
 func FromPbBrokerPartitionAssignment(self pb.ServerAddress, assignment *mq_pb.BrokerPartitionAssignment) *LocalPartition {
-	isLeaer := assignment.LeaderBroker == string(self)
-	localPartition := &LocalPartition{
-		Partition: FromPbPartition(assignment.Partition),
-		isLeader:  isLeaer,
-	}
-	if !isLeaer {
-		return localPartition
-	}
+	isLeader := assignment.LeaderBroker == string(self)
 	followers := make([]pb.ServerAddress, len(assignment.FollowerBrokers))
-	for i, follower := range assignment.FollowerBrokers {
-		followers[i] = pb.ServerAddress(follower)
+	for i, followerBroker := range assignment.FollowerBrokers {
+		followers[i] = pb.ServerAddress(followerBroker)
+	}
+	return NewLocalPartition(FromPbPartition(assignment.Partition), isLeader, followers)
+}
+
+func (p *LocalPartition) closePublishers() {
+	p.Publishers.SignalShutdown()
+	close(p.StopPublishersCh)
+}
+func (p *LocalPartition) closeSubscribers() {
+	p.Subscribers.SignalShutdown()
+}
+
+func (p *LocalPartition) WaitUntilNoPublishers() {
+	for {
+		if p.Publishers.IsEmpty() {
+			return
+		}
+		time.Sleep(113 * time.Millisecond)
 	}
-	localPartition.FollowerBrokers = followers
-	return localPartition
 }