diff options
| author | Chris Lu <chrislusf@users.noreply.github.com> | 2023-12-11 12:05:54 -0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-12-11 12:05:54 -0800 |
| commit | 580940bf8214ac467694a09436ffc6d97066b97c (patch) | |
| tree | abcc275ebd0088b817f9155b8c08f1bc3d3a0fae /weed/mq/topic/local_topic.go | |
| parent | 8784553501f6569cc2419769f18f09099cb7d30c (diff) | |
| download | seaweedfs-580940bf8214ac467694a09436ffc6d97066b97c.tar.xz seaweedfs-580940bf8214ac467694a09436ffc6d97066b97c.zip | |
Merge accumulated changes related to message queue (#5098)
* balance partitions on brokers
* prepare topic partition first and then publish, move partition
* purge unused APIs
* clean up
* adjust logs
* add BalanceTopics() grpc API
* configure topic
* configure topic command
* refactor
* repair missing partitions
* sequence of operations to ensure ordering
* proto to close publishers and consumers
* rename file
* topic partition versioned by unixTimeNs
* create local topic partition
* close publishers
* randomize the client name
* wait until no publishers
* logs
* close stop publisher channel
* send last ack
* comments
* comment
* comments
* support list of brokers
* add cli options
* Update .gitignore
* logs
* return io.eof directly
* refactor
* optionally create topic
* refactoring
* detect consumer disconnection
* sub client wait for more messages
* subscribe by time stamp
* rename
* rename to sub_balancer
* rename
* adjust comments
* rename
* fix compilation
* rename
* rename
* SubscriberToSubCoordinator
* sticky rebalance
* go fmt
* add tests
* balance partitions on brokers
* prepare topic partition first and then publish, move partition
* purge unused APIs
* clean up
* adjust logs
* add BalanceTopics() grpc API
* configure topic
* configure topic command
* refactor
* repair missing partitions
* sequence of operations to ensure ordering
* proto to close publishers and consumers
* rename file
* topic partition versioned by unixTimeNs
* create local topic partition
* close publishers
* randomize the client name
* wait until no publishers
* logs
* close stop publisher channel
* send last ack
* comments
* comment
* comments
* support list of brokers
* add cli options
* Update .gitignore
* logs
* return io.eof directly
* refactor
* optionally create topic
* refactoring
* detect consumer disconnection
* sub client wait for more messages
* subscribe by time stamp
* rename
* rename to sub_balancer
* rename
* adjust comments
* rename
* fix compilation
* rename
* rename
* SubscriberToSubCoordinator
* sticky rebalance
* go fmt
* add tests
* tracking topic=>broker
* merge
* comment
Diffstat (limited to 'weed/mq/topic/local_topic.go')
| -rw-r--r-- | weed/mq/topic/local_topic.go | 58 |
1 files changed, 58 insertions, 0 deletions
diff --git a/weed/mq/topic/local_topic.go b/weed/mq/topic/local_topic.go index ef3c0e65e..7825d2168 100644 --- a/weed/mq/topic/local_topic.go +++ b/weed/mq/topic/local_topic.go @@ -1,10 +1,19 @@ package topic +import "sync" + type LocalTopic struct { Topic Partitions []*LocalPartition } +func NewLocalTopic(topic Topic) *LocalTopic { + return &LocalTopic{ + Topic: topic, + Partitions: make([]*LocalPartition, 0), + } +} + func (localTopic *LocalTopic) findPartition(partition Partition) *LocalPartition { for _, localPartition := range localTopic.Partitions { if localPartition.Partition.Equals(partition) { @@ -27,3 +36,52 @@ func (localTopic *LocalTopic) removePartition(partition Partition) bool { localTopic.Partitions = append(localTopic.Partitions[:foundPartitionIndex], localTopic.Partitions[foundPartitionIndex+1:]...) return true } + +func (localTopic *LocalTopic) closePartitionPublishers(unixTsNs int64) bool { + var wg sync.WaitGroup + for _, localPartition := range localTopic.Partitions { + if localPartition.UnixTimeNs != unixTsNs { + continue + } + wg.Add(1) + go func(localPartition *LocalPartition) { + defer wg.Done() + localPartition.closePublishers() + }(localPartition) + } + wg.Wait() + return true +} + +func (localTopic *LocalTopic) closePartitionSubscribers(unixTsNs int64) bool { + var wg sync.WaitGroup + for _, localPartition := range localTopic.Partitions { + if localPartition.UnixTimeNs != unixTsNs { + continue + } + wg.Add(1) + go func(localPartition *LocalPartition) { + defer wg.Done() + localPartition.closeSubscribers() + }(localPartition) + } + wg.Wait() + return true +} + +func (localTopic *LocalTopic) WaitUntilNoPublishers() { + for { + var wg sync.WaitGroup + for _, localPartition := range localTopic.Partitions { + wg.Add(1) + go func(localPartition *LocalPartition) { + defer wg.Done() + localPartition.WaitUntilNoPublishers() + }(localPartition) + } + wg.Wait() + if len(localTopic.Partitions) == 0 { + return + } + } +} |