diff options
Diffstat (limited to 'weed/mq/client/sub_client')
| -rw-r--r-- | weed/mq/client/sub_client/process.go | 4 | ||||
| -rw-r--r-- | weed/mq/client/sub_client/subscribe.go | 17 | ||||
| -rw-r--r-- | weed/mq/client/sub_client/subscriber.go | 20 |
3 files changed, 33 insertions, 8 deletions
diff --git a/weed/mq/client/sub_client/process.go b/weed/mq/client/sub_client/process.go index 7717a101f..b6bdb14ee 100644 --- a/weed/mq/client/sub_client/process.go +++ b/weed/mq/client/sub_client/process.go @@ -32,6 +32,9 @@ func (sub *TopicSubscriber) doProcess() error { RangeStop: brokerPartitionAssignment.Partition.RangeStop, }, Filter: sub.ContentConfig.Filter, + Offset: &mq_pb.SubscribeRequest_InitMessage_StartTimestampNs{ + StartTimestampNs: sub.alreadyProcessedTsNs, + }, }, }, }) @@ -68,6 +71,7 @@ func (sub *TopicSubscriber) doProcess() error { if !sub.OnEachMessageFunc(m.Data.Key, m.Data.Value) { return } + sub.alreadyProcessedTsNs = m.Data.TsNs case *mq_pb.SubscribeResponse_Ctrl: if m.Ctrl.IsEndOfStream || m.Ctrl.IsEndOfTopic { return diff --git a/weed/mq/client/sub_client/subscribe.go b/weed/mq/client/sub_client/subscribe.go index 0803b2c79..370f5aa3c 100644 --- a/weed/mq/client/sub_client/subscribe.go +++ b/weed/mq/client/sub_client/subscribe.go @@ -4,17 +4,30 @@ import ( "fmt" "github.com/seaweedfs/seaweedfs/weed/util" "io" + "log" + "time" ) // Subscribe subscribes to a topic's specified partitions. // If a partition is moved to another broker, the subscriber will automatically reconnect to the new broker. func (sub *TopicSubscriber) Subscribe() error { + index := -1 util.RetryUntil("subscribe", func() error { + index++ + index = index % len(sub.bootstrapBrokers) // ask balancer for brokers of the topic - if err := sub.doLookup(sub.bootstrapBroker); err != nil { + if err := sub.doLookup(sub.bootstrapBrokers[index]); err != nil { return fmt.Errorf("lookup topic %s/%s: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, err) } + if len(sub.brokerPartitionAssignments) == 0 { + if sub.waitForMoreMessage { + time.Sleep(1 * time.Second) + return fmt.Errorf("no broker partition assignments") + } else { + return nil + } + } // treat the first broker as the topic leader // connect to the leader broker @@ -25,6 +38,8 @@ func (sub *TopicSubscriber) Subscribe() error { return nil }, func(err error) bool { if err == io.EOF { + log.Printf("subscriber %s/%s: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, err) + sub.waitForMoreMessage = false return false } return true diff --git a/weed/mq/client/sub_client/subscriber.go b/weed/mq/client/sub_client/subscriber.go index 809673de1..9b96b14cb 100644 --- a/weed/mq/client/sub_client/subscriber.go +++ b/weed/mq/client/sub_client/subscriber.go @@ -3,6 +3,7 @@ package sub_client import ( "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" "google.golang.org/grpc" + "time" ) type SubscriberConfiguration struct { @@ -19,6 +20,7 @@ type ContentConfiguration struct { Namespace string Topic string Filter string + StartTime time.Time } type OnEachMessageFunc func(key, value []byte) (shouldContinue bool) @@ -30,14 +32,18 @@ type TopicSubscriber struct { brokerPartitionAssignments []*mq_pb.BrokerPartitionAssignment OnEachMessageFunc OnEachMessageFunc OnCompletionFunc OnCompletionFunc - bootstrapBroker string + bootstrapBrokers []string + waitForMoreMessage bool + alreadyProcessedTsNs int64 } -func NewTopicSubscriber(bootstrapBroker string, subscriber *SubscriberConfiguration, content *ContentConfiguration) *TopicSubscriber { +func NewTopicSubscriber(bootstrapBrokers []string, subscriber *SubscriberConfiguration, content *ContentConfiguration) *TopicSubscriber { return &TopicSubscriber{ - SubscriberConfig: subscriber, - ContentConfig: content, - bootstrapBroker: bootstrapBroker, + SubscriberConfig: subscriber, + ContentConfig: content, + bootstrapBrokers: bootstrapBrokers, + waitForMoreMessage: true, + alreadyProcessedTsNs: content.StartTime.UnixNano(), } } @@ -45,6 +51,6 @@ func (sub *TopicSubscriber) SetEachMessageFunc(onEachMessageFn OnEachMessageFunc sub.OnEachMessageFunc = onEachMessageFn } -func (sub *TopicSubscriber) SetCompletionFunc(onCompeletionFn OnCompletionFunc) { - sub.OnCompletionFunc = onCompeletionFn +func (sub *TopicSubscriber) SetCompletionFunc(onCompletionFn OnCompletionFunc) { + sub.OnCompletionFunc = onCompletionFn } |