diff options
Diffstat (limited to 'weed/mq/client/sub_client/subscriber.go')
| -rw-r--r-- | weed/mq/client/sub_client/subscriber.go | 18 |
1 files changed, 12 insertions, 6 deletions
diff --git a/weed/mq/client/sub_client/subscriber.go b/weed/mq/client/sub_client/subscriber.go index 9b96b14cb..9adc5197e 100644 --- a/weed/mq/client/sub_client/subscriber.go +++ b/weed/mq/client/sub_client/subscriber.go @@ -7,10 +7,10 @@ import ( ) type SubscriberConfiguration struct { - ClientId string - GroupId string - GroupInstanceId string - GroupMinimumPeers int32 + ClientId string + ConsumerGroup string + ConsumerGroupInstanceId string + GroupMinimumPeers int32 GroupMaximumPeers int32 BootstrapServers []string GrpcDialOption grpc.DialOption @@ -23,12 +23,17 @@ type ContentConfiguration struct { StartTime time.Time } -type OnEachMessageFunc func(key, value []byte) (shouldContinue bool) +type ProcessorConfiguration struct { + ConcurrentPartitionLimit int // how many partitions to process concurrently +} + +type OnEachMessageFunc func(key, value []byte) (shouldContinue bool, err error) type OnCompletionFunc func() type TopicSubscriber struct { SubscriberConfig *SubscriberConfiguration ContentConfig *ContentConfiguration + ProcessorConfig *ProcessorConfiguration brokerPartitionAssignments []*mq_pb.BrokerPartitionAssignment OnEachMessageFunc OnEachMessageFunc OnCompletionFunc OnCompletionFunc @@ -37,10 +42,11 @@ type TopicSubscriber struct { alreadyProcessedTsNs int64 } -func NewTopicSubscriber(bootstrapBrokers []string, subscriber *SubscriberConfiguration, content *ContentConfiguration) *TopicSubscriber { +func NewTopicSubscriber(bootstrapBrokers []string, subscriber *SubscriberConfiguration, content *ContentConfiguration, processor ProcessorConfiguration) *TopicSubscriber { return &TopicSubscriber{ SubscriberConfig: subscriber, ContentConfig: content, + ProcessorConfig: &processor, bootstrapBrokers: bootstrapBrokers, waitForMoreMessage: true, alreadyProcessedTsNs: content.StartTime.UnixNano(), |