1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
|
package msgclient
import (
"context"
"io"
"time"
"github.com/chrislusf/seaweedfs/weed/messaging/broker"
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
"google.golang.org/grpc"
)
type Subscriber struct {
subscriberClients []messaging_pb.SeaweedMessaging_SubscribeClient
subscriberId string
}
func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, startTime time.Time) (*Subscriber, error) {
// read topic configuration
topicConfiguration := &messaging_pb.TopicConfiguration{
PartitionCount: 4,
}
subscriberClients := make([]messaging_pb.SeaweedMessaging_SubscribeClient, topicConfiguration.PartitionCount)
for i := 0; i < int(topicConfiguration.PartitionCount); i++ {
tp := broker.TopicPartition{
Namespace: namespace,
Topic: topic,
Partition: int32(i),
}
grpcClientConn, err := mc.findBroker(tp)
if err != nil {
return nil, err
}
client, err := setupSubscriberClient(grpcClientConn, tp, subscriberId, startTime)
if err != nil {
return nil, err
}
subscriberClients[i] = client
}
return &Subscriber{
subscriberClients: subscriberClients,
subscriberId: subscriberId,
}, nil
}
func setupSubscriberClient(grpcConnection *grpc.ClientConn, tp broker.TopicPartition, subscriberId string, startTime time.Time) (stream messaging_pb.SeaweedMessaging_SubscribeClient, err error) {
stream, err = messaging_pb.NewSeaweedMessagingClient(grpcConnection).Subscribe(context.Background())
if err != nil {
return
}
// send init message
err = stream.Send(&messaging_pb.SubscriberMessage{
Init: &messaging_pb.SubscriberMessage_InitMessage{
Namespace: tp.Namespace,
Topic: tp.Topic,
Partition: tp.Partition,
StartPosition: messaging_pb.SubscriberMessage_InitMessage_TIMESTAMP,
TimestampNs: startTime.UnixNano(),
SubscriberId: subscriberId,
},
})
if err != nil {
return
}
// process init response
_, err = stream.Recv()
if err != nil {
return
}
return stream, nil
}
func (s *Subscriber) doSubscribe(partition int, processFn func(m *messaging_pb.Message)) error {
for {
resp, listenErr := s.subscriberClients[partition].Recv()
if listenErr == io.EOF {
return nil
}
if listenErr != nil {
println(listenErr.Error())
return listenErr
}
if resp.Data == nil {
// this could be heartbeat from broker
continue
}
processFn(resp.Data)
}
}
// Subscribe starts goroutines to process the messages
func (s *Subscriber) Subscribe(processFn func(m *messaging_pb.Message)) {
for i := 0; i < len(s.subscriberClients); i++ {
go s.doSubscribe(i, processFn)
}
}
|
