1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
package webhook
import (
"fmt"
"net/url"
"slices"
"strconv"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
const (
queueName = "webhook"
pubSubTopicName = "webhook_topic"
deadLetterTopic = "webhook_dead_letter"
)
type eventType string
const (
eventTypeCreate eventType = "create"
eventTypeDelete eventType = "delete"
eventTypeUpdate eventType = "update"
eventTypeRename eventType = "rename"
)
func (e eventType) valid() bool {
return slices.Contains([]eventType{
eventTypeCreate,
eventTypeDelete,
eventTypeUpdate,
eventTypeRename,
},
e,
)
}
var (
pubSubHandlerNameTemplate = func(n int) string {
return "webhook_handler_" + strconv.Itoa(n)
}
)
type client interface {
sendMessage(message *webhookMessage) error
}
type webhookMessage struct {
Key string `json:"key"`
EventType string `json:"event_type"`
Notification *filer_pb.EventNotification `json:"message_data"`
}
func newWebhookMessage(key string, message proto.Message) *webhookMessage {
notification, ok := message.(*filer_pb.EventNotification)
if !ok {
return nil
}
eventType := string(detectEventType(notification))
return &webhookMessage{
Key: key,
EventType: eventType,
Notification: notification,
}
}
type config struct {
endpoint string
authBearerToken string
timeoutSeconds int
maxRetries int
backoffSeconds int
maxBackoffSeconds int
nWorkers int
bufferSize int
eventTypes []string
pathPrefixes []string
}
func newConfigWithDefaults(configuration util.Configuration, prefix string) *config {
c := &config{
endpoint: configuration.GetString(prefix + "endpoint"),
authBearerToken: configuration.GetString(prefix + "bearer_token"),
timeoutSeconds: 10,
maxRetries: 3,
backoffSeconds: 3,
maxBackoffSeconds: 30,
nWorkers: 5,
bufferSize: 10_000,
}
if bufferSize := configuration.GetInt(prefix + "buffer_size"); bufferSize > 0 {
c.bufferSize = bufferSize
}
if workers := configuration.GetInt(prefix + "workers"); workers > 0 {
c.nWorkers = workers
}
if maxRetries := configuration.GetInt(prefix + "max_retries"); maxRetries > 0 {
c.maxRetries = maxRetries
}
if backoffSeconds := configuration.GetInt(prefix + "backoff_seconds"); backoffSeconds > 0 {
c.backoffSeconds = backoffSeconds
}
if maxBackoffSeconds := configuration.GetInt(prefix + "max_backoff_seconds"); maxBackoffSeconds > 0 {
c.maxBackoffSeconds = maxBackoffSeconds
}
if timeout := configuration.GetInt(prefix + "timeout_seconds"); timeout > 0 {
c.timeoutSeconds = timeout
}
c.eventTypes = configuration.GetStringSlice(prefix + "event_types")
c.pathPrefixes = configuration.GetStringSlice(prefix + "path_prefixes")
return c
}
func (c *config) validate() error {
if c.endpoint == "" {
return fmt.Errorf("webhook endpoint is required")
}
_, err := url.Parse(c.endpoint)
if err != nil {
return fmt.Errorf("invalid webhook endpoint: %w", err)
}
if c.timeoutSeconds < 1 || c.timeoutSeconds > 300 {
return fmt.Errorf("timeout must be between 1 and 300 seconds, got %d", c.timeoutSeconds)
}
if c.maxRetries < 0 || c.maxRetries > 10 {
return fmt.Errorf("max retries must be between 0 and 10, got %d", c.maxRetries)
}
if c.backoffSeconds < 1 || c.backoffSeconds > 60 {
return fmt.Errorf("backoff seconds must be between 1 and 60, got %d", c.backoffSeconds)
}
if c.maxBackoffSeconds < c.backoffSeconds || c.maxBackoffSeconds > 300 {
return fmt.Errorf("max backoff seconds must be between %d and 300, got %d", c.backoffSeconds, c.maxBackoffSeconds)
}
if c.nWorkers < 1 || c.nWorkers > 100 {
return fmt.Errorf("workers must be between 1 and 100, got %d", c.nWorkers)
}
if c.bufferSize < 100 || c.bufferSize > 1_000_000 {
return fmt.Errorf("buffer size must be between 100 and 1,000,000, got %d", c.bufferSize)
}
return nil
}
func detectEventType(notification *filer_pb.EventNotification) eventType {
hasOldEntry := notification.OldEntry != nil
hasNewEntry := notification.NewEntry != nil
hasNewParentPath := notification.NewParentPath != ""
if !hasOldEntry && hasNewEntry {
return eventTypeCreate
}
if hasOldEntry && !hasNewEntry {
return eventTypeDelete
}
if hasOldEntry && hasNewEntry {
if hasNewParentPath {
return eventTypeRename
}
return eventTypeUpdate
}
return eventTypeUpdate
}
|
