diff options
| author | Stewart Miles <stewart@agentic.ai> | 2023-03-15 17:49:46 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-03-15 17:49:46 -0700 |
| commit | 57ab1f851656276f62124c3dc5e3fc0d15eed646 (patch) | |
| tree | e9dcc6fbf41c450a8e841a005105386a502876d2 | |
| parent | dd71f54c6b8355a35cda134173d9ebb76a53b62c (diff) | |
| download | seaweedfs-57ab1f851656276f62124c3dc5e3fc0d15eed646.tar.xz seaweedfs-57ab1f851656276f62124c3dc5e3fc0d15eed646.zip | |
Use exponential backoff to query leader. (#4313)
`topology.Leader()` was using a backoff that typically
resulted in at least a 5s delay when initially starting
a master and raft server. This changes the backoff
algorithm to use exponential backoff starting with 100ms
and waiting up to 20s for leader selection.
Related to #4307
| -rw-r--r-- | go.mod | 2 | ||||
| -rw-r--r-- | go.sum | 2 | ||||
| -rw-r--r-- | weed/topology/topology.go | 19 |
3 files changed, 10 insertions, 13 deletions
@@ -10,6 +10,7 @@ require ( github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Shopify/sarama v1.38.1 github.com/aws/aws-sdk-go v1.44.219 + github.com/cenkalti/backoff/v4 v4.2.0 github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -176,6 +177,7 @@ require ( github.com/aws/smithy-go v1.13.5 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/boltdb/bolt v1.3.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.0 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/d4l3k/messagediff v1.2.1 // indirect github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect @@ -691,10 +691,12 @@ github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgIS github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/weed/topology/topology.go b/weed/topology/topology.go index c7b492267..825c02961 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -11,6 +11,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/storage/types" + backoff "github.com/cenkalti/backoff/v4" + hashicorpRaft "github.com/hashicorp/raft" "github.com/seaweedfs/raft" @@ -96,19 +98,10 @@ func (t *Topology) IsLeader() bool { } func (t *Topology) Leader() (l pb.ServerAddress, err error) { - for count := 0; count < 3; count++ { - l, err = t.MaybeLeader() - if err != nil { - return - } - if l != "" { - break - } - - time.Sleep(time.Duration(5+count) * time.Second) - } - - return + exponentialBackoff := backoff.NewExponentialBackOff() + exponentialBackoff.InitialInterval = 100 * time.Millisecond + exponentialBackoff.MaxElapsedTime = 20 * time.Second + return backoff.RetryWithData(t.MaybeLeader, exponentialBackoff) } func (t *Topology) MaybeLeader() (l pb.ServerAddress, err error) { |