diff options
| -rw-r--r-- | weed/operation/upload_content.go | 14 | ||||
| -rw-r--r-- | weed/util/retry.go | 32 |
2 files changed, 40 insertions, 6 deletions
diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 0c3e29a43..17e767472 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -5,11 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "github.com/seaweedfs/seaweedfs/weed/security" - "github.com/seaweedfs/seaweedfs/weed/stats" - "github.com/seaweedfs/seaweedfs/weed/util" "io" "mime" "mime/multipart" @@ -19,6 +14,12 @@ import ( "path/filepath" "strings" "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/util" ) type UploadOption struct { @@ -120,7 +121,8 @@ func UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.A return true }) } else { - err = util.Retry("uploadWithRetry", doUploadFunc) + uploadErrList := []string{"transport", "is read only"} + err = util.MultiRetry("uploadWithRetry", uploadErrList, doUploadFunc) } return diff --git a/weed/util/retry.go b/weed/util/retry.go index fee6074b2..6be052f77 100644 --- a/weed/util/retry.go +++ b/weed/util/retry.go @@ -32,6 +32,29 @@ func Retry(name string, job func() error) (err error) { return err } +func MultiRetry(name string, errList []string, job func() error) (err error) { + waitTime := time.Second + hasErr := false + for waitTime < RetryWaitTime { + err = job() + if err == nil { + if hasErr { + glog.V(0).Infof("retry %s successfully", name) + } + break + } + if containErr(err.Error(), errList) { + hasErr = true + glog.V(0).Infof("retry %s: err: %v", name, err) + } else { + break + } + time.Sleep(waitTime) + waitTime += waitTime / 2 + } + return err +} + func RetryForever(name string, job func() error, onErrFn func(err error) (shouldContinue bool)) { waitTime := time.Second for { @@ -62,3 +85,12 @@ func Nvl(values ...string) string { } return "" } + +func containErr(err string, errList []string) bool { + for _, e := range errList { + if strings.Contains(err, e) { + return true + } + } + return false +} |