add RetryForever

1 files changed, 20 insertions, 0 deletions

diff --git a/weed/util/retry.go b/weed/util/retry.go
index 7b0f2d3c3..4b34d5129 100644
--- a/weed/util/retry.go
+++ b/weed/util/retry.go
@@ -32,6 +32,26 @@ func Retry(name string, job func() error) (err error) {
 	return err
 }
 
+func RetryForever(name string, job func() error, onErrFn func(err error) bool) {
+	waitTime := time.Second
+	for {
+		err = job()
+		if err == nil {
+			break
+		}
+		if onErrFn(err) {
+			if strings.Contains(err.Error(), "transport") {
+				glog.V(0).Infof("retry %s: err: %v", name, err)
+				time.Sleep(waitTime)
+				if waitTime < RetryWaitTime {
+					waitTime += waitTime / 2
+				}
+			}
+			continue
+		}
+	}
+}
+
 // return the first non empty string
 func Nvl(values ...string) string {
 	for _, s := range values {