mount:exponentially backoff if read error for about 10 minutes

2 files changed, 18 insertions, 13 deletions

diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.go
index 200fde438..0f84de4cd 100644
--- a/weed/filer/filechunk_manifest.go
+++ b/weed/filer/filechunk_manifest.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"time"
 
 	"github.com/golang/protobuf/proto"
 
@@ -89,23 +90,27 @@ func fetchChunk(lookupFileIdFn LookupFileIdFunctionType, fileId string, cipherKe
 		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
 		return nil, err
 	}
-	return fetchChunkData(urlStrings, cipherKey, isGzipped, true, 0, 0)
+	return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, true, 0, 0)
 }
 
-func fetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {
+func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {
 
 	var err error
 	var buffer bytes.Buffer
-	for _, urlString := range urlStrings {
-		err = util.ReadUrlAsStream(urlString, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
-			buffer.Write(data)
-		})
-		if err != nil {
-			glog.V(0).Infof("read %s failed, err: %v", urlString, err)
-			buffer.Reset()
-		} else {
-			break
+
+	for waitTime := time.Second; waitTime < 10*time.Minute; waitTime+=waitTime/2 {
+		for _, urlString := range urlStrings {
+			err = util.ReadUrlAsStream(urlString, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+				buffer.Write(data)
+			})
+			if err != nil {
+				glog.V(0).Infof("read %s failed, err: %v", urlString, err)
+				buffer.Reset()
+			} else {
+				break
+			}
 		}
+		time.Sleep(waitTime)
 	}
 
 	return buffer.Bytes(), err
diff --git a/weed/filer/stream.go b/weed/filer/stream.go
index e1be18f69..f6e2a7643 100644
--- a/weed/filer/stream.go
+++ b/weed/filer/stream.go
@@ -33,7 +33,7 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f
 
 		urlStrings := fileId2Url[chunkView.FileId]
 
-		data, err := fetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
+		data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
 		if err == nil {
 			return err
 		}
@@ -64,7 +64,7 @@ func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk)
 			return nil, err
 		}
 
-		data, err := fetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
+		data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
 		if err != nil {
 			return nil, err
 		}