aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorchrislu <chris.lu@gmail.com>2024-08-27 09:02:39 -0700
committerchrislu <chris.lu@gmail.com>2024-08-27 09:02:39 -0700
commitbebc385607fac28226149fcbb1db5f48d03995de (patch)
tree775ed4d30b2866df2ecc82a3e8182aea5b2269f8
parent62102d1afc713694cf9027059b45f679e9e19f5e (diff)
downloadseaweedfs-bebc385607fac28226149fcbb1db5f48d03995de.tar.xz
seaweedfs-bebc385607fac28226149fcbb1db5f48d03995de.zip
randomly pick one replica to write
-rw-r--r--weed/operation/submit.go34
1 files changed, 24 insertions, 10 deletions
diff --git a/weed/operation/submit.go b/weed/operation/submit.go
index 516478dbe..66d5542af 100644
--- a/weed/operation/submit.go
+++ b/weed/operation/submit.go
@@ -4,6 +4,7 @@ import (
"context"
"github.com/seaweedfs/seaweedfs/weed/pb"
"io"
+ "math/rand"
"mime"
"net/url"
"os"
@@ -179,11 +180,8 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
id += "_" + strconv.FormatInt(i, 10)
}
}
- fileUrl := "http://" + ret.Url + "/" + id
- if usePublicUrl {
- fileUrl = "http://" + ret.PublicUrl + "/" + id
- }
- count, e := upload_one_chunk(
+ fileUrl := genFileUrl(ret, id, usePublicUrl)
+ count, e := uploadOneChunk(
baseName+"-"+strconv.FormatInt(i+1, 10),
io.LimitReader(fi.Reader, chunkSize),
masterFn, fileUrl,
@@ -202,7 +200,7 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
)
retSize += count
}
- err = upload_chunked_file_manifest(fileUrl, &cm, jwt)
+ err = uploadChunkedFileManifest(fileUrl, &cm, jwt)
if err != nil {
// delete all uploaded chunks
cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
@@ -217,7 +215,7 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
PairMap: nil,
Jwt: jwt,
}
-
+
uploader, e := NewUploader()
if e != nil {
return 0, e
@@ -232,7 +230,23 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
return
}
-func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn,
+func genFileUrl(ret *AssignResult, id string, usePublicUrl bool) string {
+ fileUrl := "http://" + ret.Url + "/" + id
+ if usePublicUrl {
+ fileUrl = "http://" + ret.PublicUrl + "/" + id
+ }
+ for _, replica := range ret.Replicas {
+ if rand.Intn(len(ret.Replicas)+1) == 0 {
+ fileUrl = "http://" + replica.Url + "/" + id
+ if usePublicUrl {
+ fileUrl = "http://" + replica.PublicUrl + "/" + id
+ }
+ }
+ }
+ return fileUrl
+}
+
+func uploadOneChunk(filename string, reader io.Reader, masterFn GetMasterFn,
fileUrl string, jwt security.EncodedJwt,
) (size uint32, e error) {
glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
@@ -258,7 +272,7 @@ func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn,
return uploadResult.Size, nil
}
-func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt security.EncodedJwt) error {
+func uploadChunkedFileManifest(fileUrl string, manifest *ChunkManifest, jwt security.EncodedJwt) error {
buf, e := manifest.Marshal()
if e != nil {
return e
@@ -277,7 +291,7 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s
PairMap: nil,
Jwt: jwt,
}
-
+
uploader, e := NewUploader()
if e != nil {
return e