diff options
| author | Chris Lu <chrislusf@users.noreply.github.com> | 2021-04-28 10:37:18 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-04-28 10:37:18 -0700 |
| commit | ef94ff6837fe6e5700ec6cc7b5098735752fd9cf (patch) | |
| tree | 1642bdcfb53bc6a3f45ab5f836ac4e098ced516d | |
| parent | 8ae3ea4b1de528f809eadaaef1af19e46e7b3749 (diff) | |
| parent | c2269123d3e8de2ea659a87712cc44dcdc4b636b (diff) | |
| download | seaweedfs-ef94ff6837fe6e5700ec6cc7b5098735752fd9cf.tar.xz seaweedfs-ef94ff6837fe6e5700ec6cc7b5098735752fd9cf.zip | |
Merge pull request #2035 from kmlebedev/fix_chunks_etag
fix aws style Etag for chunks
| -rw-r--r-- | weed/filer/filechunks.go | 6 | ||||
| -rw-r--r-- | weed/operation/upload_content.go | 2 | ||||
| -rw-r--r-- | weed/server/filer_server_handlers_write_upload.go | 6 |
3 files changed, 9 insertions, 5 deletions
diff --git a/weed/filer/filechunks.go b/weed/filer/filechunks.go index 68f308a51..346eb3cfb 100644 --- a/weed/filer/filechunks.go +++ b/weed/filer/filechunks.go @@ -2,7 +2,6 @@ package filer import ( "bytes" - "encoding/hex" "fmt" "github.com/chrislusf/seaweedfs/weed/wdclient" "math" @@ -43,12 +42,11 @@ func ETagEntry(entry *Entry) (etag string) { func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) { if len(chunks) == 1 { - return chunks[0].ETag + return fmt.Sprintf("%x", util.Base64Md5ToBytes(chunks[0].ETag)) } md5_digests := [][]byte{} for _, c := range chunks { - md5_decoded, _ := hex.DecodeString(c.ETag) - md5_digests = append(md5_digests, md5_decoded) + md5_digests = append(md5_digests, util.Base64Md5ToBytes(c.ETag)) } return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5_digests, nil)), len(chunks)) } diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 944186eeb..8e7c6f733 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -39,7 +39,7 @@ func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *fi Offset: offset, Size: uint64(uploadResult.Size), Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + ETag: uploadResult.ContentMd5, CipherKey: uploadResult.CipherKey, IsCompressed: uploadResult.Gzip > 0, Fid: fid, diff --git a/weed/server/filer_server_handlers_write_upload.go b/weed/server/filer_server_handlers_write_upload.go index 3ab45453e..b15deb9d1 100644 --- a/weed/server/filer_server_handlers_write_upload.go +++ b/weed/server/filer_server_handlers_write_upload.go @@ -1,6 +1,7 @@ package weed_server import ( + "bytes" "crypto/md5" "hash" "io" @@ -71,6 +72,11 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque if uploadResult.Size == 0 { break } + uploadedMd5 := util.Base64Md5ToBytes(uploadResult.ContentMd5) + readedMd5 := md5Hash.Sum(nil) + if !bytes.Equal(uploadedMd5, readedMd5) { + glog.Errorf("md5 %x does not match %x uploaded chunk %s to the volume server", readedMd5, uploadedMd5, uploadResult.Name) + } // Save to chunk manifest structure fileChunks = append(fileChunks, uploadResult.ToPbFileChunk(fileId, chunkOffset)) |