diff options
| author | Chris Lu <chris.lu@gmail.com> | 2020-03-26 23:50:48 -0700 |
|---|---|---|
| committer | Chris Lu <chris.lu@gmail.com> | 2020-03-26 23:50:48 -0700 |
| commit | 2286eda575861cceb4464bf677e9885e069de385 (patch) | |
| tree | 2a65a892477c571094e480ee351e0517c402fb50 | |
| parent | 546356091218c69078c5198549d1d556a6fe9575 (diff) | |
| download | seaweedfs-2286eda575861cceb4464bf677e9885e069de385.tar.xz seaweedfs-2286eda575861cceb4464bf677e9885e069de385.zip | |
sampling whether the data can be gzipped
| -rw-r--r-- | weed/operation/upload_content.go | 13 |
1 files changed, 11 insertions, 2 deletions
diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 75775d7d0..b74663c36 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -78,9 +78,13 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i contentIsGzipped := isInputGzipped shouldGzipNow := false if !isInputGzipped { - if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); mtype == "" || iAmSure && shouldBeZipped { + if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped { shouldGzipNow = true contentIsGzipped = true + } else if len(data) > 128 { + var compressed []byte + compressed, err = util.GzipData(data[0:128]) + shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90% } } @@ -90,7 +94,12 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i // this could be double copying clearDataLen = len(data) if shouldGzipNow { - data, err = util.GzipData(data) + compressed, compressErr := util.GzipData(data) + // fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed)) + if compressErr == nil { + data = compressed + contentIsGzipped = true + } } else if isInputGzipped { // just to get the clear data length clearData, err := util.UnGzipData(data) |