diff options
| author | Chris Lu <chris.lu@gmail.com> | 2020-10-23 23:05:55 -0700 |
|---|---|---|
| committer | Chris Lu <chris.lu@gmail.com> | 2020-10-23 23:05:55 -0700 |
| commit | ef9c32ea0da7baa1b8d8a968c1776714c228c597 (patch) | |
| tree | ddfdb25db14f9718ce1efbce72675d0fc44d40bc | |
| parent | 12eafd634d564bf65d49fb2d0fa346e8f21f2e89 (diff) | |
| download | seaweedfs-ef9c32ea0da7baa1b8d8a968c1776714c228c597.tar.xz seaweedfs-ef9c32ea0da7baa1b8d8a968c1776714c228c597.zip | |
adjust compression threshold
| -rw-r--r-- | weed/filesys/dirty_page.go | 6 | ||||
| -rw-r--r-- | weed/operation/upload_content.go | 2 |
2 files changed, 5 insertions, 3 deletions
diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 6fda134aa..a7e3decdc 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -9,10 +9,12 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" ) var ( concurrentWriterLimit = runtime.NumCPU() + concurrentWriters = util.NewLimitedConcurrentExecutor(4 * concurrentWriterLimit) ) type ContinuousDirtyPages struct { @@ -107,7 +109,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, mtime := time.Now().UnixNano() pages.writeWaitGroup.Add(1) - go func() { + concurrentWriters.Execute(func() { defer pages.writeWaitGroup.Done() dir, _ := pages.f.fullpath().DirAndName() @@ -123,7 +125,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, pages.collection, pages.replication = collection, replication pages.f.addChunks([]*filer_pb.FileChunk{chunk}) glog.V(3).Infof("%s saveToStorage [%d,%d)", pages.f.fullpath(), offset, offset+size) - }() + }) } func max(x, y int64) int64 { diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 5ba9e37a0..dc92125a8 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -119,7 +119,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i } if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed { shouldGzipNow = true - } else if !iAmSure && mtype == "" && len(data) > 128 { + } else if !iAmSure && mtype == "" && len(data) > 16*1024 { var compressed []byte compressed, err = util.GzipData(data[0:128]) shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90% |