diff options
| author | Farbod <105163300+itsfarbod@users.noreply.github.com> | 2023-12-21 03:55:54 +0330 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-12-20 16:25:54 -0800 |
| commit | c278f49bca0b8253914c5490a17ac4b50b8abe2b (patch) | |
| tree | d7012b5d042f9f0476c29c816f1cec826e37fbbc | |
| parent | 06343f897645c4650f8f65e9a7ecd9b255820cfd (diff) | |
| download | seaweedfs-c278f49bca0b8253914c5490a17ac4b50b8abe2b.tar.xz seaweedfs-c278f49bca0b8253914c5490a17ac4b50b8abe2b.zip | |
Using filer.remote.sync concurrency in filer.remote.gateway (#5123)
- Chnaged ProcessEvenFn to be concurrent just like filer.remote.sync
Co-authored-by: itsfarbod <itsfarbod@hamravesh.com>
| -rw-r--r-- | weed/command/filer_remote_gateway_buckets.go | 18 |
1 files changed, 14 insertions, 4 deletions
diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go index 9694a1c9c..912607847 100644 --- a/weed/command/filer_remote_gateway_buckets.go +++ b/weed/command/filer_remote_gateway_buckets.go @@ -30,10 +30,20 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo return err } - processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error { - lastTime := time.Unix(0, lastTsNs) - glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3)) - return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, lastTsNs) + processor := NewMetadataProcessor(eachEntryFunc, 128) + + var lastLogTsNs = time.Now().UnixNano() + processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error { + processor.AddSyncJob(resp) + return nil + }, 3*time.Second, func(counter int64, lastTsNs int64) error { + if processor.processedTsWatermark == 0 { + return nil + } + now := time.Now().UnixNano() + glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, processor.processedTsWatermark), float64(counter)/(float64(now-lastLogTsNs)/1e9)) + lastLogTsNs = now + return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, processor.processedTsWatermark) }) lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo) |