diff options
| author | Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> | 2024-01-12 23:57:18 +0500 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-01-12 10:57:18 -0800 |
| commit | 1169f943103684ded4d67edac686fd94e8e78ccc (patch) | |
| tree | 107c58b62c76202d32ca7c8cd34cf1af58ce2668 /weed/command/filer_remote_sync_dir.go | |
| parent | 0e8a54f6f67e534d3af01d70ce45bd9cbfe87d42 (diff) | |
| download | seaweedfs-1169f943103684ded4d67edac686fd94e8e78ccc.tar.xz seaweedfs-1169f943103684ded4d67edac686fd94e8e78ccc.zip | |
Fix filer sync set offset (#5197)
* fix: compose 2mount with sync
* fix: DATA RACE
https://github.com/seaweedfs/seaweedfs/issues/5194
https://github.com/seaweedfs/seaweedfs/issues/5195
Diffstat (limited to 'weed/command/filer_remote_sync_dir.go')
| -rw-r--r-- | weed/command/filer_remote_sync_dir.go | 12 |
1 files changed, 6 insertions, 6 deletions
diff --git a/weed/command/filer_remote_sync_dir.go b/weed/command/filer_remote_sync_dir.go index d4305b666..00f6d7493 100644 --- a/weed/command/filer_remote_sync_dir.go +++ b/weed/command/filer_remote_sync_dir.go @@ -33,7 +33,8 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour return err } - processor := NewMetadataProcessor(eachEntryFunc, 128) + lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo) + processor := NewMetadataProcessor(eachEntryFunc, 128, lastOffsetTs.UnixNano()) var lastLogTsNs = time.Now().UnixNano() processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error { @@ -50,18 +51,17 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour processor.AddSyncJob(resp) return nil }, 3*time.Second, func(counter int64, lastTsNs int64) error { - if processor.processedTsWatermark == 0 { + offsetTsNs := processor.processedTsWatermark.Load() + if offsetTsNs == 0 { return nil } // use processor.processedTsWatermark instead of the lastTsNs from the most recent job now := time.Now().UnixNano() - glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, processor.processedTsWatermark), float64(counter)/(float64(now-lastLogTsNs)/1e9)) + glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9)) lastLogTsNs = now - return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, processor.processedTsWatermark) + return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, offsetTsNs) }) - lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo) - option.clientEpoch++ metadataFollowOption := &pb.MetadataFollowOption{ |