diff options
Diffstat (limited to 'weed/server/filer_grpc_server_sub_meta.go')
| -rw-r--r-- | weed/server/filer_grpc_server_sub_meta.go | 78 |
1 files changed, 57 insertions, 21 deletions
diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go index f4df550e6..566f24ad7 100644 --- a/weed/server/filer_grpc_server_sub_meta.go +++ b/weed/server/filer_grpc_server_sub_meta.go @@ -69,14 +69,30 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, if processedTsNs != 0 { lastReadTime = log_buffer.NewMessagePosition(processedTsNs, -2) } else { - nextDayTs := util.GetNextDayTsNano(lastReadTime.Time.UnixNano()) - position := log_buffer.NewMessagePosition(nextDayTs, -2) - found, err := fs.filer.HasPersistedLogFiles(position) - if err != nil { - return fmt.Errorf("checking persisted log files: %w", err) - } - if found { - lastReadTime = position + // No data found on disk + // Check if we previously got ResumeFromDiskError from memory, meaning we're in a gap + if errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) { + // We have a gap: requested time < earliest memory time, but no data on disk + // Skip forward to earliest memory time to avoid infinite loop + earliestTime := fs.filer.MetaAggregator.MetaLogBuffer.GetEarliestTime() + if !earliestTime.IsZero() && earliestTime.After(lastReadTime.Time) { + glog.V(3).Infof("gap detected: skipping from %v to earliest memory time %v for %v", + lastReadTime.Time, earliestTime, clientName) + // Position at earliest time; time-based reader will include it + lastReadTime = log_buffer.NewMessagePosition(earliestTime.UnixNano(), -2) + readInMemoryLogErr = nil // Clear the error since we're skipping forward + } + } else { + // First pass or no ResumeFromDiskError yet - check the next day for logs + nextDayTs := util.GetNextDayTsNano(lastReadTime.Time.UnixNano()) + position := log_buffer.NewMessagePosition(nextDayTs, -2) + found, err := fs.filer.HasPersistedLogFiles(position) + if err != nil { + return fmt.Errorf("checking persisted log files: %w", err) + } + if found { + lastReadTime = position + } } } @@ -91,12 +107,16 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, } fs.filer.MetaAggregator.ListenersLock.Lock() + atomic.AddInt64(&fs.filer.MetaAggregator.ListenersWaits, 1) fs.filer.MetaAggregator.ListenersCond.Wait() + atomic.AddInt64(&fs.filer.MetaAggregator.ListenersWaits, -1) fs.filer.MetaAggregator.ListenersLock.Unlock() return fs.hasClient(req.ClientId, req.ClientEpoch) }, eachLogEntryFn) if readInMemoryLogErr != nil { if errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) { + // Memory says data is too old - will read from disk on next iteration + // But if disk also has no data (gap in history), we'll skip forward continue } glog.Errorf("processed to %v: %v", lastReadTime, readInMemoryLogErr) @@ -166,23 +186,39 @@ func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq if processedTsNs != 0 { lastReadTime = log_buffer.NewMessagePosition(processedTsNs, -2) } else { + // No data found on disk + // Check if we previously got ResumeFromDiskError from memory, meaning we're in a gap if readInMemoryLogErr == log_buffer.ResumeFromDiskError { - time.Sleep(1127 * time.Millisecond) - continue - } - // If no persisted entries were read for this day, check the next day for logs - nextDayTs := util.GetNextDayTsNano(lastReadTime.Time.UnixNano()) - position := log_buffer.NewMessagePosition(nextDayTs, -2) - found, err := fs.filer.HasPersistedLogFiles(position) - if err != nil { - return fmt.Errorf("checking persisted log files: %w", err) - } - if found { - lastReadTime = position + // We have a gap: requested time < earliest memory time, but no data on disk + // Skip forward to earliest memory time to avoid infinite loop + earliestTime := fs.filer.LocalMetaLogBuffer.GetEarliestTime() + if !earliestTime.IsZero() && earliestTime.After(lastReadTime.Time) { + glog.V(3).Infof("gap detected: skipping from %v to earliest memory time %v for %v", + lastReadTime.Time, earliestTime, clientName) + // Position at earliest time; time-based reader will include it + lastReadTime = log_buffer.NewMessagePosition(earliestTime.UnixNano(), -2) + readInMemoryLogErr = nil // Clear the error since we're skipping forward + } else { + // No memory data yet, just wait + time.Sleep(1127 * time.Millisecond) + continue + } + } else { + // First pass or no ResumeFromDiskError yet + // Check the next day for logs + nextDayTs := util.GetNextDayTsNano(lastReadTime.Time.UnixNano()) + position := log_buffer.NewMessagePosition(nextDayTs, -2) + found, err := fs.filer.HasPersistedLogFiles(position) + if err != nil { + return fmt.Errorf("checking persisted log files: %w", err) + } + if found { + lastReadTime = position + } } } - glog.V(0).Infof("read in memory %v local subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime) + glog.V(3).Infof("read in memory %v local subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime) lastReadTime, isDone, readInMemoryLogErr = fs.filer.LocalMetaLogBuffer.LoopProcessLogData("localMeta:"+clientName, lastReadTime, req.UntilNs, func() bool { |