diff options
Diffstat (limited to 'weed/filer/reader_at.go')
| -rw-r--r-- | weed/filer/reader_at.go | 47 |
1 files changed, 41 insertions, 6 deletions
diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index 27d773f49..aeac9b34a 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -26,15 +26,42 @@ type ChunkReadAt struct { var _ = io.ReaderAt(&ChunkReadAt{}) var _ = io.Closer(&ChunkReadAt{}) +// LookupFn creates a basic volume location lookup function with simple caching. +// +// Deprecated: Use wdclient.FilerClient instead. This function has several limitations compared to wdclient.FilerClient: +// - Simple bounded cache (10k entries, no eviction policy or TTL for stale entries) +// - No singleflight deduplication (concurrent requests for same volume will duplicate work) +// - No cache history for volume moves (no fallback chain when volumes migrate) +// - No high availability (single filer address, no automatic failover) +// +// For NEW code, especially mount operations, use wdclient.FilerClient instead: +// filerClient := wdclient.NewFilerClient(filerAddresses, grpcDialOption, dataCenter, opts) +// lookupFn := filerClient.GetLookupFileIdFunction() +// +// This provides: +// - Bounded cache with configurable size +// - Singleflight deduplication of concurrent lookups +// - Cache history when volumes move +// - Battle-tested vidMap with cache chain +// +// This function is kept for backward compatibility with existing code paths +// (shell commands, streaming, etc.) but should be avoided in long-running processes +// or multi-tenant deployments where unbounded memory growth is a concern. +// +// Maximum recommended cache entries: ~10,000 volumes per process. +// Beyond this, consider migrating to wdclient.FilerClient. func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionType { vidCache := make(map[string]*filer_pb.Locations) - var vicCacheLock sync.RWMutex + var vidCacheLock sync.RWMutex + cacheSize := 0 + const maxCacheSize = 10000 // Simple bound to prevent unbounded growth + return func(ctx context.Context, fileId string) (targetUrls []string, err error) { vid := VolumeId(fileId) - vicCacheLock.RLock() + vidCacheLock.RLock() locations, found := vidCache[vid] - vicCacheLock.RUnlock() + vidCacheLock.RUnlock() if !found { util.Retry("lookup volume "+vid, func() error { @@ -51,9 +78,17 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp glog.V(0).InfofCtx(ctx, "failed to locate %s", fileId) return fmt.Errorf("failed to locate %s", fileId) } - vicCacheLock.Lock() - vidCache[vid] = locations - vicCacheLock.Unlock() + vidCacheLock.Lock() + // Simple size limit to prevent unbounded growth + // For proper cache management, use wdclient.FilerClient instead + if cacheSize < maxCacheSize { + vidCache[vid] = locations + cacheSize++ + } else if cacheSize == maxCacheSize { + glog.Warningf("filer.LookupFn cache reached limit of %d volumes, not caching new entries. Consider migrating to wdclient.FilerClient for bounded cache management.", maxCacheSize) + cacheSize++ // Only log once + } + vidCacheLock.Unlock() return nil }) |