aboutsummaryrefslogtreecommitdiff
path: root/weed/filer/empty_folder_cleanup/empty_folder_cleaner_test.go
diff options
context:
space:
mode:
Diffstat (limited to 'weed/filer/empty_folder_cleanup/empty_folder_cleaner_test.go')
-rw-r--r--weed/filer/empty_folder_cleanup/empty_folder_cleaner_test.go569
1 files changed, 569 insertions, 0 deletions
diff --git a/weed/filer/empty_folder_cleanup/empty_folder_cleaner_test.go b/weed/filer/empty_folder_cleanup/empty_folder_cleaner_test.go
new file mode 100644
index 000000000..fbc05ccf8
--- /dev/null
+++ b/weed/filer/empty_folder_cleanup/empty_folder_cleaner_test.go
@@ -0,0 +1,569 @@
+package empty_folder_cleanup
+
+import (
+ "testing"
+ "time"
+
+ "github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+ "github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+func Test_isUnderPath(t *testing.T) {
+ tests := []struct {
+ name string
+ child string
+ parent string
+ expected bool
+ }{
+ {"child under parent", "/buckets/mybucket/folder/file.txt", "/buckets", true},
+ {"child is parent", "/buckets", "/buckets", true},
+ {"child not under parent", "/other/path", "/buckets", false},
+ {"empty parent", "/any/path", "", true},
+ {"root parent", "/any/path", "/", true},
+ {"parent with trailing slash", "/buckets/mybucket", "/buckets/", true},
+ {"similar prefix but not under", "/buckets-other/file", "/buckets", false},
+ {"deeply nested", "/buckets/a/b/c/d/e/f", "/buckets/a/b", true},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := isUnderPath(tt.child, tt.parent)
+ if result != tt.expected {
+ t.Errorf("isUnderPath(%q, %q) = %v, want %v", tt.child, tt.parent, result, tt.expected)
+ }
+ })
+ }
+}
+
+func Test_isUnderBucketPath(t *testing.T) {
+ tests := []struct {
+ name string
+ directory string
+ bucketPath string
+ expected bool
+ }{
+ // Should NOT process - bucket path itself
+ {"bucket path itself", "/buckets", "/buckets", false},
+ // Should NOT process - bucket directory (immediate child)
+ {"bucket directory", "/buckets/mybucket", "/buckets", false},
+ // Should process - folder inside bucket
+ {"folder in bucket", "/buckets/mybucket/folder", "/buckets", true},
+ // Should process - nested folder
+ {"nested folder", "/buckets/mybucket/a/b/c", "/buckets", true},
+ // Should NOT process - outside buckets
+ {"outside buckets", "/other/path", "/buckets", false},
+ // Empty bucket path allows all
+ {"empty bucket path", "/any/path", "", true},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := isUnderBucketPath(tt.directory, tt.bucketPath)
+ if result != tt.expected {
+ t.Errorf("isUnderBucketPath(%q, %q) = %v, want %v", tt.directory, tt.bucketPath, result, tt.expected)
+ }
+ })
+ }
+}
+
+func TestEmptyFolderCleaner_ownsFolder(t *testing.T) {
+ // Create a LockRing with multiple servers
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+
+ servers := []pb.ServerAddress{
+ "filer1:8888",
+ "filer2:8888",
+ "filer3:8888",
+ }
+ lockRing.SetSnapshot(servers)
+
+ // Create cleaner for filer1
+ cleaner1 := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ }
+
+ // Create cleaner for filer2
+ cleaner2 := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer2:8888",
+ }
+
+ // Create cleaner for filer3
+ cleaner3 := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer3:8888",
+ }
+
+ // Test that exactly one filer owns each folder
+ testFolders := []string{
+ "/buckets/mybucket/folder1",
+ "/buckets/mybucket/folder2",
+ "/buckets/mybucket/folder3",
+ "/buckets/mybucket/a/b/c",
+ "/buckets/otherbucket/x",
+ }
+
+ for _, folder := range testFolders {
+ ownCount := 0
+ if cleaner1.ownsFolder(folder) {
+ ownCount++
+ }
+ if cleaner2.ownsFolder(folder) {
+ ownCount++
+ }
+ if cleaner3.ownsFolder(folder) {
+ ownCount++
+ }
+
+ if ownCount != 1 {
+ t.Errorf("folder %q owned by %d filers, expected exactly 1", folder, ownCount)
+ }
+ }
+}
+
+func TestEmptyFolderCleaner_ownsFolder_singleServer(t *testing.T) {
+ // Create a LockRing with a single server
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ }
+
+ // Single filer should own all folders
+ testFolders := []string{
+ "/buckets/mybucket/folder1",
+ "/buckets/mybucket/folder2",
+ "/buckets/otherbucket/x",
+ }
+
+ for _, folder := range testFolders {
+ if !cleaner.ownsFolder(folder) {
+ t.Errorf("single filer should own folder %q", folder)
+ }
+ }
+}
+
+func TestEmptyFolderCleaner_ownsFolder_emptyRing(t *testing.T) {
+ // Create an empty LockRing
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ }
+
+ // With empty ring, should own all folders
+ if !cleaner.ownsFolder("/buckets/mybucket/folder") {
+ t.Error("should own folder with empty ring")
+ }
+}
+
+func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanup(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: true,
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ stopCh: make(chan struct{}),
+ }
+
+ folder := "/buckets/mybucket/testfolder"
+ now := time.Now()
+
+ // Simulate delete event
+ cleaner.OnDeleteEvent(folder, "file.txt", false, now)
+
+ // Check that cleanup is queued
+ if cleaner.GetPendingCleanupCount() != 1 {
+ t.Errorf("expected 1 pending cleanup, got %d", cleaner.GetPendingCleanupCount())
+ }
+
+ // Simulate create event
+ cleaner.OnCreateEvent(folder, "newfile.txt", false)
+
+ // Check that cleanup is cancelled
+ if cleaner.GetPendingCleanupCount() != 0 {
+ t.Errorf("expected 0 pending cleanups after create, got %d", cleaner.GetPendingCleanupCount())
+ }
+
+ cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_OnDeleteEvent_deduplication(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: true,
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ stopCh: make(chan struct{}),
+ }
+
+ folder := "/buckets/mybucket/testfolder"
+ now := time.Now()
+
+ // Simulate multiple delete events for same folder
+ for i := 0; i < 5; i++ {
+ cleaner.OnDeleteEvent(folder, "file"+string(rune('0'+i))+".txt", false, now.Add(time.Duration(i)*time.Second))
+ }
+
+ // Check that only 1 cleanup is queued (deduplicated)
+ if cleaner.GetPendingCleanupCount() != 1 {
+ t.Errorf("expected 1 pending cleanup after deduplication, got %d", cleaner.GetPendingCleanupCount())
+ }
+
+ cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_OnDeleteEvent_multipleFolders(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: true,
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ stopCh: make(chan struct{}),
+ }
+
+ now := time.Now()
+
+ // Delete files in different folders
+ cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file.txt", false, now)
+ cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file.txt", false, now.Add(1*time.Second))
+ cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file.txt", false, now.Add(2*time.Second))
+
+ // Each folder should be queued
+ if cleaner.GetPendingCleanupCount() != 3 {
+ t.Errorf("expected 3 pending cleanups, got %d", cleaner.GetPendingCleanupCount())
+ }
+
+ cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_OnDeleteEvent_notOwner(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888", "filer2:8888"})
+
+ // Create cleaner for filer that doesn't own the folder
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: true,
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ stopCh: make(chan struct{}),
+ }
+
+ now := time.Now()
+
+ // Try many folders, looking for one that filer1 doesn't own
+ foundNonOwned := false
+ for i := 0; i < 100; i++ {
+ folder := "/buckets/mybucket/folder" + string(rune('0'+i%10)) + string(rune('0'+i/10))
+ if !cleaner.ownsFolder(folder) {
+ // This folder is not owned by filer1
+ cleaner.OnDeleteEvent(folder, "file.txt", false, now)
+ if cleaner.GetPendingCleanupCount() != 0 {
+ t.Errorf("non-owner should not queue cleanup for folder %s", folder)
+ }
+ foundNonOwned = true
+ break
+ }
+ }
+
+ if !foundNonOwned {
+ t.Skip("could not find a folder not owned by filer1")
+ }
+
+ cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_OnDeleteEvent_disabled(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: false, // Disabled
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ stopCh: make(chan struct{}),
+ }
+
+ folder := "/buckets/mybucket/testfolder"
+ now := time.Now()
+
+ // Simulate delete event
+ cleaner.OnDeleteEvent(folder, "file.txt", false, now)
+
+ // Check that no cleanup is queued when disabled
+ if cleaner.GetPendingCleanupCount() != 0 {
+ t.Errorf("disabled cleaner should not queue cleanup, got %d", cleaner.GetPendingCleanupCount())
+ }
+
+ cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_OnDeleteEvent_directoryDeletion(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: true,
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ stopCh: make(chan struct{}),
+ }
+
+ folder := "/buckets/mybucket/testfolder"
+ now := time.Now()
+
+ // Simulate directory delete event - should trigger cleanup
+ // because subdirectory deletion also makes parent potentially empty
+ cleaner.OnDeleteEvent(folder, "subdir", true, now)
+
+ // Check that cleanup IS queued for directory deletion
+ if cleaner.GetPendingCleanupCount() != 1 {
+ t.Errorf("directory deletion should trigger cleanup, got %d", cleaner.GetPendingCleanupCount())
+ }
+
+ cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_cachedCounts(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: true,
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ stopCh: make(chan struct{}),
+ }
+
+ folder := "/buckets/mybucket/testfolder"
+
+ // Initialize cached count
+ cleaner.folderCounts[folder] = &folderState{roughCount: 5}
+
+ // Simulate create events
+ cleaner.OnCreateEvent(folder, "newfile1.txt", false)
+ cleaner.OnCreateEvent(folder, "newfile2.txt", false)
+
+ // Check cached count increased
+ count, exists := cleaner.GetCachedFolderCount(folder)
+ if !exists {
+ t.Error("cached folder count should exist")
+ }
+ if count != 7 {
+ t.Errorf("expected cached count 7, got %d", count)
+ }
+
+ // Simulate delete events
+ now := time.Now()
+ cleaner.OnDeleteEvent(folder, "file1.txt", false, now)
+ cleaner.OnDeleteEvent(folder, "file2.txt", false, now.Add(1*time.Second))
+
+ // Check cached count decreased
+ count, exists = cleaner.GetCachedFolderCount(folder)
+ if !exists {
+ t.Error("cached folder count should exist")
+ }
+ if count != 5 {
+ t.Errorf("expected cached count 5, got %d", count)
+ }
+
+ cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_Stop(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: true,
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ stopCh: make(chan struct{}),
+ }
+
+ now := time.Now()
+
+ // Queue some cleanups
+ cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file1.txt", false, now)
+ cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file2.txt", false, now.Add(1*time.Second))
+ cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file3.txt", false, now.Add(2*time.Second))
+
+ // Verify cleanups are queued
+ if cleaner.GetPendingCleanupCount() < 1 {
+ t.Error("expected at least 1 pending cleanup before stop")
+ }
+
+ // Stop the cleaner
+ cleaner.Stop()
+
+ // Verify all cleanups are cancelled
+ if cleaner.GetPendingCleanupCount() != 0 {
+ t.Errorf("expected 0 pending cleanups after stop, got %d", cleaner.GetPendingCleanupCount())
+ }
+}
+
+func TestEmptyFolderCleaner_cacheEviction(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: true,
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ cacheExpiry: 100 * time.Millisecond, // Short expiry for testing
+ stopCh: make(chan struct{}),
+ }
+
+ folder1 := "/buckets/mybucket/folder1"
+ folder2 := "/buckets/mybucket/folder2"
+ folder3 := "/buckets/mybucket/folder3"
+
+ // Add some cache entries with old timestamps
+ oldTime := time.Now().Add(-1 * time.Hour)
+ cleaner.folderCounts[folder1] = &folderState{roughCount: 5, lastCheck: oldTime}
+ cleaner.folderCounts[folder2] = &folderState{roughCount: 3, lastCheck: oldTime}
+ // folder3 has recent activity
+ cleaner.folderCounts[folder3] = &folderState{roughCount: 2, lastCheck: time.Now()}
+
+ // Verify all entries exist
+ if len(cleaner.folderCounts) != 3 {
+ t.Errorf("expected 3 cache entries, got %d", len(cleaner.folderCounts))
+ }
+
+ // Run eviction
+ cleaner.evictStaleCacheEntries()
+
+ // Verify stale entries are evicted
+ if len(cleaner.folderCounts) != 1 {
+ t.Errorf("expected 1 cache entry after eviction, got %d", len(cleaner.folderCounts))
+ }
+
+ // Verify the recent entry still exists
+ if _, exists := cleaner.folderCounts[folder3]; !exists {
+ t.Error("expected folder3 to still exist in cache")
+ }
+
+ // Verify stale entries are removed
+ if _, exists := cleaner.folderCounts[folder1]; exists {
+ t.Error("expected folder1 to be evicted")
+ }
+ if _, exists := cleaner.folderCounts[folder2]; exists {
+ t.Error("expected folder2 to be evicted")
+ }
+
+ cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_cacheEviction_skipsEntriesInQueue(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: true,
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ cacheExpiry: 100 * time.Millisecond,
+ stopCh: make(chan struct{}),
+ }
+
+ folder := "/buckets/mybucket/folder"
+ oldTime := time.Now().Add(-1 * time.Hour)
+
+ // Add a stale cache entry
+ cleaner.folderCounts[folder] = &folderState{roughCount: 0, lastCheck: oldTime}
+ // Also add to cleanup queue
+ cleaner.cleanupQueue.Add(folder, time.Now())
+
+ // Run eviction
+ cleaner.evictStaleCacheEntries()
+
+ // Verify entry is NOT evicted because it's in cleanup queue
+ if _, exists := cleaner.folderCounts[folder]; !exists {
+ t.Error("expected folder to still exist in cache (is in cleanup queue)")
+ }
+
+ cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) {
+ lockRing := lock_manager.NewLockRing(5 * time.Second)
+ lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+ cleaner := &EmptyFolderCleaner{
+ lockRing: lockRing,
+ host: "filer1:8888",
+ bucketPath: "/buckets",
+ enabled: true,
+ folderCounts: make(map[string]*folderState),
+ cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+ stopCh: make(chan struct{}),
+ }
+
+ now := time.Now()
+
+ // Add folders in order
+ folders := []string{
+ "/buckets/mybucket/folder1",
+ "/buckets/mybucket/folder2",
+ "/buckets/mybucket/folder3",
+ }
+ for i, folder := range folders {
+ cleaner.OnDeleteEvent(folder, "file.txt", false, now.Add(time.Duration(i)*time.Second))
+ }
+
+ // Verify queue length
+ if cleaner.GetPendingCleanupCount() != 3 {
+ t.Errorf("expected 3 queued folders, got %d", cleaner.GetPendingCleanupCount())
+ }
+
+ // Verify time-sorted order by popping
+ for i, expected := range folders {
+ folder, ok := cleaner.cleanupQueue.Pop()
+ if !ok || folder != expected {
+ t.Errorf("expected folder %s at index %d, got %s", expected, i, folder)
+ }
+ }
+
+ cleaner.Stop()
+}
+
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 06:26:43 +0000