FUSE mount: lazy loading meta cache

2 files changed, 45 insertions, 6 deletions

diff --git a/weed/filesys/meta_cache/meta_cache.go b/weed/filesys/meta_cache/meta_cache.go
index 4c9090d42..3b04040a5 100644
--- a/weed/filesys/meta_cache/meta_cache.go
+++ b/weed/filesys/meta_cache/meta_cache.go
@@ -9,16 +9,19 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/filer2/leveldb"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/bounded_tree"
 )
 
 type MetaCache struct {
 	actualStore filer2.FilerStore
 	sync.RWMutex
+	visitedBoundary *bounded_tree.BoundedTree
 }
 
 func NewMetaCache(dbFolder string) *MetaCache {
 	return &MetaCache{
-		actualStore: openMetaStore(dbFolder),
+		actualStore:     openMetaStore(dbFolder),
+		visitedBoundary: bounded_tree.NewBoundedTree(),
 	}
 }
 
@@ -49,14 +52,24 @@ func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer2.Entry) error
 func (mc *MetaCache) AtomicUpdateEntry(ctx context.Context, oldPath util.FullPath, newEntry *filer2.Entry) error {
 	mc.Lock()
 	defer mc.Unlock()
-	if oldPath != "" {
-		if err := mc.actualStore.DeleteEntry(ctx, oldPath); err != nil {
-			return err
+
+	oldDir, _ := oldPath.DirAndName()
+	if mc.visitedBoundary.HasVisited(util.FullPath(oldDir)) {
+		if oldPath != "" {
+			if err := mc.actualStore.DeleteEntry(ctx, oldPath); err != nil {
+				return err
+			}
 		}
+	}else{
+		// println("unknown old directory:", oldDir)
 	}
+
 	if newEntry != nil {
-		if err := mc.actualStore.InsertEntry(ctx, newEntry); err != nil {
-			return err
+		newDir, _ := newEntry.DirAndName()
+		if mc.visitedBoundary.HasVisited(util.FullPath(newDir)) {
+			if err := mc.actualStore.InsertEntry(ctx, newEntry); err != nil {
+				return err
+			}
 		}
 	}
 	return nil
diff --git a/weed/filesys/meta_cache/meta_cache_init.go b/weed/filesys/meta_cache/meta_cache_init.go
index 58bf6862e..1fbc3e532 100644
--- a/weed/filesys/meta_cache/meta_cache_init.go
+++ b/weed/filesys/meta_cache/meta_cache_init.go
@@ -2,6 +2,7 @@ package meta_cache
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -10,6 +11,7 @@ import (
 )
 
 func InitMetaCache(mc *MetaCache, client filer_pb.FilerClient, path string) error {
+	return nil
 	glog.V(0).Infof("synchronizing meta data ...")
 	filer_pb.TraverseBfs(client, util.FullPath(path), func(parentPath util.FullPath, pbEntry *filer_pb.Entry) {
 		entry := filer2.FromPbEntry(string(parentPath), pbEntry)
@@ -19,3 +21,27 @@ func InitMetaCache(mc *MetaCache, client filer_pb.FilerClient, path string) erro
 	})
 	return nil
 }
+
+func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath) {
+
+	mc.visitedBoundary.EnsureVisited(dirPath, func(path util.FullPath) (childDirectories []string, err error) {
+
+		glog.V(2).Infof("ReadDirAllEntries %s ...", path)
+
+		err = filer_pb.ReadDirAllEntries(client, dirPath, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
+			entry := filer2.FromPbEntry(string(dirPath), pbEntry)
+			if err := mc.InsertEntry(context.Background(), entry); err != nil {
+				glog.V(0).Infof("read %s: %v", entry.FullPath, err)
+				return err
+			}
+			if entry.IsDirectory() {
+				childDirectories = append(childDirectories, entry.Name())
+			}
+			return nil
+		})
+		if err != nil {
+			err = fmt.Errorf("list %s: %v", dirPath, err)
+		}
+		return
+	})
+}