directory to remote storage mapping

2 files changed, 94 insertions, 12 deletions

diff --git a/weed/filer/filer_remote_storage.go b/weed/filer/filer_remote_storage.go
index 8b136a763..dbb0363a5 100644
--- a/weed/filer/filer_remote_storage.go
+++ b/weed/filer/filer_remote_storage.go
@@ -1,32 +1,39 @@
 package filer
 
 import (
-	"bytes"
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	_ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/golang/protobuf/proto"
-	"io"
 	"math"
+	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/golang/protobuf/jsonpb"
 	"github.com/viant/ptrie"
 )
 
+const REMOTE_STORAGE_CONF_SUFFIX = ".conf"
+const REMOTE_STORAGE_MOUNT_FILE = "mount.mapping"
+
 type FilerRemoteStorage struct {
-	rules ptrie.Trie
+	rules             ptrie.Trie
+	storageNameToConf map[string]*filer_pb.RemoteConf
 }
 
-func NewFilerRemoteStorage() (fc *FilerRemoteStorage) {
-	fc = &FilerRemoteStorage{
-		rules: ptrie.New(),
+func NewFilerRemoteStorage() (rs *FilerRemoteStorage) {
+	rs = &FilerRemoteStorage{
+		rules:             ptrie.New(),
+		storageNameToConf: make(map[string]*filer_pb.RemoteConf),
 	}
-	return fc
+	return rs
 }
 
-func (fc *FilerRemoteStorage) loadFromFiler(filer *Filer) (err error) {
+func (rs *FilerRemoteStorage) loadRemoteStorageConfigurations(filer *Filer) (err error) {
+	// execute this on filer
+
 	entries, _, err := filer.ListDirectoryEntries(context.Background(), DirectoryEtcRemote, "", false, math.MaxInt64, "", "", "")
 	if err != nil {
 		if err == filer_pb.ErrNotFound {
@@ -37,11 +44,56 @@ func (fc *FilerRemoteStorage) loadFromFiler(filer *Filer) (err error) {
 	}
 
 	for _, entry := range entries {
+		if entry.Name() == REMOTE_STORAGE_MOUNT_FILE {
+			rs.loadRemoteStorageMountMapping(entry.Content)
+		}
+		if !strings.HasSuffix(entry.Name(), REMOTE_STORAGE_CONF_SUFFIX) {
+			return nil
+		}
 		conf := &filer_pb.RemoteConf{}
 		if err := proto.Unmarshal(entry.Content, conf); err != nil {
-			return fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, entry.Name, err)
+			return fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, entry.Name(), err)
 		}
-		fc.MountRemoteStorage(dir, conf)
+		rs.storageNameToConf[conf.Name] = conf
 	}
 	return nil
 }
+
+func (rs *FilerRemoteStorage) loadRemoteStorageMountMapping(data []byte) (err error) {
+	mappings := &filer_pb.RemoteStorageMappingList{}
+	if err := proto.Unmarshal(data, mappings); err != nil {
+		return fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, err)
+	}
+	for _, mapping := range mappings.Mappings {
+		rs.mapDirectoryToRemoteStorage(util.FullPath(mapping.Dir), mapping.RemoteStorageName)
+	}
+	return nil
+}
+
+func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, remoteStorageName string) {
+	rs.rules.Put([]byte(dir+"/"), remoteStorageName)
+}
+
+func (rs *FilerRemoteStorage) FindRemoteStorageClient(p util.FullPath) (client remote_storage.RemoteStorageClient, found bool) {
+	var storageName string
+	rs.rules.MatchPrefix([]byte(p), func(key []byte, value interface{}) bool {
+		storageName = value.(string)
+		return true
+	})
+
+	if storageName == "" {
+		return
+	}
+
+	remoteConf, ok := rs.storageNameToConf[storageName]
+	if !ok {
+		return
+	}
+
+	var err error
+	if client, err = remote_storage.GetRemoteStorage(remoteConf); err == nil {
+		found = true
+		return
+	}
+	return
+}
diff --git a/weed/filer/filer_remote_storage_test.go b/weed/filer/filer_remote_storage_test.go
new file mode 100644
index 000000000..1a41c6e63
--- /dev/null
+++ b/weed/filer/filer_remote_storage_test.go
@@ -0,0 +1,30 @@
+package filer
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestFilerRemoteStorage_FindRemoteStorageClient(t *testing.T) {
+	conf := &filer_pb.RemoteConf{
+		Name: "s7",
+		Type: "s3",
+	}
+	rs := NewFilerRemoteStorage()
+	rs.storageNameToConf[conf.Name] = conf
+
+	rs.mapDirectoryToRemoteStorage("/a/b/c", "s7")
+
+	_, found := rs.FindRemoteStorageClient("/a/b/c/d/e/f")
+	assert.Equal(t, true, found, "find storage client")
+
+	_, found2 := rs.FindRemoteStorageClient("/a/b")
+	assert.Equal(t, false, found2, "should not find storage client")
+
+	_, found3 := rs.FindRemoteStorageClient("/a/b/c")
+	assert.Equal(t, false, found3, "should not find storage client")
+
+	_, found4 := rs.FindRemoteStorageClient("/a/b/cc")
+	assert.Equal(t, false, found4, "should not find storage client")
+}
\ No newline at end of file