From 99b599aa8a674ccd584d612e8e871fdca7670620 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 26 Jul 2021 22:53:44 -0700 Subject: remote.mount --- weed/shell/command_remote_mount.go | 200 +++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 weed/shell/command_remote_mount.go (limited to 'weed/shell/command_remote_mount.go') diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go new file mode 100644 index 000000000..22d92ef8b --- /dev/null +++ b/weed/shell/command_remote_mount.go @@ -0,0 +1,200 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" + _ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/golang/protobuf/proto" + "io" + "strings" +) + +func init() { + Commands = append(Commands, &commandRemoteMount{}) +} + +type commandRemoteMount struct { +} + +func (c *commandRemoteMount) Name() string { + return "remote.mount" +} + +func (c *commandRemoteMount) Help() string { + return `mount remote storage and pull its metadata + + # assume a remote storage is configured to name "s3_1" + remote.configure -name=s3_1 -type=s3 -access_key=xxx -secret_key=yyy + + # mount and pull one bucket + remote.mount -dir=xxx -remote=s3_1/bucket + # mount and pull one directory in the bucket + remote.mount -dir=xxx -remote=s3_1/bucket/dir1 + +` +} + +func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + remoteMountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + + dir := remoteMountCommand.String("dir", "", "a directory in filer") + nonEmpty := remoteMountCommand.Bool("nonempty", false, "allows the mounting over a non-empty directory") + remote := remoteMountCommand.String("remote", "", "a directory in remote storage, ex. //path/to/dir") + + if err = remoteMountCommand.Parse(args); err != nil { + return nil + } + + // find configuration for remote storage + remoteConf, remotePath, err := c.findRemoteStorageConfiguration(commandEnv, writer, *remote) + if err != nil { + return fmt.Errorf("find configuration for %s: %v", *remote, err) + } + + // pull metadata from remote + if err = c.pullMetadata(commandEnv, writer, *dir, *nonEmpty, remoteConf, remotePath); err != nil { + return fmt.Errorf("pull metadata: %v", err) + } + + // store a mount configuration in filer + + + return nil +} + +func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote string) (conf *filer_pb.RemoteConf, remotePath string, err error) { + + // find remote configuration + parts := strings.Split(remote, "/") + if len(parts) == 0 { + err = fmt.Errorf("wrong remote storage location %s", remote) + return + } + storageName := parts[0] + remotePath = remote[len(storageName):] + + // read storage configuration data + var confBytes []byte + err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + confBytes, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, storageName) + return err + }) + if err != nil { + err = fmt.Errorf("no remote storage configuration for %s : %v", storageName, err) + return + } + + // unmarshal storage configuration + conf = &filer_pb.RemoteConf{} + if unMarshalErr := proto.Unmarshal(confBytes, conf); unMarshalErr != nil { + err = fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, storageName, unMarshalErr) + return + } + + return +} + +func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *filer_pb.RemoteConf, remotePath string) error { + + // find existing directory, and ensure the directory is empty + var mountToDir *filer_pb.Entry + err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + parent, name := util.FullPath(dir).DirAndName() + resp, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{ + Directory: parent, + Name: name, + }) + if lookupErr != nil { + return fmt.Errorf("lookup %s: %v", dir, lookupErr) + } + mountToDir = resp.Entry + + mountToDirIsEmpty := false + listErr := filer_pb.SeaweedList(client, dir, "", func(entry *filer_pb.Entry, isLast bool) error { + mountToDirIsEmpty = false + return nil + }, "", false, 1) + + if listErr != nil { + return fmt.Errorf("list %s: %v", dir, listErr) + } + + if !mountToDirIsEmpty { + if !nonEmpty { + return fmt.Errorf("dir %s is not empty", dir) + } + } + + return nil + }) + if err != nil { + return err + } + + // visit remote storage + remoteStorage, err := remote_storage.GetRemoteStorage(remoteConf) + if err != nil { + return err + } + + err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + ctx := context.Background() + err = remoteStorage.Traverse(remotePath, func(remoteDir, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error { + localDir := dir + remoteDir + println(util.NewFullPath(localDir, name)) + + lookupResponse, lookupErr := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ + Directory: localDir, + Name: name, + }) + if lookupErr != nil { + if lookupErr != filer_pb.ErrNotFound { + return lookupErr + } + } + existingEntry := lookupResponse.Entry + + if existingEntry == nil { + _, createErr := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{ + Directory: localDir, + Entry: &filer_pb.Entry{ + Name: name, + IsDirectory: isDirectory, + Attributes: &filer_pb.FuseAttributes{ + FileSize: uint64(remoteEntry.Size), + Mtime: remoteEntry.LastModifiedAt, + FileMode: uint32(0644), + }, + RemoteEntry: remoteEntry, + }, + }) + return createErr + } else { + if existingEntry.RemoteEntry.ETag != remoteEntry.ETag { + existingEntry.RemoteEntry = remoteEntry + existingEntry.Attributes.FileSize = uint64(remoteEntry.Size) + existingEntry.Attributes.Mtime = remoteEntry.LastModifiedAt + _, updateErr := client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{ + Directory: localDir, + Entry: existingEntry, + }) + return updateErr + } + } + return nil + }) + return err + }) + + if err != nil { + return err + } + + return nil +} -- cgit v1.2.3 From 4b94b03d90a97dfd6fecc55e7091055bf5fc329c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 27 Jul 2021 01:16:28 -0700 Subject: directory to remote storage mapping --- weed/shell/command_remote_mount.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'weed/shell/command_remote_mount.go') diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 22d92ef8b..10541092a 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -7,7 +7,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/remote_storage" - _ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3" "github.com/chrislusf/seaweedfs/weed/util" "github.com/golang/protobuf/proto" "io" @@ -82,7 +81,7 @@ func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandE // read storage configuration data var confBytes []byte err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - confBytes, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, storageName) + confBytes, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, storageName+filer.REMOTE_STORAGE_CONF_SUFFIX) return err }) if err != nil { -- cgit v1.2.3 From 1752eeb53803736f72d85d35f504997744288716 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 27 Jul 2021 03:26:35 -0700 Subject: remote.mount saves the mapping --- weed/shell/command_remote_mount.go | 65 ++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 17 deletions(-) (limited to 'weed/shell/command_remote_mount.go') diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 10541092a..217be8cef 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -10,7 +10,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" "github.com/golang/protobuf/proto" "io" - "strings" ) func init() { @@ -50,33 +49,32 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io return nil } + remoteStorageLocation := remote_storage.RemoteStorageLocation(*remote) + // find configuration for remote storage - remoteConf, remotePath, err := c.findRemoteStorageConfiguration(commandEnv, writer, *remote) + // remotePath is //path/to/dir + remoteConf, err := c.findRemoteStorageConfiguration(commandEnv, writer, remoteStorageLocation) if err != nil { return fmt.Errorf("find configuration for %s: %v", *remote, err) } // pull metadata from remote - if err = c.pullMetadata(commandEnv, writer, *dir, *nonEmpty, remoteConf, remotePath); err != nil { + if err = c.pullMetadata(commandEnv, writer, *dir, *nonEmpty, remoteConf, remoteStorageLocation); err != nil { return fmt.Errorf("pull metadata: %v", err) } // store a mount configuration in filer - + if err = c.saveMountMapping(commandEnv, writer, *dir, remoteStorageLocation); err != nil { + return fmt.Errorf("save mount mapping: %v", err) + } return nil } -func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote string) (conf *filer_pb.RemoteConf, remotePath string, err error) { +func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote remote_storage.RemoteStorageLocation) (conf *filer_pb.RemoteConf, err error) { // find remote configuration - parts := strings.Split(remote, "/") - if len(parts) == 0 { - err = fmt.Errorf("wrong remote storage location %s", remote) - return - } - storageName := parts[0] - remotePath = remote[len(storageName):] + storageName, _, _ := remote.NameBucketPath() // read storage configuration data var confBytes []byte @@ -99,7 +97,7 @@ func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandE return } -func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *filer_pb.RemoteConf, remotePath string) error { +func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *filer_pb.RemoteConf, remote remote_storage.RemoteStorageLocation) error { // find existing directory, and ensure the directory is empty var mountToDir *filer_pb.Entry @@ -114,7 +112,7 @@ func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writ } mountToDir = resp.Entry - mountToDirIsEmpty := false + mountToDirIsEmpty := true listErr := filer_pb.SeaweedList(client, dir, "", func(entry *filer_pb.Entry, isLast bool) error { mountToDirIsEmpty = false return nil @@ -144,7 +142,7 @@ func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writ err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { ctx := context.Background() - err = remoteStorage.Traverse(remotePath, func(remoteDir, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error { + err = remoteStorage.Traverse(remote, func(remoteDir, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error { localDir := dir + remoteDir println(util.NewFullPath(localDir, name)) @@ -152,12 +150,14 @@ func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writ Directory: localDir, Name: name, }) + var existingEntry *filer_pb.Entry if lookupErr != nil { if lookupErr != filer_pb.ErrNotFound { return lookupErr } + } else { + existingEntry = lookupResponse.Entry } - existingEntry := lookupResponse.Entry if existingEntry == nil { _, createErr := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{ @@ -175,7 +175,7 @@ func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writ }) return createErr } else { - if existingEntry.RemoteEntry.ETag != remoteEntry.ETag { + if existingEntry.RemoteEntry == nil || existingEntry.RemoteEntry.ETag != remoteEntry.ETag { existingEntry.RemoteEntry = remoteEntry existingEntry.Attributes.FileSize = uint64(remoteEntry.Size) existingEntry.Attributes.Mtime = remoteEntry.LastModifiedAt @@ -197,3 +197,34 @@ func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writ return nil } + +func (c *commandRemoteMount) saveMountMapping(commandEnv *CommandEnv, writer io.Writer, dir string, remoteStorageLocation remote_storage.RemoteStorageLocation) (err error) { + + // read current mapping + var oldContent, newContent []byte + err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + oldContent, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE) + return err + }) + if err != nil { + if err != filer_pb.ErrNotFound { + return fmt.Errorf("read existing mapping: %v", err) + } + } + + // add new mapping + newContent, err = filer.AddMapping(oldContent, dir, remoteStorageLocation) + if err != nil { + return fmt.Errorf("add mapping %s~%s: %v", dir, remoteStorageLocation, err) + } + + // save back + err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return filer.SaveInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE, newContent) + }) + if err != nil { + return fmt.Errorf("save mapping: %v", err) + } + + return nil +} -- cgit v1.2.3 From 899963ac20556f6a48aad431eec8ec995e982dd3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Jul 2021 02:08:55 -0700 Subject: remote storage location changed to struct --- weed/shell/command_remote_mount.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'weed/shell/command_remote_mount.go') diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 217be8cef..8f16d27c8 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -49,7 +49,7 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io return nil } - remoteStorageLocation := remote_storage.RemoteStorageLocation(*remote) + remoteStorageLocation := remote_storage.ParseLocation(*remote) // find configuration for remote storage // remotePath is //path/to/dir @@ -71,33 +71,30 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io return nil } -func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote remote_storage.RemoteStorageLocation) (conf *filer_pb.RemoteConf, err error) { - - // find remote configuration - storageName, _, _ := remote.NameBucketPath() +func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote *filer_pb.RemoteStorageLocation) (conf *filer_pb.RemoteConf, err error) { // read storage configuration data var confBytes []byte err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - confBytes, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, storageName+filer.REMOTE_STORAGE_CONF_SUFFIX) + confBytes, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, remote.Name+filer.REMOTE_STORAGE_CONF_SUFFIX) return err }) if err != nil { - err = fmt.Errorf("no remote storage configuration for %s : %v", storageName, err) + err = fmt.Errorf("no remote storage configuration for %s : %v", remote.Name, err) return } // unmarshal storage configuration conf = &filer_pb.RemoteConf{} if unMarshalErr := proto.Unmarshal(confBytes, conf); unMarshalErr != nil { - err = fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, storageName, unMarshalErr) + err = fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, remote.Name, unMarshalErr) return } return } -func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *filer_pb.RemoteConf, remote remote_storage.RemoteStorageLocation) error { +func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *filer_pb.RemoteConf, remote *filer_pb.RemoteStorageLocation) error { // find existing directory, and ensure the directory is empty var mountToDir *filer_pb.Entry @@ -198,7 +195,7 @@ func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writ return nil } -func (c *commandRemoteMount) saveMountMapping(commandEnv *CommandEnv, writer io.Writer, dir string, remoteStorageLocation remote_storage.RemoteStorageLocation) (err error) { +func (c *commandRemoteMount) saveMountMapping(commandEnv *CommandEnv, writer io.Writer, dir string, remoteStorageLocation *filer_pb.RemoteStorageLocation) (err error) { // read current mapping var oldContent, newContent []byte -- cgit v1.2.3 From 9df7d16791351f85ecb364b4b11caca1ee9bcd59 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 31 Jul 2021 22:39:38 -0700 Subject: read <- remote_storage --- weed/shell/command_remote_mount.go | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) (limited to 'weed/shell/command_remote_mount.go') diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 8f16d27c8..bd6b49050 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -8,6 +8,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/remote_storage" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" "io" ) @@ -49,6 +50,10 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io return nil } + if *dir == "" { + return c.listExistingRemoteStorageMounts(commandEnv, writer) + } + remoteStorageLocation := remote_storage.ParseLocation(*remote) // find configuration for remote storage @@ -71,6 +76,34 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io return nil } +func (c *commandRemoteMount) listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) (err error) { + + // read current mapping + var oldContent []byte + err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + oldContent, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE) + return err + }) + if err != nil { + if err != filer_pb.ErrNotFound { + return fmt.Errorf("read existing mapping: %v", err) + } + } + + mappings, unmarshalErr := filer.UnmarshalRemoteStorageMappings(oldContent) + if unmarshalErr != nil { + return unmarshalErr + } + + m := jsonpb.Marshaler{ + EmitDefaults: false, + Indent: " ", + } + + return m.Marshal(writer, mappings) + +} + func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote *filer_pb.RemoteStorageLocation) (conf *filer_pb.RemoteConf, err error) { // read storage configuration data @@ -178,7 +211,7 @@ func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writ existingEntry.Attributes.Mtime = remoteEntry.LastModifiedAt _, updateErr := client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{ Directory: localDir, - Entry: existingEntry, + Entry: existingEntry, }) return updateErr } @@ -210,7 +243,7 @@ func (c *commandRemoteMount) saveMountMapping(commandEnv *CommandEnv, writer io. } // add new mapping - newContent, err = filer.AddMapping(oldContent, dir, remoteStorageLocation) + newContent, err = filer.AddRemoteStorageMapping(oldContent, dir, remoteStorageLocation) if err != nil { return fmt.Errorf("add mapping %s~%s: %v", dir, remoteStorageLocation, err) } -- cgit v1.2.3 From f6a9ad8001c439e5751fa5488fe6106b185ba8eb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 4 Aug 2021 00:31:06 -0700 Subject: fix tests --- weed/shell/command_remote_mount.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'weed/shell/command_remote_mount.go') diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index bd6b49050..55dfb42ca 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -130,17 +130,15 @@ func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandE func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *filer_pb.RemoteConf, remote *filer_pb.RemoteStorageLocation) error { // find existing directory, and ensure the directory is empty - var mountToDir *filer_pb.Entry err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { parent, name := util.FullPath(dir).DirAndName() - resp, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{ + _, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{ Directory: parent, Name: name, }) if lookupErr != nil { return fmt.Errorf("lookup %s: %v", dir, lookupErr) } - mountToDir = resp.Entry mountToDirIsEmpty := true listErr := filer_pb.SeaweedList(client, dir, "", func(entry *filer_pb.Entry, isLast bool) error { -- cgit v1.2.3 From d84c31169922d395074b1b256d05aa638462df50 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 4 Aug 2021 12:30:18 -0700 Subject: refactoring --- weed/shell/command_remote_mount.go | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) (limited to 'weed/shell/command_remote_mount.go') diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 55dfb42ca..5cd69f3b0 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -79,20 +79,9 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io func (c *commandRemoteMount) listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) (err error) { // read current mapping - var oldContent []byte - err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - oldContent, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE) - return err - }) - if err != nil { - if err != filer_pb.ErrNotFound { - return fmt.Errorf("read existing mapping: %v", err) - } - } - - mappings, unmarshalErr := filer.UnmarshalRemoteStorageMappings(oldContent) - if unmarshalErr != nil { - return unmarshalErr + mappings, readErr := remote_storage.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress) + if readErr != nil { + return readErr } m := jsonpb.Marshaler{ -- cgit v1.2.3 From b9ecf1e3a8685c62ccac80ed0fbc180ed34b48e2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 4 Aug 2021 14:56:13 -0700 Subject: refacotring --- weed/shell/command_remote_mount.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) (limited to 'weed/shell/command_remote_mount.go') diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 5cd69f3b0..35aad9498 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -95,25 +95,8 @@ func (c *commandRemoteMount) listExistingRemoteStorageMounts(commandEnv *Command func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote *filer_pb.RemoteStorageLocation) (conf *filer_pb.RemoteConf, err error) { - // read storage configuration data - var confBytes []byte - err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - confBytes, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, remote.Name+filer.REMOTE_STORAGE_CONF_SUFFIX) - return err - }) - if err != nil { - err = fmt.Errorf("no remote storage configuration for %s : %v", remote.Name, err) - return - } - - // unmarshal storage configuration - conf = &filer_pb.RemoteConf{} - if unMarshalErr := proto.Unmarshal(confBytes, conf); unMarshalErr != nil { - err = fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, remote.Name, unMarshalErr) - return - } + return remote_storage.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote.Name) - return } func (c *commandRemoteMount) pullMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *filer_pb.RemoteConf, remote *filer_pb.RemoteStorageLocation) error { -- cgit v1.2.3 From 6b743dbbf96f863e70ee80e4b32c0928f594891a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 4 Aug 2021 16:25:46 -0700 Subject: refactor client subscribe metadata --- weed/shell/command_remote_mount.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'weed/shell/command_remote_mount.go') diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 35aad9498..73a5119d5 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -9,7 +9,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/remote_storage" "github.com/chrislusf/seaweedfs/weed/util" "github.com/golang/protobuf/jsonpb" - "github.com/golang/protobuf/proto" "io" ) @@ -79,7 +78,7 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io func (c *commandRemoteMount) listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) (err error) { // read current mapping - mappings, readErr := remote_storage.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress) + mappings, readErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress) if readErr != nil { return readErr } @@ -95,7 +94,7 @@ func (c *commandRemoteMount) listExistingRemoteStorageMounts(commandEnv *Command func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote *filer_pb.RemoteStorageLocation) (conf *filer_pb.RemoteConf, err error) { - return remote_storage.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote.Name) + return filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote.Name) } -- cgit v1.2.3 From 13e45e16054d16e8d8161a8ddb02fde3cd4cde8f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 8 Aug 2021 01:21:42 -0700 Subject: filer.remote.sync can work now --- weed/shell/command_remote_mount.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'weed/shell/command_remote_mount.go') diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 73a5119d5..37b235a55 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -88,7 +88,10 @@ func (c *commandRemoteMount) listExistingRemoteStorageMounts(commandEnv *Command Indent: " ", } - return m.Marshal(writer, mappings) + err = m.Marshal(writer, mappings) + fmt.Fprintln(writer) + + return } -- cgit v1.2.3