aboutsummaryrefslogtreecommitdiff
path: root/weed/shell
diff options
context:
space:
mode:
Diffstat (limited to 'weed/shell')
-rw-r--r--weed/shell/command_remote_configure.go30
-rw-r--r--weed/shell/command_remote_meta_sync.go6
-rw-r--r--weed/shell/command_remote_mount.go10
3 files changed, 38 insertions, 8 deletions
diff --git a/weed/shell/command_remote_configure.go b/weed/shell/command_remote_configure.go
index 1bfe82673..b6c4af178 100644
--- a/weed/shell/command_remote_configure.go
+++ b/weed/shell/command_remote_configure.go
@@ -58,7 +58,7 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write
isDelete := remoteConfigureCommand.Bool("delete", false, "delete one remote storage by its name")
remoteConfigureCommand.StringVar(&conf.Name, "name", "", "a short name to identify the remote storage")
- remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", "[s3|gcs|azure|b2|aliyun|tencent|baidu|wasabi] storage type")
+ remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", "[s3|gcs|azure|b2|aliyun|tencent|baidu|wasabi|hdfs] storage type")
remoteConfigureCommand.StringVar(&conf.S3AccessKey, "s3.access_key", "", "s3 access key")
remoteConfigureCommand.StringVar(&conf.S3SecretKey, "s3.secret_key", "", "s3 secret key")
@@ -95,10 +95,27 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write
remoteConfigureCommand.StringVar(&conf.WasabiEndpoint, "wasabi.endpoint", "", "Wasabi endpoint, see https://wasabi.com/wp-content/themes/wasabi/docs/API_Guide/index.html#t=topics%2Fapidiff-intro.htm")
remoteConfigureCommand.StringVar(&conf.WasabiRegion, "wasabi.region", "", "Wasabi region")
+ var namenodes arrayFlags
+ remoteConfigureCommand.Var(&namenodes, "hdfs.namenodes", "hdfs name node and port, example: namenode1:8020,namenode2:8020")
+ remoteConfigureCommand.StringVar(&conf.HdfsUsername, "hdfs.username", "", "hdfs user name")
+ remoteConfigureCommand.StringVar(&conf.HdfsServicePrincipalName, "hdfs.servicePrincipalName", "", `Kerberos service principal name for the namenode
+
+Example: hdfs/namenode.hadoop.docker
+Namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.
+`)
+ remoteConfigureCommand.StringVar(&conf.HdfsDataTransferProtection, "hdfs.dataTransferProtection", "", "[authentication|integrity|privacy] Kerberos data transfer protection")
+
+
if err = remoteConfigureCommand.Parse(args); err != nil {
return nil
}
+ if conf.Type != "s3" {
+ // clear out the default values
+ conf.S3Region = ""
+ conf.S3ForcePathStyle = false
+ }
+
if conf.Name == "" {
return c.listExistingRemoteStorages(commandEnv, writer)
}
@@ -187,3 +204,14 @@ func (c *commandRemoteConfigure) saveRemoteStorage(commandEnv *CommandEnv, write
return nil
}
+
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+ return "my string representation"
+}
+
+func (i *arrayFlags) Set(value string) error {
+ *i = append(*i, value)
+ return nil
+}
diff --git a/weed/shell/command_remote_meta_sync.go b/weed/shell/command_remote_meta_sync.go
index b4e3534fc..17b9abdb8 100644
--- a/weed/shell/command_remote_meta_sync.go
+++ b/weed/shell/command_remote_meta_sync.go
@@ -55,7 +55,7 @@ func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer
}
mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, *dir)
- if detectErr != nil{
+ if detectErr != nil {
jsonPrintln(writer, mappings)
return detectErr
}
@@ -106,7 +106,7 @@ func detectMountInfo(commandEnv *CommandEnv, writer io.Writer, dir string) (*rem
If entry.RemoteEntry.RemoteTag != remoteEntry.RemoteTag {
the remote version is updated, need to pull meta
}
- */
+*/
func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToCache util.FullPath, remoteConf *remote_pb.RemoteConf) error {
// visit remote storage
@@ -158,7 +158,7 @@ func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util
fmt.Fprintln(writer, " (skip)")
return nil
}
- if existingEntry.RemoteEntry.RemoteETag != remoteEntry.RemoteETag {
+ if existingEntry.RemoteEntry.RemoteETag != remoteEntry.RemoteETag || existingEntry.RemoteEntry.RemoteMtime < remoteEntry.RemoteMtime {
// the remote version is updated, need to pull meta
fmt.Fprintln(writer, " (update)")
return doSaveRemoteEntry(client, string(localDir), existingEntry, remoteEntry)
diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go
index d1b282d9c..3e92428d9 100644
--- a/weed/shell/command_remote_mount.go
+++ b/weed/shell/command_remote_mount.go
@@ -60,15 +60,17 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io
return err
}
- remoteStorageLocation := remote_storage.ParseLocation(*remote)
-
// find configuration for remote storage
- // remotePath is /<bucket>/path/to/dir
- remoteConf, err := c.findRemoteStorageConfiguration(commandEnv, writer, remoteStorageLocation)
+ remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote_storage.ParseLocationName(*remote))
if err != nil {
return fmt.Errorf("find configuration for %s: %v", *remote, err)
}
+ remoteStorageLocation, err := remote_storage.ParseRemoteLocation(remoteConf.Type, *remote)
+ if err != nil {
+ return err
+ }
+
// sync metadata from remote
if err = c.syncMetadata(commandEnv, writer, *dir, *nonEmpty, remoteConf, remoteStorageLocation); err != nil {
return fmt.Errorf("pull metadata: %v", err)