aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--go.mod11
-rw-r--r--go.sum22
-rw-r--r--weed/pb/remote.proto6
-rw-r--r--weed/pb/remote_pb/remote.pb.go153
-rw-r--r--weed/remote_storage/azure/azure_storage_client.go4
-rw-r--r--weed/remote_storage/gcs/gcs_storage_client.go4
-rw-r--r--weed/remote_storage/hdfs/hdfs_kerberos.go55
-rw-r--r--weed/remote_storage/hdfs/hdfs_storage_client.go174
-rw-r--r--weed/remote_storage/hdfs/traverse_bfs.go63
-rw-r--r--weed/remote_storage/remote_storage.go42
-rw-r--r--weed/remote_storage/s3/aliyun.go4
-rw-r--r--weed/remote_storage/s3/backblaze.go4
-rw-r--r--weed/remote_storage/s3/baidu.go4
-rw-r--r--weed/remote_storage/s3/s3_storage_client.go4
-rw-r--r--weed/remote_storage/s3/tencent.go4
-rw-r--r--weed/remote_storage/s3/wasabi.go4
-rw-r--r--weed/shell/command_remote_configure.go30
-rw-r--r--weed/shell/command_remote_meta_sync.go6
-rw-r--r--weed/shell/command_remote_mount.go10
19 files changed, 539 insertions, 65 deletions
diff --git a/go.mod b/go.mod
index c57af848d..daaec16b1 100644
--- a/go.mod
+++ b/go.mod
@@ -19,6 +19,7 @@ require (
github.com/cespare/xxhash v1.1.0
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/chrislusf/raft v1.0.7
+ github.com/colinmarc/hdfs/v2 v2.2.0
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
@@ -60,7 +61,7 @@ require (
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.0.0 // indirect
- github.com/hashicorp/go-uuid v1.0.1 // indirect
+ github.com/hashicorp/go-uuid v1.0.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jinzhu/copier v0.2.8
@@ -177,6 +178,14 @@ require (
modernc.org/token v1.0.0 // indirect
)
+require (
+ github.com/jcmturner/aescts/v2 v2.0.0 // indirect
+ github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
+ github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
+ github.com/jcmturner/gokrb5/v8 v8.4.1 // indirect
+ github.com/jcmturner/rpc/v2 v2.0.2 // indirect
+)
+
// replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
// replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft
diff --git a/go.sum b/go.sum
index e18da414c..49994f764 100644
--- a/go.sum
+++ b/go.sum
@@ -154,6 +154,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636FkGM=
+github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -292,6 +294,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw=
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -360,6 +363,10 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
+github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ=
+github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
@@ -388,8 +395,9 @@ github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
+github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -405,9 +413,19 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
+github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
+github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
+github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
+github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
+github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
+github.com/jcmturner/gokrb5/v8 v8.4.1 h1:IGSJfqBzMS6TA0oJ7DxXdyzPK563QHa8T2IqER2ggyQ=
+github.com/jcmturner/gokrb5/v8 v8.4.1/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM=
+github.com/jcmturner/rpc/v2 v2.0.2 h1:gMB4IwRXYsWw4Bc6o/az2HJgFUA1ffSh90i26ZJ6Xl0=
+github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE=
github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -564,6 +582,7 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
@@ -792,6 +811,7 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
diff --git a/weed/pb/remote.proto b/weed/pb/remote.proto
index 20f4f9934..1f1c37343 100644
--- a/weed/pb/remote.proto
+++ b/weed/pb/remote.proto
@@ -46,6 +46,12 @@ message RemoteConf {
string wasabi_secret_key = 41;
string wasabi_endpoint = 42;
string wasabi_region = 43;
+
+ repeated string hdfs_namenodes = 50;
+ string hdfs_username = 51;
+ string hdfs_service_principal_name = 52;
+ string hdfs_data_transfer_protection = 53;
+
}
message RemoteStorageMapping {
diff --git a/weed/pb/remote_pb/remote.pb.go b/weed/pb/remote_pb/remote.pb.go
index 99e9db026..cc513fe26 100644
--- a/weed/pb/remote_pb/remote.pb.go
+++ b/weed/pb/remote_pb/remote.pb.go
@@ -33,35 +33,39 @@ type RemoteConf struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
- Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`
- Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
- S3AccessKey string `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"`
- S3SecretKey string `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"`
- S3Region string `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"`
- S3Endpoint string `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"`
- S3StorageClass string `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"`
- S3ForcePathStyle bool `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"`
- GcsGoogleApplicationCredentials string `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"`
- AzureAccountName string `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"`
- AzureAccountKey string `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"`
- BackblazeKeyId string `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"`
- BackblazeApplicationKey string `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"`
- BackblazeEndpoint string `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"`
- AliyunAccessKey string `protobuf:"bytes,25,opt,name=aliyun_access_key,json=aliyunAccessKey,proto3" json:"aliyun_access_key,omitempty"`
- AliyunSecretKey string `protobuf:"bytes,26,opt,name=aliyun_secret_key,json=aliyunSecretKey,proto3" json:"aliyun_secret_key,omitempty"`
- AliyunEndpoint string `protobuf:"bytes,27,opt,name=aliyun_endpoint,json=aliyunEndpoint,proto3" json:"aliyun_endpoint,omitempty"`
- AliyunRegion string `protobuf:"bytes,28,opt,name=aliyun_region,json=aliyunRegion,proto3" json:"aliyun_region,omitempty"`
- TencentSecretId string `protobuf:"bytes,30,opt,name=tencent_secret_id,json=tencentSecretId,proto3" json:"tencent_secret_id,omitempty"`
- TencentSecretKey string `protobuf:"bytes,31,opt,name=tencent_secret_key,json=tencentSecretKey,proto3" json:"tencent_secret_key,omitempty"`
- TencentEndpoint string `protobuf:"bytes,32,opt,name=tencent_endpoint,json=tencentEndpoint,proto3" json:"tencent_endpoint,omitempty"`
- BaiduAccessKey string `protobuf:"bytes,35,opt,name=baidu_access_key,json=baiduAccessKey,proto3" json:"baidu_access_key,omitempty"`
- BaiduSecretKey string `protobuf:"bytes,36,opt,name=baidu_secret_key,json=baiduSecretKey,proto3" json:"baidu_secret_key,omitempty"`
- BaiduEndpoint string `protobuf:"bytes,37,opt,name=baidu_endpoint,json=baiduEndpoint,proto3" json:"baidu_endpoint,omitempty"`
- BaiduRegion string `protobuf:"bytes,38,opt,name=baidu_region,json=baiduRegion,proto3" json:"baidu_region,omitempty"`
- WasabiAccessKey string `protobuf:"bytes,40,opt,name=wasabi_access_key,json=wasabiAccessKey,proto3" json:"wasabi_access_key,omitempty"`
- WasabiSecretKey string `protobuf:"bytes,41,opt,name=wasabi_secret_key,json=wasabiSecretKey,proto3" json:"wasabi_secret_key,omitempty"`
- WasabiEndpoint string `protobuf:"bytes,42,opt,name=wasabi_endpoint,json=wasabiEndpoint,proto3" json:"wasabi_endpoint,omitempty"`
- WasabiRegion string `protobuf:"bytes,43,opt,name=wasabi_region,json=wasabiRegion,proto3" json:"wasabi_region,omitempty"`
+ Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`
+ Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
+ S3AccessKey string `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"`
+ S3SecretKey string `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"`
+ S3Region string `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"`
+ S3Endpoint string `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"`
+ S3StorageClass string `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"`
+ S3ForcePathStyle bool `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"`
+ GcsGoogleApplicationCredentials string `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"`
+ AzureAccountName string `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"`
+ AzureAccountKey string `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"`
+ BackblazeKeyId string `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"`
+ BackblazeApplicationKey string `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"`
+ BackblazeEndpoint string `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"`
+ AliyunAccessKey string `protobuf:"bytes,25,opt,name=aliyun_access_key,json=aliyunAccessKey,proto3" json:"aliyun_access_key,omitempty"`
+ AliyunSecretKey string `protobuf:"bytes,26,opt,name=aliyun_secret_key,json=aliyunSecretKey,proto3" json:"aliyun_secret_key,omitempty"`
+ AliyunEndpoint string `protobuf:"bytes,27,opt,name=aliyun_endpoint,json=aliyunEndpoint,proto3" json:"aliyun_endpoint,omitempty"`
+ AliyunRegion string `protobuf:"bytes,28,opt,name=aliyun_region,json=aliyunRegion,proto3" json:"aliyun_region,omitempty"`
+ TencentSecretId string `protobuf:"bytes,30,opt,name=tencent_secret_id,json=tencentSecretId,proto3" json:"tencent_secret_id,omitempty"`
+ TencentSecretKey string `protobuf:"bytes,31,opt,name=tencent_secret_key,json=tencentSecretKey,proto3" json:"tencent_secret_key,omitempty"`
+ TencentEndpoint string `protobuf:"bytes,32,opt,name=tencent_endpoint,json=tencentEndpoint,proto3" json:"tencent_endpoint,omitempty"`
+ BaiduAccessKey string `protobuf:"bytes,35,opt,name=baidu_access_key,json=baiduAccessKey,proto3" json:"baidu_access_key,omitempty"`
+ BaiduSecretKey string `protobuf:"bytes,36,opt,name=baidu_secret_key,json=baiduSecretKey,proto3" json:"baidu_secret_key,omitempty"`
+ BaiduEndpoint string `protobuf:"bytes,37,opt,name=baidu_endpoint,json=baiduEndpoint,proto3" json:"baidu_endpoint,omitempty"`
+ BaiduRegion string `protobuf:"bytes,38,opt,name=baidu_region,json=baiduRegion,proto3" json:"baidu_region,omitempty"`
+ WasabiAccessKey string `protobuf:"bytes,40,opt,name=wasabi_access_key,json=wasabiAccessKey,proto3" json:"wasabi_access_key,omitempty"`
+ WasabiSecretKey string `protobuf:"bytes,41,opt,name=wasabi_secret_key,json=wasabiSecretKey,proto3" json:"wasabi_secret_key,omitempty"`
+ WasabiEndpoint string `protobuf:"bytes,42,opt,name=wasabi_endpoint,json=wasabiEndpoint,proto3" json:"wasabi_endpoint,omitempty"`
+ WasabiRegion string `protobuf:"bytes,43,opt,name=wasabi_region,json=wasabiRegion,proto3" json:"wasabi_region,omitempty"`
+ HdfsNamenodes []string `protobuf:"bytes,50,rep,name=hdfs_namenodes,json=hdfsNamenodes,proto3" json:"hdfs_namenodes,omitempty"`
+ HdfsUsername string `protobuf:"bytes,51,opt,name=hdfs_username,json=hdfsUsername,proto3" json:"hdfs_username,omitempty"`
+ HdfsServicePrincipalName string `protobuf:"bytes,52,opt,name=hdfs_service_principal_name,json=hdfsServicePrincipalName,proto3" json:"hdfs_service_principal_name,omitempty"`
+ HdfsDataTransferProtection string `protobuf:"bytes,53,opt,name=hdfs_data_transfer_protection,json=hdfsDataTransferProtection,proto3" json:"hdfs_data_transfer_protection,omitempty"`
}
func (x *RemoteConf) Reset() {
@@ -299,6 +303,34 @@ func (x *RemoteConf) GetWasabiRegion() string {
return ""
}
+func (x *RemoteConf) GetHdfsNamenodes() []string {
+ if x != nil {
+ return x.HdfsNamenodes
+ }
+ return nil
+}
+
+func (x *RemoteConf) GetHdfsUsername() string {
+ if x != nil {
+ return x.HdfsUsername
+ }
+ return ""
+}
+
+func (x *RemoteConf) GetHdfsServicePrincipalName() string {
+ if x != nil {
+ return x.HdfsServicePrincipalName
+ }
+ return ""
+}
+
+func (x *RemoteConf) GetHdfsDataTransferProtection() string {
+ if x != nil {
+ return x.HdfsDataTransferProtection
+ }
+ return ""
+}
+
type RemoteStorageMapping struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -413,7 +445,7 @@ var File_remote_proto protoreflect.FileDescriptor
var file_remote_proto_rawDesc = []byte{
0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09,
- 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0xbe, 0x09, 0x0a, 0x0a, 0x52, 0x65,
+ 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0x8c, 0x0b, 0x0a, 0x0a, 0x52, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
@@ -489,30 +521,43 @@ var file_remote_proto_rawDesc = []byte{
0x2a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x45, 0x6e, 0x64,
0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f,
0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x61,
- 0x73, 0x61, 0x62, 0x69, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x22, 0xc0, 0x01, 0x0a, 0x14, 0x52,
- 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70,
- 0x69, 0x6e, 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18,
- 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70,
- 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d,
- 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45,
- 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x5d,
- 0x0a, 0x0d, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12,
- 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65,
- 0x79, 0x12, 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
- 0x32, 0x20, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d,
- 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69,
- 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a,
- 0x15, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f,
- 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75,
- 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b,
- 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
- 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65,
- 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65,
- 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
- 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61,
- 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x72,
- 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+ 0x73, 0x61, 0x62, 0x69, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x68, 0x64,
+ 0x66, 0x73, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x32, 0x20, 0x03,
+ 0x28, 0x09, 0x52, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65,
+ 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61,
+ 0x6d, 0x65, 0x18, 0x33, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x64, 0x66, 0x73, 0x55, 0x73,
+ 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x1b, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x73,
+ 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, 0x6c,
+ 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x34, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x68, 0x64, 0x66,
+ 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61,
+ 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x1d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x64, 0x61,
+ 0x74, 0x61, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x74,
+ 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x35, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x68, 0x64,
+ 0x66, 0x73, 0x44, 0x61, 0x74, 0x61, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x72,
+ 0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc0, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x6d,
+ 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e,
+ 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20,
+ 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e,
+ 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70,
+ 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74,
+ 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x5d, 0x0a, 0x0d,
+ 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
+ 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
+ 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20,
+ 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74,
+ 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+ 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, 0x15, 0x52,
+ 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61,
+ 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01,
+ 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b,
+ 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74,
+ 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
+ 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66,
+ 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50,
+ 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
+ 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65,
+ 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x72, 0x65, 0x6d,
+ 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
diff --git a/weed/remote_storage/azure/azure_storage_client.go b/weed/remote_storage/azure/azure_storage_client.go
index c2e1416a5..21b8606c3 100644
--- a/weed/remote_storage/azure/azure_storage_client.go
+++ b/weed/remote_storage/azure/azure_storage_client.go
@@ -22,6 +22,10 @@ func init() {
type azureRemoteStorageMaker struct{}
+func (s azureRemoteStorageMaker) HasBucket() bool {
+ return true
+}
+
func (s azureRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
client := &azureRemoteStorageClient{
diff --git a/weed/remote_storage/gcs/gcs_storage_client.go b/weed/remote_storage/gcs/gcs_storage_client.go
index 44d41f4fd..828d62978 100644
--- a/weed/remote_storage/gcs/gcs_storage_client.go
+++ b/weed/remote_storage/gcs/gcs_storage_client.go
@@ -22,6 +22,10 @@ func init() {
type gcsRemoteStorageMaker struct{}
+func (s gcsRemoteStorageMaker) HasBucket() bool {
+ return true
+}
+
func (s gcsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
client := &gcsRemoteStorageClient{
conf: conf,
diff --git a/weed/remote_storage/hdfs/hdfs_kerberos.go b/weed/remote_storage/hdfs/hdfs_kerberos.go
new file mode 100644
index 000000000..50abc0ad5
--- /dev/null
+++ b/weed/remote_storage/hdfs/hdfs_kerberos.go
@@ -0,0 +1,55 @@
+package hdfs
+
+import (
+ "fmt"
+ "os"
+ "os/user"
+ "strings"
+
+ krb "github.com/jcmturner/gokrb5/v8/client"
+ "github.com/jcmturner/gokrb5/v8/config"
+ "github.com/jcmturner/gokrb5/v8/credentials"
+)
+
+// copy-paste from https://github.com/colinmarc/hdfs/blob/master/cmd/hdfs/kerberos.go
+func getKerberosClient() (*krb.Client, error) {
+ configPath := os.Getenv("KRB5_CONFIG")
+ if configPath == "" {
+ configPath = "/etc/krb5.conf"
+ }
+
+ cfg, err := config.Load(configPath)
+ if err != nil {
+ return nil, err
+ }
+
+ // Determine the ccache location from the environment, falling back to the
+ // default location.
+ ccachePath := os.Getenv("KRB5CCNAME")
+ if strings.Contains(ccachePath, ":") {
+ if strings.HasPrefix(ccachePath, "FILE:") {
+ ccachePath = strings.SplitN(ccachePath, ":", 2)[1]
+ } else {
+ return nil, fmt.Errorf("unusable ccache: %s", ccachePath)
+ }
+ } else if ccachePath == "" {
+ u, err := user.Current()
+ if err != nil {
+ return nil, err
+ }
+
+ ccachePath = fmt.Sprintf("/tmp/krb5cc_%s", u.Uid)
+ }
+
+ ccache, err := credentials.LoadCCache(ccachePath)
+ if err != nil {
+ return nil, err
+ }
+
+ client, err := krb.NewFromCCache(ccache, cfg)
+ if err != nil {
+ return nil, err
+ }
+
+ return client, nil
+}
diff --git a/weed/remote_storage/hdfs/hdfs_storage_client.go b/weed/remote_storage/hdfs/hdfs_storage_client.go
new file mode 100644
index 000000000..0c5f5a45d
--- /dev/null
+++ b/weed/remote_storage/hdfs/hdfs_storage_client.go
@@ -0,0 +1,174 @@
+package hdfs
+
+import (
+ "fmt"
+ "github.com/chrislusf/seaweedfs/weed/glog"
+ "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+ "github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+ "github.com/chrislusf/seaweedfs/weed/remote_storage"
+ "github.com/chrislusf/seaweedfs/weed/util"
+ "github.com/colinmarc/hdfs/v2"
+ "io"
+ "os"
+ "path"
+)
+
+func init() {
+ remote_storage.RemoteStorageClientMakers["hdfs"] = new(hdfsRemoteStorageMaker)
+}
+
+type hdfsRemoteStorageMaker struct{}
+
+func (s hdfsRemoteStorageMaker) HasBucket() bool {
+ return false
+}
+
+func (s hdfsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+ client := &hdfsRemoteStorageClient{
+ conf: conf,
+ }
+
+ options := hdfs.ClientOptions{
+ Addresses: conf.HdfsNamenodes,
+ UseDatanodeHostname: false,
+ }
+
+ if conf.HdfsServicePrincipalName != "" {
+ var err error
+ options.KerberosClient, err = getKerberosClient()
+ if err != nil {
+ return nil, fmt.Errorf("get kerberos authentication: %s", err)
+ }
+ options.KerberosServicePrincipleName = conf.HdfsServicePrincipalName
+
+ if conf.HdfsDataTransferProtection != "" {
+ options.DataTransferProtection = conf.HdfsDataTransferProtection
+ }
+ } else {
+ options.User = conf.HdfsUsername
+ }
+
+ c, err := hdfs.NewClient(options)
+ if err != nil {
+ return nil, err
+ }
+
+ client.client = c
+ return client, nil
+}
+
+type hdfsRemoteStorageClient struct {
+ conf *remote_pb.RemoteConf
+ client *hdfs.Client
+}
+
+var _ = remote_storage.RemoteStorageClient(&hdfsRemoteStorageClient{})
+
+func (c *hdfsRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+
+ return TraverseBfs(func(parentDir util.FullPath, visitFn remote_storage.VisitFunc) error {
+ children, err := c.client.ReadDir(string(parentDir))
+ if err != nil {
+ return err
+ }
+ for _, child := range children {
+ if err := visitFn(string(parentDir), child.Name(), child.IsDir(), &filer_pb.RemoteEntry{
+ StorageName: c.conf.Name,
+ LastLocalSyncTsNs: 0,
+ RemoteETag: "",
+ RemoteMtime: child.ModTime().Unix(),
+ RemoteSize: child.Size(),
+ }); err != nil {
+ return nil
+ }
+ }
+ return nil
+ }, util.FullPath(loc.Path), visitFn)
+
+}
+func (c *hdfsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
+
+ f, err := c.client.Open(loc.Path)
+ if err != nil {
+ return
+ }
+ defer f.Close()
+ data = make([]byte, size)
+ _, err = f.ReadAt(data, offset)
+
+ return
+
+}
+
+func (c *hdfsRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
+ return c.client.MkdirAll(loc.Path, os.FileMode(entry.Attributes.FileMode))
+}
+
+func (c *hdfsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
+
+ dirname := path.Dir(loc.Path)
+
+ // ensure parent directory
+ if err = c.client.MkdirAll(dirname, 0755); err != nil {
+ return
+ }
+
+ // remove existing file
+ info, err := c.client.Stat(loc.Path)
+ if err == nil {
+ err = c.client.Remove(loc.Path)
+ if err != nil {
+ return
+ }
+ }
+
+ // create new file
+ out, err := c.client.Create(loc.Path)
+ if err != nil {
+ return
+ }
+
+ cleanup := func() {
+ if removeErr := c.client.Remove(loc.Path); removeErr != nil {
+ glog.Errorf("clean up %s%s: %v", loc.Name, loc.Path, removeErr)
+ }
+ }
+
+ if _, err = io.Copy(out, reader); err != nil {
+ cleanup()
+ return
+ }
+
+ if err = out.Close(); err != nil {
+ cleanup()
+ return
+ }
+
+ info, err = c.client.Stat(loc.Path)
+ if err != nil {
+ return
+ }
+
+ return &filer_pb.RemoteEntry{
+ RemoteMtime: info.ModTime().Unix(),
+ RemoteSize: info.Size(),
+ RemoteETag: "",
+ StorageName: c.conf.Name,
+ }, nil
+
+}
+
+func (c *hdfsRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
+ if oldEntry.Attributes.FileMode != newEntry.Attributes.FileMode {
+ if err := c.client.Chmod(loc.Path, os.FileMode(newEntry.Attributes.FileMode)); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+func (c *hdfsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
+ if err = c.client.Remove(loc.Path); err != nil {
+ return fmt.Errorf("hdfs delete %s: %v", loc.Path, err)
+ }
+ return
+}
diff --git a/weed/remote_storage/hdfs/traverse_bfs.go b/weed/remote_storage/hdfs/traverse_bfs.go
new file mode 100644
index 000000000..755771283
--- /dev/null
+++ b/weed/remote_storage/hdfs/traverse_bfs.go
@@ -0,0 +1,63 @@
+package hdfs
+
+import (
+ "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+ "github.com/chrislusf/seaweedfs/weed/remote_storage"
+ "github.com/chrislusf/seaweedfs/weed/util"
+ "sync"
+ "time"
+)
+
+type ListDirectoryFunc func(parentDir util.FullPath, visitFn remote_storage.VisitFunc) error
+
+func TraverseBfs(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn remote_storage.VisitFunc) (err error) {
+ K := 5
+
+ var dirQueueWg sync.WaitGroup
+ dirQueue := util.NewQueue()
+ dirQueueWg.Add(1)
+ dirQueue.Enqueue(parentPath)
+ var isTerminating bool
+
+ for i := 0; i < K; i++ {
+ go func() {
+ for {
+ if isTerminating {
+ break
+ }
+ t := dirQueue.Dequeue()
+ if t == nil {
+ time.Sleep(329 * time.Millisecond)
+ continue
+ }
+ dir := t.(util.FullPath)
+ processErr := processOneDirectory(listDirFn, dir, visitFn, dirQueue, &dirQueueWg)
+ if processErr != nil {
+ err = processErr
+ }
+ dirQueueWg.Done()
+ }
+ }()
+ }
+
+ dirQueueWg.Wait()
+ isTerminating = true
+ return
+
+}
+
+func processOneDirectory(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn remote_storage.VisitFunc, dirQueue *util.Queue, dirQueueWg *sync.WaitGroup) (error) {
+
+ return listDirFn(parentPath, func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error {
+ if err := visitFn(dir, name, isDirectory, remoteEntry); err != nil {
+ return err
+ }
+ if !isDirectory {
+ return nil
+ }
+ dirQueueWg.Add(1)
+ dirQueue.Enqueue(parentPath.Child(name))
+ return nil
+ })
+
+}
diff --git a/weed/remote_storage/remote_storage.go b/weed/remote_storage/remote_storage.go
index c9bef8c9b..a4caef0d4 100644
--- a/weed/remote_storage/remote_storage.go
+++ b/weed/remote_storage/remote_storage.go
@@ -9,7 +9,18 @@ import (
"sync"
)
-func ParseLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
+func ParseLocationName(remote string) (locationName string) {
+ if strings.HasSuffix(string(remote), "/") {
+ remote = remote[:len(remote)-1]
+ }
+ parts := strings.SplitN(string(remote), "/", 2)
+ if len(parts) >= 1 {
+ return parts[0]
+ }
+ return
+}
+
+func parseBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
loc = &remote_pb.RemoteStorageLocation{}
if strings.HasSuffix(string(remote), "/") {
remote = remote[:len(remote)-1]
@@ -28,6 +39,22 @@ func ParseLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
return
}
+func parseNoBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
+ loc = &remote_pb.RemoteStorageLocation{}
+ if strings.HasSuffix(string(remote), "/") {
+ remote = remote[:len(remote)-1]
+ }
+ parts := strings.SplitN(string(remote), "/", 2)
+ if len(parts) >= 1 {
+ loc.Name = parts[0]
+ }
+ loc.Path = string(remote[len(loc.Name):])
+ if loc.Path == "" {
+ loc.Path = "/"
+ }
+ return
+}
+
func FormatLocation(loc *remote_pb.RemoteStorageLocation) string {
return fmt.Sprintf("%s/%s%s", loc.Name, loc.Bucket, loc.Path)
}
@@ -45,6 +72,7 @@ type RemoteStorageClient interface {
type RemoteStorageClientMaker interface {
Make(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error)
+ HasBucket() bool
}
var (
@@ -53,6 +81,18 @@ var (
remoteStorageClientsLock sync.Mutex
)
+func ParseRemoteLocation(remoteConfType string, remote string) (remoteStorageLocation *remote_pb.RemoteStorageLocation, err error) {
+ maker, found := RemoteStorageClientMakers[remoteConfType]
+ if !found {
+ return nil, fmt.Errorf("remote storage type %s not found", remoteConfType)
+ }
+
+ if !maker.HasBucket() {
+ return parseNoBucketLocation(remote), nil
+ }
+ return parseBucketLocation(remote), nil
+}
+
func makeRemoteStorageClient(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) {
maker, found := RemoteStorageClientMakers[remoteConf.Type]
if !found {
diff --git a/weed/remote_storage/s3/aliyun.go b/weed/remote_storage/s3/aliyun.go
index 3a681369a..567c74299 100644
--- a/weed/remote_storage/s3/aliyun.go
+++ b/weed/remote_storage/s3/aliyun.go
@@ -18,6 +18,10 @@ func init() {
type AliyunRemoteStorageMaker struct{}
+func (s AliyunRemoteStorageMaker) HasBucket() bool {
+ return true
+}
+
func (s AliyunRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
client := &s3RemoteStorageClient{
conf: conf,
diff --git a/weed/remote_storage/s3/backblaze.go b/weed/remote_storage/s3/backblaze.go
index e4833999e..914f0ca44 100644
--- a/weed/remote_storage/s3/backblaze.go
+++ b/weed/remote_storage/s3/backblaze.go
@@ -16,6 +16,10 @@ func init() {
type BackBlazeRemoteStorageMaker struct{}
+func (s BackBlazeRemoteStorageMaker) HasBucket() bool {
+ return true
+}
+
func (s BackBlazeRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
client := &s3RemoteStorageClient{
conf: conf,
diff --git a/weed/remote_storage/s3/baidu.go b/weed/remote_storage/s3/baidu.go
index 8c5bf7d1b..dfcf32512 100644
--- a/weed/remote_storage/s3/baidu.go
+++ b/weed/remote_storage/s3/baidu.go
@@ -18,6 +18,10 @@ func init() {
type BaiduRemoteStorageMaker struct{}
+func (s BaiduRemoteStorageMaker) HasBucket() bool {
+ return true
+}
+
func (s BaiduRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
client := &s3RemoteStorageClient{
conf: conf,
diff --git a/weed/remote_storage/s3/s3_storage_client.go b/weed/remote_storage/s3/s3_storage_client.go
index 5fadcbc3b..a210683aa 100644
--- a/weed/remote_storage/s3/s3_storage_client.go
+++ b/weed/remote_storage/s3/s3_storage_client.go
@@ -23,6 +23,10 @@ func init() {
type s3RemoteStorageMaker struct{}
+func (s s3RemoteStorageMaker) HasBucket() bool {
+ return true
+}
+
func (s s3RemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
client := &s3RemoteStorageClient{
conf: conf,
diff --git a/weed/remote_storage/s3/tencent.go b/weed/remote_storage/s3/tencent.go
index e2591ca8c..9df72a7e2 100644
--- a/weed/remote_storage/s3/tencent.go
+++ b/weed/remote_storage/s3/tencent.go
@@ -18,6 +18,10 @@ func init() {
type TencentRemoteStorageMaker struct{}
+func (s TencentRemoteStorageMaker) HasBucket() bool {
+ return true
+}
+
func (s TencentRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
client := &s3RemoteStorageClient{
conf: conf,
diff --git a/weed/remote_storage/s3/wasabi.go b/weed/remote_storage/s3/wasabi.go
index 6f8fc3ca8..29cdf7395 100644
--- a/weed/remote_storage/s3/wasabi.go
+++ b/weed/remote_storage/s3/wasabi.go
@@ -18,6 +18,10 @@ func init() {
type WasabiRemoteStorageMaker struct{}
+func (s WasabiRemoteStorageMaker) HasBucket() bool {
+ return true
+}
+
func (s WasabiRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
client := &s3RemoteStorageClient{
conf: conf,
diff --git a/weed/shell/command_remote_configure.go b/weed/shell/command_remote_configure.go
index 1bfe82673..b6c4af178 100644
--- a/weed/shell/command_remote_configure.go
+++ b/weed/shell/command_remote_configure.go
@@ -58,7 +58,7 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write
isDelete := remoteConfigureCommand.Bool("delete", false, "delete one remote storage by its name")
remoteConfigureCommand.StringVar(&conf.Name, "name", "", "a short name to identify the remote storage")
- remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", "[s3|gcs|azure|b2|aliyun|tencent|baidu|wasabi] storage type")
+ remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", "[s3|gcs|azure|b2|aliyun|tencent|baidu|wasabi|hdfs] storage type")
remoteConfigureCommand.StringVar(&conf.S3AccessKey, "s3.access_key", "", "s3 access key")
remoteConfigureCommand.StringVar(&conf.S3SecretKey, "s3.secret_key", "", "s3 secret key")
@@ -95,10 +95,27 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write
remoteConfigureCommand.StringVar(&conf.WasabiEndpoint, "wasabi.endpoint", "", "Wasabi endpoint, see https://wasabi.com/wp-content/themes/wasabi/docs/API_Guide/index.html#t=topics%2Fapidiff-intro.htm")
remoteConfigureCommand.StringVar(&conf.WasabiRegion, "wasabi.region", "", "Wasabi region")
+ var namenodes arrayFlags
+ remoteConfigureCommand.Var(&namenodes, "hdfs.namenodes", "hdfs name node and port, example: namenode1:8020,namenode2:8020")
+ remoteConfigureCommand.StringVar(&conf.HdfsUsername, "hdfs.username", "", "hdfs user name")
+ remoteConfigureCommand.StringVar(&conf.HdfsServicePrincipalName, "hdfs.servicePrincipalName", "", `Kerberos service principal name for the namenode
+
+Example: hdfs/namenode.hadoop.docker
+Namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.
+`)
+ remoteConfigureCommand.StringVar(&conf.HdfsDataTransferProtection, "hdfs.dataTransferProtection", "", "[authentication|integrity|privacy] Kerberos data transfer protection")
+
+
if err = remoteConfigureCommand.Parse(args); err != nil {
return nil
}
+ if conf.Type != "s3" {
+ // clear out the default values
+ conf.S3Region = ""
+ conf.S3ForcePathStyle = false
+ }
+
if conf.Name == "" {
return c.listExistingRemoteStorages(commandEnv, writer)
}
@@ -187,3 +204,14 @@ func (c *commandRemoteConfigure) saveRemoteStorage(commandEnv *CommandEnv, write
return nil
}
+
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+ return "my string representation"
+}
+
+func (i *arrayFlags) Set(value string) error {
+ *i = append(*i, value)
+ return nil
+}
diff --git a/weed/shell/command_remote_meta_sync.go b/weed/shell/command_remote_meta_sync.go
index b4e3534fc..17b9abdb8 100644
--- a/weed/shell/command_remote_meta_sync.go
+++ b/weed/shell/command_remote_meta_sync.go
@@ -55,7 +55,7 @@ func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer
}
mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, *dir)
- if detectErr != nil{
+ if detectErr != nil {
jsonPrintln(writer, mappings)
return detectErr
}
@@ -106,7 +106,7 @@ func detectMountInfo(commandEnv *CommandEnv, writer io.Writer, dir string) (*rem
If entry.RemoteEntry.RemoteTag != remoteEntry.RemoteTag {
the remote version is updated, need to pull meta
}
- */
+*/
func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToCache util.FullPath, remoteConf *remote_pb.RemoteConf) error {
// visit remote storage
@@ -158,7 +158,7 @@ func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util
fmt.Fprintln(writer, " (skip)")
return nil
}
- if existingEntry.RemoteEntry.RemoteETag != remoteEntry.RemoteETag {
+ if existingEntry.RemoteEntry.RemoteETag != remoteEntry.RemoteETag || existingEntry.RemoteEntry.RemoteMtime < remoteEntry.RemoteMtime {
// the remote version is updated, need to pull meta
fmt.Fprintln(writer, " (update)")
return doSaveRemoteEntry(client, string(localDir), existingEntry, remoteEntry)
diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go
index d1b282d9c..3e92428d9 100644
--- a/weed/shell/command_remote_mount.go
+++ b/weed/shell/command_remote_mount.go
@@ -60,15 +60,17 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io
return err
}
- remoteStorageLocation := remote_storage.ParseLocation(*remote)
-
// find configuration for remote storage
- // remotePath is /<bucket>/path/to/dir
- remoteConf, err := c.findRemoteStorageConfiguration(commandEnv, writer, remoteStorageLocation)
+ remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote_storage.ParseLocationName(*remote))
if err != nil {
return fmt.Errorf("find configuration for %s: %v", *remote, err)
}
+ remoteStorageLocation, err := remote_storage.ParseRemoteLocation(remoteConf.Type, *remote)
+ if err != nil {
+ return err
+ }
+
// sync metadata from remote
if err = c.syncMetadata(commandEnv, writer, *dir, *nonEmpty, remoteConf, remoteStorageLocation); err != nil {
return fmt.Errorf("pull metadata: %v", err)
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 07:01:06 +0000