diff options
| -rw-r--r-- | .github/workflows/binaries_release4.yml | 4 | ||||
| -rw-r--r-- | .github/workflows/container_release4.yml | 2 | ||||
| -rw-r--r-- | .github/workflows/container_release5.yml | 2 | ||||
| -rw-r--r-- | .github/workflows/go.yml | 4 | ||||
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | weed/command/imports.go | 1 | ||||
| -rw-r--r-- | weed/pb/remote.proto | 5 | ||||
| -rw-r--r-- | weed/remote_storage/hdfs/doc.go | 9 | ||||
| -rw-r--r-- | weed/remote_storage/hdfs/hdfs_kerberos.go | 58 | ||||
| -rw-r--r-- | weed/remote_storage/hdfs/hdfs_storage_client.go | 194 | ||||
| -rw-r--r-- | weed/shell/command_remote_configure.go | 21 |
11 files changed, 7 insertions, 295 deletions
diff --git a/.github/workflows/binaries_release4.yml b/.github/workflows/binaries_release4.yml index 7621c32e2..71434e6a4 100644 --- a/.github/workflows/binaries_release4.yml +++ b/.github/workflows/binaries_release4.yml @@ -36,7 +36,7 @@ jobs: goos: ${{ matrix.goos }} goarch: ${{ matrix.goarch }} overwrite: true - build_flags: -tags elastic,ydb,gocdk,hdfs + build_flags: -tags elastic,ydb,gocdk pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 # build_flags: -tags 5BytesOffset # optional, default is ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} @@ -52,7 +52,7 @@ jobs: goarch: ${{ matrix.goarch }} overwrite: true pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 - build_flags: -tags 5BytesOffset,elastic,ydb,gocdk,hdfs + build_flags: -tags 5BytesOffset,elastic,ydb,gocdk ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} # Where to run `go build .` project_path: weed diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index 0788abeca..1bba0ee9c 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -52,7 +52,7 @@ jobs: context: ./docker push: ${{ github.event_name != 'pull_request' }} file: ./docker/Dockerfile.go_build - build-args: TAGS=elastic,ydb,gocdk,hdfs + build-args: TAGS=elastic,ydb,gocdk platforms: linux/amd64 tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index e536a20b7..181e86fc4 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -52,7 +52,7 @@ jobs: context: ./docker push: ${{ github.event_name != 'pull_request' }} file: ./docker/Dockerfile.go_build - build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,hdfs + build-args: TAGS=5BytesOffset,elastic,ydb,gocdk platforms: linux/amd64 tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9350dc738..b5c407315 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -34,7 +34,7 @@ jobs: cd weed; go get -v -t -d ./... - name: Build - run: cd weed; go build -tags "elastic gocdk sqlite hdfs ydb" -v . + run: cd weed; go build -tags "elastic gocdk sqlite ydb" -v . - name: Test - run: cd weed; go test -tags "elastic gocdk sqlite hdfs ydb" -v ./... + run: cd weed; go test -tags "elastic gocdk sqlite ydb" -v ./... @@ -8,4 +8,4 @@ install: cd weed; go install full_install: - cd weed; go install -tags "elastic gocdk sqlite hdfs ydb" + cd weed; go install -tags "elastic gocdk sqlite ydb" diff --git a/weed/command/imports.go b/weed/command/imports.go index 04079b162..bada94975 100644 --- a/weed/command/imports.go +++ b/weed/command/imports.go @@ -5,7 +5,6 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/remote_storage/azure" _ "github.com/chrislusf/seaweedfs/weed/remote_storage/gcs" - _ "github.com/chrislusf/seaweedfs/weed/remote_storage/hdfs" _ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink" diff --git a/weed/pb/remote.proto b/weed/pb/remote.proto index bdecf4dbe..13f7a878b 100644 --- a/weed/pb/remote.proto +++ b/weed/pb/remote.proto @@ -49,11 +49,6 @@ message RemoteConf { string wasabi_endpoint = 42; string wasabi_region = 43; - repeated string hdfs_namenodes = 50; - string hdfs_username = 51; - string hdfs_service_principal_name = 52; - string hdfs_data_transfer_protection = 53; - string filebase_access_key = 60; string filebase_secret_key = 61; string filebase_endpoint = 62; diff --git a/weed/remote_storage/hdfs/doc.go b/weed/remote_storage/hdfs/doc.go deleted file mode 100644 index 086c9de3f..000000000 --- a/weed/remote_storage/hdfs/doc.go +++ /dev/null @@ -1,9 +0,0 @@ -/* - -Package hdfs is for remote hdfs storage. - -The referenced "github.com/colinmarc/hdfs/v2" library is too big when compiled. -So this is only compiled in "make full_install". - -*/ -package hdfs diff --git a/weed/remote_storage/hdfs/hdfs_kerberos.go b/weed/remote_storage/hdfs/hdfs_kerberos.go deleted file mode 100644 index ba152020a..000000000 --- a/weed/remote_storage/hdfs/hdfs_kerberos.go +++ /dev/null @@ -1,58 +0,0 @@ -//go:build hdfs -// +build hdfs - -package hdfs - -import ( - "fmt" - "os" - "os/user" - "strings" - - krb "github.com/jcmturner/gokrb5/v8/client" - "github.com/jcmturner/gokrb5/v8/config" - "github.com/jcmturner/gokrb5/v8/credentials" -) - -// copy-paste from https://github.com/colinmarc/hdfs/blob/master/cmd/hdfs/kerberos.go -func getKerberosClient() (*krb.Client, error) { - configPath := os.Getenv("KRB5_CONFIG") - if configPath == "" { - configPath = "/etc/krb5.conf" - } - - cfg, err := config.Load(configPath) - if err != nil { - return nil, err - } - - // Determine the ccache location from the environment, falling back to the - // default location. - ccachePath := os.Getenv("KRB5CCNAME") - if strings.Contains(ccachePath, ":") { - if strings.HasPrefix(ccachePath, "FILE:") { - ccachePath = strings.SplitN(ccachePath, ":", 2)[1] - } else { - return nil, fmt.Errorf("unusable ccache: %s", ccachePath) - } - } else if ccachePath == "" { - u, err := user.Current() - if err != nil { - return nil, err - } - - ccachePath = fmt.Sprintf("/tmp/krb5cc_%s", u.Uid) - } - - ccache, err := credentials.LoadCCache(ccachePath) - if err != nil { - return nil, err - } - - client, err := krb.NewFromCCache(ccache, cfg) - if err != nil { - return nil, err - } - - return client, nil -} diff --git a/weed/remote_storage/hdfs/hdfs_storage_client.go b/weed/remote_storage/hdfs/hdfs_storage_client.go deleted file mode 100644 index 3b71958fd..000000000 --- a/weed/remote_storage/hdfs/hdfs_storage_client.go +++ /dev/null @@ -1,194 +0,0 @@ -//go:build hdfs -// +build hdfs - -package hdfs - -import ( - "fmt" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" - "github.com/chrislusf/seaweedfs/weed/remote_storage" - "github.com/chrislusf/seaweedfs/weed/util" - hdfs "github.com/colinmarc/hdfs/v2" - "io" - "os" - "path" -) - -func init() { - remote_storage.RemoteStorageClientMakers["hdfs"] = new(hdfsRemoteStorageMaker) -} - -type hdfsRemoteStorageMaker struct{} - -func (s hdfsRemoteStorageMaker) HasBucket() bool { - return false -} - -func (s hdfsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { - client := &hdfsRemoteStorageClient{ - conf: conf, - } - - options := hdfs.ClientOptions{ - Addresses: conf.HdfsNamenodes, - UseDatanodeHostname: false, - } - - if conf.HdfsServicePrincipalName != "" { - var err error - options.KerberosClient, err = getKerberosClient() - if err != nil { - return nil, fmt.Errorf("get kerberos authentication: %s", err) - } - options.KerberosServicePrincipleName = conf.HdfsServicePrincipalName - - if conf.HdfsDataTransferProtection != "" { - options.DataTransferProtection = conf.HdfsDataTransferProtection - } - } else { - options.User = conf.HdfsUsername - } - - c, err := hdfs.NewClient(options) - if err != nil { - return nil, err - } - - client.client = c - return client, nil -} - -type hdfsRemoteStorageClient struct { - conf *remote_pb.RemoteConf - client *hdfs.Client -} - -var _ = remote_storage.RemoteStorageClient(&hdfsRemoteStorageClient{}) - -func (c *hdfsRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) { - - return remote_storage.TraverseBfs(func(parentDir util.FullPath, visitFn remote_storage.VisitFunc) error { - children, err := c.client.ReadDir(string(parentDir)) - if err != nil { - return err - } - for _, child := range children { - if err := visitFn(string(parentDir), child.Name(), child.IsDir(), &filer_pb.RemoteEntry{ - StorageName: c.conf.Name, - LastLocalSyncTsNs: 0, - RemoteETag: "", - RemoteMtime: child.ModTime().Unix(), - RemoteSize: child.Size(), - }); err != nil { - return nil - } - } - return nil - }, util.FullPath(loc.Path), visitFn) - -} -func (c *hdfsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) { - - f, err := c.client.Open(loc.Path) - if err != nil { - return - } - defer f.Close() - data = make([]byte, size) - _, err = f.ReadAt(data, offset) - - return - -} - -func (c *hdfsRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) { - return c.client.MkdirAll(loc.Path, os.FileMode(entry.Attributes.FileMode)) -} - -func (c *hdfsRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) { - return c.client.RemoveAll(loc.Path) -} - -func (c *hdfsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) { - - dirname := path.Dir(loc.Path) - - // ensure parent directory - if err = c.client.MkdirAll(dirname, 0755); err != nil { - return - } - - // remove existing file - info, err := c.client.Stat(loc.Path) - if err == nil { - err = c.client.Remove(loc.Path) - if err != nil { - return - } - } - - // create new file - out, err := c.client.Create(loc.Path) - if err != nil { - return - } - - cleanup := func() { - if removeErr := c.client.Remove(loc.Path); removeErr != nil { - glog.Errorf("clean up %s%s: %v", loc.Name, loc.Path, removeErr) - } - } - - if _, err = io.Copy(out, reader); err != nil { - cleanup() - return - } - - if err = out.Close(); err != nil { - cleanup() - return - } - - info, err = c.client.Stat(loc.Path) - if err != nil { - return - } - - return &filer_pb.RemoteEntry{ - RemoteMtime: info.ModTime().Unix(), - RemoteSize: info.Size(), - RemoteETag: "", - StorageName: c.conf.Name, - }, nil - -} - -func (c *hdfsRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error { - if oldEntry.Attributes.FileMode != newEntry.Attributes.FileMode { - if err := c.client.Chmod(loc.Path, os.FileMode(newEntry.Attributes.FileMode)); err != nil { - return err - } - } - return nil -} - -func (c *hdfsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) { - if err = c.client.Remove(loc.Path); err != nil { - return fmt.Errorf("hdfs delete %s: %v", loc.Path, err) - } - return -} - -func (c *hdfsRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) { - return -} - -func (c *hdfsRemoteStorageClient) CreateBucket(name string) (err error) { - return -} - -func (c *hdfsRemoteStorageClient) DeleteBucket(name string) (err error) { - return -} diff --git a/weed/shell/command_remote_configure.go b/weed/shell/command_remote_configure.go index c892c3443..e15090190 100644 --- a/weed/shell/command_remote_configure.go +++ b/weed/shell/command_remote_configure.go @@ -108,16 +108,6 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write remoteConfigureCommand.StringVar(&conf.StorjSecretKey, "storj.secret_key", "", "Storj secret key") remoteConfigureCommand.StringVar(&conf.StorjEndpoint, "storj.endpoint", "", "Storj endpoint") - var namenodes arrayFlags - remoteConfigureCommand.Var(&namenodes, "hdfs.namenodes", "hdfs name node and port, example: namenode1:8020,namenode2:8020") - remoteConfigureCommand.StringVar(&conf.HdfsUsername, "hdfs.username", "", "hdfs user name") - remoteConfigureCommand.StringVar(&conf.HdfsServicePrincipalName, "hdfs.servicePrincipalName", "", `Kerberos service principal name for the namenode - -Example: hdfs/namenode.hadoop.docker -Namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'. -`) - remoteConfigureCommand.StringVar(&conf.HdfsDataTransferProtection, "hdfs.dataTransferProtection", "", "[authentication|integrity|privacy] Kerberos data transfer protection") - if err = remoteConfigureCommand.Parse(args); err != nil { return nil } @@ -223,14 +213,3 @@ func (c *commandRemoteConfigure) saveRemoteStorage(commandEnv *CommandEnv, write return nil } - -type arrayFlags []string - -func (i *arrayFlags) String() string { - return "my string representation" -} - -func (i *arrayFlags) Set(value string) error { - *i = append(*i, value) - return nil -} |