aboutsummaryrefslogtreecommitdiff
path: root/weed/command
diff options
context:
space:
mode:
Diffstat (limited to 'weed/command')
-rw-r--r--weed/command/backup.go2
-rw-r--r--weed/command/benchmark.go2
-rw-r--r--weed/command/filer.go3
-rw-r--r--weed/command/filer_copy.go2
-rw-r--r--weed/command/filer_meta_tail.go72
-rw-r--r--weed/command/filer_meta_tail_elastic.go82
-rw-r--r--weed/command/filer_meta_tail_non_elastic.go14
-rw-r--r--weed/command/imports.go2
-rw-r--r--weed/command/scaffold/filer.toml23
-rw-r--r--weed/command/scaffold/master.toml1
-rw-r--r--weed/command/server.go1
-rw-r--r--weed/command/shell.go1
12 files changed, 130 insertions, 75 deletions
diff --git a/weed/command/backup.go b/weed/command/backup.go
index ba1b0d287..c43b0d351 100644
--- a/weed/command/backup.go
+++ b/weed/command/backup.go
@@ -120,7 +120,7 @@ func runBackup(cmd *Command, args []string) bool {
}
if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
- if err = v.Compact2(30*1024*1024*1024, 0, nil); err != nil {
+ if err = v.Compact2(0, 0, nil); err != nil {
fmt.Printf("Compact Volume before synchronizing %v\n", err)
return true
}
diff --git a/weed/command/benchmark.go b/weed/command/benchmark.go
index 7091463cc..82821f579 100644
--- a/weed/command/benchmark.go
+++ b/weed/command/benchmark.go
@@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
defer pprof.StopCPUProfile()
}
- b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
+ b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
go b.masterClient.KeepConnectedToMaster()
b.masterClient.WaitUntilConnected()
diff --git a/weed/command/filer.go b/weed/command/filer.go
index 4dbc04a0c..0935feb76 100644
--- a/weed/command/filer.go
+++ b/weed/command/filer.go
@@ -37,6 +37,7 @@ type FilerOptions struct {
port *int
portGrpc *int
publicPort *int
+ filerGroup *string
collection *string
defaultReplicaPlacement *string
disableDirListing *bool
@@ -59,6 +60,7 @@ type FilerOptions struct {
func init() {
cmdFiler.Run = runFiler // break init cycle
f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
+ f.filerGroup = cmdFiler.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
@@ -201,6 +203,7 @@ func (fo *FilerOptions) startFiler() {
fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
Masters: fo.masters,
+ FilerGroup: *fo.filerGroup,
Collection: *fo.collection,
DefaultReplication: *fo.defaultReplicaPlacement,
DisableDirListing: *fo.disableDirListing,
diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go
index 06bb82319..9a41dd933 100644
--- a/weed/command/filer_copy.go
+++ b/weed/command/filer_copy.go
@@ -71,7 +71,7 @@ var cmdFilerCopy = &Command{
It can copy one or a list of files or folders.
If copying a whole folder recursively:
- All files under the folder and subfolders will be copyed.
+ All files under the folder and sub folders will be copied.
Optional parameter "-include" allows you to specify the file name patterns.
If "maxMB" is set to a positive number, files larger than it would be split into chunks.
diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go
index 51c4e7128..7dbeee444 100644
--- a/weed/command/filer_meta_tail.go
+++ b/weed/command/filer_meta_tail.go
@@ -1,12 +1,9 @@
package command
import (
- "context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/golang/protobuf/jsonpb"
- jsoniter "github.com/json-iterator/go"
- elastic "github.com/olivere/elastic/v7"
"os"
"path/filepath"
"strings"
@@ -124,72 +121,3 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
return true
}
-
-type EsDocument struct {
- Dir string `json:"dir,omitempty"`
- Name string `json:"name,omitempty"`
- IsDirectory bool `json:"isDir,omitempty"`
- Size uint64 `json:"size,omitempty"`
- Uid uint32 `json:"uid,omitempty"`
- Gid uint32 `json:"gid,omitempty"`
- UserName string `json:"userName,omitempty"`
- Collection string `json:"collection,omitempty"`
- Crtime int64 `json:"crtime,omitempty"`
- Mtime int64 `json:"mtime,omitempty"`
- Mime string `json:"mime,omitempty"`
-}
-
-func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) {
- entry := event.NewEntry
- dir, name := event.NewParentPath, entry.Name
- id := util.Md5String([]byte(util.NewFullPath(dir, name)))
- esEntry := &EsDocument{
- Dir: dir,
- Name: name,
- IsDirectory: entry.IsDirectory,
- Size: entry.Attributes.FileSize,
- Uid: entry.Attributes.Uid,
- Gid: entry.Attributes.Gid,
- UserName: entry.Attributes.UserName,
- Collection: entry.Attributes.Collection,
- Crtime: entry.Attributes.Crtime,
- Mtime: entry.Attributes.Mtime,
- Mime: entry.Attributes.Mime,
- }
- return esEntry, id
-}
-
-func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
- options := []elastic.ClientOptionFunc{}
- options = append(options, elastic.SetURL(strings.Split(servers, ",")...))
- options = append(options, elastic.SetSniff(false))
- options = append(options, elastic.SetHealthcheck(false))
- client, err := elastic.NewClient(options...)
- if err != nil {
- return nil, err
- }
- return func(resp *filer_pb.SubscribeMetadataResponse) error {
- event := resp.EventNotification
- if event.OldEntry != nil &&
- (event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) {
- // delete or not update the same file
- dir, name := resp.Directory, event.OldEntry.Name
- id := util.Md5String([]byte(util.NewFullPath(dir, name)))
- println("delete", id)
- _, err := client.Delete().Index(esIndex).Id(id).Do(context.Background())
- return err
- }
- if event.NewEntry != nil {
- // add a new file or update the same file
- esEntry, id := toEsEntry(event)
- value, err := jsoniter.Marshal(esEntry)
- if err != nil {
- return err
- }
- println(string(value))
- _, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background())
- return err
- }
- return nil
- }, nil
-}
diff --git a/weed/command/filer_meta_tail_elastic.go b/weed/command/filer_meta_tail_elastic.go
new file mode 100644
index 000000000..4c5b606a3
--- /dev/null
+++ b/weed/command/filer_meta_tail_elastic.go
@@ -0,0 +1,82 @@
+//go:build elastic
+// +build elastic
+
+package command
+
+import (
+ "context"
+ "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+ "github.com/chrislusf/seaweedfs/weed/util"
+ jsoniter "github.com/json-iterator/go"
+ elastic "github.com/olivere/elastic/v7"
+ "strings"
+)
+
+type EsDocument struct {
+ Dir string `json:"dir,omitempty"`
+ Name string `json:"name,omitempty"`
+ IsDirectory bool `json:"isDir,omitempty"`
+ Size uint64 `json:"size,omitempty"`
+ Uid uint32 `json:"uid,omitempty"`
+ Gid uint32 `json:"gid,omitempty"`
+ UserName string `json:"userName,omitempty"`
+ Collection string `json:"collection,omitempty"`
+ Crtime int64 `json:"crtime,omitempty"`
+ Mtime int64 `json:"mtime,omitempty"`
+ Mime string `json:"mime,omitempty"`
+}
+
+func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) {
+ entry := event.NewEntry
+ dir, name := event.NewParentPath, entry.Name
+ id := util.Md5String([]byte(util.NewFullPath(dir, name)))
+ esEntry := &EsDocument{
+ Dir: dir,
+ Name: name,
+ IsDirectory: entry.IsDirectory,
+ Size: entry.Attributes.FileSize,
+ Uid: entry.Attributes.Uid,
+ Gid: entry.Attributes.Gid,
+ UserName: entry.Attributes.UserName,
+ Collection: entry.Attributes.Collection,
+ Crtime: entry.Attributes.Crtime,
+ Mtime: entry.Attributes.Mtime,
+ Mime: entry.Attributes.Mime,
+ }
+ return esEntry, id
+}
+
+func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
+ options := []elastic.ClientOptionFunc{}
+ options = append(options, elastic.SetURL(strings.Split(servers, ",")...))
+ options = append(options, elastic.SetSniff(false))
+ options = append(options, elastic.SetHealthcheck(false))
+ client, err := elastic.NewClient(options...)
+ if err != nil {
+ return nil, err
+ }
+ return func(resp *filer_pb.SubscribeMetadataResponse) error {
+ event := resp.EventNotification
+ if event.OldEntry != nil &&
+ (event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) {
+ // delete or not update the same file
+ dir, name := resp.Directory, event.OldEntry.Name
+ id := util.Md5String([]byte(util.NewFullPath(dir, name)))
+ println("delete", id)
+ _, err := client.Delete().Index(esIndex).Id(id).Do(context.Background())
+ return err
+ }
+ if event.NewEntry != nil {
+ // add a new file or update the same file
+ esEntry, id := toEsEntry(event)
+ value, err := jsoniter.Marshal(esEntry)
+ if err != nil {
+ return err
+ }
+ println(string(value))
+ _, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background())
+ return err
+ }
+ return nil
+ }, nil
+}
diff --git a/weed/command/filer_meta_tail_non_elastic.go b/weed/command/filer_meta_tail_non_elastic.go
new file mode 100644
index 000000000..f78f3ee09
--- /dev/null
+++ b/weed/command/filer_meta_tail_non_elastic.go
@@ -0,0 +1,14 @@
+//go:build !elastic
+// +build !elastic
+
+package command
+
+import (
+ "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
+ return func(resp *filer_pb.SubscribeMetadataResponse) error {
+ return nil
+ }, nil
+}
diff --git a/weed/command/imports.go b/weed/command/imports.go
index 3792c45c4..04079b162 100644
--- a/weed/command/imports.go
+++ b/weed/command/imports.go
@@ -15,6 +15,7 @@ import (
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
+ _ "github.com/chrislusf/seaweedfs/weed/filer/arangodb"
_ "github.com/chrislusf/seaweedfs/weed/filer/cassandra"
_ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7"
_ "github.com/chrislusf/seaweedfs/weed/filer/etcd"
@@ -31,4 +32,5 @@ import (
_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"
_ "github.com/chrislusf/seaweedfs/weed/filer/redis3"
_ "github.com/chrislusf/seaweedfs/weed/filer/sqlite"
+ _ "github.com/chrislusf/seaweedfs/weed/filer/ydb"
)
diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml
index 5d4513c36..595fb2e62 100644
--- a/weed/command/scaffold/filer.toml
+++ b/weed/command/scaffold/filer.toml
@@ -285,6 +285,29 @@ healthcheck_enabled = false
index.max_result_window = 10000
+[arangodb] # in development dont use it
+enabled = false
+db_name = "seaweedfs"
+servers=["http://localhost:8529"] # list of servers to connect to
+# only basic auth supported for now
+username=""
+password=""
+# skip tls cert validation
+insecure_skip_verify = true
+
+[ydb] # https://ydb.tech/
+enabled = false
+dsn = "grpc://localhost:2136?database=/local"
+prefix = "seaweedfs"
+useBucketPrefix = true # Fast Bucket Deletion
+poolSizeLimit = 50
+dialTimeOut = 10
+
+# Authenticate produced with one of next environment variables:
+# YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=<path/to/sa_key_file> — used service account key file by path
+# YDB_ANONYMOUS_CREDENTIALS="1" — used for authenticate with anonymous access. Anonymous access needs for connect to testing YDB installation
+# YDB_METADATA_CREDENTIALS="1" — used metadata service for authenticate to YDB from yandex cloud virtual machine or from yandex function
+# YDB_ACCESS_TOKEN_CREDENTIALS=<access_token> — used for authenticate to YDB with short-life access token. For example, access token may be IAM token
##########################
##########################
diff --git a/weed/command/scaffold/master.toml b/weed/command/scaffold/master.toml
index ff0d90d1d..10d9d1914 100644
--- a/weed/command/scaffold/master.toml
+++ b/weed/command/scaffold/master.toml
@@ -36,6 +36,7 @@ aws_secret_access_key = "" # if empty, loads from the shared credentials fil
region = "us-east-2"
bucket = "your_bucket_name" # an existing bucket
endpoint = ""
+storage_class = "STANDARD_IA"
# create this number of logical volumes if no more writable volumes
# count_x means how many copies of data.
diff --git a/weed/command/server.go b/weed/command/server.go
index e3aec67d1..d26376c1a 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -101,6 +101,7 @@ func init() {
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
+ filerOptions.filerGroup = cmdServer.Flag.String("filer.filerGroup", "", "share metadata with other filers in the same filerGroup")
filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
filerOptions.portGrpc = cmdServer.Flag.Int("filer.port.grpc", 0, "filer server grpc listen port")
diff --git a/weed/command/shell.go b/weed/command/shell.go
index 3ba51aec1..c32a8e614 100644
--- a/weed/command/shell.go
+++ b/weed/command/shell.go
@@ -18,6 +18,7 @@ var (
func init() {
cmdShell.Run = runShell // break init cycle
shellOptions.Masters = cmdShell.Flag.String("master", "", "comma-separated master servers, e.g. localhost:9333")
+ shellOptions.FilerGroup = cmdShell.Flag.String("filerGroup", "", "filerGroup for the filers")
shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port, e.g. localhost:8888")
shellCluster = cmdShell.Flag.String("cluster", "", "cluster defined in shell.toml")
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-24 12:19:55 +0000