aboutsummaryrefslogtreecommitdiff
path: root/weed/command
diff options
context:
space:
mode:
Diffstat (limited to 'weed/command')
-rw-r--r--weed/command/command.go1
-rw-r--r--weed/command/copy.go147
-rw-r--r--weed/command/server.go3
-rw-r--r--weed/command/upload.go10
4 files changed, 155 insertions, 6 deletions
diff --git a/weed/command/command.go b/weed/command/command.go
index d654f57cd..c451936e5 100644
--- a/weed/command/command.go
+++ b/weed/command/command.go
@@ -11,6 +11,7 @@ var Commands = []*Command{
cmdBenchmark,
cmdBackup,
cmdCompact,
+ cmdCopy,
cmdFix,
cmdServer,
cmdMaster,
diff --git a/weed/command/copy.go b/weed/command/copy.go
new file mode 100644
index 000000000..0e7109daf
--- /dev/null
+++ b/weed/command/copy.go
@@ -0,0 +1,147 @@
+package command
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/chrislusf/seaweedfs/weed/operation"
+ filer_operation "github.com/chrislusf/seaweedfs/weed/operation/filer"
+ "github.com/chrislusf/seaweedfs/weed/security"
+)
+
+var (
+ copy CopyOptions
+)
+
+type CopyOptions struct {
+ master *string
+ include *string
+ replication *string
+ collection *string
+ ttl *string
+ maxMB *int
+ secretKey *string
+
+ secret security.Secret
+}
+
+func init() {
+ cmdCopy.Run = runCopy // break init cycle
+ cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information")
+ copy.master = cmdCopy.Flag.String("master", "localhost:9333", "SeaweedFS master location")
+ copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
+ copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
+ copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
+ copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
+ copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
+ copy.secretKey = cmdCopy.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+}
+
+var cmdCopy = &Command{
+ UsageLine: "copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/",
+ Short: "copy one or a list of files to a filer folder",
+ Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder
+
+ It can copy one or a list of files or folders.
+
+ If copying a whole folder recursively:
+ All files under the folder and subfolders will be copyed.
+ Optional parameter "-include" allows you to specify the file name patterns.
+
+ If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
+ This can save volume server's gzipped processing and allow customizable gzip compression level.
+ The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
+
+ If "maxMB" is set to a positive number, files larger than it would be split into chunks and copyed separatedly.
+ The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
+
+ `,
+}
+
+func runCopy(cmd *Command, args []string) bool {
+ copy.secret = security.Secret(*copy.secretKey)
+ if len(args) <= 1 {
+ return false
+ }
+ filerDestination := args[len(args)-1]
+ fileOrDirs := args[0 : len(args)-1]
+
+ filerUrl, err := url.Parse(filerDestination)
+ if err != nil {
+ fmt.Printf("The last argument should be a URL on filer: %v\n", err)
+ return false
+ }
+ if len(fileOrDirs) > 1 && !strings.HasSuffix(filerUrl.Path, "/") {
+ fmt.Println("Can not copy multiple items to a file. The last argument must be a folder ended with \"/\"")
+ return false
+ }
+
+ for _, fileOrDir := range fileOrDirs {
+ if !doEachCopy(fileOrDir, filerUrl.Host, filerUrl.Path) {
+ return false
+ }
+ }
+ return true
+}
+
+func doEachCopy(fileOrDir string, host string, path string) bool {
+ f, err := os.Open(fileOrDir)
+ if err != nil {
+ fmt.Printf("Failed to open file %s: %v", fileOrDir, err)
+ return false
+ }
+ defer f.Close()
+
+ fi, err := f.Stat()
+ if err != nil {
+ fmt.Printf("Failed to get stat for file %s: %v", fileOrDir, err)
+ return false
+ }
+
+ mode := fi.Mode()
+ if mode.IsDir() {
+ files, _ := ioutil.ReadDir(fileOrDir)
+ for _, subFileOrDir := range files {
+ if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), host, path+fi.Name()+"/") {
+ return false
+ }
+ }
+ return true
+ }
+
+ // this is a regular file
+ if *copy.include != "" {
+ if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
+ return true
+ }
+ }
+
+ parts, err := operation.NewFileParts([]string{fileOrDir})
+ if err != nil {
+ fmt.Printf("Failed to read file %s: %v", fileOrDir, err)
+ }
+
+ results, err := operation.SubmitFiles(*copy.master, parts,
+ *copy.replication, *copy.collection,
+ *copy.ttl, *copy.maxMB, copy.secret)
+ if err != nil {
+ fmt.Printf("Failed to submit file %s: %v", fileOrDir, err)
+ }
+
+ if strings.HasSuffix(path, "/") {
+ path = path + fi.Name()
+ }
+
+ if err = filer_operation.RegisterFile(host, path, results[0].Fid, copy.secret); err != nil {
+ fmt.Printf("Failed to register file %s on %s: %v", fileOrDir, host, err)
+ return false
+ }
+
+ fmt.Printf("Copy %s => http://%s/%s\n", fileOrDir, host, path)
+
+ return true
+}
diff --git a/weed/command/server.go b/weed/command/server.go
index 6ed1e5228..1211c7137 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -82,7 +82,7 @@ func init() {
filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
- filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified")
+ filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -dir is specified")
filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
@@ -164,6 +164,7 @@ func runServer(cmd *Command, args []string) bool {
if *isStartingFiler {
go func() {
+ time.Sleep(1 * time.Second)
r := http.NewServeMux()
_, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection,
*filerOptions.defaultReplicaPlacement,
diff --git a/weed/command/upload.go b/weed/command/upload.go
index 0dfa115bb..1f0696f70 100644
--- a/weed/command/upload.go
+++ b/weed/command/upload.go
@@ -15,7 +15,7 @@ var (
)
type UploadOptions struct {
- server *string
+ master *string
dir *string
include *string
replication *string
@@ -28,7 +28,7 @@ type UploadOptions struct {
func init() {
cmdUpload.Run = runUpload // break init cycle
cmdUpload.IsDebug = cmdUpload.Flag.Bool("debug", false, "verbose debug information")
- upload.server = cmdUpload.Flag.String("server", "localhost:9333", "SeaweedFS master location")
+ upload.master = cmdUpload.Flag.String("master", "localhost:9333", "SeaweedFS master location")
upload.dir = cmdUpload.Flag.String("dir", "", "Upload the whole folder recursively if specified.")
upload.include = cmdUpload.Flag.String("include", "", "pattens of files to upload, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
upload.replication = cmdUpload.Flag.String("replication", "", "replication type")
@@ -39,7 +39,7 @@ func init() {
}
var cmdUpload = &Command{
- UsageLine: "upload -server=localhost:9333 file1 [file2 file3]\n weed upload -server=localhost:9333 -dir=one_directory -include=*.pdf",
+ UsageLine: "upload -master=localhost:9333 file1 [file2 file3]\n weed upload -server=localhost:9333 -dir=one_directory -include=*.pdf",
Short: "upload one or a list of files",
Long: `upload one or a list of files, or batch upload one whole folder recursively.
@@ -79,7 +79,7 @@ func runUpload(cmd *Command, args []string) bool {
if e != nil {
return e
}
- results, e := operation.SubmitFiles(*upload.server, parts,
+ results, e := operation.SubmitFiles(*upload.master, parts,
*upload.replication, *upload.collection,
*upload.ttl, *upload.maxMB, secret)
bytes, _ := json.Marshal(results)
@@ -98,7 +98,7 @@ func runUpload(cmd *Command, args []string) bool {
if e != nil {
fmt.Println(e.Error())
}
- results, _ := operation.SubmitFiles(*upload.server, parts,
+ results, _ := operation.SubmitFiles(*upload.master, parts,
*upload.replication, *upload.collection,
*upload.ttl, *upload.maxMB, secret)
bytes, _ := json.Marshal(results)
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 09:32:07 +0000