aboutsummaryrefslogtreecommitdiff
path: root/weed/storage/backend
diff options
context:
space:
mode:
Diffstat (limited to 'weed/storage/backend')
-rw-r--r--weed/storage/backend/rclone_backend/rclone_backend.go36
1 files changed, 33 insertions, 3 deletions
diff --git a/weed/storage/backend/rclone_backend/rclone_backend.go b/weed/storage/backend/rclone_backend/rclone_backend.go
index 825640a3d..55fde6456 100644
--- a/weed/storage/backend/rclone_backend/rclone_backend.go
+++ b/weed/storage/backend/rclone_backend/rclone_backend.go
@@ -1,12 +1,14 @@
package rclone_backend
import (
+ "bytes"
"context"
"fmt"
"github.com/rclone/rclone/fs/config/configfile"
"github.com/seaweedfs/seaweedfs/weed/util"
"io"
"os"
+ "text/template"
"time"
"github.com/google/uuid"
@@ -38,15 +40,22 @@ func (factory *RcloneBackendFactory) BuildStorage(configuration backend.StringPr
}
type RcloneBackendStorage struct {
- id string
- remoteName string
- fs fs.Fs
+ id string
+ remoteName string
+ keyTemplate *template.Template
+ keyTemplateText string
+ fs fs.Fs
}
func newRcloneBackendStorage(configuration backend.StringProperties, configPrefix string, id string) (s *RcloneBackendStorage, err error) {
s = &RcloneBackendStorage{}
s.id = id
s.remoteName = configuration.GetString(configPrefix + "remote_name")
+ s.keyTemplateText = configuration.GetString(configPrefix + "key_template")
+ s.keyTemplate, err = template.New("keyTemplate").Parse(s.keyTemplateText)
+ if err != nil {
+ return
+ }
ctx := context.TODO()
accounting.Start(ctx)
@@ -65,9 +74,25 @@ func newRcloneBackendStorage(configuration backend.StringProperties, configPrefi
func (s *RcloneBackendStorage) ToProperties() map[string]string {
m := make(map[string]string)
m["remote_name"] = s.remoteName
+ if len(s.keyTemplateText) > 0 {
+ m["key_template"] = s.keyTemplateText
+ }
return m
}
+func formatKey(key string, storage RcloneBackendStorage) (fKey string, err error) {
+ var b bytes.Buffer
+ if len(storage.keyTemplateText) == 0 {
+ fKey = key
+ } else {
+ err = storage.keyTemplate.Execute(&b, key)
+ if err == nil {
+ fKey = b.String()
+ }
+ }
+ return
+}
+
func (s *RcloneBackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile {
f := &RcloneBackendStorageFile{
backendStorage: s,
@@ -85,6 +110,11 @@ func (s *RcloneBackendStorage) CopyFile(f *os.File, fn func(progressed int64, pe
}
key = randomUuid.String()
+ key, err = formatKey(key, *s)
+ if err != nil {
+ return key, 0, err
+ }
+
glog.V(1).Infof("copy dat file of %s to remote rclone.%s as %s", f.Name(), s.id, key)
util.Retry("upload via Rclone", func() error {