1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
package remote_storage
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"google.golang.org/protobuf/proto"
"io"
"sort"
"strings"
"sync"
"time"
)
const slash = "/"
func ParseLocationName(remote string) (locationName string) {
remote = strings.TrimSuffix(remote, slash)
parts := strings.SplitN(remote, slash, 2)
if len(parts) >= 1 {
return parts[0]
}
return
}
func parseBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
loc = &remote_pb.RemoteStorageLocation{}
remote = strings.TrimSuffix(remote, slash)
parts := strings.SplitN(remote, slash, 3)
if len(parts) >= 1 {
loc.Name = parts[0]
}
if len(parts) >= 2 {
loc.Bucket = parts[1]
}
loc.Path = remote[len(loc.Name)+1+len(loc.Bucket):]
if loc.Path == "" {
loc.Path = slash
}
return
}
func parseNoBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
loc = &remote_pb.RemoteStorageLocation{}
remote = strings.TrimSuffix(remote, slash)
parts := strings.SplitN(remote, slash, 2)
if len(parts) >= 1 {
loc.Name = parts[0]
}
loc.Path = remote[len(loc.Name):]
if loc.Path == "" {
loc.Path = slash
}
return
}
func FormatLocation(loc *remote_pb.RemoteStorageLocation) string {
if loc.Bucket == "" {
return fmt.Sprintf("%s%s", loc.Name, loc.Path)
}
return fmt.Sprintf("%s/%s%s", loc.Name, loc.Bucket, loc.Path)
}
type VisitFunc func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error
type Bucket struct {
Name string
CreatedAt time.Time
}
type RemoteStorageClient interface {
Traverse(loc *remote_pb.RemoteStorageLocation, visitFn VisitFunc) error
ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error)
WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error)
RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error)
WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error)
UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error)
DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error)
ListBuckets() ([]*Bucket, error)
CreateBucket(name string) (err error)
DeleteBucket(name string) (err error)
}
type RemoteStorageClientMaker interface {
Make(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error)
HasBucket() bool
}
type CachedRemoteStorageClient struct {
*remote_pb.RemoteConf
RemoteStorageClient
}
var (
RemoteStorageClientMakers = make(map[string]RemoteStorageClientMaker)
remoteStorageClients = make(map[string]CachedRemoteStorageClient)
remoteStorageClientsLock sync.Mutex
)
func GetAllRemoteStorageNames() string {
var storageNames []string
for k := range RemoteStorageClientMakers {
storageNames = append(storageNames, k)
}
sort.Strings(storageNames)
return strings.Join(storageNames, "|")
}
func GetRemoteStorageNamesHasBucket() string {
var storageNames []string
for k, m := range RemoteStorageClientMakers {
if m.HasBucket() {
storageNames = append(storageNames, k)
}
}
sort.Strings(storageNames)
return strings.Join(storageNames, "|")
}
func ParseRemoteLocation(remoteConfType string, remote string) (remoteStorageLocation *remote_pb.RemoteStorageLocation, err error) {
maker, found := RemoteStorageClientMakers[remoteConfType]
if !found {
return nil, fmt.Errorf("remote storage type %s not found", remoteConfType)
}
if !maker.HasBucket() {
return parseNoBucketLocation(remote), nil
}
return parseBucketLocation(remote), nil
}
func makeRemoteStorageClient(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) {
maker, found := RemoteStorageClientMakers[remoteConf.Type]
if !found {
return nil, fmt.Errorf("remote storage type %s not found", remoteConf.Type)
}
return maker.Make(remoteConf)
}
func GetRemoteStorage(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) {
remoteStorageClientsLock.Lock()
defer remoteStorageClientsLock.Unlock()
existingRemoteStorageClient, found := remoteStorageClients[remoteConf.Name]
if found && proto.Equal(existingRemoteStorageClient.RemoteConf, remoteConf) {
return existingRemoteStorageClient.RemoteStorageClient, nil
}
newRemoteStorageClient, err := makeRemoteStorageClient(remoteConf)
if err != nil {
return nil, fmt.Errorf("make remote storage client %s: %v", remoteConf.Name, err)
}
remoteStorageClients[remoteConf.Name] = CachedRemoteStorageClient{
RemoteConf: remoteConf,
RemoteStorageClient: newRemoteStorageClient,
}
return newRemoteStorageClient, nil
}
|
