1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
package arangodb
import (
"context"
"crypto/md5"
"encoding/binary"
"encoding/hex"
"io"
"strings"
"github.com/arangodb/go-driver"
"github.com/chrislusf/seaweedfs/weed/util"
)
//convert a string into arango-key safe hex bytes hash
func hashString(dir string) string {
h := md5.New()
io.WriteString(h, dir)
b := h.Sum(nil)
return hex.EncodeToString(b)
}
// convert slice of bytes into slice of uint64
// the first uint64 indicates the length in bytes
func bytesToArray(bs []byte) []uint64 {
out := make([]uint64, 0, 2+len(bs)/8)
out = append(out, uint64(len(bs)))
for len(bs)%8 != 0 {
bs = append(bs, 0)
}
for i := 0; i < len(bs); i = i + 8 {
out = append(out, binary.BigEndian.Uint64(bs[i:]))
}
return out
}
// convert from slice of uint64 back to bytes
// if input length is 0 or 1, will return nil
func arrayToBytes(xs []uint64) []byte {
if len(xs) < 2 {
return nil
}
first := xs[0]
out := make([]byte, len(xs)*8) // i think this can actually be len(xs)*8-8, but i dont think an extra 8 bytes hurts...
for i := 1; i < len(xs); i = i + 1 {
binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i])
}
return out[:first]
}
// gets the collection the bucket points to from filepath
func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpath util.FullPath) (c driver.Collection, err error) {
bucket, _ := extractBucket(fullpath)
if bucket == "" {
bucket = DEFAULT_COLLECTION
}
c, err = store.ensureBucket(ctx, bucket)
if err != nil {
return nil, err
}
return c, err
}
// called by extractBucketCollection
func extractBucket(fullpath util.FullPath) (string, string) {
if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") {
return "", string(fullpath)
}
if strings.Count(string(fullpath), "/") < 3 {
return "", string(fullpath)
}
bucketAndObjectKey := string(fullpath)[len(BUCKET_PREFIX+"/"):]
t := strings.Index(bucketAndObjectKey, "/")
bucket := bucketAndObjectKey
shortPath := "/"
if t > 0 {
bucket = bucketAndObjectKey[:t]
shortPath = string(util.FullPath(bucketAndObjectKey[t:]))
}
return bucket, shortPath
}
// get bucket collection from cache. if not exist, creates the buckets collection and grab it
func (store *ArangodbStore) ensureBucket(ctx context.Context, bucket string) (bc driver.Collection, err error) {
var ok bool
store.mu.RLock()
bc, ok = store.buckets[bucket]
store.mu.RUnlock()
if ok {
return bc, nil
}
store.mu.Lock()
defer store.mu.Unlock()
store.buckets[bucket], err = store.ensureCollection(ctx, bucket)
if err != nil {
return nil, err
}
return store.buckets[bucket], nil
}
// creates collection if not exist, ensures indices if not exist
func (store *ArangodbStore) ensureCollection(ctx context.Context, name string) (c driver.Collection, err error) {
ok, err := store.database.CollectionExists(ctx, name)
if err != nil {
return
}
if ok {
c, err = store.database.Collection(ctx, name)
} else {
c, err = store.database.CreateCollection(ctx, name, &driver.CreateCollectionOptions{})
}
if err != nil {
return
}
// ensure indices
if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory", "name"},
&driver.EnsurePersistentIndexOptions{
Name: "directory_name_multi", Unique: true,
}); err != nil {
return
}
if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory"},
&driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil {
return
}
if _, _, err = c.EnsureTTLIndex(ctx, "ttl", 1,
&driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil {
return
}
if _, _, err = c.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{
Name: "IDX_name",
}); err != nil {
return
}
return c, nil
}
|
