aboutsummaryrefslogtreecommitdiff
path: root/weed/storage
diff options
context:
space:
mode:
Diffstat (limited to 'weed/storage')
-rw-r--r--weed/storage/file_id.go10
-rw-r--r--weed/storage/needle.go12
-rw-r--r--weed/storage/needle/btree_map.go2
-rw-r--r--weed/storage/needle/compact_map.go2
-rw-r--r--weed/storage/needle/compact_map_perf_test.go8
-rw-r--r--weed/storage/needle/compact_map_test.go2
-rw-r--r--weed/storage/needle/needle_value.go2
-rw-r--r--weed/storage/needle_map.go4
-rw-r--r--weed/storage/needle_map_boltdb.go4
-rw-r--r--weed/storage/needle_map_leveldb.go2
-rw-r--r--weed/storage/needle_map_memory.go2
-rw-r--r--weed/storage/needle_map_metric.go4
-rw-r--r--weed/storage/needle_map_metric_test.go6
-rw-r--r--weed/storage/needle_parse_multipart.go6
-rw-r--r--weed/storage/needle_read_write.go32
-rw-r--r--weed/storage/needle_test.go2
-rw-r--r--weed/storage/store.go2
-rw-r--r--weed/storage/types/needle_id_type.go6
-rw-r--r--weed/storage/types/needle_types.go12
-rw-r--r--weed/storage/volume_vacuum.go2
-rw-r--r--weed/storage/volume_vacuum_test.go2
21 files changed, 64 insertions, 60 deletions
diff --git a/weed/storage/file_id.go b/weed/storage/file_id.go
index 0871bfb25..37dcb7c70 100644
--- a/weed/storage/file_id.go
+++ b/weed/storage/file_id.go
@@ -20,11 +20,15 @@ func NewFileId(VolumeId VolumeId, key uint64, cookie uint32) *FileId {
}
func (n *FileId) String() string {
+ return n.VolumeId.String() + "," + formatNeedleIdCookie(n.Key, n.Cookie)
+}
+
+func formatNeedleIdCookie(key NeedleId, cookie Cookie) string {
bytes := make([]byte, NeedleIdSize+CookieSize)
- NeedleIdToBytes(bytes[0:NeedleIdSize], n.Key)
- CookieToBytes(bytes[NeedleIdSize:NeedleIdSize+CookieSize], n.Cookie)
+ NeedleIdToBytes(bytes[0:NeedleIdSize], key)
+ CookieToBytes(bytes[NeedleIdSize:NeedleIdSize+CookieSize], cookie)
nonzero_index := 0
for ; bytes[nonzero_index] == 0; nonzero_index++ {
}
- return n.VolumeId.String() + "," + hex.EncodeToString(bytes[nonzero_index:])
+ return hex.EncodeToString(bytes[nonzero_index:])
}
diff --git a/weed/storage/needle.go b/weed/storage/needle.go
index 46ba933ca..31bada091 100644
--- a/weed/storage/needle.go
+++ b/weed/storage/needle.go
@@ -29,12 +29,12 @@ type Needle struct {
DataSize uint32 `comment:"Data size"` //version2
Data []byte `comment:"The actual file data"`
- Flags byte `comment:"boolean flags"` //version2
- NameSize uint8 //version2
+ Flags byte `comment:"boolean flags"` //version2
+ NameSize uint8 //version2
Name []byte `comment:"maximum 256 characters"` //version2
- MimeSize uint8 //version2
+ MimeSize uint8 //version2
Mime []byte `comment:"maximum 256 characters"` //version2
- PairsSize uint16 //version2
+ PairsSize uint16 //version2
Pairs []byte `comment:"additional name value pairs, json format, maximum 64kB"`
LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
Ttl *TTL
@@ -44,7 +44,7 @@ type Needle struct {
}
func (n *Needle) String() (str string) {
- str = fmt.Sprintf("Cookie:%d, Id:%d, Size:%d, DataSize:%d, Name: %s, Mime: %s", n.Cookie, n.Id, n.Size, n.DataSize, n.Name, n.Mime)
+ str = fmt.Sprintf("%s Size:%d, DataSize:%d, Name:%s, Mime:%s", formatNeedleIdCookie(n.Id, n.Cookie), n.Size, n.DataSize, n.Name, n.Mime)
return
}
@@ -134,7 +134,7 @@ func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
dotSep := strings.LastIndex(r.URL.Path, ".")
fid := r.URL.Path[commaSep+1:]
if dotSep > 0 {
- fid = r.URL.Path[commaSep+1: dotSep]
+ fid = r.URL.Path[commaSep+1 : dotSep]
}
e = n.ParsePath(fid)
diff --git a/weed/storage/needle/btree_map.go b/weed/storage/needle/btree_map.go
index a7c9982ac..d688b802e 100644
--- a/weed/storage/needle/btree_map.go
+++ b/weed/storage/needle/btree_map.go
@@ -1,8 +1,8 @@
package needle
import (
- "github.com/google/btree"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "github.com/google/btree"
)
//This map assumes mostly inserting increasing keys
diff --git a/weed/storage/needle/compact_map.go b/weed/storage/needle/compact_map.go
index 7b653d838..9852dca74 100644
--- a/weed/storage/needle/compact_map.go
+++ b/weed/storage/needle/compact_map.go
@@ -1,8 +1,8 @@
package needle
import (
- "sync"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "sync"
)
type CompactSection struct {
diff --git a/weed/storage/needle/compact_map_perf_test.go b/weed/storage/needle/compact_map_perf_test.go
index 7908e8d1d..a66836ac8 100644
--- a/weed/storage/needle/compact_map_perf_test.go
+++ b/weed/storage/needle/compact_map_perf_test.go
@@ -6,8 +6,8 @@ import (
"testing"
"github.com/chrislusf/seaweedfs/weed/glog"
- "github.com/chrislusf/seaweedfs/weed/util"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "github.com/chrislusf/seaweedfs/weed/util"
)
func TestMemoryUsage(t *testing.T) {
@@ -30,9 +30,9 @@ func loadNewNeedleMap(file *os.File) {
}
for count > 0 && e == nil {
for i := 0; i < count; i += 16 {
- key := BytesToNeedleId(bytes[i:i+NeedleIdSize])
- offset := BytesToOffset(bytes[i+NeedleIdSize:i+NeedleIdSize+OffsetSize])
- size := util.BytesToUint32(bytes[i+NeedleIdSize+OffsetSize:i+NeedleIdSize+OffsetSize+SizeSize])
+ key := BytesToNeedleId(bytes[i : i+NeedleIdSize])
+ offset := BytesToOffset(bytes[i+NeedleIdSize : i+NeedleIdSize+OffsetSize])
+ size := util.BytesToUint32(bytes[i+NeedleIdSize+OffsetSize : i+NeedleIdSize+OffsetSize+SizeSize])
if offset > 0 {
m.Set(NeedleId(key), offset, size)
diff --git a/weed/storage/needle/compact_map_test.go b/weed/storage/needle/compact_map_test.go
index 11d98eab3..b4cbb446a 100644
--- a/weed/storage/needle/compact_map_test.go
+++ b/weed/storage/needle/compact_map_test.go
@@ -1,8 +1,8 @@
package needle
import (
- "testing"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "testing"
)
func TestIssue52(t *testing.T) {
diff --git a/weed/storage/needle/needle_value.go b/weed/storage/needle/needle_value.go
index 8fd7b1b1c..b15d25245 100644
--- a/weed/storage/needle/needle_value.go
+++ b/weed/storage/needle/needle_value.go
@@ -1,8 +1,8 @@
package needle
import (
- "github.com/google/btree"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "github.com/google/btree"
)
const (
diff --git a/weed/storage/needle_map.go b/weed/storage/needle_map.go
index ce2079e89..6d815679b 100644
--- a/weed/storage/needle_map.go
+++ b/weed/storage/needle_map.go
@@ -57,8 +57,8 @@ func (nm *baseNeedleMapper) IndexFileName() string {
func IdxFileEntry(bytes []byte) (key NeedleId, offset Offset, size uint32) {
key = BytesToNeedleId(bytes[:NeedleIdSize])
- offset = BytesToOffset(bytes[NeedleIdSize:NeedleIdSize+OffsetSize])
- size = util.BytesToUint32(bytes[NeedleIdSize+OffsetSize:NeedleIdSize+OffsetSize+SizeSize])
+ offset = BytesToOffset(bytes[NeedleIdSize : NeedleIdSize+OffsetSize])
+ size = util.BytesToUint32(bytes[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])
return
}
func (nm *baseNeedleMapper) appendToIndexFile(key NeedleId, offset Offset, size uint32) error {
diff --git a/weed/storage/needle_map_boltdb.go b/weed/storage/needle_map_boltdb.go
index d5062a1b7..fd335fd00 100644
--- a/weed/storage/needle_map_boltdb.go
+++ b/weed/storage/needle_map_boltdb.go
@@ -6,11 +6,11 @@ import (
"github.com/boltdb/bolt"
+ "errors"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
- "errors"
)
type BoltDbNeedleMap struct {
@@ -101,7 +101,7 @@ func (m *BoltDbNeedleMap) Get(key NeedleId) (element *needle.NeedleValue, ok boo
}
offset = BytesToOffset(data[0:OffsetSize])
- size = util.BytesToUint32(data[OffsetSize:OffsetSize+SizeSize])
+ size = util.BytesToUint32(data[OffsetSize : OffsetSize+SizeSize])
return nil
})
diff --git a/weed/storage/needle_map_leveldb.go b/weed/storage/needle_map_leveldb.go
index 1af88e545..1580bb005 100644
--- a/weed/storage/needle_map_leveldb.go
+++ b/weed/storage/needle_map_leveldb.go
@@ -81,7 +81,7 @@ func (m *LevelDbNeedleMap) Get(key NeedleId) (element *needle.NeedleValue, ok bo
return nil, false
}
offset := BytesToOffset(data[0:OffsetSize])
- size := util.BytesToUint32(data[OffsetSize:OffsetSize+SizeSize])
+ size := util.BytesToUint32(data[OffsetSize : OffsetSize+SizeSize])
return &needle.NeedleValue{Key: NeedleId(key), Offset: offset, Size: size}, true
}
diff --git a/weed/storage/needle_map_memory.go b/weed/storage/needle_map_memory.go
index 690ddd737..fa5576c2b 100644
--- a/weed/storage/needle_map_memory.go
+++ b/weed/storage/needle_map_memory.go
@@ -88,7 +88,7 @@ func WalkIndexFile(r *os.File, fn func(key NeedleId, offset Offset, size uint32)
for count > 0 && e == nil || e == io.EOF {
for i = 0; i+NeedleEntrySize <= count; i += NeedleEntrySize {
- key, offset, size = IdxFileEntry(bytes[i: i+NeedleEntrySize])
+ key, offset, size = IdxFileEntry(bytes[i : i+NeedleEntrySize])
if e = fn(key, offset, size); e != nil {
return e
}
diff --git a/weed/storage/needle_map_metric.go b/weed/storage/needle_map_metric.go
index 793a9ea10..3bcb140f1 100644
--- a/weed/storage/needle_map_metric.go
+++ b/weed/storage/needle_map_metric.go
@@ -2,10 +2,10 @@ package storage
import (
"fmt"
- "os"
- "github.com/willf/bloom"
"github.com/chrislusf/seaweedfs/weed/glog"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "github.com/willf/bloom"
+ "os"
)
type mapMetric struct {
diff --git a/weed/storage/needle_map_metric_test.go b/weed/storage/needle_map_metric_test.go
index 400e655a7..539f83a87 100644
--- a/weed/storage/needle_map_metric_test.go
+++ b/weed/storage/needle_map_metric_test.go
@@ -1,11 +1,11 @@
package storage
import (
- "testing"
- "io/ioutil"
- "math/rand"
"github.com/chrislusf/seaweedfs/weed/glog"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "io/ioutil"
+ "math/rand"
+ "testing"
)
func TestFastLoadingNeedleMapMetrics(t *testing.T) {
diff --git a/weed/storage/needle_parse_multipart.go b/weed/storage/needle_parse_multipart.go
index 112ec32d4..af12b994d 100644
--- a/weed/storage/needle_parse_multipart.go
+++ b/weed/storage/needle_parse_multipart.go
@@ -1,12 +1,12 @@
package storage
import (
- "mime"
- "github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/glog"
+ "github.com/chrislusf/seaweedfs/weed/operation"
+ "io/ioutil"
+ "mime"
"net/http"
"path"
- "io/ioutil"
"strings"
)
diff --git a/weed/storage/needle_read_write.go b/weed/storage/needle_read_write.go
index a62539afc..bfd325475 100644
--- a/weed/storage/needle_read_write.go
+++ b/weed/storage/needle_read_write.go
@@ -6,8 +6,8 @@ import (
"io"
"os"
- . "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/glog"
+ . "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
@@ -59,7 +59,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
actualSize = NeedleEntrySize + int64(n.Size)
padding := NeedlePaddingSize - ((NeedleEntrySize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
- _, err = w.Write(header[0: NeedleChecksumSize+padding])
+ _, err = w.Write(header[0 : NeedleChecksumSize+padding])
return
case Version2:
header := make([]byte, NeedleEntrySize)
@@ -123,7 +123,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
}
if n.HasLastModifiedDate() {
util.Uint64toBytes(header[0:8], n.LastModified)
- if _, err = w.Write(header[8-LastModifiedBytesLength: 8]); err != nil {
+ if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil {
return
}
}
@@ -145,7 +145,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
}
padding := NeedlePaddingSize - ((NeedleEntrySize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
- _, err = w.Write(header[0: NeedleChecksumSize+padding])
+ _, err = w.Write(header[0 : NeedleChecksumSize+padding])
return n.DataSize, getActualSize(n.Size), err
}
@@ -169,14 +169,14 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
}
switch version {
case Version1:
- n.Data = bytes[NeedleEntrySize: NeedleEntrySize+size]
+ n.Data = bytes[NeedleEntrySize : NeedleEntrySize+size]
case Version2:
- n.readNeedleDataVersion2(bytes[NeedleEntrySize: NeedleEntrySize+int(n.Size)])
+ n.readNeedleDataVersion2(bytes[NeedleEntrySize : NeedleEntrySize+int(n.Size)])
}
if size == 0 {
return nil
}
- checksum := util.BytesToUint32(bytes[NeedleEntrySize+size: NeedleEntrySize+size+NeedleChecksumSize])
+ checksum := util.BytesToUint32(bytes[NeedleEntrySize+size : NeedleEntrySize+size+NeedleChecksumSize])
newChecksum := NewCRC(n.Data)
if checksum != newChecksum.Value() {
return errors.New("CRC error! Data On Disk Corrupted")
@@ -187,21 +187,21 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
func (n *Needle) ParseNeedleHeader(bytes []byte) {
n.Cookie = BytesToCookie(bytes[0:CookieSize])
- n.Id = BytesToNeedleId(bytes[CookieSize:CookieSize+NeedleIdSize])
- n.Size = util.BytesToUint32(bytes[CookieSize+NeedleIdSize:NeedleEntrySize])
+ n.Id = BytesToNeedleId(bytes[CookieSize : CookieSize+NeedleIdSize])
+ n.Size = util.BytesToUint32(bytes[CookieSize+NeedleIdSize : NeedleEntrySize])
}
func (n *Needle) readNeedleDataVersion2(bytes []byte) {
index, lenBytes := 0, len(bytes)
if index < lenBytes {
- n.DataSize = util.BytesToUint32(bytes[index: index+4])
+ n.DataSize = util.BytesToUint32(bytes[index : index+4])
index = index + 4
if int(n.DataSize)+index > lenBytes {
// this if clause is due to bug #87 and #93, fixed in v0.69
// remove this clause later
return
}
- n.Data = bytes[index: index+int(n.DataSize)]
+ n.Data = bytes[index : index+int(n.DataSize)]
index = index + int(n.DataSize)
n.Flags = bytes[index]
index = index + 1
@@ -209,25 +209,25 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) {
if index < lenBytes && n.HasName() {
n.NameSize = uint8(bytes[index])
index = index + 1
- n.Name = bytes[index: index+int(n.NameSize)]
+ n.Name = bytes[index : index+int(n.NameSize)]
index = index + int(n.NameSize)
}
if index < lenBytes && n.HasMime() {
n.MimeSize = uint8(bytes[index])
index = index + 1
- n.Mime = bytes[index: index+int(n.MimeSize)]
+ n.Mime = bytes[index : index+int(n.MimeSize)]
index = index + int(n.MimeSize)
}
if index < lenBytes && n.HasLastModifiedDate() {
- n.LastModified = util.BytesToUint64(bytes[index: index+LastModifiedBytesLength])
+ n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
index = index + LastModifiedBytesLength
}
if index < lenBytes && n.HasTtl() {
- n.Ttl = LoadTTLFromBytes(bytes[index: index+TtlBytesLength])
+ n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
index = index + TtlBytesLength
}
if index < lenBytes && n.HasPairs() {
- n.PairsSize = util.BytesToUint16(bytes[index: index+2])
+ n.PairsSize = util.BytesToUint16(bytes[index : index+2])
index += 2
end := index + int(n.PairsSize)
n.Pairs = bytes[index:end]
diff --git a/weed/storage/needle_test.go b/weed/storage/needle_test.go
index 4dce3f123..65036409c 100644
--- a/weed/storage/needle_test.go
+++ b/weed/storage/needle_test.go
@@ -1,8 +1,8 @@
package storage
import (
- "testing"
"github.com/chrislusf/seaweedfs/weed/storage/types"
+ "testing"
)
func TestParseKeyHash(t *testing.T) {
diff --git a/weed/storage/store.go b/weed/storage/store.go
index ef055ee59..ac125ef4b 100644
--- a/weed/storage/store.go
+++ b/weed/storage/store.go
@@ -5,9 +5,9 @@ import (
"strconv"
"strings"
- . "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+ . "github.com/chrislusf/seaweedfs/weed/storage/types"
)
const (
diff --git a/weed/storage/types/needle_id_type.go b/weed/storage/types/needle_id_type.go
index d9e7074b3..cb19a6f45 100644
--- a/weed/storage/types/needle_id_type.go
+++ b/weed/storage/types/needle_id_type.go
@@ -1,9 +1,9 @@
package types
import (
+ "fmt"
"github.com/chrislusf/seaweedfs/weed/util"
"strconv"
- "fmt"
)
type NeedleId uint64
@@ -21,11 +21,11 @@ func NeedleIdToUint64(needleId NeedleId) uint64 {
return uint64(needleId)
}
-func Uint64ToNeedleId(needleId uint64) (NeedleId) {
+func Uint64ToNeedleId(needleId uint64) NeedleId {
return NeedleId(needleId)
}
-func BytesToNeedleId(bytes []byte) (NeedleId) {
+func BytesToNeedleId(bytes []byte) NeedleId {
return NeedleId(util.BytesToUint64(bytes))
}
diff --git a/weed/storage/types/needle_types.go b/weed/storage/types/needle_types.go
index 09ff727e6..8a2054fc5 100644
--- a/weed/storage/types/needle_types.go
+++ b/weed/storage/types/needle_types.go
@@ -1,10 +1,10 @@
package types
import (
- "math"
+ "fmt"
"github.com/chrislusf/seaweedfs/weed/util"
+ "math"
"strconv"
- "fmt"
)
type Offset uint32
@@ -23,11 +23,11 @@ const (
func CookieToBytes(bytes []byte, cookie Cookie) {
util.Uint32toBytes(bytes, uint32(cookie))
}
-func Uint32ToCookie(cookie uint32) (Cookie) {
+func Uint32ToCookie(cookie uint32) Cookie {
return Cookie(cookie)
}
-func BytesToCookie(bytes []byte) (Cookie) {
+func BytesToCookie(bytes []byte) Cookie {
return Cookie(util.BytesToUint32(bytes[0:4]))
}
@@ -43,10 +43,10 @@ func OffsetToBytes(bytes []byte, offset Offset) {
util.Uint32toBytes(bytes, uint32(offset))
}
-func Uint32ToOffset(offset uint32) (Offset) {
+func Uint32ToOffset(offset uint32) Offset {
return Offset(offset)
}
-func BytesToOffset(bytes []byte) (Offset) {
+func BytesToOffset(bytes []byte) Offset {
return Offset(util.BytesToUint32(bytes[0:4]))
}
diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go
index 34b4cfe0d..58ecc73cb 100644
--- a/weed/storage/volume_vacuum.go
+++ b/weed/storage/volume_vacuum.go
@@ -5,8 +5,8 @@ import (
"os"
"time"
- . "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/glog"
+ . "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
diff --git a/weed/storage/volume_vacuum_test.go b/weed/storage/volume_vacuum_test.go
index 896540621..464d52618 100644
--- a/weed/storage/volume_vacuum_test.go
+++ b/weed/storage/volume_vacuum_test.go
@@ -1,11 +1,11 @@
package storage
import (
+ "github.com/chrislusf/seaweedfs/weed/storage/types"
"io/ioutil"
"math/rand"
"os"
"testing"
- "github.com/chrislusf/seaweedfs/weed/storage/types"
)
/*
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-24 17:33:43 +0000