aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--weed/command/server.go2
-rw-r--r--weed/command/volume.go8
-rw-r--r--weed/storage/needle_map.go6
-rw-r--r--weed/storage/needle_map_boltdb.go185
-rw-r--r--weed/storage/needle_map_leveldb.go7
-rw-r--r--weed/storage/volume_loading.go31
6 files changed, 34 insertions, 205 deletions
diff --git a/weed/command/server.go b/weed/command/server.go
index 1638a7218..a56944b48 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -90,7 +90,7 @@ func init() {
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
- serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
+ serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
diff --git a/weed/command/volume.go b/weed/command/volume.go
index 2ee6bb11a..88aee95ae 100644
--- a/weed/command/volume.go
+++ b/weed/command/volume.go
@@ -58,7 +58,7 @@ func init() {
v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
- v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
+ v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
@@ -142,10 +142,10 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
switch *v.indexType {
case "leveldb":
volumeNeedleMapKind = storage.NeedleMapLevelDb
- case "boltdb":
+ case "leveldbMedium":
+ volumeNeedleMapKind = storage.NeedleMapBoltDb
+ case "leveldbLarge":
volumeNeedleMapKind = storage.NeedleMapBoltDb
- case "btree":
- volumeNeedleMapKind = storage.NeedleMapBtree
}
masters := *v.masters
diff --git a/weed/storage/needle_map.go b/weed/storage/needle_map.go
index 6d815679b..ee31b06f6 100644
--- a/weed/storage/needle_map.go
+++ b/weed/storage/needle_map.go
@@ -14,8 +14,10 @@ import (
type NeedleMapType int
const (
- NeedleMapInMemory NeedleMapType = iota
- NeedleMapLevelDb
+ NeedleMapInMemory NeedleMapType = iota
+ NeedleMapLevelDb // small memory footprint, 4MB total, 1 write buffer, 3 block buffer
+ NeedleMapLevelDbMedium // medium memory footprint, 8MB total, 3 write buffer, 5 block buffer
+ NeedleMapLevelDbLarge // large memory footprint, 12MB total, 4write buffer, 8 block buffer
NeedleMapBoltDb
NeedleMapBtree
)
diff --git a/weed/storage/needle_map_boltdb.go b/weed/storage/needle_map_boltdb.go
deleted file mode 100644
index e2e4d22f7..000000000
--- a/weed/storage/needle_map_boltdb.go
+++ /dev/null
@@ -1,185 +0,0 @@
-package storage
-
-import (
- "fmt"
- "os"
-
- "github.com/boltdb/bolt"
-
- "errors"
- "github.com/chrislusf/seaweedfs/weed/glog"
- "github.com/chrislusf/seaweedfs/weed/storage/needle"
- . "github.com/chrislusf/seaweedfs/weed/storage/types"
- "github.com/chrislusf/seaweedfs/weed/util"
-)
-
-type BoltDbNeedleMap struct {
- dbFileName string
- db *bolt.DB
- baseNeedleMapper
-}
-
-var boltdbBucket = []byte("weed")
-
-var NotFound = errors.New("not found")
-
-func NewBoltDbNeedleMap(dbFileName string, indexFile *os.File) (m *BoltDbNeedleMap, err error) {
- m = &BoltDbNeedleMap{dbFileName: dbFileName}
- m.indexFile = indexFile
- if !isBoltDbFresh(dbFileName, indexFile) {
- glog.V(0).Infof("Start to Generate %s from %s", dbFileName, indexFile.Name())
- generateBoltDbFile(dbFileName, indexFile)
- glog.V(0).Infof("Finished Generating %s from %s", dbFileName, indexFile.Name())
- }
- glog.V(1).Infof("Opening %s...", dbFileName)
- if m.db, err = bolt.Open(dbFileName, 0644, nil); err != nil {
- return
- }
- glog.V(1).Infof("Loading %s...", indexFile.Name())
- mm, indexLoadError := newNeedleMapMetricFromIndexFile(indexFile)
- if indexLoadError != nil {
- return nil, indexLoadError
- }
- m.mapMetric = *mm
- return
-}
-
-func isBoltDbFresh(dbFileName string, indexFile *os.File) bool {
- // normally we always write to index file first
- dbLogFile, err := os.Open(dbFileName)
- if err != nil {
- return false
- }
- defer dbLogFile.Close()
- dbStat, dbStatErr := dbLogFile.Stat()
- indexStat, indexStatErr := indexFile.Stat()
- if dbStatErr != nil || indexStatErr != nil {
- glog.V(0).Infof("Can not stat file: %v and %v", dbStatErr, indexStatErr)
- return false
- }
-
- return dbStat.ModTime().After(indexStat.ModTime())
-}
-
-func generateBoltDbFile(dbFileName string, indexFile *os.File) error {
- db, err := bolt.Open(dbFileName, 0644, nil)
- if err != nil {
- return err
- }
- defer db.Close()
- return WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size uint32) error {
- if !offset.IsZero() && size != TombstoneFileSize {
- boltDbWrite(db, key, offset, size)
- } else {
- boltDbDelete(db, key)
- }
- return nil
- })
-}
-
-func (m *BoltDbNeedleMap) Get(key NeedleId) (element *needle.NeedleValue, ok bool) {
- var offset Offset
- var size uint32
- bytes := make([]byte, NeedleIdSize)
- NeedleIdToBytes(bytes, key)
- err := m.db.View(func(tx *bolt.Tx) error {
- bucket := tx.Bucket(boltdbBucket)
- if bucket == nil {
- return fmt.Errorf("Bucket %q not found!", boltdbBucket)
- }
-
- data := bucket.Get(bytes)
-
- if len(data) == 0 {
- return NotFound
- }
-
- if len(data) != OffsetSize+SizeSize {
- glog.V(0).Infof("key:%v has wrong data length: %d", key, len(data))
- return fmt.Errorf("key:%v has wrong data length: %d", key, len(data))
- }
-
- offset = BytesToOffset(data[0:OffsetSize])
- size = util.BytesToUint32(data[OffsetSize : OffsetSize+SizeSize])
-
- return nil
- })
-
- if err != nil {
- return nil, false
- }
- return &needle.NeedleValue{Key: key, Offset: offset, Size: size}, true
-}
-
-func (m *BoltDbNeedleMap) Put(key NeedleId, offset Offset, size uint32) error {
- var oldSize uint32
- if oldNeedle, ok := m.Get(key); ok {
- oldSize = oldNeedle.Size
- }
- m.logPut(key, oldSize, size)
- // write to index file first
- if err := m.appendToIndexFile(key, offset, size); err != nil {
- return fmt.Errorf("cannot write to indexfile %s: %v", m.indexFile.Name(), err)
- }
- return boltDbWrite(m.db, key, offset, size)
-}
-
-func boltDbWrite(db *bolt.DB,
- key NeedleId, offset Offset, size uint32) error {
-
- bytes := make([]byte, NeedleIdSize+OffsetSize+SizeSize)
- NeedleIdToBytes(bytes[0:NeedleIdSize], key)
- OffsetToBytes(bytes[NeedleIdSize:NeedleIdSize+OffsetSize], offset)
- util.Uint32toBytes(bytes[NeedleIdSize+OffsetSize:NeedleIdSize+OffsetSize+SizeSize], size)
-
- return db.Update(func(tx *bolt.Tx) error {
- bucket, err := tx.CreateBucketIfNotExists(boltdbBucket)
- if err != nil {
- return err
- }
-
- err = bucket.Put(bytes[0:NeedleIdSize], bytes[NeedleIdSize:NeedleIdSize+OffsetSize+SizeSize])
- if err != nil {
- return err
- }
- return nil
- })
-}
-func boltDbDelete(db *bolt.DB, key NeedleId) error {
- bytes := make([]byte, NeedleIdSize)
- NeedleIdToBytes(bytes, key)
- return db.Update(func(tx *bolt.Tx) error {
- bucket, err := tx.CreateBucketIfNotExists(boltdbBucket)
- if err != nil {
- return err
- }
-
- err = bucket.Delete(bytes)
- if err != nil {
- return err
- }
- return nil
- })
-}
-
-func (m *BoltDbNeedleMap) Delete(key NeedleId, offset Offset) error {
- if oldNeedle, ok := m.Get(key); ok {
- m.logDelete(oldNeedle.Size)
- }
- // write to index file first
- if err := m.appendToIndexFile(key, offset, TombstoneFileSize); err != nil {
- return err
- }
- return boltDbDelete(m.db, key)
-}
-
-func (m *BoltDbNeedleMap) Close() {
- m.indexFile.Close()
- m.db.Close()
-}
-
-func (m *BoltDbNeedleMap) Destroy() error {
- m.Close()
- os.Remove(m.indexFile.Name())
- return os.Remove(m.dbFileName)
-}
diff --git a/weed/storage/needle_map_leveldb.go b/weed/storage/needle_map_leveldb.go
index c5c4f22eb..4d5280938 100644
--- a/weed/storage/needle_map_leveldb.go
+++ b/weed/storage/needle_map_leveldb.go
@@ -19,7 +19,7 @@ type LevelDbNeedleMap struct {
baseNeedleMapper
}
-func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedleMap, err error) {
+func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File, opts *opt.Options) (m *LevelDbNeedleMap, err error) {
m = &LevelDbNeedleMap{dbFileName: dbFileName}
m.indexFile = indexFile
if !isLevelDbFresh(dbFileName, indexFile) {
@@ -28,9 +28,8 @@ func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedl
glog.V(0).Infof("Finished Generating %s from %s", dbFileName, indexFile.Name())
}
glog.V(1).Infof("Opening %s...", dbFileName)
- if m.db, err = leveldb.OpenFile(dbFileName, &opt.Options{
- BlockCacheCapacity: -1, // default value is 8MiB
- }); err != nil {
+
+ if m.db, err = leveldb.OpenFile(dbFileName, opts); err != nil {
return
}
glog.V(1).Infof("Loading %s...", indexFile.Name())
diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go
index 37a6e07b2..14013b302 100644
--- a/weed/storage/volume_loading.go
+++ b/weed/storage/volume_loading.go
@@ -2,6 +2,7 @@ package storage
import (
"fmt"
+ "github.com/syndtr/goleveldb/leveldb/opt"
"os"
"time"
@@ -82,18 +83,30 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
}
case NeedleMapLevelDb:
glog.V(0).Infoln("loading leveldb", fileName+".ldb")
- if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile); e != nil {
+ opts := &opt.Options{
+ BlockCacheCapacity: 2 * 1024 * 1024, // default value is 8MiB
+ WriteBuffer: 1 * 1024 * 1024, // default value is 4MiB
+ }
+ if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
}
- case NeedleMapBoltDb:
- glog.V(0).Infoln("loading boltdb", fileName+".bdb")
- if v.nm, e = NewBoltDbNeedleMap(fileName+".bdb", indexFile); e != nil {
- glog.V(0).Infof("loading boltdb %s error: %v", fileName+".bdb", e)
+ case NeedleMapLevelDbMedium:
+ glog.V(0).Infoln("loading leveldb medium", fileName+".ldb")
+ opts := &opt.Options{
+ BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB
+ WriteBuffer: 2 * 1024 * 1024, // default value is 4MiB
+ }
+ if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
+ glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
}
- case NeedleMapBtree:
- glog.V(0).Infoln("loading index", fileName+".idx", "to btree readonly", v.readOnly)
- if v.nm, e = LoadBtreeNeedleMap(indexFile); e != nil {
- glog.V(0).Infof("loading index %s to btree error: %v", fileName+".idx", e)
+ case NeedleMapLevelDbLarge:
+ glog.V(0).Infoln("loading leveldb large", fileName+".ldb")
+ opts := &opt.Options{
+ BlockCacheCapacity: 8 * 1024 * 1024, // default value is 8MiB
+ WriteBuffer: 4 * 1024 * 1024, // default value is 4MiB
+ }
+ if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
+ glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
}
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-22 01:35:26 +0000