aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--unmaintained/repeated_vacuum/repeated_vacuum.go2
-rw-r--r--weed/command/server.go9
-rw-r--r--weed/command/volume.go26
-rw-r--r--weed/server/volume_server.go2
-rw-r--r--weed/storage/disk_location.go14
-rw-r--r--weed/storage/store.go4
-rw-r--r--weed/util/bytes.go22
-rw-r--r--weed/util/bytes_test.go26
-rw-r--r--weed/util/minfreespace.go90
-rw-r--r--weed/util/minfreespace_test.go29
10 files changed, 147 insertions, 77 deletions
diff --git a/unmaintained/repeated_vacuum/repeated_vacuum.go b/unmaintained/repeated_vacuum/repeated_vacuum.go
index bff5becc1..d85e45af0 100644
--- a/unmaintained/repeated_vacuum/repeated_vacuum.go
+++ b/unmaintained/repeated_vacuum/repeated_vacuum.go
@@ -52,7 +52,7 @@ func main() {
}
func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, string) {
- assignResult, err := operation.Assign(*master, grpcDialOption, &operation.VolumeAssignRequest{
+ assignResult, err := operation.Assign(func() string { return *master }, grpcDialOption, &operation.VolumeAssignRequest{
Count: 1,
Replication: *replication,
})
diff --git a/weed/command/server.go b/weed/command/server.go
index 6eb3bf97c..d0020d33b 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -58,7 +58,8 @@ var (
serverDisableHttp = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
- volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")
+ volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
+ volumeMinFreeSpace = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
serverMetricsHttpPort = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
// pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
@@ -244,8 +245,8 @@ func runServer(cmd *Command, args []string) bool {
// start volume server
if *isStartingVolumeServer {
- go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent)
-
+ minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
+ go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, minFreeSpaces)
}
if *isStartingMasterServer {
@@ -253,6 +254,4 @@ func runServer(cmd *Command, args []string) bool {
}
select {}
-
- return true
}
diff --git a/weed/command/volume.go b/weed/command/volume.go
index 93a19102d..139a3791e 100644
--- a/weed/command/volume.go
+++ b/weed/command/volume.go
@@ -57,7 +57,6 @@ type VolumeServerOptions struct {
compactionMBPerSecond *int
fileSizeLimitMB *int
concurrentUploadLimitMB *int
- minFreeSpaces []float32
pprof *bool
preStopSeconds *int
metricsHttpPort *int
@@ -121,13 +120,13 @@ func runVolume(cmd *Command, args []string) bool {
go stats_collect.StartMetricsServer(*v.metricsHttpPort)
- v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption,
- util.EmptyTo(*minFreeSpace, *minFreeSpacePercent))
+ minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
+ v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces)
return true
}
-func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpace string) {
+func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption string, minFreeSpaces []util.MinFreeSpace) {
// Set multiple folders and each folder's max volume count limit'
v.folders = strings.Split(volumeFolders, ",")
@@ -155,22 +154,13 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
}
- // set minFreeSpacePercent
- minFreeSpaceStrings := strings.Split(minFreeSpace, ",")
- for _, freeString := range minFreeSpaceStrings {
- if vv, e := util.ParseMinFreeSpace(freeString); e == nil {
- v.minFreeSpaces = append(v.minFreeSpaces, vv)
- } else {
- glog.Fatalf("The value specified in -minFreeSpace not a valid value %s", freeString)
- }
- }
- if len(v.minFreeSpaces) == 1 && len(v.folders) > 1 {
+ if len(minFreeSpaces) == 1 && len(v.folders) > 1 {
for i := 0; i < len(v.folders)-1; i++ {
- v.minFreeSpaces = append(v.minFreeSpaces, v.minFreeSpaces[0])
+ minFreeSpaces = append(minFreeSpaces, minFreeSpaces[0])
}
}
- if len(v.folders) != len(v.minFreeSpaces) {
- glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(v.minFreeSpaces))
+ if len(v.folders) != len(minFreeSpaces) {
+ glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(minFreeSpaces))
}
// set disk types
@@ -233,7 +223,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
*v.ip, *v.port, *v.publicUrl,
- v.folders, v.folderMaxLimits, v.minFreeSpaces, diskTypes,
+ v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
*v.idxFolder,
volumeNeedleMapKind,
strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go
index f29f12148..f7359ea6b 100644
--- a/weed/server/volume_server.go
+++ b/weed/server/volume_server.go
@@ -43,7 +43,7 @@ type VolumeServer struct {
func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
port int, publicUrl string,
- folders []string, maxCounts []int, minFreeSpaces []float32, diskTypes []types.DiskType,
+ folders []string, maxCounts []int, minFreeSpaces []util.MinFreeSpace, diskTypes []types.DiskType,
idxFolder string,
needleMapKind storage.NeedleMapKind,
masterNodes []string, pulseSeconds int,
diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go
index 3246718c8..33dd272ce 100644
--- a/weed/storage/disk_location.go
+++ b/weed/storage/disk_location.go
@@ -23,10 +23,9 @@ type DiskLocation struct {
DiskType types.DiskType
MaxVolumeCount int
OriginalMaxVolumeCount int
- // MinFreeSpace limits the minimum free space (<=100 as percentage, > 100 as bytes)
- MinFreeSpace float32
- volumes map[needle.VolumeId]*Volume
- volumesLock sync.RWMutex
+ MinFreeSpace util.MinFreeSpace
+ volumes map[needle.VolumeId]*Volume
+ volumesLock sync.RWMutex
// erasure coding
ecVolumes map[needle.VolumeId]*erasure_coding.EcVolume
@@ -35,7 +34,7 @@ type DiskLocation struct {
isDiskSpaceLow bool
}
-func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace float32, idxDir string, diskType types.DiskType) *DiskLocation {
+func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation {
dir = util.ResolvePath(dir)
if idxDir == "" {
idxDir = dir
@@ -363,7 +362,7 @@ func (l *DiskLocation) CheckDiskSpace() {
stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used))
stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free))
- isLow := l.MinFreeSpace < 100 && s.PercentFree < l.MinFreeSpace || s.Free < uint64(l.MinFreeSpace)
+ isLow, desc := l.MinFreeSpace.IsLow(s.Free, s.PercentFree)
if isLow != l.isDiskSpaceLow {
l.isDiskSpaceLow = !l.isDiskSpaceLow
}
@@ -373,8 +372,7 @@ func (l *DiskLocation) CheckDiskSpace() {
logLevel = glog.Level(0)
}
- glog.V(logLevel).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v",
- dir, s.PercentFree, l.MinFreeSpace, l.isDiskSpaceLow)
+ glog.V(logLevel).Infof("dir %s %s", dir, desc)
}
time.Sleep(time.Minute)
}
diff --git a/weed/storage/store.go b/weed/storage/store.go
index 18496890c..f27f2412f 100644
--- a/weed/storage/store.go
+++ b/weed/storage/store.go
@@ -2,6 +2,7 @@ package storage
import (
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/util"
"path/filepath"
"strings"
"sync/atomic"
@@ -52,7 +53,8 @@ func (s *Store) String() (str string) {
return
}
-func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, minFreeSpaces []float32, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType) (s *Store) {
+func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int,
+ minFreeSpaces []util.MinFreeSpace, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType) (s *Store) {
s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapKind: needleMapKind}
s.Locations = make([]*DiskLocation, 0)
for i := 0; i < len(dirnames); i++ {
diff --git a/weed/util/bytes.go b/weed/util/bytes.go
index 260e5067e..26da91033 100644
--- a/weed/util/bytes.go
+++ b/weed/util/bytes.go
@@ -5,7 +5,6 @@ import (
"crypto/md5"
"crypto/rand"
"encoding/base64"
- "errors"
"fmt"
"io"
"math"
@@ -176,23 +175,12 @@ func EmptyTo(s, to string) string {
return s
}
-var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid")
-
-// ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G
-func ParseMinFreeSpace(s string) (float32, error) {
- if value, e := strconv.ParseFloat(s, 32); e == nil {
- if value < 0 || value > 100 {
- return 0, ErrMinFreeSpaceBadValue
- }
- return float32(value), nil
- } else if directSize, e2 := ParseBytes(s); e2 == nil {
- if directSize <= 100 {
- return 0, ErrMinFreeSpaceBadValue
- }
- return float32(directSize), nil
+// IfElse works like b ? this : that.
+func IfElse(b bool, this, that string) string {
+ if b {
+ return this
}
-
- return 0, ErrMinFreeSpaceBadValue
+ return that
}
// ParseBytes parses a string representation of bytes into the number
diff --git a/weed/util/bytes_test.go b/weed/util/bytes_test.go
index 4a9c25e52..d9269cadb 100644
--- a/weed/util/bytes_test.go
+++ b/weed/util/bytes_test.go
@@ -2,32 +2,6 @@ package util
import "testing"
-func TestParseMinFreeSpace(t *testing.T) {
- tests := []struct {
- in string
- ok bool
- value float32
- }{
- {in: "42", ok: true, value: 42},
- {in: "-1", ok: false, value: 0},
- {in: "101", ok: false, value: 0},
- {in: "100B", ok: false, value: 0},
- {in: "100Ki", ok: true, value: 100 * 1024},
- {in: "100GiB", ok: true, value: 100 * 1024 * 1024 * 1024},
- {in: "42M", ok: true, value: 42 * 1000 * 1000},
- }
-
- for _, p := range tests {
- got, err := ParseMinFreeSpace(p.in)
- if p.ok != (err == nil) {
- t.Errorf("failed to test %v", p.in)
- }
- if p.ok && err == nil && got != p.value {
- t.Errorf("failed to test %v", p.in)
- }
- }
-}
-
func TestByteParsing(t *testing.T) {
tests := []struct {
in string
diff --git a/weed/util/minfreespace.go b/weed/util/minfreespace.go
new file mode 100644
index 000000000..c802bf6dd
--- /dev/null
+++ b/weed/util/minfreespace.go
@@ -0,0 +1,90 @@
+package util
+
+import (
+ "errors"
+ "fmt"
+ "github.com/chrislusf/seaweedfs/weed/glog"
+ "strconv"
+ "strings"
+)
+
+// MinFreeSpaceType is the type of MinFreeSpace.
+type MinFreeSpaceType int
+
+const (
+ // AsPercent set the MinFreeSpaceType to a percentage value from 0 to 100.
+ AsPercent MinFreeSpaceType = iota
+ // AsBytes set the MinFreeSpaceType to a absolute value bytes.
+ AsBytes
+)
+
+// MinFreeSpace is type that defines the limit for the minimum free space.
+type MinFreeSpace struct {
+ Type MinFreeSpaceType
+ Bytes uint64
+ Percent float32
+ Raw string
+}
+
+// IsLow tells whether the free space is low or not.
+func (s MinFreeSpace) IsLow(freeBytes uint64, freePercent float32) (yes bool, desc string) {
+ switch s.Type {
+ case AsPercent:
+ yes = freePercent < s.Percent
+ op := IfElse(yes, "<", ">=")
+ return yes, fmt.Sprintf("disk free %.2f%% %s required %.2f%%", freePercent, op, s.Percent)
+ case AsBytes:
+ yes = freeBytes < s.Bytes
+ op := IfElse(yes, "<", ">=")
+ return yes, fmt.Sprintf("disk free %s %s required %s",
+ BytesToHumanReadable(freeBytes), op, BytesToHumanReadable(s.Bytes))
+ }
+
+ return false, ""
+}
+
+// String returns a string representation of MinFreeSpace.
+func (s MinFreeSpace) String() string {
+ switch s.Type {
+ case AsPercent:
+ return fmt.Sprintf("%.2f%%", s.Percent)
+ default:
+ return s.Raw
+ }
+}
+
+// MustParseMinFreeSpace parses comma-separated argument for min free space setting.
+// minFreeSpace has the high priority than minFreeSpacePercent if it is set.
+func MustParseMinFreeSpace(minFreeSpace string, minFreeSpacePercent string) (spaces []MinFreeSpace) {
+ ss := strings.Split(EmptyTo(minFreeSpace, minFreeSpacePercent), ",")
+ for _, freeString := range ss {
+ if vv, e := ParseMinFreeSpace(freeString); e == nil {
+ spaces = append(spaces, *vv)
+ } else {
+ glog.Fatalf("The value specified in -minFreeSpace not a valid value %s", freeString)
+ }
+ }
+
+ return spaces
+}
+
+var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid")
+
+// ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G
+func ParseMinFreeSpace(s string) (*MinFreeSpace, error) {
+ if percent, e := strconv.ParseFloat(s, 32); e == nil {
+ if percent < 0 || percent > 100 {
+ return nil, ErrMinFreeSpaceBadValue
+ }
+ return &MinFreeSpace{Type: AsPercent, Percent: float32(percent), Raw: s}, nil
+ }
+
+ if directSize, e := ParseBytes(s); e == nil {
+ if directSize <= 100 {
+ return nil, ErrMinFreeSpaceBadValue
+ }
+ return &MinFreeSpace{Type: AsBytes, Bytes: directSize, Raw: s}, nil
+ }
+
+ return nil, ErrMinFreeSpaceBadValue
+}
diff --git a/weed/util/minfreespace_test.go b/weed/util/minfreespace_test.go
new file mode 100644
index 000000000..eec1942dd
--- /dev/null
+++ b/weed/util/minfreespace_test.go
@@ -0,0 +1,29 @@
+package util
+
+import "testing"
+
+func TestParseMinFreeSpace(t *testing.T) {
+ tests := []struct {
+ in string
+ ok bool
+ value *MinFreeSpace
+ }{
+ {in: "42", ok: true, value: &MinFreeSpace{Type: AsPercent, Percent: 42, Raw: "42"}},
+ {in: "-1", ok: false, value: nil},
+ {in: "101", ok: false, value: nil},
+ {in: "100B", ok: false, value: nil},
+ {in: "100Ki", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 100 * 1024, Raw: "100Ki"}},
+ {in: "100GiB", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 100 * 1024 * 1024 * 1024, Raw: "100GiB"}},
+ {in: "42M", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 42 * 1000 * 1000, Raw: "42M"}},
+ }
+
+ for _, p := range tests {
+ got, err := ParseMinFreeSpace(p.in)
+ if p.ok != (err == nil) {
+ t.Errorf("failed to test %v", p.in)
+ }
+ if p.ok && err == nil && *got != *p.value {
+ t.Errorf("failed to test %v", p.in)
+ }
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-20 23:20:07 +0000