Merge pull request #2025 from bingoohuang/master

improvement for minFreeSpace argument

4 files changed, 273 insertions, 0 deletions

diff --git a/weed/util/bytes.go b/weed/util/bytes.go
index c2a4df108..26da91033 100644
--- a/weed/util/bytes.go
+++ b/weed/util/bytes.go
@@ -7,6 +7,10 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
+	"math"
+	"strconv"
+	"strings"
+	"unicode"
 )
 
 // BytesToHumanReadable returns the converted human readable representation of the bytes.
@@ -161,3 +165,94 @@ func NewBytesReader(b []byte) *BytesReader {
 		Reader: bytes.NewReader(b),
 	}
 }
+
+// EmptyTo returns to if s is empty.
+func EmptyTo(s, to string) string {
+	if s == "" {
+		return to
+	}
+
+	return s
+}
+
+// IfElse works like b ? this : that.
+func IfElse(b bool, this, that string) string {
+	if b {
+		return this
+	}
+	return that
+}
+
+// ParseBytes parses a string representation of bytes into the number
+// of bytes it represents.
+//
+// See Also: Bytes, IBytes.
+//
+// ParseBytes("42MB") -> 42000000, nil
+// ParseBytes("42 MB") -> 42000000, nil
+// ParseBytes("42 mib") -> 44040192, nil
+func ParseBytes(s string) (uint64, error) {
+	lastDigit := 0
+	hasComma := false
+	for _, r := range s {
+		if !(unicode.IsDigit(r) || r == '.' || r == ',') {
+			break
+		}
+		if r == ',' {
+			hasComma = true
+		}
+		lastDigit++
+	}
+
+	num := s[:lastDigit]
+	if hasComma {
+		num = strings.Replace(num, ",", "", -1)
+	}
+
+	f, err := strconv.ParseFloat(num, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	extra := strings.ToLower(strings.TrimSpace(s[lastDigit:]))
+	if m, ok := bytesSizeTable[extra]; ok {
+		f *= float64(m)
+		if f >= math.MaxUint64 {
+			return 0, fmt.Errorf("too large: %v", s)
+		}
+		return uint64(f), nil
+	}
+
+	return 0, fmt.Errorf("unhandled size name: %v", extra)
+}
+
+var bytesSizeTable = map[string]uint64{
+	"b": Byte, "kib": KiByte, "kb": KByte, "mib": MiByte, "mb": MByte, "gib": GiByte, "gb": GByte,
+	"tib": TiByte, "tb": TByte, "pib": PiByte, "pb": PByte, "eib": EiByte, "eb": EByte,
+	// Without suffix
+	"": Byte, "ki": KiByte, "k": KByte, "mi": MiByte, "m": MByte, "gi": GiByte, "g": GByte,
+	"ti": TiByte, "t": TByte, "pi": PiByte, "p": PByte, "ei": EiByte, "e": EByte,
+}
+
+// IEC Sizes.
+// kibis of bits
+const (
+	Byte = 1 << (iota * 10)
+	KiByte
+	MiByte
+	GiByte
+	TiByte
+	PiByte
+	EiByte
+)
+
+// SI Sizes.
+const (
+	IByte = 1
+	KByte = IByte * 1000
+	MByte = KByte * 1000
+	GByte = MByte * 1000
+	TByte = GByte * 1000
+	PByte = TByte * 1000
+	EByte = PByte * 1000
+)
diff --git a/weed/util/bytes_test.go b/weed/util/bytes_test.go
new file mode 100644
index 000000000..d9269cadb
--- /dev/null
+++ b/weed/util/bytes_test.go
@@ -0,0 +1,59 @@
+package util
+
+import "testing"
+
+func TestByteParsing(t *testing.T) {
+	tests := []struct {
+		in  string
+		exp uint64
+	}{
+		{"42", 42},
+		{"42MB", 42000000},
+		{"42MiB", 44040192},
+		{"42mb", 42000000},
+		{"42mib", 44040192},
+		{"42MIB", 44040192},
+		{"42 MB", 42000000},
+		{"42 MiB", 44040192},
+		{"42 mb", 42000000},
+		{"42 mib", 44040192},
+		{"42 MIB", 44040192},
+		{"42.5MB", 42500000},
+		{"42.5MiB", 44564480},
+		{"42.5 MB", 42500000},
+		{"42.5 MiB", 44564480},
+		// No need to say B
+		{"42M", 42000000},
+		{"42Mi", 44040192},
+		{"42m", 42000000},
+		{"42mi", 44040192},
+		{"42MI", 44040192},
+		{"42 M", 42000000},
+		{"42 Mi", 44040192},
+		{"42 m", 42000000},
+		{"42 mi", 44040192},
+		{"42 MI", 44040192},
+		{"42.5M", 42500000},
+		{"42.5Mi", 44564480},
+		{"42.5 M", 42500000},
+		{"42.5 Mi", 44564480},
+		// Bug #42
+		{"1,005.03 MB", 1005030000},
+		// Large testing, breaks when too much larger than
+		// this.
+		{"12.5 EB", uint64(12.5 * float64(EByte))},
+		{"12.5 E", uint64(12.5 * float64(EByte))},
+		{"12.5 EiB", uint64(12.5 * float64(EiByte))},
+	}
+
+	for _, p := range tests {
+		got, err := ParseBytes(p.in)
+		if err != nil {
+			t.Errorf("Couldn't parse %v: %v", p.in, err)
+		}
+		if got != p.exp {
+			t.Errorf("Expected %v for %v, got %v",
+				p.exp, p.in, got)
+		}
+	}
+}
diff --git a/weed/util/minfreespace.go b/weed/util/minfreespace.go
new file mode 100644
index 000000000..c802bf6dd
--- /dev/null
+++ b/weed/util/minfreespace.go
@@ -0,0 +1,90 @@
+package util
+
+import (
+	"errors"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"strconv"
+	"strings"
+)
+
+// MinFreeSpaceType is the type of MinFreeSpace.
+type MinFreeSpaceType int
+
+const (
+	// AsPercent set the MinFreeSpaceType to a percentage value from 0 to 100.
+	AsPercent MinFreeSpaceType = iota
+	// AsBytes set the MinFreeSpaceType to a absolute value bytes.
+	AsBytes
+)
+
+// MinFreeSpace is type that defines the limit for the minimum free space.
+type MinFreeSpace struct {
+	Type    MinFreeSpaceType
+	Bytes   uint64
+	Percent float32
+	Raw     string
+}
+
+// IsLow tells whether the free space is low or not.
+func (s MinFreeSpace) IsLow(freeBytes uint64, freePercent float32) (yes bool, desc string) {
+	switch s.Type {
+	case AsPercent:
+		yes = freePercent < s.Percent
+		op := IfElse(yes, "<", ">=")
+		return yes, fmt.Sprintf("disk free %.2f%% %s required %.2f%%", freePercent, op, s.Percent)
+	case AsBytes:
+		yes = freeBytes < s.Bytes
+		op := IfElse(yes, "<", ">=")
+		return yes, fmt.Sprintf("disk free %s %s required %s",
+			BytesToHumanReadable(freeBytes), op, BytesToHumanReadable(s.Bytes))
+	}
+
+	return false, ""
+}
+
+// String returns a string representation of MinFreeSpace.
+func (s MinFreeSpace) String() string {
+	switch s.Type {
+	case AsPercent:
+		return fmt.Sprintf("%.2f%%", s.Percent)
+	default:
+		return s.Raw
+	}
+}
+
+// MustParseMinFreeSpace parses comma-separated argument for min free space setting.
+// minFreeSpace has the high priority than minFreeSpacePercent if it is set.
+func MustParseMinFreeSpace(minFreeSpace string, minFreeSpacePercent string) (spaces []MinFreeSpace) {
+	ss := strings.Split(EmptyTo(minFreeSpace, minFreeSpacePercent), ",")
+	for _, freeString := range ss {
+		if vv, e := ParseMinFreeSpace(freeString); e == nil {
+			spaces = append(spaces, *vv)
+		} else {
+			glog.Fatalf("The value specified in -minFreeSpace not a valid value %s", freeString)
+		}
+	}
+
+	return spaces
+}
+
+var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid")
+
+// ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G
+func ParseMinFreeSpace(s string) (*MinFreeSpace, error) {
+	if percent, e := strconv.ParseFloat(s, 32); e == nil {
+		if percent < 0 || percent > 100 {
+			return nil, ErrMinFreeSpaceBadValue
+		}
+		return &MinFreeSpace{Type: AsPercent, Percent: float32(percent), Raw: s}, nil
+	}
+
+	if directSize, e := ParseBytes(s); e == nil {
+		if directSize <= 100 {
+			return nil, ErrMinFreeSpaceBadValue
+		}
+		return &MinFreeSpace{Type: AsBytes, Bytes: directSize, Raw: s}, nil
+	}
+
+	return nil, ErrMinFreeSpaceBadValue
+}
diff --git a/weed/util/minfreespace_test.go b/weed/util/minfreespace_test.go
new file mode 100644
index 000000000..eec1942dd
--- /dev/null
+++ b/weed/util/minfreespace_test.go
@@ -0,0 +1,29 @@
+package util
+
+import "testing"
+
+func TestParseMinFreeSpace(t *testing.T) {
+	tests := []struct {
+		in    string
+		ok    bool
+		value *MinFreeSpace
+	}{
+		{in: "42", ok: true, value: &MinFreeSpace{Type: AsPercent, Percent: 42, Raw: "42"}},
+		{in: "-1", ok: false, value: nil},
+		{in: "101", ok: false, value: nil},
+		{in: "100B", ok: false, value: nil},
+		{in: "100Ki", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 100 * 1024, Raw: "100Ki"}},
+		{in: "100GiB", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 100 * 1024 * 1024 * 1024, Raw: "100GiB"}},
+		{in: "42M", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 42 * 1000 * 1000, Raw: "42M"}},
+	}
+
+	for _, p := range tests {
+		got, err := ParseMinFreeSpace(p.in)
+		if p.ok != (err == nil) {
+			t.Errorf("failed to test %v", p.in)
+		}
+		if p.ok && err == nil && *got != *p.value {
+			t.Errorf("failed to test %v", p.in)
+		}
+	}
+}