add a snowflake sequencer as more robust fid generator, but less compressable than small auto-inc id

3 files changed, 52 insertions, 1 deletions

diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index e990bb3ea..52b547563 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -506,7 +506,7 @@ default = "localhost:8888"    # used by maintenance scripts if the scripts needs
 
 
 [master.sequencer]
-type = "raft"     # Choose [raft|etcd] type for storing the file id sequence
+type = "raft"     # Choose [raft|etcd|snowflake] type for storing the file id sequence
 # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
 # example : http://127.0.0.1:2379,http://127.0.0.1:2389
 sequencer_etcd_urls = "http://127.0.0.1:2379"
diff --git a/weed/sequence/snowflake_sequencer.go b/weed/sequence/snowflake_sequencer.go
new file mode 100644
index 000000000..d6479fc68
--- /dev/null
+++ b/weed/sequence/snowflake_sequencer.go
@@ -0,0 +1,43 @@
+package sequence
+
+import (
+	"fmt"
+	"hash/fnv"
+
+	"github.com/bwmarrin/snowflake"
+)
+
+// a simple snowflake Sequencer
+type SnowflakeSequencer struct {
+	node *snowflake.Node
+}
+
+func NewSnowflakeSequencer(nodeid string) (*SnowflakeSequencer, error) {
+	node, err := snowflake.NewNode(int64(hash(nodeid) & 0x3ff))
+	if err != nil {
+		fmt.Println(err)
+		return nil, err
+	}
+
+	sequencer := &SnowflakeSequencer{node: node}
+	return sequencer, nil
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(s))
+	return h.Sum32()
+}
+
+func (m *SnowflakeSequencer) NextFileId(count uint64) uint64 {
+	return uint64(m.node.Generate().Int64())
+}
+
+// ignore setmax as we are snowflake
+func (m *SnowflakeSequencer) SetMax(seenValue uint64) {
+}
+
+// return a new id as no Peek is stored
+func (m *SnowflakeSequencer) Peek() uint64 {
+	return uint64(m.node.Generate().Int64())
+}
diff --git a/weed/server/master_server.go b/weed/server/master_server.go
index 9404081b4..f5bad53fa 100644
--- a/weed/server/master_server.go
+++ b/weed/server/master_server.go
@@ -277,6 +277,14 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer
 			glog.Error(err)
 			seq = nil
 		}
+	case "snowflake":
+		var err error
+		glog.V(0).Infof("use a snowfalke seq id, nodeid %s:%d", option.Host, option.Port)
+		seq, err = sequence.NewSnowflakeSequencer(fmt.Sprintf("%s:%d", option.Host, option.Port))
+		if err != nil {
+			glog.Error(err)
+			seq = nil
+		}
 	default:
 		seq = sequence.NewMemorySequencer()
 	}