diff options
| -rw-r--r-- | weed-fs/bin/.gitignore | 1 | ||||
| -rw-r--r-- | weed-fs/note/replication.txt | 59 | ||||
| -rw-r--r-- | weed-fs/src/cmd/weed/shell.go | 54 | ||||
| -rw-r--r-- | weed-fs/src/cmd/weed/weed.go | 39 | ||||
| -rw-r--r-- | weed-fs/src/pkg/topology/data_center.go | 6 | ||||
| -rw-r--r-- | weed-fs/src/pkg/topology/node.go | 16 | ||||
| -rw-r--r-- | weed-fs/src/pkg/topology/rack.go | 10 | ||||
| -rw-r--r-- | weed-fs/src/pkg/util/config.go | 128 |
8 files changed, 297 insertions, 16 deletions
diff --git a/weed-fs/bin/.gitignore b/weed-fs/bin/.gitignore new file mode 100644 index 000000000..aa3146581 --- /dev/null +++ b/weed-fs/bin/.gitignore @@ -0,0 +1 @@ +/weed diff --git a/weed-fs/note/replication.txt b/weed-fs/note/replication.txt new file mode 100644 index 000000000..2a203f6e8 --- /dev/null +++ b/weed-fs/note/replication.txt @@ -0,0 +1,59 @@ +1. each file can choose the replication factor +2. replication granularity is in volume level +3. if not enough spaces, we can automatically decrease some volume's the replication factor, especially for cold data +4. support migrating data to cheaper storage +5. manual volume placement, access-based volume placement, auction based volume placement + +When a new volume server is started, it reports + 1. how many volumes it can hold + 2. current list of existing volumes +Each volume server remembers: + 1. current volume ids, replica locations + +The master assign volume ids based on + 1. replication factor + data center, rack + 2. concurrent write support +On master, stores the replication configuration +{ + replication:{ + {factor:1, min_volume_count:3, weight:10}, + {factor:2, min_volume_count:2, weight:20}, + {factor:3, min_volume_count:3, weight:30} + }, + port:9333, + +} +Or manually via command line + 1. add volume with specified replication factor + 2. add volume with specified volume id + + +If duplicated volume ids are reported from different volume servers, +the master determines the replication factor of the volume, +if less than the replication factor, the volume is in readonly mode +if more than the replication factor, the volume will purge the smallest/oldest volume +if equal, the volume will function as usual + +maybe use gossip to send the volumeServer~volumes information + + +Use cases: + on volume server + 1. weed volume -mserver="xx.xx.xx.xx:9333" -publicUrl="good.com:8080" -dir="/tmp" -volumes=50 + on weed master + 1. weed master -port=9333 + generate a default json configuration file if doesn't exist + +Bootstrap + 1. at the very beginning, the system has no volumes at all. + 2. if maxReplicationFactor==1, always initialize volumes right away + 3. if nServersHasFreeSpaces >= maxReplicationFactor, auto initialize + 4. if maxReplicationFactor>1 + weed shell + > disable_auto_initialize + > enable_auto_initialize + > assign_free_volume vid "server1:port","server2:port","server3:port" + > status + 5. +
\ No newline at end of file diff --git a/weed-fs/src/cmd/weed/shell.go b/weed-fs/src/cmd/weed/shell.go new file mode 100644 index 000000000..78a4b9eb1 --- /dev/null +++ b/weed-fs/src/cmd/weed/shell.go @@ -0,0 +1,54 @@ +package main + +import ( + "bufio" + "os" + "fmt" +) + +func init() { + cmdShell.Run = runShell // break init cycle +} + +var cmdShell = &Command{ + UsageLine: "shell", + Short: "run interactive commands, now just echo", + Long: `run interactive commands. + + `, +} + +var ( +) + +func runShell(command *Command, args []string) bool { + r := bufio.NewReader(os.Stdin) + o := bufio.NewWriter(os.Stdout) + e := bufio.NewWriter(os.Stderr) + prompt := func () { + o.WriteString("> ") + o.Flush() + }; + readLine := func () string { + ret, err := r.ReadString('\n') + if err != nil { + fmt.Fprint(e,err); + os.Exit(1) + } + return ret + } + execCmd := func (cmd string) int { + if cmd != "" { + o.WriteString(cmd) + } + return 0 + } + + cmd := "" + for { + prompt() + cmd = readLine() + execCmd(cmd) + } + return true +} diff --git a/weed-fs/src/cmd/weed/weed.go b/weed-fs/src/cmd/weed/weed.go index 72ce4c09c..a6f857023 100644 --- a/weed-fs/src/cmd/weed/weed.go +++ b/weed-fs/src/cmd/weed/weed.go @@ -1,12 +1,12 @@ package main import ( - "encoding/json" + "encoding/json" "flag" "fmt" - "net/http" "io" "log" + "net/http" "os" "strings" "sync" @@ -22,7 +22,8 @@ var port *int var commands = []*Command{ cmdFix, cmdMaster, - cmdUpload, + cmdUpload, + cmdShell, cmdVersion, cmdVolume, } @@ -50,6 +51,12 @@ func main() { if args[0] == "help" { help(args[1:]) + for _, cmd := range commands { + if cmd.Name() == args[1] && cmd.Run != nil { + fmt.Fprintf(os.Stderr, "Default Parameters:\n") + cmd.Flag.PrintDefaults() + } + } return } @@ -59,10 +66,10 @@ func main() { cmd.Flag.Parse(args[1:]) args = cmd.Flag.Args() if !cmd.Run(cmd, args) { + fmt.Fprintf(os.Stderr, "\n") + cmd.Flag.Usage() fmt.Fprintf(os.Stderr, "Default Parameters:\n") cmd.Flag.PrintDefaults() - fmt.Fprintf(os.Stderr, "\n") - cmd.Flag.Usage() } exit() return @@ -173,15 +180,15 @@ func exitIfErrors() { } } func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) { - w.Header().Set("Content-Type", "application/javascript") - bytes, _ := json.Marshal(obj) - callback := r.FormValue("callback") - if callback == "" { - w.Write(bytes) - } else { - w.Write([]uint8(callback)) - w.Write([]uint8("(")) - fmt.Fprint(w, string(bytes)) - w.Write([]uint8(")")) - } + w.Header().Set("Content-Type", "application/javascript") + bytes, _ := json.Marshal(obj) + callback := r.FormValue("callback") + if callback == "" { + w.Write(bytes) + } else { + w.Write([]uint8(callback)) + w.Write([]uint8("(")) + fmt.Fprint(w, string(bytes)) + w.Write([]uint8(")")) + } } diff --git a/weed-fs/src/pkg/topology/data_center.go b/weed-fs/src/pkg/topology/data_center.go new file mode 100644 index 000000000..022f733cc --- /dev/null +++ b/weed-fs/src/pkg/topology/data_center.go @@ -0,0 +1,6 @@ +package topology + +import ( + +) + diff --git a/weed-fs/src/pkg/topology/node.go b/weed-fs/src/pkg/topology/node.go new file mode 100644 index 000000000..cf48b0d67 --- /dev/null +++ b/weed-fs/src/pkg/topology/node.go @@ -0,0 +1,16 @@ +package topology + +import ( + +) + +type VolumeInfo struct { + Id uint32 + Size int64 +} +type Node struct { + volumes map[uint64]VolumeInfo + volumeLimit int + Port int + PublicUrl string +} diff --git a/weed-fs/src/pkg/topology/rack.go b/weed-fs/src/pkg/topology/rack.go new file mode 100644 index 000000000..69895a243 --- /dev/null +++ b/weed-fs/src/pkg/topology/rack.go @@ -0,0 +1,10 @@ +package topology + +import ( + +) + +type Rack struct { + nodes map[uint64]*Node + IpRanges []string +} diff --git a/weed-fs/src/pkg/util/config.go b/weed-fs/src/pkg/util/config.go new file mode 100644 index 000000000..6ac8a3a65 --- /dev/null +++ b/weed-fs/src/pkg/util/config.go @@ -0,0 +1,128 @@ +// Copyright 2011 Numerotron Inc. +// Use of this source code is governed by an MIT-style license +// that can be found in the LICENSE file. +// +// Developed at www.stathat.com by Patrick Crosby +// Contact us on twitter with any questions: twitter.com/stat_hat + +// The jconfig package provides a simple, basic configuration file parser using JSON. +package util + +import ( + "bytes" + "encoding/json" + "log" + "os" +) + +type Config struct { + data map[string]interface{} + filename string +} + +func newConfig() *Config { + result := new(Config) + result.data = make(map[string]interface{}) + return result +} + +// Loads config information from a JSON file +func LoadConfig(filename string) *Config { + result := newConfig() + result.filename = filename + err := result.parse() + if err != nil { + log.Fatalf("error loading config file %s: %s", filename, err) + } + return result +} + +// Loads config information from a JSON string +func LoadConfigString(s string) *Config { + result := newConfig() + err := json.Unmarshal([]byte(s), &result.data) + if err != nil { + log.Fatalf("error parsing config string %s: %s", s, err) + } + return result +} + +func (c *Config) StringMerge(s string) { + next := LoadConfigString(s) + c.merge(next.data) +} + +func (c *Config) LoadMerge(filename string) { + next := LoadConfig(filename) + c.merge(next.data) +} + +func (c *Config) merge(ndata map[string]interface{}) { + for k, v := range ndata { + c.data[k] = v + } +} + +func (c *Config) parse() error { + f, err := os.Open(c.filename) + if err != nil { + return err + } + defer f.Close() + b := new(bytes.Buffer) + _, err = b.ReadFrom(f) + if err != nil { + return err + } + err = json.Unmarshal(b.Bytes(), &c.data) + if err != nil { + return err + } + + return nil +} + +// Returns a string for the config variable key +func (c *Config) GetString(key string) string { + result, present := c.data[key] + if !present { + return "" + } + return result.(string) +} + +// Returns an int for the config variable key +func (c *Config) GetInt(key string) int { + x, ok := c.data[key] + if !ok { + return -1 + } + return int(x.(float64)) +} + +// Returns a float for the config variable key +func (c *Config) GetFloat(key string) float64 { + x, ok := c.data[key] + if !ok { + return -1 + } + return x.(float64) +} + +// Returns a bool for the config variable key +func (c *Config) GetBool(key string) bool { + x, ok := c.data[key] + if !ok { + return false + } + return x.(bool) +} + +// Returns an array for the config variable key +func (c *Config) GetArray(key string) []interface{} { + result, present := c.data[key] + if !present { + return []interface{}(nil) + } + return result.([]interface{}) +} |