aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--go/weed/filer.go79
-rw-r--r--go/weed/server.go83
-rw-r--r--go/weed/weed.go1
-rw-r--r--go/weed/weed_server/filer_server.go10
4 files changed, 138 insertions, 35 deletions
diff --git a/go/weed/filer.go b/go/weed/filer.go
new file mode 100644
index 000000000..d7d028d50
--- /dev/null
+++ b/go/weed/filer.go
@@ -0,0 +1,79 @@
+package main
+
+import (
+ "code.google.com/p/weed-fs/go/glog"
+ "code.google.com/p/weed-fs/go/util"
+ "code.google.com/p/weed-fs/go/weed/weed_server"
+ "net/http"
+ "os"
+ "strconv"
+ "time"
+)
+
+var (
+ f FilerOptions
+)
+
+type FilerOptions struct {
+ master *string
+ port *int
+ collection *string
+ defaultReplicaPlacement *string
+ dir *string
+}
+
+func init() {
+ cmdFiler.Run = runFiler // break init cycle
+ f.master = cmdFiler.Flag.String("master", "localhost:9333", "master server location")
+ f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
+ f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
+ f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data")
+ f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "Default replication type if not specified.")
+}
+
+var cmdFiler = &Command{
+ UsageLine: "filer -port=8888 -dir=/tmp -master=<ip:port>",
+ Short: "start a file server that points to a master server",
+ Long: `start a file server which accepts REST operation for any files.
+
+ //create or overwrite the file, the directories /path/to will be automatically created
+ POST /path/to/file
+ //get the file content
+ GET /path/to/file
+ //create or overwrite the file, the filename in the multipart request will be used
+ POST /path/to/
+ //return a json format subdirectory and files listing
+ GET /path/to/
+
+ Current <fullpath~fileid> mapping metadata store is local embedded leveldb.
+ It should be highly scalable to hundreds of millions of files on a modest machine.
+
+ Future we will ensure it can avoid of being SPOF.
+
+ `,
+}
+
+func runFiler(cmd *Command, args []string) bool {
+ if err := util.TestFolderWritable(*f.dir); err != nil {
+ glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err)
+ }
+
+ r := http.NewServeMux()
+ _, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection)
+ if nfs_err != nil {
+ glog.Fatalf(nfs_err.Error())
+ }
+ glog.V(0).Infoln("Start Weed Filer", util.VERSION, "at port", strconv.Itoa(*f.port))
+ filerListener, e := util.NewListener(
+ ":"+strconv.Itoa(*f.port),
+ time.Duration(10)*time.Second,
+ )
+ if e != nil {
+ glog.Fatalf(e.Error())
+ }
+ if e := http.Serve(filerListener, r); e != nil {
+ glog.Fatalf("Filer Fail to serve:%s", e.Error())
+ }
+
+ return true
+}
diff --git a/go/weed/server.go b/go/weed/server.go
index c3dca3f4b..87b541fd3 100644
--- a/go/weed/server.go
+++ b/go/weed/server.go
@@ -14,6 +14,10 @@ import (
"time"
)
+var (
+ filer FilerOptions
+)
+
func init() {
cmdServer.Run = runServer // break init cycle
}
@@ -28,10 +32,6 @@ var cmdServer = &Command{
The servers are the same as starting them separately.
So other volume servers can use this embedded master server also.
- However, this may change very soon.
- The target is to start both volume server and embedded master server on all instances,
- and use a leader election process to auto choose a master server.
-
`,
}
@@ -53,23 +53,32 @@ var (
volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
volumeMaxDataVolumeCounts = cmdServer.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...")
volumePulse = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
+ isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer")
serverWhiteList []string
)
+func init() {
+ filer.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
+ filer.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
+ filer.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
+ filer.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified")
+ filer.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
+}
+
func runServer(cmd *Command, args []string) bool {
+
+ *filer.master = *serverIp + ":" + strconv.Itoa(*masterPort)
+
+ if *filer.defaultReplicaPlacement == "" {
+ *filer.defaultReplicaPlacement = *masterDefaultReplicaPlacement
+ }
+
if *serverMaxCpu < 1 {
*serverMaxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*serverMaxCpu)
- if *masterMetaFolder == "" {
- *masterMetaFolder = *volumeDataFolders
- }
- if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
- glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *masterMetaFolder, err)
- }
-
folders := strings.Split(*volumeDataFolders, ",")
maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",")
maxCounts := make([]int, 0)
@@ -89,6 +98,20 @@ func runServer(cmd *Command, args []string) bool {
}
}
+ if *masterMetaFolder == "" {
+ *masterMetaFolder = folders[0]
+ }
+ if *filer.dir == "" {
+ *filer.dir = *masterMetaFolder + "/filer"
+ os.MkdirAll(*filer.dir, 0700)
+ }
+ if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
+ glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
+ }
+ if err := util.TestFolderWritable(*filer.dir); err != nil {
+ glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filer.dir, err)
+ }
+
if *volumePublicUrl == "" {
*volumePublicUrl = *serverIp + ":" + strconv.Itoa(*volumePort)
}
@@ -96,24 +119,26 @@ func runServer(cmd *Command, args []string) bool {
serverWhiteList = strings.Split(*serverWhiteListOption, ",")
}
- go func() {
- r := http.NewServeMux()
- _, nfs_err := weed_server.NewFilerServer(r, *serverIp+":"+strconv.Itoa(*masterPort), *volumeDataFolders)
- if nfs_err != nil {
- glog.Fatalf(nfs_err.Error())
- }
- glog.V(0).Infoln("Start Weed Filer", util.VERSION, "at port", *serverIp+":"+strconv.Itoa(8888))
- filerListener, e := util.NewListener(
- *serverIp+":"+strconv.Itoa(8888),
- time.Duration(*serverTimeout)*time.Second,
- )
- if e != nil {
- glog.Fatalf(e.Error())
- }
- if e := http.Serve(filerListener, r); e != nil {
- glog.Fatalf("Master Fail to serve:%s", e.Error())
- }
- }()
+ if *isStartingFiler {
+ go func() {
+ r := http.NewServeMux()
+ _, nfs_err := weed_server.NewFilerServer(r, *filer.port, *filer.master, *filer.dir, *filer.collection)
+ if nfs_err != nil {
+ glog.Fatalf(nfs_err.Error())
+ }
+ glog.V(0).Infoln("Start Weed Filer", util.VERSION, "at port", strconv.Itoa(*filer.port))
+ filerListener, e := util.NewListener(
+ ":"+strconv.Itoa(*filer.port),
+ time.Duration(10)*time.Second,
+ )
+ if e != nil {
+ glog.Fatalf(e.Error())
+ }
+ if e := http.Serve(filerListener, r); e != nil {
+ glog.Fatalf("Filer Fail to serve:%s", e.Error())
+ }
+ }()
+ }
var raftWaitForMaster sync.WaitGroup
var volumeWait sync.WaitGroup
diff --git a/go/weed/weed.go b/go/weed/weed.go
index 2a5a040c1..441ac9e61 100644
--- a/go/weed/weed.go
+++ b/go/weed/weed.go
@@ -21,6 +21,7 @@ var server *string
var commands = []*Command{
cmdBenchmark,
cmdCompact,
+ cmdFiler,
cmdFix,
cmdServer,
cmdMaster,
diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go
index c80b3b56f..7e9c644dd 100644
--- a/go/weed/weed_server/filer_server.go
+++ b/go/weed/weed_server/filer_server.go
@@ -1,11 +1,11 @@
package weed_server
import (
- "code.google.com/p/weed-fs/go/glog"
"errors"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
"net/http"
+ "strconv"
"strings"
)
@@ -25,11 +25,11 @@ type FilerServer struct {
db *leveldb.DB
}
-func NewFilerServer(r *http.ServeMux, master string, dir string) (fs *FilerServer, err error) {
+func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string) (fs *FilerServer, err error) {
fs = &FilerServer{
master: master,
- collection: "",
- port: ":8888",
+ collection: collection,
+ port: ":" + strconv.Itoa(port),
}
if fs.db, err = leveldb.OpenFile(dir, nil); err != nil {
@@ -38,8 +38,6 @@ func NewFilerServer(r *http.ServeMux, master string, dir string) (fs *FilerServe
r.HandleFunc("/", fs.filerHandler)
- glog.V(0).Infoln("file server started on port ", fs.port)
-
return fs, nil
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 21:19:22 +0000