aboutsummaryrefslogtreecommitdiff
path: root/go/weed
diff options
context:
space:
mode:
Diffstat (limited to 'go/weed')
-rw-r--r--go/weed/filer.go6
-rw-r--r--go/weed/master.go4
-rw-r--r--go/weed/server.go19
-rw-r--r--go/weed/volume.go102
-rw-r--r--go/weed/weed_server/common.go2
-rw-r--r--go/weed/weed_server/filer_server.go2
-rw-r--r--go/weed/weed_server/master_server_handlers_admin.go2
-rw-r--r--go/weed/weed_server/volume_server.go26
-rw-r--r--go/weed/weed_server/volume_server_handlers_helper.go2
9 files changed, 95 insertions, 70 deletions
diff --git a/go/weed/filer.go b/go/weed/filer.go
index 5b3fd2b67..4e7191e34 100644
--- a/go/weed/filer.go
+++ b/go/weed/filer.go
@@ -77,7 +77,7 @@ func runFiler(cmd *Command, args []string) bool {
*f.redis_server, *f.redis_database,
)
if nfs_err != nil {
- glog.Fatalf(nfs_err.Error())
+ glog.Fatalf("Filer startup error: %v", nfs_err)
}
glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*f.port))
filerListener, e := util.NewListener(
@@ -85,10 +85,10 @@ func runFiler(cmd *Command, args []string) bool {
time.Duration(10)*time.Second,
)
if e != nil {
- glog.Fatalf(e.Error())
+ glog.Fatalf("Filer listener error: %v", e)
}
if e := http.Serve(filerListener, r); e != nil {
- glog.Fatalf("Filer Fail to serve:%s", e.Error())
+ glog.Fatalf("Filer Fail to serve: %v", e)
}
return true
diff --git a/go/weed/master.go b/go/weed/master.go
index de4b5cb4b..13f6d7c43 100644
--- a/go/weed/master.go
+++ b/go/weed/master.go
@@ -71,7 +71,7 @@ func runMaster(cmd *Command, args []string) bool {
listener, e := util.NewListener(listeningAddress, time.Duration(*mTimeout)*time.Second)
if e != nil {
- glog.Fatalf(e.Error())
+ glog.Fatalf("Master startup error: %v", e)
}
go func() {
@@ -93,7 +93,7 @@ func runMaster(cmd *Command, args []string) bool {
}()
if e := http.Serve(listener, r); e != nil {
- glog.Fatalf("Fail to serve:%s", e.Error())
+ glog.Fatalf("Fail to serve: %v", e)
}
return true
}
diff --git a/go/weed/server.go b/go/weed/server.go
index 0b973f7e1..980c545b4 100644
--- a/go/weed/server.go
+++ b/go/weed/server.go
@@ -81,10 +81,10 @@ func init() {
filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified")
filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
- filerOptions.cassandra_server = cmdFiler.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server")
- filerOptions.cassandra_keyspace = cmdFiler.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
+ filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server")
+ filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379")
- filerOptions.redis_database = cmdFiler.Flag.Int("filer.redis.database", 0, "the database on the redis server")
+ filerOptions.redis_database = cmdServer.Flag.Int("filer.redis.database", 0, "the database on the redis server")
}
@@ -167,7 +167,7 @@ func runServer(cmd *Command, args []string) bool {
"", 0,
)
if nfs_err != nil {
- glog.Fatalf(nfs_err.Error())
+ glog.Fatalf("Filer startup error: %v", nfs_err)
}
glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*filerOptions.port))
filerListener, e := util.NewListener(
@@ -175,10 +175,11 @@ func runServer(cmd *Command, args []string) bool {
time.Duration(10)*time.Second,
)
if e != nil {
+ glog.Fatalf("Filer listener error: %v", e)
glog.Fatalf(e.Error())
}
if e := http.Serve(filerListener, r); e != nil {
- glog.Fatalf("Filer Fail to serve:%s", e.Error())
+ glog.Fatalf("Filer Fail to serve: %v", e)
}
}()
}
@@ -199,7 +200,7 @@ func runServer(cmd *Command, args []string) bool {
glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), time.Duration(*serverTimeout)*time.Second)
if e != nil {
- glog.Fatalf(e.Error())
+ glog.Fatalf("Master startup error: %v", e)
}
go func() {
@@ -224,7 +225,7 @@ func runServer(cmd *Command, args []string) bool {
volumeWait.Wait()
time.Sleep(100 * time.Millisecond)
r := http.NewServeMux()
- volumeServer := weed_server.NewVolumeServer(r, *serverIp, *volumePort, *serverPublicIp, folders, maxCounts,
+ volumeServer := weed_server.NewVolumeServer(r, r, *serverIp, *volumePort, *serverPublicIp, folders, maxCounts,
*serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack,
serverWhiteList, *volumeFixJpgOrientation,
)
@@ -235,7 +236,7 @@ func runServer(cmd *Command, args []string) bool {
time.Duration(*serverTimeout)*time.Second,
)
if e != nil {
- glog.Fatalf(e.Error())
+ glog.Fatalf("Volume server listener error: %v", e)
}
OnInterrupt(func() {
@@ -244,7 +245,7 @@ func runServer(cmd *Command, args []string) bool {
})
if e := http.Serve(volumeListener, r); e != nil {
- glog.Fatalf("Fail to serve:%s", e.Error())
+ glog.Fatalf("Volume server fail to serve:%v", e)
}
return true
diff --git a/go/weed/volume.go b/go/weed/volume.go
index 1683e1927..1dfd88576 100644
--- a/go/weed/volume.go
+++ b/go/weed/volume.go
@@ -13,8 +13,42 @@ import (
"github.com/chrislusf/weed-fs/go/weed/weed_server"
)
+var (
+ v VolumeServerOptions
+)
+
+type VolumeServerOptions struct {
+ port *int
+ adminPort *int
+ folders []string
+ folderMaxLimits []int
+ ip *string
+ publicIp *string
+ bindIp *string
+ master *string
+ pulseSeconds *int
+ idleConnectionTimeout *int
+ maxCpu *int
+ dataCenter *string
+ rack *string
+ whiteList []string
+ fixJpgOrientation *bool
+}
+
func init() {
cmdVolume.Run = runVolume // break init cycle
+ v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
+ v.adminPort = cmdVolume.Flag.Int("port.admin", 8443, "https admin port, active when SSL certs are specified. Not ready yet.")
+ v.ip = cmdVolume.Flag.String("ip", "", "ip or server name")
+ v.publicIp = cmdVolume.Flag.String("publicIp", "", "Publicly accessible <ip|server_name>")
+ v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+ v.master = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
+ v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
+ v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds")
+ v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
+ v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
+ v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
+ v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", true, "Adjust jpg orientation when uploading.")
}
var cmdVolume = &Command{
@@ -26,76 +60,66 @@ var cmdVolume = &Command{
}
var (
- vport = cmdVolume.Flag.Int("port", 8080, "http listen port")
- volumeSecurePort = cmdVolume.Flag.Int("port.secure", 8443, "https listen port, active when SSL certs are specified. Not ready yet.")
volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...")
- ip = cmdVolume.Flag.String("ip", "", "ip or server name")
- publicIp = cmdVolume.Flag.String("publicIp", "", "Publicly accessible <ip|server_name>")
- volumeBindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
- masterNode = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
- vpulse = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
- vTimeout = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds")
- vMaxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
- dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
- rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
- fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", true, "Adjust jpg orientation when uploading.")
-
- volumeWhiteList []string
)
func runVolume(cmd *Command, args []string) bool {
- if *vMaxCpu < 1 {
- *vMaxCpu = runtime.NumCPU()
+ if *v.maxCpu < 1 {
+ *v.maxCpu = runtime.NumCPU()
}
- runtime.GOMAXPROCS(*vMaxCpu)
- folders := strings.Split(*volumeFolders, ",")
+ runtime.GOMAXPROCS(*v.maxCpu)
+
+ //Set multiple folders and each folder's max volume count limit'
+ v.folders = strings.Split(*volumeFolders, ",")
maxCountStrings := strings.Split(*maxVolumeCounts, ",")
- maxCounts := make([]int, 0)
for _, maxString := range maxCountStrings {
if max, e := strconv.Atoi(maxString); e == nil {
- maxCounts = append(maxCounts, max)
+ v.folderMaxLimits = append(v.folderMaxLimits, max)
} else {
glog.Fatalf("The max specified in -max not a valid number %s", maxString)
}
}
- if len(folders) != len(maxCounts) {
- glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
+ if len(v.folders) != len(v.folderMaxLimits) {
+ glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
}
- for _, folder := range folders {
+ for _, folder := range v.folders {
if err := util.TestFolderWritable(folder); err != nil {
glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
}
}
- if *publicIp == "" {
- if *ip == "" {
- *ip = "127.0.0.1"
- *publicIp = "localhost"
+ //security related white list configuration
+ if *volumeWhiteListOption != "" {
+ v.whiteList = strings.Split(*volumeWhiteListOption, ",")
+ }
+
+ //derive default public ip address
+ if *v.publicIp == "" {
+ if *v.ip == "" {
+ *v.ip = "127.0.0.1"
+ *v.publicIp = "localhost"
} else {
- *publicIp = *ip
+ *v.publicIp = *v.ip
}
}
- if *volumeWhiteListOption != "" {
- volumeWhiteList = strings.Split(*volumeWhiteListOption, ",")
- }
r := http.NewServeMux()
- volumeServer := weed_server.NewVolumeServer(r, *ip, *vport, *publicIp, folders, maxCounts,
- *masterNode, *vpulse, *dataCenter, *rack,
- volumeWhiteList,
- *fixJpgOrientation,
+ volumeServer := weed_server.NewVolumeServer(r, r, *v.ip, *v.port, *v.publicIp, v.folders, v.folderMaxLimits,
+ *v.master, *v.pulseSeconds, *v.dataCenter, *v.rack,
+ v.whiteList,
+ *v.fixJpgOrientation,
)
- listeningAddress := *volumeBindIp + ":" + strconv.Itoa(*vport)
+ listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress)
- listener, e := util.NewListener(listeningAddress, time.Duration(*vTimeout)*time.Second)
+ listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
if e != nil {
- glog.Fatalf(e.Error())
+ glog.Fatalf("Volume server listener error:%v", e)
}
OnInterrupt(func() {
@@ -103,7 +127,7 @@ func runVolume(cmd *Command, args []string) bool {
})
if e := http.Serve(listener, r); e != nil {
- glog.Fatalf("Fail to serve:%s", e.Error())
+ glog.Fatalf("Volume server fail to serve: %v", e)
}
return true
}
diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go
index 44ffcce47..d259aa660 100644
--- a/go/weed/weed_server/common.go
+++ b/go/weed/weed_server/common.go
@@ -61,7 +61,7 @@ func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj inter
// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) {
if err := writeJson(w, r, httpStatus, obj); err != nil {
- glog.V(0).Infof("error writing JSON %s: %s", obj, err.Error())
+ glog.V(0).Infof("error writing JSON %s: %v", obj, err)
}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) {
diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go
index 18a02b5e0..b43e1965b 100644
--- a/go/weed/weed_server/filer_server.go
+++ b/go/weed/weed_server/filer_server.go
@@ -45,7 +45,7 @@ func NewFilerServer(r *http.ServeMux, port int, master string, dir string, colle
fs.filer = flat_namespace.NewFlatNamesapceFiler(master, redis_store)
} else {
if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil {
- glog.Fatalf("Can not start filer in dir %s : %v", err)
+ glog.Fatalf("Can not start filer in dir %s : %v", dir, err)
return
}
diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go
index 437044eee..4d304efcb 100644
--- a/go/weed/weed_server/master_server_handlers_admin.go
+++ b/go/weed/weed_server/master_server_handlers_admin.go
@@ -121,7 +121,7 @@ func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request)
if machines != nil && len(machines) > 0 {
http.Redirect(w, r, "http://"+machines[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently)
} else {
- writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("volume id %s not found.", volumeId))
+ writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("volume id %d not found.", volumeId))
}
}
diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go
index 9ceeb0149..0eb9daa0e 100644
--- a/go/weed/weed_server/volume_server.go
+++ b/go/weed/weed_server/volume_server.go
@@ -22,7 +22,7 @@ type VolumeServer struct {
FixJpgOrientation bool
}
-func NewVolumeServer(r *http.ServeMux, ip string, port int, publicIp string, folders []string, maxCounts []int,
+func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, port int, publicIp string, folders []string, maxCounts []int,
masterNode string, pulseSeconds int,
dataCenter string, rack string,
whiteList []string,
@@ -39,18 +39,18 @@ func NewVolumeServer(r *http.ServeMux, ip string, port int, publicIp string, fol
vs.guard = security.NewGuard(whiteList, "")
- r.HandleFunc("/status", vs.guard.Secure(vs.statusHandler))
- r.HandleFunc("/admin/assign_volume", vs.guard.Secure(vs.assignVolumeHandler))
- r.HandleFunc("/admin/vacuum_volume_check", vs.guard.Secure(vs.vacuumVolumeCheckHandler))
- r.HandleFunc("/admin/vacuum_volume_compact", vs.guard.Secure(vs.vacuumVolumeCompactHandler))
- r.HandleFunc("/admin/vacuum_volume_commit", vs.guard.Secure(vs.vacuumVolumeCommitHandler))
- r.HandleFunc("/admin/freeze_volume", vs.guard.Secure(vs.freezeVolumeHandler))
- r.HandleFunc("/admin/delete_collection", vs.guard.Secure(vs.deleteCollectionHandler))
- r.HandleFunc("/stats/counter", vs.guard.Secure(statsCounterHandler))
- r.HandleFunc("/stats/memory", vs.guard.Secure(statsMemoryHandler))
- r.HandleFunc("/stats/disk", vs.guard.Secure(vs.statsDiskHandler))
- r.HandleFunc("/delete", vs.guard.Secure(vs.batchDeleteHandler))
- r.HandleFunc("/", vs.storeHandler)
+ adminMux.HandleFunc("/status", vs.guard.Secure(vs.statusHandler))
+ adminMux.HandleFunc("/admin/assign_volume", vs.guard.Secure(vs.assignVolumeHandler))
+ adminMux.HandleFunc("/admin/vacuum_volume_check", vs.guard.Secure(vs.vacuumVolumeCheckHandler))
+ adminMux.HandleFunc("/admin/vacuum_volume_compact", vs.guard.Secure(vs.vacuumVolumeCompactHandler))
+ adminMux.HandleFunc("/admin/vacuum_volume_commit", vs.guard.Secure(vs.vacuumVolumeCommitHandler))
+ adminMux.HandleFunc("/admin/freeze_volume", vs.guard.Secure(vs.freezeVolumeHandler))
+ adminMux.HandleFunc("/admin/delete_collection", vs.guard.Secure(vs.deleteCollectionHandler))
+ adminMux.HandleFunc("/stats/counter", vs.guard.Secure(statsCounterHandler))
+ adminMux.HandleFunc("/stats/memory", vs.guard.Secure(statsMemoryHandler))
+ adminMux.HandleFunc("/stats/disk", vs.guard.Secure(vs.statsDiskHandler))
+ publicMux.HandleFunc("/delete", vs.guard.Secure(vs.batchDeleteHandler))
+ publicMux.HandleFunc("/", vs.storeHandler)
go func() {
connected := true
diff --git a/go/weed/weed_server/volume_server_handlers_helper.go b/go/weed/weed_server/volume_server_handlers_helper.go
index 6d21ffb62..2bab35e45 100644
--- a/go/weed/weed_server/volume_server_handlers_helper.go
+++ b/go/weed/weed_server/volume_server_handlers_helper.go
@@ -93,7 +93,7 @@ func (w *countingWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}
-// rangesMIMESize returns the nunber of bytes it takes to encode the
+// rangesMIMESize returns the number of bytes it takes to encode the
// provided ranges as a multipart response.
func rangesMIMESize(ranges []httpRange, contentType string, contentSize int64) (encSize int64) {
var w countingWriter
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-24 10:00:47 +0000