* Fix undefined http serve behaiver (#6943)

7 files changed, 62 insertions, 61 deletions

diff --git a/weed/command/filer.go b/weed/command/filer.go
index aa4f878ad..1b7065a73 100644
--- a/weed/command/filer.go
+++ b/weed/command/filer.go
@@ -5,7 +5,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net"
 	"net/http"
 	"os"
@@ -14,6 +13,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/spf13/viper"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
+	"google.golang.org/grpc/reflection"
+
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -22,10 +26,7 @@ import (
 	weed_server "github.com/seaweedfs/seaweedfs/weed/server"
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"github.com/spf13/viper"
-	"google.golang.org/grpc/credentials/tls/certprovider"
-	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
-	"google.golang.org/grpc/reflection"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 )
 
 var (
@@ -372,7 +373,6 @@ func (fo *FilerOptions) startFiler() {
 	}
 	go grpcS.Serve(grpcL)
 
-	httpS := &http.Server{Handler: defaultMux}
 	if runtime.GOOS != "windows" {
 		localSocket := *fo.localSocket
 		if localSocket == "" {
@@ -387,7 +387,7 @@ func (fo *FilerOptions) startFiler() {
 			if err != nil {
 				glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
 			}
-			httpS.Serve(filerSocketListener)
+			newHttpServer(defaultMux, nil).Serve(filerSocketListener)
 		}()
 	}
 
@@ -420,31 +420,33 @@ func (fo *FilerOptions) startFiler() {
 			clientAuth = tls.RequireAndVerifyClientCert
 		}
 
-		httpS.TLSConfig = &tls.Config{
+		tlsConfig := &tls.Config{
 			GetCertificate: fo.GetCertificateWithUpdate,
 			ClientAuth:     clientAuth,
 			ClientCAs:      caCertPool,
 		}
 
+		security.FixTlsConfig(util.GetViper(), tlsConfig)
+
 		if filerLocalListener != nil {
 			go func() {
-				if err := httpS.ServeTLS(filerLocalListener, "", ""); err != nil {
+				if err := newHttpServer(defaultMux, tlsConfig).ServeTLS(filerLocalListener, "", ""); err != nil {
 					glog.Errorf("Filer Fail to serve: %v", e)
 				}
 			}()
 		}
-		if err := httpS.ServeTLS(filerListener, "", ""); err != nil {
+		if err := newHttpServer(defaultMux, tlsConfig).ServeTLS(filerListener, "", ""); err != nil {
 			glog.Fatalf("Filer Fail to serve: %v", e)
 		}
 	} else {
 		if filerLocalListener != nil {
 			go func() {
-				if err := httpS.Serve(filerLocalListener); err != nil {
+				if err := newHttpServer(defaultMux, nil).Serve(filerLocalListener); err != nil {
 					glog.Errorf("Filer Fail to serve: %v", e)
 				}
 			}()
 		}
-		if err := httpS.Serve(filerListener); err != nil {
+		if err := newHttpServer(defaultMux, nil).Serve(filerListener); err != nil {
 			glog.Fatalf("Filer Fail to serve: %v", e)
 		}
 	}
diff --git a/weed/command/iam.go b/weed/command/iam.go
index 1839bba2c..c484ed18d 100644
--- a/weed/command/iam.go
+++ b/weed/command/iam.go
@@ -3,7 +3,6 @@ package command
 import (
 	"context"
 	"fmt"
-	"net/http"
 
 	"github.com/seaweedfs/seaweedfs/weed/util/version"
 
@@ -88,8 +87,6 @@ func (iamopt *IamOptions) startIamServer() bool {
 		glog.Fatalf("IAM API Server startup error: %v", iamApiServer_err)
 	}
 
-	httpS := &http.Server{Handler: router}
-
 	listenAddress := fmt.Sprintf(":%d", *iamopt.port)
 	iamApiListener, iamApiLocalListener, err := util.NewIpAndLocalListeners(*iamopt.ip, *iamopt.port, time.Duration(10)*time.Second)
 	if err != nil {
@@ -99,12 +96,12 @@ func (iamopt *IamOptions) startIamServer() bool {
 	glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", version.Version(), *iamopt.port)
 	if iamApiLocalListener != nil {
 		go func() {
-			if err = httpS.Serve(iamApiLocalListener); err != nil {
+			if err = newHttpServer(router, nil).Serve(iamApiLocalListener); err != nil {
 				glog.Errorf("IAM API Server Fail to serve: %v", err)
 			}
 		}()
 	}
-	if err = httpS.Serve(iamApiListener); err != nil {
+	if err = newHttpServer(router, nil).Serve(iamApiListener); err != nil {
 		glog.Fatalf("IAM API Server Fail to serve: %v", err)
 	}
 
diff --git a/weed/command/master.go b/weed/command/master.go
index 6421d321d..8e10d25a2 100644
--- a/weed/command/master.go
+++ b/weed/command/master.go
@@ -2,6 +2,7 @@ package command
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"os"
@@ -264,19 +265,20 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		clientCertFile = viper.GetString("https.master.ca")
 	}
 
-	httpS := &http.Server{Handler: r}
 	if masterLocalListener != nil {
-		go httpS.Serve(masterLocalListener)
+		go newHttpServer(r, nil).Serve(masterLocalListener)
 	}
 
+	var tlsConfig *tls.Config
 	if useMTLS {
-		httpS.TLSConfig = security.LoadClientTLSHTTP(clientCertFile)
+		tlsConfig = security.LoadClientTLSHTTP(clientCertFile)
+		security.FixTlsConfig(util.GetViper(), tlsConfig)
 	}
 
 	if useTLS {
-		go httpS.ServeTLS(masterListener, certFile, keyFile)
+		go newHttpServer(r, tlsConfig).ServeTLS(masterListener, certFile, keyFile)
 	} else {
-		go httpS.Serve(masterListener)
+		go newHttpServer(r, nil).Serve(masterListener)
 	}
 
 	grace.OnInterrupt(ms.Shutdown)
diff --git a/weed/command/master_follower.go b/weed/command/master_follower.go
index 17beaa7ec..43707946d 100644
--- a/weed/command/master_follower.go
+++ b/weed/command/master_follower.go
@@ -3,19 +3,19 @@ package command
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
-	"net/http"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/gorilla/mux"
+	"google.golang.org/grpc/reflection"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	weed_server "github.com/seaweedfs/seaweedfs/weed/server"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"google.golang.org/grpc/reflection"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 )
 
 var (
@@ -144,11 +144,10 @@ func startMasterFollower(masterOptions MasterOptions) {
 	go ms.MasterClient.KeepConnectedToMaster(context.Background())
 
 	// start http server
-	httpS := &http.Server{Handler: r}
 	if masterLocalListener != nil {
-		go httpS.Serve(masterLocalListener)
+		go newHttpServer(r, nil).Serve(masterLocalListener)
 	}
-	go httpS.Serve(masterListener)
+	go newHttpServer(r, nil).Serve(masterListener)
 
 	select {}
 }
diff --git a/weed/command/s3.go b/weed/command/s3.go
index f955c4222..25b75e9da 100644
--- a/weed/command/s3.go
+++ b/weed/command/s3.go
@@ -7,30 +7,26 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
-	"net/http"
 	"os"
 	"runtime"
 	"strings"
 	"time"
 
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
-
-	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"github.com/gorilla/mux"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
 	"google.golang.org/grpc/reflection"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
-	"github.com/seaweedfs/seaweedfs/weed/security"
-
-	"github.com/gorilla/mux"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"github.com/seaweedfs/seaweedfs/weed/security"
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 )
 
 var (
@@ -251,8 +247,6 @@ func (s3opt *S3Options) startS3Server() bool {
 		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
 	}
 
-	httpS := &http.Server{Handler: router}
-
 	if *s3opt.portGrpc == 0 {
 		*s3opt.portGrpc = 10000 + *s3opt.port
 	}
@@ -274,7 +268,7 @@ func (s3opt *S3Options) startS3Server() bool {
 			if err != nil {
 				glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
 			}
-			httpS.Serve(s3SocketListener)
+			newHttpServer(router, nil).Serve(s3SocketListener)
 		}()
 	}
 
@@ -331,12 +325,12 @@ func (s3opt *S3Options) startS3Server() bool {
 			clientAuth = tls.RequireAndVerifyClientCert
 		}
 
-		httpS.TLSConfig = &tls.Config{
+		tlsConfig := &tls.Config{
 			GetCertificate: s3opt.GetCertificateWithUpdate,
 			ClientAuth:     clientAuth,
 			ClientCAs:      caCertPool,
 		}
-		err = security.FixTlsConfig(util.GetViper(), httpS.TLSConfig)
+		err = security.FixTlsConfig(util.GetViper(), tlsConfig)
 		if err != nil {
 			glog.Fatalf("error with tls config: %v", err)
 		}
@@ -344,12 +338,12 @@ func (s3opt *S3Options) startS3Server() bool {
 			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.port)
 			if s3ApiLocalListener != nil {
 				go func() {
-					if err = httpS.ServeTLS(s3ApiLocalListener, "", ""); err != nil {
+					if err = newHttpServer(router, tlsConfig).ServeTLS(s3ApiLocalListener, "", ""); err != nil {
 						glog.Fatalf("S3 API Server Fail to serve: %v", err)
 					}
 				}()
 			}
-			if err = httpS.ServeTLS(s3ApiListener, "", ""); err != nil {
+			if err = newHttpServer(router, tlsConfig).ServeTLS(s3ApiListener, "", ""); err != nil {
 				glog.Fatalf("S3 API Server Fail to serve: %v", err)
 			}
 		} else {
@@ -358,13 +352,13 @@ func (s3opt *S3Options) startS3Server() bool {
 				*s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
 			if s3ApiLocalListenerHttps != nil {
 				go func() {
-					if err = httpS.ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil {
+					if err = newHttpServer(router, tlsConfig).ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil {
 						glog.Fatalf("S3 API Server Fail to serve: %v", err)
 					}
 				}()
 			}
 			go func() {
-				if err = httpS.ServeTLS(s3ApiListenerHttps, "", ""); err != nil {
+				if err = newHttpServer(router, tlsConfig).ServeTLS(s3ApiListenerHttps, "", ""); err != nil {
 					glog.Fatalf("S3 API Server Fail to serve: %v", err)
 				}
 			}()
@@ -374,12 +368,12 @@ func (s3opt *S3Options) startS3Server() bool {
 		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", version.Version(), *s3opt.port)
 		if s3ApiLocalListener != nil {
 			go func() {
-				if err = httpS.Serve(s3ApiLocalListener); err != nil {
+				if err = newHttpServer(router, nil).Serve(s3ApiLocalListener); err != nil {
 					glog.Fatalf("S3 API Server Fail to serve: %v", err)
 				}
 			}()
 		}
-		if err = httpS.Serve(s3ApiListener); err != nil {
+		if err = newHttpServer(router, nil).Serve(s3ApiListener); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}
 	}
diff --git a/weed/command/server.go b/weed/command/server.go
index 9d7626e78..02641bbe2 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -1,16 +1,16 @@
 package command
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"os"
 	"strings"
 	"time"
 
-	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
-
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 )
@@ -358,3 +358,13 @@ func runServer(cmd *Command, args []string) bool {
 
 	select {}
 }
+
+func newHttpServer(h http.Handler, tlsConfig *tls.Config) *http.Server {
+	s := &http.Server{
+		Handler: h,
+	}
+	if tlsConfig != nil {
+		s.TLSConfig = tlsConfig.Clone()
+	}
+	return s
+}
diff --git a/weed/command/volume.go b/weed/command/volume.go
index 97986b500..c18ed3222 100644
--- a/weed/command/volume.go
+++ b/weed/command/volume.go
@@ -2,7 +2,6 @@ package command
 
 import (
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	httppprof "net/http/pprof"
 	"os"
@@ -11,26 +10,23 @@ import (
 	"strings"
 	"time"
 
-	"github.com/seaweedfs/seaweedfs/weed/storage/types"
-
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
-
-	"github.com/seaweedfs/seaweedfs/weed/util/grace"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/security"
-	"github.com/seaweedfs/seaweedfs/weed/server/constants"
-	"github.com/seaweedfs/seaweedfs/weed/util/httpdown"
-
 	"google.golang.org/grpc/reflection"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
 	weed_server "github.com/seaweedfs/seaweedfs/weed/server"
+	"github.com/seaweedfs/seaweedfs/weed/server/constants"
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/grace"
+	"github.com/seaweedfs/seaweedfs/weed/util/httpdown"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 )
 
 var (
@@ -398,6 +394,7 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd
 	if viper.GetString("https.volume.ca") != "" {
 		clientCertFile := viper.GetString("https.volume.ca")
 		httpS.TLSConfig = security.LoadClientTLSHTTP(clientCertFile)
+		security.FixTlsConfig(util.GetViper(), httpS.TLSConfig)
 	}
 
 	clusterHttpServer := httpDown.Serve(httpS, listener)