diff --git a/pkg/agent/https/https.go b/pkg/agent/https/https.go index df91db4542..c531716b53 100644 --- a/pkg/agent/https/https.go +++ b/pkg/agent/https/https.go @@ -2,19 +2,13 @@ package https import ( "context" - "net/http" "strconv" "sync" "github.com/gorilla/mux" "github.com/k3s-io/k3s/pkg/daemons/config" - "github.com/k3s-io/api/pkg/generated/clientset/versioned/scheme" + "github.com/k3s-io/k3s/pkg/server/auth" "github.com/k3s-io/k3s/pkg/util" - "github.com/k3s-io/k3s/pkg/version" - "k8s.io/apiserver/pkg/authentication/authenticator" - "k8s.io/apiserver/pkg/authorization/authorizer" - genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" - apirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" ) @@ -32,7 +26,7 @@ var err error func Start(ctx context.Context, nodeConfig *config.Node, runtime *config.ControlRuntime) (*mux.Router, error) { once.Do(func() { router = mux.NewRouter().SkipClean(true) - config := server.Config{} + config := &server.Config{} if runtime == nil { // If we do not have an existing handler, set up a new listener @@ -62,31 +56,7 @@ func Start(ctx context.Context, nodeConfig *config.Node, runtime *config.Control runtime.Handler = router } - authn := options.NewDelegatingAuthenticationOptions() - authn.DisableAnonymous = true - authn.SkipInClusterLookup = true - authn.ClientCert = options.ClientCertAuthenticationOptions{ - ClientCA: nodeConfig.AgentConfig.ClientCA, - } - authn.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet - if applyErr := authn.ApplyTo(&config.Authentication, config.SecureServing, nil); applyErr != nil { - err = applyErr - return - } - - authz := options.NewDelegatingAuthorizationOptions() - authz.AlwaysAllowPaths = []string{ // skip authz for paths that should not use SubjectAccessReview; basically everything that will use this router other than metrics - "/v1-" + version.Program + "/p2p", // spegel libp2p peer discovery - "/v2/*", // spegel registry mirror - "/debug/pprof/*", // profiling - } - authz.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet - if applyErr := authz.ApplyTo(&config.Authorization); applyErr != nil { - err = applyErr - return - } - - router.Use(filterChain(config.Authentication.Authenticator, config.Authorization.Authorizer)) + router.Use(auth.Delegated(nodeConfig.AgentConfig.ClientCA, nodeConfig.AgentConfig.KubeConfigKubelet, config)) if config.SecureServing != nil { _, _, err = config.SecureServing.Serve(router, 0, ctx.Done()) @@ -95,16 +65,3 @@ func Start(ctx context.Context, nodeConfig *config.Node, runtime *config.Control return router, err } - -// filterChain runs the kubernetes authn/authz filter chain using the mux middleware API -func filterChain(authn authenticator.Request, authz authorizer.Authorizer) mux.MiddlewareFunc { - return func(handler http.Handler) http.Handler { - requestInfoResolver := &apirequest.RequestInfoFactory{} - failedHandler := genericapifilters.Unauthorized(scheme.Codecs) - handler = genericapifilters.WithAuthorization(handler, authz, scheme.Codecs) - handler = genericapifilters.WithAuthentication(handler, authn, failedHandler, nil, nil) - handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver) - handler = genericapifilters.WithCacheControl(handler) - return handler - } -} diff --git a/pkg/cluster/https.go b/pkg/cluster/https.go index 5705384440..df5446b98a 100644 --- a/pkg/cluster/https.go +++ b/pkg/cluster/https.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "errors" + "fmt" "io" "log" "net" @@ -90,19 +91,26 @@ func (c *Cluster) filterCN(cn ...string) []string { // initClusterAndHTTPS sets up the dynamic tls listener, request router, // and cluster database. Once the database is up, it starts the supervisor http server. func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error { - // Set up dynamiclistener TLS listener and request handler - listener, handler, err := c.newListener(ctx) + // Set up dynamiclistener TLS listener and request handler. + // The dynamiclistener request handler is always called first as a middleware to add TLS SANs for host headers. + // It does not actually do any request handling or send a response. + listener, certHandler, err := c.newListener(ctx) if err != nil { return err } - // Get the base request handler - handler, err = c.getHandler(handler) - if err != nil { - return err - } + // Create a stub request handler that returns a Service Unavailable response + // if the core request handlers have not yet been started yet. + var handler http.Handler = http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + if c.config.Runtime.Handler != nil { + c.config.Runtime.Handler.ServeHTTP(rw, req) + } else { + util.SendError(fmt.Errorf("starting"), rw, req, http.StatusServiceUnavailable) + } + }) - // Register database request handlers and controller callbacks + // Register database request handlers and controller callbacks. + // The database handler wraps the stub handler, and calls it for any requests not related to database bootstrapping. handler, err = c.registerDBHandlers(handler) if err != nil { return err @@ -110,7 +118,10 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error { // Create a HTTP server with the registered request handlers, using logrus for logging server := http.Server{ - Handler: handler, + Handler: http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + certHandler.ServeHTTP(rw, req) + handler.ServeHTTP(rw, req) + }), } if logrus.IsLevelEnabled(logrus.DebugLevel) { diff --git a/pkg/cluster/router.go b/pkg/cluster/router.go deleted file mode 100644 index 39dc5e2164..0000000000 --- a/pkg/cluster/router.go +++ /dev/null @@ -1,31 +0,0 @@ -package cluster - -import ( - "fmt" - "net/http" - - "github.com/k3s-io/k3s/pkg/util" -) - -// getHandler returns a basic request handler that processes requests through -// the cluster's request router chain. -func (c *Cluster) getHandler(handler http.Handler) (http.Handler, error) { - next := c.router() - - return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - handler.ServeHTTP(rw, req) - next.ServeHTTP(rw, req) - }), nil -} - -// router is a stub request router that returns a Service Unavailable response -// if no additional handlers are available. -func (c *Cluster) router() http.Handler { - return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - if c.config.Runtime.Handler != nil { - c.config.Runtime.Handler.ServeHTTP(rw, req) - } else { - util.SendError(fmt.Errorf("starting"), rw, req, http.StatusServiceUnavailable) - } - }) -} diff --git a/pkg/server/auth/auth.go b/pkg/server/auth/auth.go index 9e1eb1e84a..151cbfbc7a 100644 --- a/pkg/server/auth/auth.go +++ b/pkg/server/auth/auth.go @@ -8,8 +8,19 @@ import ( "github.com/gorilla/mux" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" "github.com/sirupsen/logrus" - "k8s.io/apiserver/pkg/endpoints/request" + genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" + apirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/server" + genericfilters "k8s.io/apiserver/pkg/server/filters" + "k8s.io/apiserver/pkg/server/options" + "k8s.io/client-go/kubernetes/scheme" +) + +var ( + requestInfoResolver = &apirequest.RequestInfoFactory{} + failedHandler = genericapifilters.Unauthorized(scheme.Codecs) ) func hasRole(mustRoles []string, roles []string) bool { @@ -48,7 +59,7 @@ func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw return } - ctx := request.WithUser(req.Context(), resp.User) + ctx := apirequest.WithUser(req.Context(), resp.User) req = req.WithContext(ctx) next.ServeHTTP(rw, req) } @@ -77,3 +88,52 @@ func IsLocalOrHasRole(serverConfig *config.Control, roles ...string) mux.Middlew }) } } + +// Delegated returns a middleware function that uses core Kubernetes +// authentication/authorization via client certificate auth and the SubjectAccessReview API +func Delegated(clientCA, kubeConfig string, config *server.Config) mux.MiddlewareFunc { + if config == nil { + config = &server.Config{} + } + + authn := options.NewDelegatingAuthenticationOptions() + authn.DisableAnonymous = true + authn.SkipInClusterLookup = true + authn.ClientCert = options.ClientCertAuthenticationOptions{ + ClientCA: clientCA, + } + authn.RemoteKubeConfigFile = kubeConfig + if err := authn.ApplyTo(&config.Authentication, config.SecureServing, nil); err != nil { + logrus.Fatalf("Failed to apply authentication configuration: %v", err) + } + + authz := options.NewDelegatingAuthorizationOptions() + authz.AlwaysAllowPaths = []string{ // skip authz for paths that should not use SubjectAccessReview; basically everything that will use this router other than metrics + "/v1-" + version.Program + "/p2p", // spegel libp2p peer discovery + "/v2/*", // spegel registry mirror + "/debug/pprof/*", // profiling + } + authz.RemoteKubeConfigFile = kubeConfig + if err := authz.ApplyTo(&config.Authorization); err != nil { + logrus.Fatalf("Failed to apply authorization configuration: %v", err) + } + + return func(handler http.Handler) http.Handler { + handler = genericapifilters.WithAuthorization(handler, config.Authorization.Authorizer, scheme.Codecs) + handler = genericapifilters.WithAuthentication(handler, config.Authentication.Authenticator, failedHandler, nil, nil) + handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver) + handler = genericapifilters.WithCacheControl(handler) + return handler + } +} + +// MaxInFlight returns a middleware function that limits the number of requests that are executed concurrently. +// This is not strictly auth related, but it also uses the core Kubernetes request filters. +func MaxInFlight(nonMutatingLimit, mutatingLimit int) mux.MiddlewareFunc { + return func(handler http.Handler) http.Handler { + handler = genericfilters.WithMaxInFlightLimit(handler, nonMutatingLimit, mutatingLimit, nil) + handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver) + handler = genericapifilters.WithCacheControl(handler) + return handler + } +} diff --git a/pkg/server/handlers/router.go b/pkg/server/handlers/router.go index eef6ffe7d5..a787aa08f9 100644 --- a/pkg/server/handlers/router.go +++ b/pkg/server/handlers/router.go @@ -19,13 +19,21 @@ const ( staticURL = "/static/" ) +var ( + // When starting, each agent sequentially makes requests for certs, config, and apiservers, and will poll the readyz endpoint + // before starting kube-proxy. These limits effectively cap the number of agents that can join simultaneously. + // Agents will automatically retry with jitter when rate-limited. + maxNonMutatingAgentRequests = 20 // max concurrent get/list/watch requests + maxMutatingAgentRequests = 10 // max concurrent other requests; cert generation with client-provided private key uses post. +) + func NewHandler(ctx context.Context, control *config.Control, cfg *cmds.Server) http.Handler { nodeAuth := nodepassword.GetNodeAuthValidator(ctx, control) prefix := "/v1-{program}" authed := mux.NewRouter().SkipClean(true) authed.NotFoundHandler = APIServer(control, cfg) - authed.Use(auth.HasRole(control, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup)) + authed.Use(auth.HasRole(control, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup), auth.MaxInFlight(maxNonMutatingAgentRequests, maxMutatingAgentRequests)) authed.Handle(prefix+"/serving-kubelet.crt", ServingKubeletCert(control, nodeAuth)) authed.Handle(prefix+"/client-kubelet.crt", ClientKubeletCert(control, nodeAuth)) authed.Handle(prefix+"/client-kube-proxy.crt", ClientKubeProxyCert(control)) diff --git a/pkg/spegel/spegel.go b/pkg/spegel/spegel.go index 7816cd1000..e0b1ba094a 100644 --- a/pkg/spegel/spegel.go +++ b/pkg/spegel/spegel.go @@ -18,6 +18,7 @@ import ( "github.com/k3s-io/k3s/pkg/agent/https" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/server/auth" "github.com/k3s-io/k3s/pkg/version" "github.com/rancher/dynamiclistener/cert" "k8s.io/apimachinery/pkg/util/wait" @@ -56,6 +57,11 @@ var ( P2pEnableLatestEnv = version.ProgramUpper + "_P2P_ENABLE_LATEST" resolveLatestTag = false + + // Agents request a list of peers when joining, and then again periodically afterwards. + // Limit the number of concurrent peer list requests that will be served simultaneously. + maxNonMutatingPeerInfoRequests = 20 // max concurrent get/list/watch requests + maxMutatingPeerInfoRequests = 0 // max concurrent other requests; not used ) // Config holds fields for a distributed registry @@ -231,7 +237,9 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error { return err } mRouter.PathPrefix("/v2").Handler(regSvr.Handler) - mRouter.PathPrefix("/v1-{program}/p2p").Handler(c.peerInfo()) + sRouter := mRouter.PathPrefix("/v1-{program}/p2p").Subrouter() + sRouter.Use(auth.MaxInFlight(maxNonMutatingPeerInfoRequests, maxMutatingPeerInfoRequests)) + sRouter.Handle("", c.peerInfo()) // Wait up to 5 seconds for the p2p network to find peers. This will return // immediately if the node is bootstrapping from itself.