diff --git a/go.mod b/go.mod index b4614476c0..888572b035 100644 --- a/go.mod +++ b/go.mod @@ -128,11 +128,11 @@ require ( github.com/prometheus/client_golang v1.21.1 github.com/prometheus/common v0.62.0 github.com/rancher/dynamiclistener v0.6.0-rc1 - github.com/rancher/lasso v0.0.0-20250109193533-00757eec2dbd + github.com/rancher/lasso v0.2.1 github.com/rancher/permissions v0.0.0-20240523180510-4001d3d637f7 github.com/rancher/remotedialer v0.4.1 github.com/rancher/wharfie v0.6.9 - github.com/rancher/wrangler/v3 v3.2.0-rc.1 + github.com/rancher/wrangler/v3 v3.2.1-0.20250404112553-6674f9c57bfe github.com/robfig/cron/v3 v3.0.1 github.com/rootless-containers/rootlesskit v1.1.1 github.com/sirupsen/logrus v1.9.3 @@ -150,7 +150,7 @@ require ( go.uber.org/mock v0.5.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.37.0 - golang.org/x/mod v0.22.0 + golang.org/x/mod v0.23.0 golang.org/x/net v0.39.0 golang.org/x/sync v0.13.0 golang.org/x/sys v0.32.0 @@ -257,7 +257,7 @@ require ( github.com/emicklei/go-restful v2.16.0+incompatible // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect - github.com/evanphx/json-patch v5.9.0+incompatible // indirect + github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fatih/camelcase v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -322,7 +322,6 @@ require ( github.com/karrick/godirwalk v1.17.0 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/koron/go-ssdp v0.0.4 // indirect - github.com/kylelemons/godebug v1.1.0 // indirect github.com/libopenstorage/openstorage v1.0.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect @@ -473,7 +472,7 @@ require ( golang.org/x/term v0.30.0 // indirect golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.10.0 // indirect - golang.org/x/tools v0.28.0 // indirect + golang.org/x/tools v0.30.0 // indirect golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 // indirect gonum.org/v1/gonum v0.15.0 // indirect diff --git a/go.sum b/go.sum index 8f75ccb182..d296c69ad6 100644 --- a/go.sum +++ b/go.sum @@ -454,8 +454,8 @@ github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83 h1:ngHdSomn2My github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83/go.mod h1:v6o7m/E9bfvm79dE1iFiF+3T7zLBnrjYjkWMa1J+Hv0= github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= -github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= -github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= +github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= @@ -1204,16 +1204,16 @@ github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6 github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/rancher/dynamiclistener v0.6.0-rc1 h1:Emwf9o7PMLdQNv4lvFx7xJKxDuDa4Y69GvVEGU9U9Js= github.com/rancher/dynamiclistener v0.6.0-rc1/go.mod h1:BIPgJ8xFSUyuTyGvRMVt++S1qjD3+7Ptvq1TXl6hcTM= -github.com/rancher/lasso v0.0.0-20250109193533-00757eec2dbd h1:W4hWtKB8A2SSJlxkphvzwA+RkiOrgHg9Gytc78S0pvk= -github.com/rancher/lasso v0.0.0-20250109193533-00757eec2dbd/go.mod h1:IxgTBO55lziYhTEETyVKiT8/B5Rg92qYiRmcIIYoPgI= +github.com/rancher/lasso v0.2.1 h1:SZTqMVQn8cAOqvwGBd1/EYOIJ/MGN+UfJrOWvHd4jHU= +github.com/rancher/lasso v0.2.1/go.mod h1:KSV3jBXfdXqdCuMm2uC8kKB9q/wuDYb3h0eHZoRjShM= github.com/rancher/permissions v0.0.0-20240523180510-4001d3d637f7 h1:0Kg2SGoMeU1ll4xPi4DE0+qNHLFO/U5MwtK0WrIdK+o= github.com/rancher/permissions v0.0.0-20240523180510-4001d3d637f7/go.mod h1:fsbs0YOsGn1ofPD5p+BuI4qDhbMbSJtTegKt6Ucna+c= github.com/rancher/remotedialer v0.4.1 h1:jwOf2kPRjBBpSFofv1OuZHWaYHeC9Eb6/XgDvbkoTgc= github.com/rancher/remotedialer v0.4.1/go.mod h1:Ys004RpJuTLSm+k4aYUCoFiOOad37ubYev3TkOFg/5w= github.com/rancher/wharfie v0.6.9 h1:w0xf8F/LqjuXBYYJRMWTcHWIF8p4zVKmFOOEqz74VhQ= github.com/rancher/wharfie v0.6.9/go.mod h1:PpyuDiBgnFVd56kKtNfSS3cMDTVcRJZVYfNwxcqv50I= -github.com/rancher/wrangler/v3 v3.2.0-rc.1 h1:c0r5aBVUwWUqfeVFP9S4JGalsLra5zHZtvyteSuKHtI= -github.com/rancher/wrangler/v3 v3.2.0-rc.1/go.mod h1:8DV2nXOOiUrsoRviL3WmzjELR5B6AS8CtMGEP34E0DE= +github.com/rancher/wrangler/v3 v3.2.1-0.20250404112553-6674f9c57bfe h1:sRkWWOzFJFkoqWknniWes0S9KpYLHOUqBdzPpIpWipM= +github.com/rancher/wrangler/v3 v3.2.1-0.20250404112553-6674f9c57bfe/go.mod h1:0C5QyvSrQOff8gQQzpB/L/FF03EQycjR3unSJcKCHno= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= @@ -1508,8 +1508,8 @@ golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= +golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1678,8 +1678,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= -golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= +golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= +golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/agent/https/https.go b/pkg/agent/https/https.go index 33e001be97..c531716b53 100644 --- a/pkg/agent/https/https.go +++ b/pkg/agent/https/https.go @@ -2,20 +2,13 @@ package https import ( "context" - "net/http" "strconv" "sync" "github.com/gorilla/mux" "github.com/k3s-io/k3s/pkg/daemons/config" - "github.com/k3s-io/api/pkg/generated/clientset/versioned/scheme" + "github.com/k3s-io/k3s/pkg/server/auth" "github.com/k3s-io/k3s/pkg/util" - "github.com/k3s-io/k3s/pkg/version" - "k8s.io/apiserver/pkg/apis/apiserver" - "k8s.io/apiserver/pkg/authentication/authenticator" - "k8s.io/apiserver/pkg/authorization/authorizer" - genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" - apirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" ) @@ -33,7 +26,7 @@ var err error func Start(ctx context.Context, nodeConfig *config.Node, runtime *config.ControlRuntime) (*mux.Router, error) { once.Do(func() { router = mux.NewRouter().SkipClean(true) - config := server.Config{} + config := &server.Config{} if runtime == nil { // If we do not have an existing handler, set up a new listener @@ -63,33 +56,7 @@ func Start(ctx context.Context, nodeConfig *config.Node, runtime *config.Control runtime.Handler = router } - authn := options.NewDelegatingAuthenticationOptions() - authn.Anonymous = &apiserver.AnonymousAuthConfig{ - Enabled: false, - } - authn.SkipInClusterLookup = true - authn.ClientCert = options.ClientCertAuthenticationOptions{ - ClientCA: nodeConfig.AgentConfig.ClientCA, - } - authn.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet - if applyErr := authn.ApplyTo(&config.Authentication, config.SecureServing, nil); applyErr != nil { - err = applyErr - return - } - - authz := options.NewDelegatingAuthorizationOptions() - authz.AlwaysAllowPaths = []string{ // skip authz for paths that should not use SubjectAccessReview; basically everything that will use this router other than metrics - "/v1-" + version.Program + "/p2p", // spegel libp2p peer discovery - "/v2/*", // spegel registry mirror - "/debug/pprof/*", // profiling - } - authz.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet - if applyErr := authz.ApplyTo(&config.Authorization); applyErr != nil { - err = applyErr - return - } - - router.Use(filterChain(config.Authentication.Authenticator, config.Authorization.Authorizer)) + router.Use(auth.Delegated(nodeConfig.AgentConfig.ClientCA, nodeConfig.AgentConfig.KubeConfigKubelet, config)) if config.SecureServing != nil { _, _, err = config.SecureServing.Serve(router, 0, ctx.Done()) @@ -98,16 +65,3 @@ func Start(ctx context.Context, nodeConfig *config.Node, runtime *config.Control return router, err } - -// filterChain runs the kubernetes authn/authz filter chain using the mux middleware API -func filterChain(authn authenticator.Request, authz authorizer.Authorizer) mux.MiddlewareFunc { - return func(handler http.Handler) http.Handler { - requestInfoResolver := &apirequest.RequestInfoFactory{} - failedHandler := genericapifilters.Unauthorized(scheme.Codecs) - handler = genericapifilters.WithAuthorization(handler, authz, scheme.Codecs) - handler = genericapifilters.WithAuthentication(handler, authn, failedHandler, nil, nil) - handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver) - handler = genericapifilters.WithCacheControl(handler) - return handler - } -} diff --git a/pkg/cluster/https.go b/pkg/cluster/https.go index 5705384440..df5446b98a 100644 --- a/pkg/cluster/https.go +++ b/pkg/cluster/https.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "errors" + "fmt" "io" "log" "net" @@ -90,19 +91,26 @@ func (c *Cluster) filterCN(cn ...string) []string { // initClusterAndHTTPS sets up the dynamic tls listener, request router, // and cluster database. Once the database is up, it starts the supervisor http server. func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error { - // Set up dynamiclistener TLS listener and request handler - listener, handler, err := c.newListener(ctx) + // Set up dynamiclistener TLS listener and request handler. + // The dynamiclistener request handler is always called first as a middleware to add TLS SANs for host headers. + // It does not actually do any request handling or send a response. + listener, certHandler, err := c.newListener(ctx) if err != nil { return err } - // Get the base request handler - handler, err = c.getHandler(handler) - if err != nil { - return err - } + // Create a stub request handler that returns a Service Unavailable response + // if the core request handlers have not yet been started yet. + var handler http.Handler = http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + if c.config.Runtime.Handler != nil { + c.config.Runtime.Handler.ServeHTTP(rw, req) + } else { + util.SendError(fmt.Errorf("starting"), rw, req, http.StatusServiceUnavailable) + } + }) - // Register database request handlers and controller callbacks + // Register database request handlers and controller callbacks. + // The database handler wraps the stub handler, and calls it for any requests not related to database bootstrapping. handler, err = c.registerDBHandlers(handler) if err != nil { return err @@ -110,7 +118,10 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error { // Create a HTTP server with the registered request handlers, using logrus for logging server := http.Server{ - Handler: handler, + Handler: http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + certHandler.ServeHTTP(rw, req) + handler.ServeHTTP(rw, req) + }), } if logrus.IsLevelEnabled(logrus.DebugLevel) { diff --git a/pkg/cluster/router.go b/pkg/cluster/router.go deleted file mode 100644 index 39dc5e2164..0000000000 --- a/pkg/cluster/router.go +++ /dev/null @@ -1,31 +0,0 @@ -package cluster - -import ( - "fmt" - "net/http" - - "github.com/k3s-io/k3s/pkg/util" -) - -// getHandler returns a basic request handler that processes requests through -// the cluster's request router chain. -func (c *Cluster) getHandler(handler http.Handler) (http.Handler, error) { - next := c.router() - - return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - handler.ServeHTTP(rw, req) - next.ServeHTTP(rw, req) - }), nil -} - -// router is a stub request router that returns a Service Unavailable response -// if no additional handlers are available. -func (c *Cluster) router() http.Handler { - return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - if c.config.Runtime.Handler != nil { - c.config.Runtime.Handler.ServeHTTP(rw, req) - } else { - util.SendError(fmt.Errorf("starting"), rw, req, http.StatusServiceUnavailable) - } - }) -} diff --git a/pkg/server/auth/auth.go b/pkg/server/auth/auth.go index 9e1eb1e84a..17994aa04b 100644 --- a/pkg/server/auth/auth.go +++ b/pkg/server/auth/auth.go @@ -8,8 +8,20 @@ import ( "github.com/gorilla/mux" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" "github.com/sirupsen/logrus" - "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/apis/apiserver" + genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" + apirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/server" + genericfilters "k8s.io/apiserver/pkg/server/filters" + "k8s.io/apiserver/pkg/server/options" + "k8s.io/client-go/kubernetes/scheme" +) + +var ( + requestInfoResolver = &apirequest.RequestInfoFactory{} + failedHandler = genericapifilters.Unauthorized(scheme.Codecs) ) func hasRole(mustRoles []string, roles []string) bool { @@ -48,7 +60,7 @@ func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw return } - ctx := request.WithUser(req.Context(), resp.User) + ctx := apirequest.WithUser(req.Context(), resp.User) req = req.WithContext(ctx) next.ServeHTTP(rw, req) } @@ -77,3 +89,54 @@ func IsLocalOrHasRole(serverConfig *config.Control, roles ...string) mux.Middlew }) } } + +// Delegated returns a middleware function that uses core Kubernetes +// authentication/authorization via client certificate auth and the SubjectAccessReview API +func Delegated(clientCA, kubeConfig string, config *server.Config) mux.MiddlewareFunc { + if config == nil { + config = &server.Config{} + } + + authn := options.NewDelegatingAuthenticationOptions() + authn.Anonymous = &apiserver.AnonymousAuthConfig{ + Enabled: false, + } + authn.SkipInClusterLookup = true + authn.ClientCert = options.ClientCertAuthenticationOptions{ + ClientCA: clientCA, + } + authn.RemoteKubeConfigFile = kubeConfig + if err := authn.ApplyTo(&config.Authentication, config.SecureServing, nil); err != nil { + logrus.Fatalf("Failed to apply authentication configuration: %v", err) + } + + authz := options.NewDelegatingAuthorizationOptions() + authz.AlwaysAllowPaths = []string{ // skip authz for paths that should not use SubjectAccessReview; basically everything that will use this router other than metrics + "/v1-" + version.Program + "/p2p", // spegel libp2p peer discovery + "/v2/*", // spegel registry mirror + "/debug/pprof/*", // profiling + } + authz.RemoteKubeConfigFile = kubeConfig + if err := authz.ApplyTo(&config.Authorization); err != nil { + logrus.Fatalf("Failed to apply authorization configuration: %v", err) + } + + return func(handler http.Handler) http.Handler { + handler = genericapifilters.WithAuthorization(handler, config.Authorization.Authorizer, scheme.Codecs) + handler = genericapifilters.WithAuthentication(handler, config.Authentication.Authenticator, failedHandler, nil, nil) + handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver) + handler = genericapifilters.WithCacheControl(handler) + return handler + } +} + +// MaxInFlight returns a middleware function that limits the number of requests that are executed concurrently. +// This is not strictly auth related, but it also uses the core Kubernetes request filters. +func MaxInFlight(nonMutatingLimit, mutatingLimit int) mux.MiddlewareFunc { + return func(handler http.Handler) http.Handler { + handler = genericfilters.WithMaxInFlightLimit(handler, nonMutatingLimit, mutatingLimit, nil) + handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver) + handler = genericapifilters.WithCacheControl(handler) + return handler + } +} diff --git a/pkg/server/handlers/router.go b/pkg/server/handlers/router.go index eef6ffe7d5..a787aa08f9 100644 --- a/pkg/server/handlers/router.go +++ b/pkg/server/handlers/router.go @@ -19,13 +19,21 @@ const ( staticURL = "/static/" ) +var ( + // When starting, each agent sequentially makes requests for certs, config, and apiservers, and will poll the readyz endpoint + // before starting kube-proxy. These limits effectively cap the number of agents that can join simultaneously. + // Agents will automatically retry with jitter when rate-limited. + maxNonMutatingAgentRequests = 20 // max concurrent get/list/watch requests + maxMutatingAgentRequests = 10 // max concurrent other requests; cert generation with client-provided private key uses post. +) + func NewHandler(ctx context.Context, control *config.Control, cfg *cmds.Server) http.Handler { nodeAuth := nodepassword.GetNodeAuthValidator(ctx, control) prefix := "/v1-{program}" authed := mux.NewRouter().SkipClean(true) authed.NotFoundHandler = APIServer(control, cfg) - authed.Use(auth.HasRole(control, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup)) + authed.Use(auth.HasRole(control, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup), auth.MaxInFlight(maxNonMutatingAgentRequests, maxMutatingAgentRequests)) authed.Handle(prefix+"/serving-kubelet.crt", ServingKubeletCert(control, nodeAuth)) authed.Handle(prefix+"/client-kubelet.crt", ClientKubeletCert(control, nodeAuth)) authed.Handle(prefix+"/client-kube-proxy.crt", ClientKubeProxyCert(control)) diff --git a/pkg/spegel/spegel.go b/pkg/spegel/spegel.go index d2c7d524db..f657827d1a 100644 --- a/pkg/spegel/spegel.go +++ b/pkg/spegel/spegel.go @@ -18,6 +18,7 @@ import ( "github.com/k3s-io/k3s/pkg/agent/https" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/server/auth" "github.com/k3s-io/k3s/pkg/version" "github.com/rancher/dynamiclistener/cert" "k8s.io/apimachinery/pkg/util/wait" @@ -56,6 +57,11 @@ var ( P2pEnableLatestEnv = version.ProgramUpper + "_P2P_ENABLE_LATEST" resolveLatestTag = false + + // Agents request a list of peers when joining, and then again periodically afterwards. + // Limit the number of concurrent peer list requests that will be served simultaneously. + maxNonMutatingPeerInfoRequests = 20 // max concurrent get/list/watch requests + maxMutatingPeerInfoRequests = 0 // max concurrent other requests; not used ) // Config holds fields for a distributed registry @@ -231,7 +237,9 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error { return err } mRouter.PathPrefix("/v2").Handler(regSvr.Handler) - mRouter.PathPrefix("/v1-{program}/p2p").Handler(c.peerInfo()) + sRouter := mRouter.PathPrefix("/v1-{program}/p2p").Subrouter() + sRouter.Use(auth.MaxInFlight(maxNonMutatingPeerInfoRequests, maxMutatingPeerInfoRequests)) + sRouter.Handle("", c.peerInfo()) // Wait up to 5 seconds for the p2p network to find peers. This will return // immediately if the node is bootstrapping from itself.