Use existing server-CA and hash if available

Also wraps errors along the cluster prepare path to improve tracability.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

pkg/clientaccess/token.go

@@ -67,8 +67,29 @@ type Info struct {
 // ValidationOption is a callback to mutate the token prior to use
 type ValidationOption func(*Info)
 
+// WithCACertificate overrides the CA cert and hash with certs loaded from the
+// provided file. It is not an error if the file doesn't exist; the client
+// will just follow the normal hash validation steps if so.
+func WithCACertificate(certFile string) ValidationOption {
+	return func(i *Info) {
+		cacerts, err := os.ReadFile(certFile)
+		if err != nil {
+			return
+		}
+
+		digest, _ := hashCA(cacerts)
+		if i.caHash != "" && i.caHash != digest {
+			return
+		}
+
+		i.caHash = digest
+		i.CACerts = cacerts
+	}
+}
+
 // WithClientCertificate configures certs and keys to be used
-// to authenticate the request.
+// to authenticate the request. It is not an error if the files do not
+// exist, client cert auth will not be attempted if so.
 func WithClientCertificate(certFile, keyFile string) ValidationOption {
 	return func(i *Info) {
 		i.CertFile = certFile
@@ -338,7 +359,8 @@ func (i *Info) Post(path string, body []byte, options ...any) ([]byte, error) {
 }
 
 // setServer sets the BaseURL and CACerts fields of the Info by connecting to the server
-// and storing the CA bundle.
+// and storing the CA bundle. If CACerts has already been set via ValidationOption,
+// retrieval is skipped.
 func (i *Info) setServer(server string) error {
 	url, err := url.Parse(server)
 	if err != nil {
@@ -353,13 +375,15 @@ func (i *Info) setServer(server string) error {
 		url.Path = url.Path[:len(url.Path)-1]
 	}
 
-	cacerts, err := getCACerts(*url)
-	if err != nil {
-		return err
+	if len(i.CACerts) == 0 {
+		cacerts, err := getCACerts(*url)
+		if err != nil {
+			return err
+		}
+		i.CACerts = cacerts
 	}
 
 	i.BaseURL = url.String()
-	i.CACerts = cacerts
 	return nil
 }
 

pkg/cluster/bootstrap.go

@@ -34,12 +34,12 @@ import (
 // ControlRuntimeBootstrap struct, either via HTTP or from the datastore.
 func (c *Cluster) Bootstrap(ctx context.Context, clusterReset bool) error {
 	if err := c.assignManagedDriver(ctx); err != nil {
-		return err
+		return errors.Wrap(err, "failed to set datastore driver")
 	}
 
 	shouldBootstrap, isInitialized, err := c.shouldBootstrapLoad(ctx)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to check if bootstrap data has been initialized")
 	}
 	c.shouldBootstrap = shouldBootstrap
 
@@ -80,6 +80,11 @@ func (c *Cluster) Bootstrap(ctx context.Context, clusterReset bool) error {
 // indicating that the server has or has not been initialized, if etcd. This is controlled by a stamp file on
 // disk that records successful bootstrap using a hash of the join token.
 func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
+	opts := []clientaccess.ValidationOption{
+		clientaccess.WithUser("server"),
+		clientaccess.WithCACertificate(c.config.Runtime.ServerCA),
+	}
+
 	// Non-nil managedDB indicates that the database is either initialized, initializing, or joining
 	if c.managedDB != nil {
 		c.config.Runtime.HTTPBootstrap = c.serveBootstrap()
@@ -96,7 +101,7 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
 			// etcd is promoted from learner. Odds are we won't need this info, and we don't want to fail startup
 			// due to failure to retrieve it as this will break cold cluster restart, so we ignore any errors.
 			if c.config.JoinURL != "" && c.config.Token != "" {
-				c.clientAccessInfo, _ = clientaccess.ParseAndValidateToken(c.config.JoinURL, c.config.Token, clientaccess.WithUser("server"))
+				c.clientAccessInfo, _ = clientaccess.ParseAndValidateToken(c.config.JoinURL, c.config.Token, opts...)
 			}
 			return false, true, nil
 		} else if c.config.JoinURL == "" {
@@ -105,15 +110,16 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
 			return false, false, nil
 		} else {
 			// Not initialized, but have a Join URL - fail if there's no token; if there is then validate it.
+			// Note that this is the path taken by control-plane-only nodes every startup, as they have a non-nil managedDB that is never initialized.
 			if c.config.Token == "" {
-				return false, false, errors.New(version.ProgramUpper + "_TOKEN is required to join a cluster")
+				return false, false, errors.New("token is required to join a cluster")
 			}
 
 			// Fail if the token isn't syntactically valid, or if the CA hash on the remote server doesn't match
 			// the hash in the token. The password isn't actually checked until later when actually bootstrapping.
-			info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, c.config.Token, clientaccess.WithUser("server"))
+			info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, c.config.Token, opts...)
 			if err != nil {
-				return false, false, err
+				return false, false, errors.Wrap(err, "failed to validate token")
 			}
 
 			logrus.Infof("Managed %s cluster not yet initialized", c.managedDB.EndpointName())
@@ -451,11 +457,16 @@ func (c *Cluster) bootstrap(ctx context.Context) error {
 
 // compareConfig verifies that the config of the joining control plane node coincides with the cluster's config
 func (c *Cluster) compareConfig() error {
+	opts := []clientaccess.ValidationOption{
+		clientaccess.WithUser("node"),
+		clientaccess.WithCACertificate(c.config.Runtime.ServerCA),
+	}
+
 	token := c.config.AgentToken
 	if token == "" {
 		token = c.config.Token
 	}
-	agentClientAccessInfo, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token, clientaccess.WithUser("node"))
+	agentClientAccessInfo, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token, opts...)
 	if err != nil {
 		return err
 	}

pkg/daemons/control/server.go

@@ -287,17 +287,16 @@ func defaults(config *config.Control) {
 }
 
 func prepare(ctx context.Context, config *config.Control) error {
-	var err error
-
 	defaults(config)
 
 	if err := os.MkdirAll(config.DataDir, 0700); err != nil {
 		return err
 	}
 
-	config.DataDir, err = filepath.Abs(config.DataDir)
-	if err != nil {
+	if dataDir, err := filepath.Abs(config.DataDir); err != nil {
 		return err
+	} else {
+		config.DataDir = dataDir
 	}
 
 	os.MkdirAll(filepath.Join(config.DataDir, "etc"), 0700)
@@ -308,19 +307,19 @@ func prepare(ctx context.Context, config *config.Control) error {
 
 	cluster := cluster.New(config)
 	if err := cluster.Bootstrap(ctx, config.ClusterReset); err != nil {
-		return err
+		return errors.Wrap(err, "failed to bootstrap cluster data")
 	}
 
 	if err := deps.GenServerDeps(config); err != nil {
-		return err
+		return errors.Wrap(err, "failed to generate server dependencies")
 	}
 
-	ready, err := cluster.Start(ctx)
-	if err != nil {
-		return err
+	if ready, err := cluster.Start(ctx); err != nil {
+		return errors.Wrap(err, "failed to start cluster")
+	} else {
+		config.Runtime.ETCDReady = ready
 	}
 
-	config.Runtime.ETCDReady = ready
 	return nil
 }