From 401eec3bae363bbdb4e08f1db6fae79b2e30ec7f Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Tue, 14 Jul 2026 23:08:28 -0500 Subject: [PATCH 01/18] feat: support separate client TLS certificates - Support for separate server and client certificates in dual client/server TLSConfigManager objects. - Certificate monitoring will now certificate loaders using the same certificate to avoid overly noisy logs. - Server, client, and client/server TLSConfigManager objects are now distinct and provide better checks based on their role. - General refactoring of TLSConfigManager and support classes. --- cmd/influxd/run/server.go | 31 +- pkg/testing/helper/helpers.go | 1 + pkg/tlsconfig/certconfig.go | 565 +++++------ pkg/tlsconfig/certconfig_test.go | 302 ++++-- pkg/tlsconfig/certmonitor.go | 453 +++++++++ pkg/tlsconfig/configmanager.go | 582 ++++++++--- pkg/tlsconfig/configmanager_test.go | 1085 ++++++++++++--------- pkg/tlsconfig/loadcert.go | 246 +++++ services/httpd/config_test.go | 9 +- services/httpd/service.go | 14 +- services/httpd/service_test.go | 37 +- services/opentsdb/service.go | 16 +- services/opentsdb/service_test.go | 49 +- services/subscriber/mtls_internal_test.go | 23 +- services/subscriber/service.go | 26 +- services/subscriber/service_test.go | 29 +- 16 files changed, 2345 insertions(+), 1123 deletions(-) create mode 100644 pkg/tlsconfig/certmonitor.go create mode 100644 pkg/tlsconfig/loadcert.go diff --git a/cmd/influxd/run/server.go b/cmd/influxd/run/server.go index 6782a481fce..298b9f429db 100644 --- a/cmd/influxd/run/server.go +++ b/cmd/influxd/run/server.go @@ -23,6 +23,7 @@ import ( "github.com/influxdata/influxdb/logger" "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/monitor" + "github.com/influxdata/influxdb/pkg/tlsconfig" prometheus2 "github.com/influxdata/influxdb/prometheus" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxdb/query/control" @@ -80,6 +81,10 @@ type Server struct { err chan error closing chan struct{} + // certMonitor is the TLS certificate monitor all TLSConfigManager objects + // in influxd should use. + certMonitor *tlsconfig.TLSCertMonitor + BindAddress string Listener net.Listener @@ -197,6 +202,8 @@ func NewServer(c *Config, buildInfo *BuildInfo) (*Server, error) { reportingDisabled: c.ReportingDisabled, + certMonitor: tlsconfig.NewTLSCertMonitor(), + httpAPIAddr: c.HTTPD.BindAddress, httpUseTLS: c.HTTPD.HTTPSEnabled, @@ -217,7 +224,7 @@ func NewServer(c *Config, buildInfo *BuildInfo) (*Server, error) { s.TSDBStore.EngineOptions.IndexVersion = c.Data.Index // Create the Subscriber service - s.Subscriber = subscriber.NewService(c.Subscriber) + s.Subscriber = subscriber.NewService(c.Subscriber, s.certMonitor) // Initialize points writer. s.PointsWriter = coordinator.NewPointsWriter() @@ -308,11 +315,11 @@ func (s *Server) appendRetentionPolicyService(c retention.Config) { s.Services = append(s.Services, srv) } -func (s *Server) appendHTTPDService(c httpd.Config) error { +func (s *Server) appendHTTPDService(c httpd.Config, certMonitor *tlsconfig.TLSCertMonitor) error { if !c.Enabled { return nil } - srv := httpd.NewService(c) + srv := httpd.NewService(c, certMonitor) srv.Handler.MetaClient = s.MetaClient authorizer := meta.NewQueryAuthorizer(s.MetaClient) srv.Handler.QueryAuthorizer = authorizer @@ -361,11 +368,11 @@ func (s *Server) appendCollectdService(c collectd.Config) { s.Services = append(s.Services, srv) } -func (s *Server) appendOpenTSDBService(c opentsdb.Config) error { +func (s *Server) appendOpenTSDBService(c opentsdb.Config, certMonitor *tlsconfig.TLSCertMonitor) error { if !c.Enabled { return nil } - srv, err := opentsdb.NewService(c) + srv, err := opentsdb.NewService(c, certMonitor) if err != nil { return err } @@ -437,6 +444,13 @@ func (s *Server) Open() error { return err } + // Configure and start certificate monitor. Do this early so we don't block + // on any of its channels. + s.certMonitor.SetLogger(s.Logger) + if err := s.certMonitor.Open(); err != nil { + return fmt.Errorf("error starting TLS certificate monitor: %w", err) + } + // Open shared TCP connection. ln, err := net.Listen("tcp", s.BindAddress) if err != nil { @@ -453,7 +467,7 @@ func (s *Server) Open() error { s.appendPrecreatorService(s.config.Precreator) s.appendSnapshotterService() s.appendContinuousQueryService(s.config.ContinuousQuery) - if err := s.appendHTTPDService(s.config.HTTPD); err != nil { + if err := s.appendHTTPDService(s.config.HTTPD, s.certMonitor); err != nil { return err } s.appendRetentionPolicyService(s.config.Retention) @@ -466,7 +480,7 @@ func (s *Server) Open() error { s.appendCollectdService(i) } for _, i := range s.config.OpenTSDBInputs { - if err := s.appendOpenTSDBService(i); err != nil { + if err := s.appendOpenTSDBService(i, s.certMonitor); err != nil { return err } } @@ -542,6 +556,9 @@ func (s *Server) Open() error { func (s *Server) Close() error { s.stopProfile() + // Close the TLS certificate monitor. + s.certMonitor.Close() + // Close the listener first to stop any new connections if s.Listener != nil { s.Listener.Close() diff --git a/pkg/testing/helper/helpers.go b/pkg/testing/helper/helpers.go index 84ea5ae9e7e..62a9420e10d 100644 --- a/pkg/testing/helper/helpers.go +++ b/pkg/testing/helper/helpers.go @@ -15,6 +15,7 @@ import ( // can be replaced by: // defer CheckedClose(t, c)() func CheckedClose(t require.TestingT, c io.Closer) func() { + require.NotNil(t, c, "CheckedClose cannot be used with a nil io.Closer") return func() { if h, ok := t.(interface{ Helper() }); ok { h.Helper() diff --git a/pkg/tlsconfig/certconfig.go b/pkg/tlsconfig/certconfig.go index 092bd249925..516d4ed670e 100644 --- a/pkg/tlsconfig/certconfig.go +++ b/pkg/tlsconfig/certconfig.go @@ -5,13 +5,10 @@ import ( "crypto/x509" "errors" "fmt" - "io" - "os" "sync" "time" "github.com/influxdata/influxdb/logger" - "github.com/influxdata/influxdb/pkg/file" "go.uber.org/zap" ) @@ -32,285 +29,217 @@ const ( var ( ErrCertificateNil = errors.New("TLS certificate is nil") ErrCertificateEmpty = errors.New("TLS certificate is empty") + ErrCertificateInvalid = errors.New("TLS certificate is invalid") ErrCertificateRequestInfoNil = errors.New("CertificateRequestInfo is nil") ErrLoadedCertificateInvalid = errors.New("LoadedCertificate is invalid") + ErrNoCertificateMonitor = errors.New("no certificate monitor") ErrPathEmpty = errors.New("empty path") + ErrSingleRoleRequired = errors.New("single role required (Server or Client)") ) -// LoadedCertificate encapsulates information about a loaded certificate. -type LoadedCertificate struct { - // valid indicates if this object is valid. - valid bool - - // CertPath is the path the certificate was loaded from. - CertificatePath string - - // KeyPath is the path the private key was loaded from. - KeyPath string - - // Certificate is the certificate that was loaded. - Certificate *tls.Certificate - - // Leaf is the parsed x509 certificate of Certificate's leaf certificate. - Leaf *x509.Certificate -} - -func (lc *LoadedCertificate) IsValid() bool { - return lc.valid -} - -// loadCertificateConfig is an internal config for LoadCertificate. -type loadCertificateConfig struct { - // ignoreFilePermissions indicates if file permissions should be ignored during load. - ignoreFilePermissions bool -} - -// LoadCertificateOpt are functions to change the behavior of LoadCertificate. -type LoadCertificateOpt func(*loadCertificateConfig) - -// WithLoadCertificateIgnoreFilePermissions instructs LoadCertificate to ignore file permissions -// if ignore is true. -func WithLoadCertificateIgnoreFilePermissions(ignore bool) LoadCertificateOpt { - return func(c *loadCertificateConfig) { - c.ignoreFilePermissions = ignore - } -} - -// LoadCertificate loads a key pair from certPath and keyPath, performing several checks -// along the way. If any checks fail or an error occurs loading the files, then an error is returned. -// If keyPath is empty, then certPath is assumed to contain both the certificate and the private key. -// Only trusted input (standard configuration files) should be used for certPath and keyPath. -func LoadCertificate(certPath, keyPath string, opts ...LoadCertificateOpt) (LoadedCertificate, error) { - fail := func(err error) (LoadedCertificate, error) { return LoadedCertificate{valid: false}, err } +// TLSCertLoader handles loading TLS certificates, providing them to a tls.Config, and +// monitoring the certificate for expiration. +type TLSCertLoader struct { + // All fields before mu can only be set at construction time. - config := loadCertificateConfig{} - for _, o := range opts { - o(&config) - } + // role specifies how a certificate will be used. Since a TLSCertLoader + // only handles a server or client certificate but not both, Server and + // Client are the only accepted values. ServerAndClient is an error. + role Role - if certPath == "" { - return fail(fmt.Errorf("LoadCertificate: certificate: %w", ErrPathEmpty)) - } + // monitor is the certificate monitor for loader. + monitor *TLSCertMonitor - if keyPath == "" { - // Assume key is combined with certificate. - keyPath = certPath - } + // logger is the logger used for logging status. It can only be + // set at construction time using WithCertLoaderLogger. + logger *zap.Logger - wipeData := func(d []byte) { - for i := range d { - d[i] = 0 - } - } + // usage is the descriptive usage string for logging. It can only + // be set at construction time using WIthCertLoaderUsage. + usage string - // Load the certificate and private key from their files. - loadFile := func(path string, maxPerms os.FileMode) (rData []byte, rErr error) { - f, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("LoadCertificate: error opening %q for reading: %w", path, err) - } - defer func() { - if err := f.Close(); err != nil { - wipeData(rData) - rData = nil - rErr = errors.Join(rErr, fmt.Errorf("LoadCertificate: error closing file %q: %w", path, err)) - } - }() + // closeCh is used to trigger closing the monitor. + closeCh chan struct{} - if !config.ignoreFilePermissions { - if err := file.VerifyFilePermissivenessF(f, maxPerms); err != nil { - // VerifyFilePermissivenessF includes a lot context in its errors. No need to add duplicate here. - return nil, fmt.Errorf("LoadCertificate: %w", err) - } - } - data, err := io.ReadAll(f) - if err != nil { - return nil, fmt.Errorf("LoadCertificate: error data from %q: %w", path, err) - } - return data, nil - } - certData, err := loadFile(certPath, CertMaxPermissions) - defer wipeData(certData) - if err != nil { - return fail(err) - } + // mu protects all members below. All fields below can be set at construction + // time or with Reconfigure. + mu sync.RWMutex - keyData, err := loadFile(keyPath, KeyMaxPermissions) - defer wipeData(keyData) - if err != nil { - return fail(err) - } + // cert is the currently active certificate. + cert LoadedCertificate - // Create key pair from loaded data - cert, err := tls.X509KeyPair(certData, keyData) - if err != nil { - return fail(fmt.Errorf("error loading x509 key pair (%q / %q): %w", certPath, keyPath, err)) - } + // config is the current configuration of the loader. + config *tlsCertLoaderConfig - // Parse the first X509 certificate in cert's chain. - // X509KeyPair() guarantees that cert.Certificate is not empty. - leaf, err := x509.ParseCertificate(cert.Certificate[0]) - if err != nil { - // This should be impossible to reach because `tls.X509KeyPair` will fail - // if the leaf certificate can't be parsed. - return fail(fmt.Errorf("error parsing leaf certificate (%q / %q): %w", certPath, keyPath, err)) - } - if leaf == nil { - // This shouldn't happen, but we should be extra careful with TLS certs. - return fail(fmt.Errorf("error parsing leaf certificate (%q / %q): %w", certPath, keyPath, ErrCertificateNil)) - } - - return LoadedCertificate{ - valid: true, - CertificatePath: certPath, - KeyPath: keyPath, - Certificate: &cert, - Leaf: leaf, - }, nil + // ignorePermissions is true if file permission checks should be bypassed. + ignorePermissions bool } -// TLSCertLoader handles loading TLS certificates, providing them to a tls.Config, and -// monitoring the certificate for expiration. -type TLSCertLoader struct { - // logger is the logger used for logging status. - logger *zap.Logger +// tlsCertLoaderConfig holds configuration data for TLSCertLoader. It is the actual +// struct loaded by the WithCertLoader* functions. +type tlsCertLoaderConfig struct { + // certPath is the certificate path to load. + certPath string - // expirationAdvanced determines how long before a certificate expires a warning is issued. - expirationAdvanced time.Duration + // keyPath is the key path to load. + keyPath string - // certificateCheckInterval determines the duration between each certificate check. - certificateCheckInterval time.Duration + // usage is the descriptive usage of the cert loader. + usage string // ignoreFilePermissions is true if file permission checks should be bypassed. + // It is during a reconfiguration if ignoreFilePermissionsSet is false. ignoreFilePermissions bool - // closeOnce is used to close closeCh exactly one time. - closeOnce sync.Once - - // closeCh is used to trigger closing the monitor. - closeCh chan struct{} - - // monitorStartWg can be used to detect if the monitor goroutine has started. - monitorStartWg sync.WaitGroup - - // mu protects all members below. - mu sync.Mutex - - // certPath is the path to the TLS certificate PEM file. - certPath string - - // keyPath is the path to the TLS certificate key file. - keyPath string + // ignoreFilePermissionsSet indicates if ignoreFilePermissions was set or is + // simply the default value. + ignoreFilePermissionsSet bool - // cert is the TLS certificate. - cert *tls.Certificate + // logger is the logger to use for logging. It is ignored during reconfiguration + // if loggerSet is false. + logger *zap.Logger - // leaf is the parsed leaf certificate. - leaf *x509.Certificate + // loggerSet is only true if WithCertLoaderLogger was used. + loggerSet bool } -type TLSCertLoaderOpt func(*TLSCertLoader) +// TLSCertLoaderOpt is a function to configure a TLSCertLoader. +type TLSCertLoaderOpt func(*tlsCertLoaderConfig) -// WithCertLoaderExpirationAdvanced sets the how far ahead a CertLoader will -// warn about a certificate that is about to expire. -func WithCertLoaderExpirationAdvanced(d time.Duration) TLSCertLoaderOpt { - return func(cl *TLSCertLoader) { - cl.expirationAdvanced = d +// WithCertLoaderCertificate sets the certificate and key for the cert loader +// to load. +func WithCertLoaderCertificate(certPath string, keyPath string) TLSCertLoaderOpt { + return func(c *tlsCertLoaderConfig) { + c.certPath = certPath + c.keyPath = keyPath } } -// WithCertLoaderCertificateCheckInterval sets how often to check for certificate expiration. -func WithCertLoaderCertificateCheckInterval(d time.Duration) TLSCertLoaderOpt { - return func(cl *TLSCertLoader) { - cl.certificateCheckInterval = d +// WithCertLoaderLogger assigns a logger for to use. +func WithCertLoaderLogger(logger *zap.Logger) TLSCertLoaderOpt { + return func(c *tlsCertLoaderConfig) { + c.logger = logger + c.loggerSet = true } } -// WithCertLoaderLogger assigns a logger for to use. -func WithCertLoaderLogger(logger *zap.Logger) TLSCertLoaderOpt { - return func(cl *TLSCertLoader) { - cl.logger = logger +// WithCertLoaderUsage assigns the descriptive usage of the cert loader. +func WithCertLoaderUsage(usage string) TLSCertLoaderOpt { + return func(c *tlsCertLoaderConfig) { + c.usage = usage } } // WithCertLoaderIgnoreFilePermissions skips file permission checking when loading certificates. func WithCertLoaderIgnoreFilePermissions(ignore bool) TLSCertLoaderOpt { - return func(cl *TLSCertLoader) { - cl.ignoreFilePermissions = ignore + return func(c *tlsCertLoaderConfig) { + c.ignoreFilePermissions = ignore + c.ignoreFilePermissionsSet = true } } -// NewTLSCertLoader creates a TLSCertLoader loaded with the certifcate found in certPath and keyPath. +// NewTLSCertLoader creates a TLSCertLoader loaded with the certificate found in certPath and keyPath. // Only trusted input (standard configuration files) should be used for certPath and keyPath. // If the certificate can not be loaded, an error is returned. On success, a monitor is setup to // periodically check the certificate for expiration. -func NewTLSCertLoader(certPath, keyPath string, opts ...TLSCertLoaderOpt) (rCertLoader *TLSCertLoader, rErr error) { +func NewTLSCertLoader(role Role, monitor *TLSCertMonitor, opts ...TLSCertLoaderOpt) (rCertLoader *TLSCertLoader, rErr error) { cl := &TLSCertLoader{ - expirationAdvanced: DefaultExpirationWarnTime, - certificateCheckInterval: DefaultCertificateCheckTime, - closeCh: make(chan struct{}), + role: role, + monitor: monitor, } // Configure options. + config := &tlsCertLoaderConfig{} for _, o := range opts { - o(cl) + o(config) } - // Make sure there is a valid logger. - if cl.logger == nil { - cl.logger = zap.NewNop() + // Copy some config over. + cl.usage = config.usage + cl.config = config + + certPath := config.certPath + keyPath := config.keyPath + + // Check for configuration issues. + if !cl.role.IsSingleRole() { + return nil, fmt.Errorf("NewTLSCertLoader: usage=%q, cert=%q, key=%q: %w", cl.usage, certPath, keyPath, ErrSingleRoleRequired) } - // Perform initial certificate load. - if err := cl.Load(certPath, keyPath); err != nil { - return nil, err + if cl.monitor == nil { + return nil, fmt.Errorf("NewTLSCertLoader: usage=%q, cert=%q, key=%q: %w", cl.usage, certPath, keyPath, ErrNoCertificateMonitor) + } + + // On construction we set the logger even if none was configured to ensure we have a valid logger. + cl.setLogger(config.logger) + + // Perform initial certificate load, if needed. + if certPath != "" || keyPath != "" { + if err := cl.Load(certPath, keyPath); err != nil { + return nil, fmt.Errorf("NewTLSCertLoader: usage=%q: error loading certificate: %w", cl.usage, err) + } } // Start monitoring certificate. - cl.monitorStartWg.Add(1) - go cl.monitorCert(&cl.monitorStartWg) + cl.monitor.registerCertLoader(cl) return cl, nil } -// SetCertificate sets the currently loaded certificate from a LoadedCertificate. -// Will also log any warnings about certificate (e.g. expired, about to expire, etc.). -func (cl *TLSCertLoader) SetCertificate(l LoadedCertificate) error { - err := func() error { - cl.mu.Lock() - defer cl.mu.Unlock() - return cl.setCertificate(l) - }() - if err != nil { - return err +// setLogger sets the current logger and adds context to it. +func (cl *TLSCertLoader) setLogger(logger *zap.Logger) { + cl.logger = logger + if cl.logger == nil { + cl.logger = zap.NewNop() } - cl.checkCurrentCert() - return nil + // Add usage to logger. + cl.logger = cl.logger.With(zap.String(logUsageContext, cl.usage)) } -// setCertificate is an internal method used to set the current certificate information -// from a LoadedCertficate. cl.mu must be held when calling. -func (cl *TLSCertLoader) setCertificate(l LoadedCertificate) error { - if !l.valid { - return fmt.Errorf("setCertificate: %w", ErrLoadedCertificateInvalid) - } +// ignoreFilePermissions indicates if file permissions should be ignored on load. +func (cl *TLSCertLoader) ignoreFilePermissions() bool { + cl.mu.RLock() + defer cl.mu.RUnlock() + return cl.config.ignoreFilePermissions +} - cl.certPath = l.CertificatePath - cl.keyPath = l.KeyPath - cl.cert = l.Certificate - cl.leaf = l.Leaf +// SetIgnoreFilePermissions sets if file permissions should be ignored on load. +func (cl *TLSCertLoader) SetIgnoreFilePermissions(ignore bool) { + cl.mu.Lock() + defer cl.mu.Unlock() + cl.ignorePermissions = ignore +} - return nil +// Usage is the descriptive usage set using WithCertLoaderUsage. +func (cl *TLSCertLoader) Usage() string { + return cl.usage } -// Certificate returns the currently loaded certificate, which may be nil. -func (cl *TLSCertLoader) Certificate() *tls.Certificate { +// Clear clears the loaded certificate. +func (cl *TLSCertLoader) Clear() { cl.mu.Lock() defer cl.mu.Unlock() + cl.cert = LoadedCertificate{} + cl.config.certPath = "" + cl.config.keyPath = "" +} + +// LoadedCertificate returns the currently loaded certificate, which may be +// invalid or empty. +func (cl *TLSCertLoader) LoadedCertificate() LoadedCertificate { + cl.mu.RLock() + defer cl.mu.RUnlock() return cl.cert } +// Certificate returns the currently loaded certificate, which may be nil. +func (cl *TLSCertLoader) Certificate() *tls.Certificate { + cl.mu.RLock() + defer cl.mu.RUnlock() + return cl.cert.Certificate +} + // SetupTLSConfig modifies tlsConfig to use cl for server and client certificates. // tlsConfig may be nil. If other fields like tlsConfig.Certificates or // tlsConfig.NameToCertificate have been set, then cl's certificate may not be used @@ -319,8 +248,14 @@ func (cl *TLSCertLoader) SetupTLSConfig(tlsConfig *tls.Config) { if tlsConfig == nil { return } - tlsConfig.GetCertificate = cl.GetCertificate - tlsConfig.GetClientCertificate = cl.GetClientCertificate + if cl.LoadedCertificate().IsEmpty() { + return + } + if cl.role.IsServerRole() { + tlsConfig.GetCertificate = cl.GetCertificate + } else if cl.role.IsClientRole() { + tlsConfig.GetClientCertificate = cl.GetClientCertificate + } } // GetCertificate is for use with a tls.Config's GetCertificate member. This allows a @@ -364,61 +299,94 @@ func (cl *TLSCertLoader) GetClientCertificate(cri *tls.CertificateRequestInfo) ( // Leaf returns the parsed x509 certificate of the currently loaded certificate. // If no certificate is loaded then nil is returned. func (cl *TLSCertLoader) Leaf() *x509.Certificate { - cl.mu.Lock() - defer cl.mu.Unlock() - return cl.leaf + cl.mu.RLock() + defer cl.mu.RUnlock() + return cl.cert.Leaf } func (cl *TLSCertLoader) loadCertificate(certPath, keyPath string) (LoadedCertificate, error) { - return LoadCertificate(certPath, keyPath, WithLoadCertificateIgnoreFilePermissions(cl.ignoreFilePermissions)) + return LoadCertificate(certPath, keyPath, WithLoadCertificateIgnoreFilePermissions(cl.ignoreFilePermissions())) } // Load loads the certificate at the given certificate path and private keyfile path. // Only trusted input (standard configuration files) should be used for certPath and keyPath. -func (cl *TLSCertLoader) Load(certPath, keyPath string) (rErr error) { - log, logEnd := logger.NewOperation(cl.logger, "Loading TLS certificate", "tls_load_cert", zap.String("cert", certPath), zap.String("key", keyPath)) - defer logEnd() +func (cl *TLSCertLoader) Load(certPath, keyPath string) error { + if apply, err := cl.PrepareLoad(WithCertLoaderCertificate(certPath, keyPath)); err != nil { + return err + } else if err := apply(); err != nil { + return err + } - loadedCert, err := cl.loadCertificate(certPath, keyPath) + return nil +} - cl.mu.Lock() - defer cl.mu.Unlock() - if err == nil { - if err := cl.setCertificate(loadedCert); err != nil { - // There shouldn't be a way to get here. - log.Error("error setting certificate after load", zap.Error(err)) - return err - } - cl.logX509CertIssues(log, loadedCert.Leaf) - log.Info("Successfully loaded TLS certificate", zap.String("cert", certPath), zap.String("key", keyPath)) - } else if cl.cert != nil { - // This case shouldn't be possible, but we can't be too careful with TLS certificates. - log.Error("Error loading TLS certificate, continuing to use previously loaded certificate", - zap.Error(err), - zap.String("failedCert", certPath), zap.String("failedKey", keyPath), - zap.String("activeCert", cl.certPath), zap.String("activeKey", cl.keyPath)) - } else { - log.Error("Error loading TLS certificate, no previously loaded TLS certificate is available", - zap.Error(err), - zap.String("failedCert", certPath), zap.String("failedKey", keyPath)) - } +// copyCurrentConfig creates a copy of the current configuration. +func (cl *TLSCertLoader) copyCurrentConfig() *tlsCertLoaderConfig { + cl.mu.RLock() + defer cl.mu.RUnlock() - return err + c := &tlsCertLoaderConfig{} + *c = *cl.config + return c } // PrepareLoad verifies that the certificate at certPath and keyPath will load without error. // If the certificate can be loaded, a function that will apply the certificate reload is // returned. Otherwise, an error is returned. -func (cl *TLSCertLoader) PrepareLoad(certPath, keyPath string) (func() error, error) { - loadedCert, err := cl.loadCertificate(certPath, keyPath) +func (cl *TLSCertLoader) PrepareLoad(opts ...TLSCertLoaderOpt) (func() error, error) { + // Start with the current config so any options that are not overridden with opts + // will not change. + c := cl.copyCurrentConfig() + for _, o := range opts { + o(c) + } + + log, logEnd := logger.NewOperation(cl.logger, "Loading TLS certificate", "tls_load_cert", + zap.String(logCertContext, c.certPath), zap.String(logKeyContext, c.keyPath), zap.String(logUsageContext, cl.usage)) + defer logEnd() + + logLoadError := func(err error) { + activeCert := cl.LoadedCertificate() + if !activeCert.IsEmpty() { + // The leaf should be good, but you can't be too careful with TLS certificates. + log.Error("Error loading TLS certificate, continuing to use previously loaded certificate", + zap.Error(err), + zap.String("failedCert", c.certPath), zap.String("failedKey", c.keyPath), + zap.String("activeCert", activeCert.CertificatePath), zap.String("activeKey", activeCert.KeyPath), + zap.String("activeCertSerial", activeCert.Serial())) + } else { + log.Error("Error loading TLS certificate, no previously loaded TLS certificate is available", + zap.Error(err), + zap.String("failedCert", c.certPath), zap.String("failedKey", c.keyPath)) + } + } + + ignoreFilePermissions := cl.ignoreFilePermissions() + if c.ignoreFilePermissionsSet { + ignoreFilePermissions = c.ignoreFilePermissions + } + loadedCert, err := LoadCertificate(c.certPath, c.keyPath, WithLoadCertificateIgnoreFilePermissions(ignoreFilePermissions)) if err != nil { + logLoadError(err) return nil, err } + + if loadedCert.IsEmpty() && cl.role.IsServerRole() { + err := fmt.Errorf("%s: can not use an empty certificate for a server: %w", cl.usage, ErrCertificateEmpty) + logLoadError(err) + return nil, err + } + + loadedCert.WithLogContext(log).Info("Successfully loaded TLS certificate") + return func() error { - if err := cl.SetCertificate(loadedCert); err != nil { - cl.logger.Error("error applying new certificate after VerifyLoad success", zap.Error(err)) - return err - } + func() { + cl.mu.Lock() + defer cl.mu.Unlock() + cl.cert = loadedCert + cl.config = c + }() + cl.monitor.QueueWarnIssues(cl) return nil }, nil } @@ -427,107 +395,18 @@ func (cl *TLSCertLoader) PrepareLoad(certPath, keyPath string) (func() error, er // Even after the monitoring goroutine is shutdown, Load and GetCertificate // will continue to work normally. func (cl *TLSCertLoader) Close() error { - cl.closeOnce.Do(func() { - close(cl.closeCh) - }) - - return nil -} - -// isCertPremature determines if an x509 cert is premature (not valid yet). -// Returns true if certificate is premature, false otherwise. cert must not be nil. -func (cl *TLSCertLoader) isCertPremature(cert *x509.Certificate) bool { - return time.Now().Before(cert.NotBefore) -} - -// isCertExpired determines if an x509 cert is expired. Returns if true if certificate -// is expired, false otherwise. cert must not be nil. -func (cl *TLSCertLoader) isCertExpired(cert *x509.Certificate) bool { - return time.Now().After(cert.NotAfter) -} - -// certExpiresSoon determines if an x509 cert is about to expire, as well as returning -// how long until the cert expires if we are within the expiration warn window. -// cert must not be nil. -func (cl *TLSCertLoader) certExpiresSoon(cert *x509.Certificate) (bool, time.Duration) { - untilExpires := time.Until(cert.NotAfter) - if untilExpires < cl.expirationAdvanced { - return true, untilExpires - } - return false, 0 -} - -// logX509CertIssues logs issues with an x509.Certificate to log. Included issues are: -// - expired certificate -// - certificates that are about to expire -// - certificate that is not valid yet -func (cl *TLSCertLoader) logX509CertIssues(log *zap.Logger, x509Cert *x509.Certificate) { - if log == nil || x509Cert == nil { - return - } - - if cl.isCertExpired(x509Cert) { - log.Warn("Certificate is expired", zap.Time("NotAfter", x509Cert.NotAfter)) - } else if cl.isCertPremature(x509Cert) { - log.Warn("Certificate is not valid yet", zap.Time("NotBefore", x509Cert.NotBefore)) - } else if expiresSoon, timeUntilExpires := cl.certExpiresSoon(x509Cert); expiresSoon { - log.Warn("Certificate will expire soon", zap.Time("NotAfter", x509Cert.NotAfter), zap.Duration("untilExpires", timeUntilExpires)) + // unregisterCertLoader is safe to call multiple times. + if cl.monitor != nil { + cl.monitor.unregisterCertLoader(cl) } + return nil } // Paths returns the path of the currently loaded certificate and private key. // The keyPath will be the file containing the private key, even if no keyPath // was provided to NewTLSCertLoader / Load. func (cl *TLSCertLoader) Paths() (certPath, keyPath string) { - cl.mu.Lock() - defer cl.mu.Unlock() - return cl.certPath, cl.keyPath -} - -// checkCurrentCert logs errors with the currently loaded leaf certificate. -func (cl *TLSCertLoader) checkCurrentCert() { - leaf, log := func() (*x509.Certificate, *zap.Logger) { - cl.mu.Lock() - defer cl.mu.Unlock() - log := cl.logger.With(zap.String("cert", cl.certPath), zap.String("key", cl.keyPath)) - return cl.leaf, log - }() - if leaf != nil { - cl.logX509CertIssues(log, leaf) - } else { - // There shouldn't be a way to get here because we don't return the CertLoader if the - // initial certificate load fails, and we also don't replace the certificate if Load - // fails. - log.Error("No certificate loaded when TLS certificate check performed", zap.Error(ErrCertificateNil)) - } -} - -// WaitForMonitorStart will wait for the certificate monitor goroutine to start. This is mainly useful -// for tests to avoid race conditions. -func (cl *TLSCertLoader) WaitForMonitorStart() { - cl.monitorStartWg.Wait() -} - -// monitorCert periodically logs errors with the currently loaded certificate. -func (cl *TLSCertLoader) monitorCert(wg *sync.WaitGroup) { - cl.logger.Info("Starting TLS certificate monitor") - - ticker := time.NewTicker(cl.certificateCheckInterval) - defer ticker.Stop() - - if wg != nil { - wg.Done() - } - - for { - select { - case <-ticker.C: - cl.checkCurrentCert() - - case <-cl.closeCh: - certPath, keyPath := cl.Paths() - cl.logger.Info("Closing TLS certificate monitor", zap.String("cert", certPath), zap.String("key", keyPath)) - return - } - } + cl.mu.RLock() + defer cl.mu.RUnlock() + return cl.cert.CertificatePath, cl.cert.KeyPath } diff --git a/pkg/tlsconfig/certconfig_test.go b/pkg/tlsconfig/certconfig_test.go index b818b6ca26a..bc106b3faf7 100644 --- a/pkg/tlsconfig/certconfig_test.go +++ b/pkg/tlsconfig/certconfig_test.go @@ -10,11 +10,13 @@ import ( "testing" "time" - "github.com/influxdata/influxdb/pkg/testing/selfsigned" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest/observer" + + th "github.com/influxdata/influxdb/pkg/testing/helper" + "github.com/influxdata/influxdb/pkg/testing/selfsigned" ) func TestTLSCertLoader_HappyPath(t *testing.T) { @@ -24,29 +26,41 @@ func TestTLSCertLoader_HappyPath(t *testing.T) { core, logs := observer.New(zapcore.InfoLevel) logger := zap.New(core) + certMonitor := newTestCertMonitor(t, WithMonitorLogger(logger)) + defer th.CheckedClose(t, certMonitor)() + + // We should be able to call WaitForMonitorStart multiple times without issues. + certMonitor.WaitForMonitorStart() + certMonitor.WaitForMonitorStart() + // Start cert loader - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath, WithCertLoaderLogger(logger)) + usage := "data.server" + cl, err := NewTLSCertLoader( + ServerOnlyRole, + certMonitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath), + WithCertLoaderLogger(logger), + WithCertLoaderUsage(usage)) require.NoError(t, err) require.NotNil(t, cl) - defer func() { - require.NoError(t, cl.Close()) - }() - cl.WaitForMonitorStart() - cl.WaitForMonitorStart() // should be able to safely call multiple times + defer th.CheckedClose(t, cl)() { // Check for expected log output - require.Equal(t, 4, logs.Len()) + require.Equal(t, 5, logs.Len()) logLines := logs.TakeAll() - require.Equal(t, "Loading TLS certificate (start)", logLines[0].Message) - require.Equal(t, "Successfully loaded TLS certificate", logLines[1].Message) - require.Equal(t, "Loading TLS certificate (end)", logLines[2].Message) - require.Equal(t, "Starting TLS certificate monitor", logLines[3].Message) - for _, l := range logLines[:3] { // "Starting TLS certificate monitor" doesn't include the cert name and key + require.Equal(t, "Starting TLS certificate monitor", logLines[0].Message) + require.Equal(t, "Loading TLS certificate (start)", logLines[1].Message) + require.Equal(t, "Successfully loaded TLS certificate", logLines[2].Message) + require.Equal(t, "Loading TLS certificate (end)", logLines[3].Message) + require.Equal(t, "Registered certificate loader", logLines[4].Message) + for _, l := range logLines[1:3] { // "Starting TLS certificate monitor" doesn't include the cert name and key cm := l.ContextMap() + require.Equal(t, usage, cm["usage"]) require.Equal(t, ss.CertPath, cm["cert"]) require.Equal(t, ss.KeyPath, cm["key"]) } + require.Equal(t, usage, logLines[4].ContextMap()["usage"]) // Get certificate and do some checks on it. cp, kp := cl.Paths() @@ -82,6 +96,7 @@ func TestTLSCertLoader_HappyPath(t *testing.T) { cm := l.ContextMap() require.Equal(t, ss.CertPath, cm["cert"]) require.Equal(t, ss.KeyPath, cm["key"]) + require.Equal(t, usage, cm["usage"]) } cp, kp := cl.Paths() @@ -100,6 +115,25 @@ func TestTLSCertLoader_HappyPath(t *testing.T) { require.Equal(t, []string{DNSName2}, x509Cert.DNSNames) require.Equal(t, x509Cert, cl.Leaf()) } + + { + // Close everything and check for proper logs. + logs.TakeAll() + require.NoError(t, cl.Close()) + require.NoError(t, certMonitor.Close()) + + // Should be able to call WaitForMonitorStop multiple times. + certMonitor.WaitForMonitorStop() + certMonitor.WaitForMonitorStop() + + require.Equal(t, 2, logs.Len()) + logLines := logs.TakeAll() + require.Equal(t, "Unregistered certificate loader", logLines[0].Message) + cm := logLines[0].ContextMap() + require.Equal(t, usage, cm["usage"]) + + require.Equal(t, "Stopping TLS certificate monitor", logLines[1].Message) + } } func TestTLSCertLoader_GoodCertPersists(t *testing.T) { @@ -109,14 +143,20 @@ func TestTLSCertLoader_GoodCertPersists(t *testing.T) { core, logs := observer.New(zapcore.InfoLevel) logger := zap.New(core) + certMonitor := newTestCertMonitor(t, WithMonitorLogger(logger)) + defer th.CheckedClose(t, certMonitor)() + // Start cert loader - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath, WithCertLoaderLogger(logger)) + cl, err := NewTLSCertLoader( + ServerOnlyRole, + certMonitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath), + WithCertLoaderLogger(logger)) require.NoError(t, err) require.NotNil(t, cl) defer func() { require.NoError(t, cl.Close()) }() - cl.WaitForMonitorStart() var goodSerial big.Int { @@ -163,6 +203,8 @@ func TestTLSCertLoader_GoodCertPersists(t *testing.T) { require.NotNil(t, x509Cert) require.NotNil(t, x509Cert.SerialNumber) require.Equal(t, goodSerial, *x509Cert.SerialNumber) + + // TODO: Check log lines } } @@ -170,17 +212,31 @@ func TestTLSCertLoader_GoodCertPersists(t *testing.T) { func TestTLSCertLoader_EmptyPaths(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - cl, err := NewTLSCertLoader("", ss.KeyPath) - require.ErrorIs(t, err, ErrPathEmpty) - require.Nil(t, cl) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() - cl, err = NewTLSCertLoader("", "") + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate("", ss.KeyPath)) require.ErrorIs(t, err, ErrPathEmpty) require.Nil(t, cl) + // This is no longer an error on instantiation for a server role, + // only on a PrepareLoad. + cl, err = NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate("", "")) + require.NoError(t, err) + require.NotNil(t, cl) + // For this case, the loader will assume CertPath also contains, which // it does not, so this will fail. - cl, err = NewTLSCertLoader(ss.CertPath, "") + cl, err = NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, "")) require.ErrorContains(t, err, "found a certificate rather than a key") require.Nil(t, cl) } @@ -188,18 +244,30 @@ func TestTLSCertLoader_EmptyPaths(t *testing.T) { func TestTLSCertLoader_FileNotFound(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + // Non-existent certificate file - cl, err := NewTLSCertLoader("/nonexistent/path/to/cert.pem", ss.KeyPath) + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate("/nonexistent/path/to/cert.pem", ss.KeyPath)) require.ErrorContains(t, err, "no such file or directory") require.Nil(t, cl) // Non-existent key file - cl, err = NewTLSCertLoader(ss.CertPath, "/nonexistent/path/to/key.pem") + cl, err = NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, "/nonexistent/path/to/key.pem")) require.ErrorContains(t, err, "no such file or directory") require.Nil(t, cl) // Both files non-existent - cl, err = NewTLSCertLoader("/nonexistent/cert.pem", "/nonexistent/key.pem") + cl, err = NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate("/nonexistent/cert.pem", "/nonexistent/key.pem")) require.ErrorContains(t, err, "no such file or directory") require.Nil(t, cl) } @@ -209,14 +277,23 @@ func TestTLSCertLoader_MismatchedCertAndKey(t *testing.T) { ss1 := selfsigned.NewSelfSignedCert(t, selfsigned.WithDNSName("cert1.influxdata.edge")) ss2 := selfsigned.NewSelfSignedCert(t, selfsigned.WithDNSName("cert2.influxdata.edge")) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + // Try to load cert from first pair with key from second pair - cl, err := NewTLSCertLoader(ss1.CertPath, ss2.KeyPath) + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss1.CertPath, ss2.KeyPath)) require.ErrorContains(t, err, "error loading x509 key pair") require.ErrorContains(t, err, "tls: private key does not match public key") require.Nil(t, cl) // Try to load cert from second pair with key from first pair - cl, err = NewTLSCertLoader(ss2.CertPath, ss1.KeyPath) + cl, err = NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss2.CertPath, ss1.KeyPath)) require.ErrorContains(t, err, "error loading x509 key pair") require.ErrorContains(t, err, "tls: private key does not match public key") require.Nil(t, cl) @@ -229,8 +306,14 @@ func TestTLSCertLoader_CombinedFile(t *testing.T) { // Verify that CertPath and KeyPath point to the same file require.Equal(t, ss.CertPath, ss.KeyPath, "expected CertPath and KeyPath to be the same for combined file") + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + // Start cert loader with the combined file. Let the cert loader infer that the key is combined with the cert. - cl, err := NewTLSCertLoader(ss.CertPath, "") + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, "")) require.NoError(t, err) require.NotNil(t, cl) defer func() { @@ -251,8 +334,14 @@ func TestTLSCertLoader_CombinedFile(t *testing.T) { func TestTLSLoader_CertPermissionsTooOpen(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + require.NoError(t, os.Chmod(ss.CertPath, 0660)) - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath) + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) require.ErrorContains(t, err, fmt.Sprintf("LoadCertificate: file permissions are too open: for %q, maximum is 0644 (-rw-r--r--) but found 0660 (-rw-rw----); extra permissions: 0020 (-----w----)", ss.CertPath)) require.Nil(t, cl) } @@ -260,8 +349,14 @@ func TestTLSLoader_CertPermissionsTooOpen(t *testing.T) { func TestTLSLoader_KeyPermissionsTooOpen(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + require.NoError(t, os.Chmod(ss.KeyPath, 0644)) - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath) + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) require.ErrorContains(t, err, fmt.Sprintf("LoadCertificate: file permissions are too open: for %q, maximum is 0600 (-rw-------) but found 0644 (-rw-r--r--); extra permissions: 0044 (----r--r--)", ss.KeyPath)) require.Nil(t, cl) } @@ -274,8 +369,22 @@ const ( // it should be more than testCheckTime, but less than 2 * testCheckTime. Furthermore, it should be at least // 100 ms more than testCheckCapture time and more than 100 ms less than 2 * testCheckTime. testCheckCapture = 500 * time.Millisecond + + // testWarnWaitTime is the time to wait for a warning to be logged for a triggered warning. + testWarnWaitTime = 50 * time.Millisecond ) +func newTestCertMonitor(t *testing.T, opts ...TLSCertMonitorOpt) *TLSCertMonitor { + // Put default test options first so opts can override them. + combinedOpts := append([]TLSCertMonitorOpt{WithMonitorCheckInterval(testCheckTime)}, opts...) + certMonitor := NewTLSCertMonitor(combinedOpts...) + require.NotNil(t, certMonitor) + + require.NoError(t, certMonitor.Open()) + certMonitor.WaitForMonitorStart() + return certMonitor +} + func TestTLSCertLoader_PrematureCertificateLogging(t *testing.T) { notBefore := time.Now().UTC().Truncate(time.Hour).Add(7 * 24 * time.Hour) notAfter := notBefore.Add(7 * 24 * time.Hour) @@ -284,13 +393,19 @@ func TestTLSCertLoader_PrematureCertificateLogging(t *testing.T) { core, logs := observer.New(zapcore.InfoLevel) logger := zap.New(core) - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath, WithCertLoaderLogger(logger), WithCertLoaderCertificateCheckInterval(testCheckTime)) + monitor := newTestCertMonitor(t, WithMonitorLogger(logger), WithMonitorTriggerDelay(1)) + defer th.CheckedClose(t, monitor)() + + usage := "httpd.server" + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath), + WithCertLoaderLogger(logger), + WithCertLoaderUsage(usage)) require.NoError(t, err) require.NotNil(t, cl) - defer func() { - require.NoError(t, cl.Close()) - }() - cl.WaitForMonitorStart() + defer th.CheckedClose(t, cl)() checkWarning := func(t *testing.T) { warning := logs.FilterMessage("Certificate is not valid yet").TakeAll() @@ -299,8 +414,10 @@ func TestTLSCertLoader_PrematureCertificateLogging(t *testing.T) { require.Equal(t, ss.CertPath, warning[0].ContextMap()["cert"]) require.Equal(t, ss.KeyPath, warning[0].ContextMap()["key"]) require.Equal(t, notBefore, warning[0].ContextMap()["NotBefore"]) + require.Equal(t, []any{usage}, warning[0].ContextMap()["usages"]) logs.TakeAll() // dump all logs } + time.Sleep(testWarnWaitTime) checkWarning(t) // Check for warning during monitor @@ -317,13 +434,19 @@ func TestTLSCertLoader_ExpiredCertificateLogging(t *testing.T) { core, logs := observer.New(zapcore.InfoLevel) logger := zap.New(core) - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath, WithCertLoaderLogger(logger), WithCertLoaderCertificateCheckInterval(testCheckTime)) + monitor := newTestCertMonitor(t, WithMonitorLogger(logger), WithMonitorTriggerDelay(0)) + defer th.CheckedClose(t, monitor)() + + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath), + WithCertLoaderLogger(logger)) require.NoError(t, err) require.NotNil(t, cl) defer func() { require.NoError(t, cl.Close()) }() - cl.WaitForMonitorStart() checkWarning := func(t *testing.T) { warning := logs.FilterMessage("Certificate is expired").TakeAll() @@ -334,6 +457,7 @@ func TestTLSCertLoader_ExpiredCertificateLogging(t *testing.T) { require.Equal(t, notAfter, warning[0].ContextMap()["NotAfter"]) logs.TakeAll() // dump all logs } + time.Sleep(testWarnWaitTime) checkWarning(t) // Check for warning during monitor @@ -351,16 +475,17 @@ func TestTLSCertLoader_CertificateExpiresSoonLogging(t *testing.T) { core, logs := observer.New(zapcore.InfoLevel) logger := zap.New(core) - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath, - WithCertLoaderLogger(logger), - WithCertLoaderCertificateCheckInterval(testCheckTime), - WithCertLoaderExpirationAdvanced(2*24*time.Hour)) + monitor := newTestCertMonitor(t, WithMonitorExpirationAdvanced(2*24*time.Hour), WithMonitorLogger(logger), WithMonitorTriggerDelay(0)) + defer th.CheckedClose(t, monitor)() + + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath), + WithCertLoaderLogger(logger)) require.NoError(t, err) require.NotNil(t, cl) - defer func() { - require.NoError(t, cl.Close()) - }() - cl.WaitForMonitorStart() + defer th.CheckedClose(t, cl)() checkWarning := func(t *testing.T) { warning := logs.FilterMessage("Certificate will expire soon").TakeAll() @@ -375,6 +500,7 @@ func TestTLSCertLoader_CertificateExpiresSoonLogging(t *testing.T) { require.WithinDuration(t, notAfter, timeExpires, 2*time.Minute, "untilExpires varies more than expected") logs.TakeAll() // dump all logs } + time.Sleep(testWarnWaitTime) checkWarning(t) // Check for warning during monitor @@ -383,63 +509,19 @@ func TestTLSCertLoader_CertificateExpiresSoonLogging(t *testing.T) { checkWarning(t) } -func TestTLSCertLoader_SetCertificate(t *testing.T) { - // Initial setup - ss := selfsigned.NewSelfSignedCert(t) - - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath) - require.NoError(t, err) - require.NotNil(t, cl) - defer func() { - require.NoError(t, cl.Close()) - }() - - cert1, err := cl.GetCertificate(nil) - require.NoError(t, err) - require.NotNil(t, cert1) - leaf := cl.Leaf() - require.NotNil(t, leaf) - - // Make sure that setting an invalid LoadedCertificate returns an error and doesn't - // change the currently loaded certificate. - { - require.ErrorIs(t, cl.SetCertificate(LoadedCertificate{}), ErrLoadedCertificateInvalid) - cp, kp := cl.Paths() - require.Equal(t, ss.CertPath, cp) - require.Equal(t, ss.KeyPath, kp) - actualCert, err := cl.GetCertificate(nil) - require.NoError(t, err) - require.Equal(t, cert1, actualCert) - require.Equal(t, leaf, cl.Leaf()) - } - - // Check that a valid LoadedCertificate works properly. - ss2 := selfsigned.NewSelfSignedCert(t) - { - lc2, err := LoadCertificate(ss2.CertPath, ss2.KeyPath) - require.NoError(t, err) - require.True(t, lc2.IsValid()) - require.NoError(t, cl.SetCertificate(lc2)) - - cp, kp := cl.Paths() - require.Equal(t, ss2.CertPath, cp) - require.Equal(t, ss2.KeyPath, kp) - actualCert, err := cl.GetCertificate(nil) - require.NoError(t, err) - require.Equal(t, lc2.Certificate, actualCert) - require.Equal(t, lc2.Leaf, cl.Leaf()) - } -} - func TestTLSCertLoader_VerifyLoad(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) require.NotNil(t, cl) - defer func() { - require.NoError(t, cl.Close()) - }() + defer th.CheckedClose(t, cl)() cert1, err := cl.GetCertificate(nil) require.NoError(t, err) @@ -453,7 +535,7 @@ func TestTLSCertLoader_VerifyLoad(t *testing.T) { // Test VerifyLoad with a cert pair that will not load properly. { - apply, err := cl.PrepareLoad(ss2.CACertPath, ss2.KeyPath) // mismatched cert and key + apply, err := cl.PrepareLoad(WithCertLoaderCertificate(ss2.CACertPath, ss2.KeyPath)) // mismatched cert and key require.ErrorContains(t, err, "private key does not match public key") require.Nil(t, apply) @@ -474,7 +556,7 @@ func TestTLSCertLoader_VerifyLoad(t *testing.T) { require.NotEmpty(t, sn2) require.NotEqual(t, sn1, sn2) - apply, err := cl.PrepareLoad(ss2.CertPath, ss2.KeyPath) + apply, err := cl.PrepareLoad(WithCertLoaderCertificate(ss2.CertPath, ss2.KeyPath)) require.NoError(t, err) require.NotNil(t, apply) @@ -503,12 +585,16 @@ func TestTLSCertLoader_VerifyLoad(t *testing.T) { func TestTLSCertLoader_GetClientCertificate(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t, selfsigned.WithDNSName("client.influxdata.edge")) - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) require.NotNil(t, cl) - defer func() { - require.NoError(t, cl.Close()) - }() + defer th.CheckedClose(t, cl)() // Test happy path: certificate supports the request. // The selfsigned package creates RSA certificates, so we use RSA signature schemes. @@ -614,12 +700,16 @@ func TestTLSCertLoader_GetClientCertificate(t *testing.T) { func TestTLSCertLoader_SetupTLSConfig(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - cl, err := NewTLSCertLoader(ss.CertPath, ss.KeyPath) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) require.NotNil(t, cl) - defer func() { - require.NoError(t, cl.Close()) - }() + defer th.CheckedClose(t, cl)() t.Run("nil config", func(t *testing.T) { require.NotPanics(t, func() { @@ -636,6 +726,6 @@ func TestTLSCertLoader_SetupTLSConfig(t *testing.T) { cl.SetupTLSConfig(tlsConfig) require.NotNil(t, tlsConfig.GetCertificate) - require.NotNil(t, tlsConfig.GetClientCertificate) + require.Nil(t, tlsConfig.GetClientCertificate) }) } diff --git a/pkg/tlsconfig/certmonitor.go b/pkg/tlsconfig/certmonitor.go new file mode 100644 index 00000000000..9bc17e47250 --- /dev/null +++ b/pkg/tlsconfig/certmonitor.go @@ -0,0 +1,453 @@ +package tlsconfig + +import ( + "cmp" + "maps" + "slices" + "sync" + "time" + + "go.uber.org/zap" +) + +const ( + DefaultTriggerDelay = 5 * time.Second +) + +// certMonitor is an internal class that implements periodic certificate +// monitoring. It avoids logging about a single certificate / key pair +// multiple times, as well as the number of goroutines required to monitor +// certificates. +type TLSCertMonitor struct { + // log is the logger to use. + log *zap.Logger + + // startOnce is used to start the background monitor goroutine one time. + startOnce sync.Once + + // triggerDelay is a timer used to send for delayed issue warning triggers. + // This allows multiple certificate loads to be batched together when + // certificate loading is complete. + triggerDelay *time.Timer + + // triggerResetCh is a channel used to reset triggerDelay. This is required + // because go does not reliably allow calling Reset() on a timer whose + // .C is actively being waited upon in a select statement. + triggerResetCh chan time.Duration + + // checkIntervalResetCh is a channel used to reset the check interval timer. + // This is required because go does not reliably allow calling Reset() on a + // timer whose .C is actively being waited upon in a select statement. It + // also necessary since the ticker is itself only visible in the monitor + // goroutine. + checkIntervalResetCh chan time.Duration + + // closeOnce is used to stop the background monitor goroutine one time only. + closeOnce sync.Once + + // closeCh is used to trigger closing the monitor. + closeCh chan struct{} + + // monitorStartWg can be used to detect if the monitor goroutine has started. + monitorStartWg sync.WaitGroup + + // monitorStopWg can be used to detect if the monitor goroutine has ended. + monitorStopWg sync.WaitGroup + + // mu protects all members below. + mu sync.RWMutex + + // certLoaders is a set of TLSCertLoader objects whose certificates we monitor. + // certLoaders handle registering and unregistering themselves. + certLoaders map[*TLSCertLoader]struct{} + + // queuedCertLoaders is a list of TLSCertLoader objects which have queued themselves + // for issue warning when the next requested warn trigger activates. It is essentially + // a filter for which CertLoaders will have issues logged, so that only TLSCertLoader + // objects with newly loaded certificates will have their issues logged. It does not + // apply for the regularly scheduled issue warning. + queuedCertLoaders []*TLSCertLoader + + // certExpirationAdvanced determines how long before a certificate expires a warning is issued. + certExpirationAdvanced time.Duration + + // certCheckInterval determines the duration between each certificate check. + certCheckInterval time.Duration + + // triggerDelayInterval is the interval after starting triggerDelay to fire + // the after function. + triggerDelayInterval time.Duration +} + +// TLSCertMonitorOpt is an option for NewTLSCertMonitor. +type TLSCertMonitorOpt func(*TLSCertMonitor) + +// WithMonitorCheckInternal sets the initial check interval for the monitor. +// It can be changed later with SetCheckInterval. +func WithMonitorCheckInterval(d time.Duration) TLSCertMonitorOpt { + return func(m *TLSCertMonitor) { + m.certCheckInterval = d + } +} + +// WithMonitorExpirationWarn sets the initial expiration warn time for the +// monitor. It can be changed later with SetExpirationWarn. +func WithMonitorExpirationAdvanced(d time.Duration) TLSCertMonitorOpt { + return func(m *TLSCertMonitor) { + m.certExpirationAdvanced = d + } +} + +// WithMonitorLogger sets the logger for the monitor. It can be changed later +// with SetLogger. +func WithMonitorLogger(log *zap.Logger) TLSCertMonitorOpt { + return func(m *TLSCertMonitor) { + m.log = log + } +} + +// WithMonitorTriggerDelay sets the initial trigger delay interval. This +// +// can be changed later with SetTriggerDelay. +func WithMonitorTriggerDelay(d time.Duration) TLSCertMonitorOpt { + return func(m *TLSCertMonitor) { + m.triggerDelayInterval = d + } +} + +// NewTLSCertMonitor creates a new TLSCertMonitor and starts its worker +// goroutine. +func NewTLSCertMonitor(opts ...TLSCertMonitorOpt) *TLSCertMonitor { + // chanDepth should be high enough we don't need to block during startup. + // It shouldn't matter as long as the monitor is running, but it'll + // prevent any small pauses. + const chanDepth = 20 + + m := &TLSCertMonitor{ + closeCh: make(chan struct{}), + triggerResetCh: make(chan time.Duration, chanDepth), + checkIntervalResetCh: make(chan time.Duration, chanDepth), + + certLoaders: make(map[*TLSCertLoader]struct{}), + triggerDelayInterval: DefaultTriggerDelay, + } + + for _, o := range opts { + o(m) + } + + if m.log == nil { + m.log = zap.NewNop() + } + + if m.certCheckInterval <= 0 { + m.certCheckInterval = DefaultCertificateCheckTime + } + if m.certExpirationAdvanced <= 0 { + m.certExpirationAdvanced = DefaultExpirationWarnTime + } + + // Setup up the after function for trigger delay, but don't let it + // fire. + m.triggerDelay = time.NewTimer(m.triggerDelayInterval) + m.triggerDelay.Stop() + + // Increment start wait group so that WaitForMonitorStart won't + // return immediately if Open hasn't been called yet. + m.monitorStartWg.Add(1) + m.monitorStopWg.Add(1) + + return m +} + +// Open starts the certificate monitor background goroutine. The certificate +// monitor won't do anything until this is called. It is safe to call +// multiple times, but it will only start one monitor goroutine. +func (m *TLSCertMonitor) Open() error { + m.startOnce.Do(func() { + go m.monitorCerts(&m.monitorStartWg, &m.monitorStopWg) + }) + return nil +} + +// Close stops the certificate monitor background routine. After calling +// this, certificates will no longer be monitored. It is safe to call +// multiple times, but it will only stop the monitor goroutine once. +func (m *TLSCertMonitor) Close() error { + m.closeOnce.Do(func() { + close(m.closeCh) + }) + return nil +} + +// WaitForMonitorStop waits for the certificate monitor goroutine to stop. +// This is mainly useful for tests to avoid race conditions. +func (m *TLSCertMonitor) WaitForMonitorStop() { + m.monitorStopWg.Wait() +} + +// WaitForMonitorStart will wait for the certificate monitor goroutine to +// start. This is mainly useful for tests to avoid race conditions. +func (m *TLSCertMonitor) WaitForMonitorStart() { + m.monitorStartWg.Wait() +} + +func (m *TLSCertMonitor) logger() *zap.Logger { + m.mu.RLock() + defer m.mu.RUnlock() + return m.log +} + +func (m *TLSCertMonitor) SetLogger(log *zap.Logger) { + m.mu.Lock() + defer m.mu.Unlock() + m.log = log +} + +// isClosed returns true if the monitor has been closed. +func (m *TLSCertMonitor) isClosed() bool { + select { + case <-m.closeCh: + return true + default: + return false + } +} + +// resetTrigger resets the triggerDelay timer to d. +func (m *TLSCertMonitor) resetTrigger(d time.Duration) { + if !m.isClosed() { + m.triggerResetCh <- d + } +} + +// resetCheckInterval resets the check interval ticker to d. +func (m *TLSCertMonitor) resetCheckInterval(d time.Duration) { + if !m.isClosed() { + m.checkIntervalResetCh <- d + } +} + +func (m *TLSCertMonitor) SetCheckInterval(checkInterval time.Duration) { + m.mu.Lock() + defer m.mu.Unlock() + m.certCheckInterval = checkInterval + m.resetCheckInterval(m.certCheckInterval) +} + +func (m *TLSCertMonitor) checkInterval() time.Duration { + m.mu.RLock() + defer m.mu.RUnlock() + return m.certCheckInterval +} + +func (m *TLSCertMonitor) SetExpirationAdvanced(expirationAdvanced time.Duration) { + m.mu.Lock() + defer m.mu.Unlock() + m.certExpirationAdvanced = expirationAdvanced + m.resetTrigger(m.triggerDelayInterval) +} + +func (m *TLSCertMonitor) expirationAdvanced() time.Duration { + m.mu.RLock() + defer m.mu.RUnlock() + return m.certExpirationAdvanced +} + +// SetTriggerDelay sets the trigger delay interval to d. +func (m *TLSCertMonitor) SetTriggerDelay(d time.Duration) { + m.mu.Lock() + defer m.mu.Unlock() + m.triggerDelayInterval = d +} + +// QueueWarnIssues allows a TLSCertLoader to queue itself for issue warning. +// When the warn issues trigger occurs, all queued certificate loggers will +// have their issues logged. This also sets / resets the delayed trigger +// timer so that when a reload occurs issue warning won't occur until +// have certificates have been reloaded. +func (m *TLSCertMonitor) QueueWarnIssues(cl *TLSCertLoader) { + m.mu.Lock() + defer m.mu.Unlock() + + m.queuedCertLoaders = append(m.queuedCertLoaders, cl) + m.resetTrigger(m.triggerDelayInterval) +} + +// takeQueuedCertLoaders returns and clears TLSCertLoader objects queued +// through QueueWarnIssues since the last takeQueuedCertLoaders call. +func (m *TLSCertMonitor) takeQueuedCertLoaders() []*TLSCertLoader { + m.mu.Lock() + defer m.mu.Unlock() + + cls := m.queuedCertLoaders + m.queuedCertLoaders = nil + return cls +} + +// registerCertLoader adds cl to the set of TLSCertLoader objects +// to monitor. +func (m *TLSCertMonitor) registerCertLoader(cl *TLSCertLoader) { + m.mu.Lock() + defer m.mu.Unlock() + m.certLoaders[cl] = struct{}{} + // Don't use logger() here to avoid a deadlock. + m.log.Info("Registered certificate loader", zap.String(logUsageContext, cl.Usage())) +} + +// unregisterCertLoader removes cl from the set of TLSCertLoader objects +// to monitor. +func (m *TLSCertMonitor) unregisterCertLoader(cl *TLSCertLoader) { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.certLoaders[cl]; ok { + // Don't use logger() here to avoid a deadlock. + m.log.Info("Unregistered certificate loader", zap.String(logUsageContext, cl.Usage())) + } + delete(m.certLoaders, cl) +} + +// logIssues logs issues found with lc. +func (m *TLSCertMonitor) logIssues(log *zap.Logger, lc LoadedCertificate) { + if log == nil || lc.IsEmpty() { + return + } + + log = lc.WithLogContext(log) + if xc, err := lc.GetLeaf(); err == nil { + xc.logIssues(log, m.expirationAdvanced()) + } else { + log.Error("error logging certificate issues", zap.Error(err)) + } +} + +// logIssues logs issues with a given certLoaders +// gatherCertLoaders gathers all registered TLSCertLoader objects and +// returns them as a slice. +func (m *TLSCertMonitor) gatherCertLoaders() []*TLSCertLoader { + m.mu.RLock() + defer m.mu.RUnlock() + return slices.Collect(maps.Keys(m.certLoaders)) +} + +// checkCerts checks all registered TLSCertLoader objects for warnings on +// their certificates. Duplicate certificates are merged to avoid overly +// verbose output. +func (m *TLSCertMonitor) checkCerts(filter []*TLSCertLoader) { + // checkCerts does not lock m.mu itself, but it does call functions + // which take read locks. + + // Get the list of all cert loaders tracked by this monitor. + certLoaders := m.gatherCertLoaders() + + // certKey is the key to identify identical certificates. Not only + // must the paths match, but also the certificate serial number. + type certKey struct { + cert, key, serial string + } + makeKey := func(lc LoadedCertificate) certKey { + return certKey{ + cert: lc.CertificatePath, + key: lc.KeyPath, + serial: lc.Serial(), + } + } + + // Create a set containing all registered cert loaders along with + // all those listed in filter. This allows logging warnings on + // cert loaders that queued a warning check before they registered + // themselves. This is mainly a race condition found in tests where + // the trigger delay may be set to 0. + allCertLoaders := make(map[*TLSCertLoader]struct{}, len(certLoaders)+len(filter)) + for _, cl := range append(certLoaders, filter...) { + allCertLoaders[cl] = struct{}{} + } + + // Collect all described usages of a unique key. + usageListMap := make(map[certKey][]string) + lcMap := make(map[certKey]LoadedCertificate) + for cl := range allCertLoaders { + loaded := cl.LoadedCertificate() + if !loaded.IsEmpty() { + key := makeKey(loaded) + usageListMap[key] = append(usageListMap[key], cl.Usage()) + lcMap[key] = loaded + } + } + + // Log issues with unique keys, including the described usages in the + // log messages. Sort unique keys so output is in predictable order. + // This is also where we decide on the filter. If filter is empty, then + // all keys in usageListMap are used for the keys. Otherwise, the filter + // list is processed to removed duplicate LoadedCertificates and then sorted. + var keys []certKey + if len(filter) == 0 { + keys = slices.Collect(maps.Keys(usageListMap)) + } else { + uniqueFilterKeys := make(map[certKey]struct{}) + for _, cl := range filter { + loaded := cl.LoadedCertificate() + if !loaded.IsEmpty() { + uniqueFilterKeys[makeKey(loaded)] = struct{}{} + } + } + keys = slices.Collect(maps.Keys(uniqueFilterKeys)) + } + slices.SortFunc(keys, + func(a, b certKey) int { + if c := cmp.Compare(a.cert, b.cert); c != 0 { + return c + } + if c := cmp.Compare(a.key, b.key); c != 0 { + return c + } + return cmp.Compare(a.serial, b.serial) + }) + + log := m.logger() + for _, k := range keys { + usageList := usageListMap[k] + // Sort to usages to present then in a predictable order. + slices.Sort(usageList) + lc := lcMap[k] + m.logIssues(log.With(zap.Strings(logUsagesContext, usageList)), lc) + } +} + +// monitor periodically logs certificate errors with the currently registered +// TLSCertLoader objects. +func (m *TLSCertMonitor) monitorCerts(startWg *sync.WaitGroup, stopWg *sync.WaitGroup) { + m.log.Info("Starting TLS certificate monitor") + + ticker := time.NewTicker(m.checkInterval()) + defer ticker.Stop() + + if startWg != nil { + startWg.Done() + } + + for { + select { + case <-ticker.C: + m.checkCerts(nil) + + case d := <-m.checkIntervalResetCh: + ticker.Reset(d) + + case <-m.triggerDelay.C: + m.checkCerts(m.takeQueuedCertLoaders()) + + case d := <-m.triggerResetCh: + m.triggerDelay.Reset(d) + + case <-m.closeCh: + m.log.Info("Stopping TLS certificate monitor") + ticker.Stop() + m.triggerDelay.Stop() + if stopWg != nil { + stopWg.Done() + } + return + } + } +} diff --git a/pkg/tlsconfig/configmanager.go b/pkg/tlsconfig/configmanager.go index 04e20efbf11..16f103fa469 100644 --- a/pkg/tlsconfig/configmanager.go +++ b/pkg/tlsconfig/configmanager.go @@ -7,23 +7,129 @@ import ( "fmt" "net" "os" - "time" + "sync" "go.uber.org/zap" ) +const ( + // logCertContext is the key for log context with the certificate usages list. + // Use literals instead of constants in tests to ensure no one changes + // the constant unintentionally. + logUsagesContext = "usages" + + // logCertContext is the key for log context with the certificate usage (singular). + // Use literals instead of constants in tests to ensure no one changes + // the constant unintentionally. + logUsageContext = "usage" +) + var ( + // ErrConfigureDisabledManager is returned when an attempt is made to reconfigure + // a disabled config managaer. + ErrConfigureDisabledManager = errors.New("cannot configure disabled TLS manager") + + // ErrClientListen is returned when attempt is made to have a client role manager + // create a listener. + ErrClientListen = errors.New("client TLS manager cannot Listen") + // ErrNoCertLoader indicates that an operation requiring a TLSCertLoader did not have one available. // This can happen if the TLSConfigManager was created without a certificate for client-side use only. ErrNoCertLoader = errors.New("no TLSCertLoader available") + + // ErrNoRole indicates that a configuration manager or other object was not initialized + // with a valid Role. It is generally due to a misuse of an internal API. + ErrNoRole = errors.New("no role specified for TLS certificate") + + // ErrNotSupportedServer indicates that an operation is not supported by a server role + // config manager. + ErrNotSupportedServer = errors.New("operation not supported by server role TLS manager") + + // ErrServerDial is returned when attempt is made to have a server role manager + // dial a connection. + ErrServerDial = errors.New("server TLS manager cannot Dial / DialWithDialer") +) + +// Role is an enum that specifies how a config manager or cert loader +// will be used. +type Role int + +const ( + // InvalidRole is an invalid role. It is the zero value so IsValid can be used + // to determine if a role was properly initialized. + InvalidRole Role = iota + + // ServerOnlyRole specifies that a config manager or certificate is only for + // server use through Listen. + ServerOnlyRole + + // ClientOnlyRole specifies that a config manager or certificate is only for + // client use through Dial. + ClientOnlyRole + + // ServerAndClientRole specifies that a config manager or certificate is for + // use for both a server (through Listen) and a client (through Dial). + ServerAndClientRole ) +// IsValid returns true if r is valid. It can be used to determine if r +// was initialized properly. +func (r Role) IsValid() bool { + return r != InvalidRole +} + +// IsSingleRole returns true if role specifies a single role, Server or Client. +// This is used to check for valid values in places where only a single role +// is accepted. +func (r Role) IsSingleRole() bool { + return r == ServerOnlyRole || r == ClientOnlyRole +} + +// IsServerRole returns true if role specifies a server role. +func (r Role) IsServerRole() bool { + return r == ServerOnlyRole || r == ServerAndClientRole +} + +// IsClientRole returns true if role specifies a client role. +func (r Role) IsClientRole() bool { + return r == ClientOnlyRole || r == ServerAndClientRole +} + // TLSConfigManager will manage a TLS configuration and make sure that only one instance of its tls.Config exists. // Different TLSConfigManager objects will have different configurations, even if they are instantiated in exactly // the same way. No struct member is modified once the NewTLSConfigManager constructor is finished. type TLSConfigManager struct { - tlsConfig *tls.Config - certLoader *TLSCertLoader + // Fields above mu are not protected by mu and can only be set at construction time. + + // disabled indicates if this config manager is disabled. A disabled configuration manager has + // TLS disabled and does not support reconfiguration. A disabled configuration manager + // is mainly used by tests. + disabled bool + + // role is the specified role for this config manager. + role Role + + // usage is the descriptive usage for this config manager. + usage string + + // serverCertLoader is the cert loader for server certificates. It is only + // created if role is a server role. It is only modified on instantiation and + // does not require a mutex because it has its own internal locking. + serverCertLoader *TLSCertLoader + + // clientCertLoader is the cert loader for client certificates. It is only + // created if role is a client role. It is only modified on instantiation and + // does not require a mutex because it has its own internal locking. + clientCertLoader *TLSCertLoader + + // mu protects all fields below. + mu sync.RWMutex + + // config is the currently configured tlsConfigManagerConfig. + config *tlsConfigManagerConfig + + // tlsConfig is the tls.Config object returned when needed. + tlsConfig *tls.Config } // CAConfig configures a CA certificate pool: the PEM files to trust and whether @@ -109,17 +215,39 @@ func (c *caConfig) newCertPool() (*x509.CertPool, error) { // tlsConfigManagerConfig holds all options for a TLSConfigManager. type tlsConfigManagerConfig struct { + // monitor is the certificate monitor the cert loaders use. + monitor *TLSCertMonitor + + // role is the Role for this config manager. All enum values are allowed. + role Role + + // usage is the descriptive usage for this config manager. + usage string + + // logger is the logger to use for this config manager and its cert loaders. + logger *zap.Logger + // useTLS indicates if TLS should be used. If not, the rest of the configuration is ignored. useTLS bool // baseConfig is the *tls.Config to use as the basis for the manager's *tls.Config. baseConfig *tls.Config - // certPath is the path to the server certificate. - certPath string + // serverCertPath is the path to the server certificate. It is also used a s fallback for + // the client certificate. + serverCertPath string + + // serverKeyPath is the path to the server private key. It is also used a fallback for + // the client private key. + serverKeyPath string - // keyPath is the path to the server private key. - keyPath string + // clientCertPath is the path to the client certificate. certPath is used as a + // fallback if both clientCertPath and clientKeyPath are unset. + clientCertPath string + + // clientKeyPath is the path to the client private key. keyPath is used as a fallback + // if both clientCertPath and clientKeyPath are unset. + clientKeyPath string // allowInsecure indicates if certificate checks should be ignored. allowInsecure bool @@ -138,13 +266,36 @@ type tlsConfigManagerConfig struct { // (tls.NoClientCert). clientAuth *tls.ClientAuthType - // certLoaderOpts are options for the underlying TLSCertLoader. - certLoaderOpts []TLSCertLoaderOpt + // ignoreFilePermissions indicates if cert loaders should ignore file permissions. + ignoreFilePermissions bool } -// addCertLoaderOpt adds a TLSCertLoaderOpt to the configuration for the TLSCertLoader. -func (cl *tlsConfigManagerConfig) addCertLoaderOpt(o TLSCertLoaderOpt) { - cl.certLoaderOpts = append(cl.certLoaderOpts, o) +// commonCertLoaderOpts returns a common list of options for the TLSCertLoader. +func (c *tlsConfigManagerConfig) commonCertLoaderOpts() []TLSCertLoaderOpt { + return []TLSCertLoaderOpt{ + WithCertLoaderLogger(c.logger), + WithCertLoaderIgnoreFilePermissions(c.ignoreFilePermissions), + } +} + +// serverCertLoaderOpts returns the options needed for the server TLSCertLoader. +func (c *tlsConfigManagerConfig) serverCertLoaderOpts() []TLSCertLoaderOpt { + return append(c.commonCertLoaderOpts(), + WithCertLoaderCertificate(c.serverCertPath, c.serverKeyPath), + ) +} + +// clientCertLoaderOpts returns the options needed for the client TLSCertLoader. +func (c *tlsConfigManagerConfig) clientCertLoaderOpts() []TLSCertLoaderOpt { + certPath := c.clientCertPath + keyPath := c.clientKeyPath + if certPath == "" && keyPath == "" { + certPath = c.serverCertPath + keyPath = c.serverKeyPath + } + return append(c.commonCertLoaderOpts(), + WithCertLoaderCertificate(certPath, keyPath), + ) } // TLSConfigManagerOpt is an option for use with NewTLSConfigManager and related constructors. @@ -164,11 +315,22 @@ func WithBaseConfig(baseConfig *tls.Config) TLSConfigManagerOpt { } } -// WithCertificate sets the config manager's certificate and private key path. -func WithCertificate(certPath, keyPath string) TLSConfigManagerOpt { +// WithServerCertificate sets the config manager's server certificate and private +// key path. These will also be used as fallbacks for a client if no client +// certificate is configured. +func WithServerCertificate(certPath, keyPath string) TLSConfigManagerOpt { return func(cp *tlsConfigManagerConfig) { - cp.certPath = certPath - cp.keyPath = keyPath + cp.serverCertPath = certPath + cp.serverKeyPath = keyPath + } +} + +// WithServerCertificate sets the config manager's client certificate and private +// key path. These will also be used as fallbacks for a client if no client +func WithClientCertificate(certPath, keyPath string) TLSConfigManagerOpt { + return func(cp *tlsConfigManagerConfig) { + cp.clientCertPath = certPath + cp.clientKeyPath = keyPath } } @@ -224,38 +386,48 @@ func WithClientAuthPtr(clientAuthPtr *tls.ClientAuthType) TLSConfigManagerOpt { } } -// WithExpirationAdvanced sets the how far ahead the underlying CertLoader will -// warn about a certificate that is about to expire. -func WithExpirationAdvanced(d time.Duration) TLSConfigManagerOpt { - return func(cp *tlsConfigManagerConfig) { - cp.addCertLoaderOpt(WithCertLoaderExpirationAdvanced(d)) +// WithLogger assigns a logger for to use. +func WithLogger(logger *zap.Logger) TLSConfigManagerOpt { + return func(cl *tlsConfigManagerConfig) { + cl.logger = logger } } -// WithCertificateCheckInterval sets how often to check for certificate expiration. -func WithCertificateCheckInterval(d time.Duration) TLSConfigManagerOpt { +// WithIgnoreFilePermissions ignores file permissions when loading certificates. +func WithIgnoreFilePermissions(ignore bool) TLSConfigManagerOpt { return func(cl *tlsConfigManagerConfig) { - cl.addCertLoaderOpt(WithCertLoaderCertificateCheckInterval(d)) + cl.ignoreFilePermissions = ignore } } -// WithLogger assigns a logger for to use. -func WithLogger(logger *zap.Logger) TLSConfigManagerOpt { +// WithUsage sets the config manager descriptive usage. +func WithUsage(usage string) TLSConfigManagerOpt { + return func(c *tlsConfigManagerConfig) { + c.usage = usage + } +} + +// withMonitor sets the TLSCertMonitor for a config manager to use. It can only +// be set at construction time. This is an internal function because the public +// constructors have it as a required positional parameter, which is then +// converted to a TLSConfigManagerOpt internally. +func withMonitor(monitor *TLSCertMonitor) TLSConfigManagerOpt { return func(cl *tlsConfigManagerConfig) { - cl.addCertLoaderOpt(WithCertLoaderLogger(logger)) + cl.monitor = monitor } } -// WithIgnoreFilePermissions ignores file permissions when loading certificates. -func WithIgnoreFilePermissions(ignore bool) TLSConfigManagerOpt { +// withRole sets the role for this config manager. The role impacts what +// configurations are required and what operations are allowed. +func withRole(role Role) TLSConfigManagerOpt { return func(cl *tlsConfigManagerConfig) { - cl.addCertLoaderOpt(WithCertLoaderIgnoreFilePermissions(ignore)) + cl.role = role } } -// errCATrustsNothing is returned by resolveCA when a configured CA pool would +// ErrCATrustsNothing is returned by resolveCA when a configured CA pool would // trust no certificates. Callers wrap it with root/client context. -var errCATrustsNothing = errors.New("trusts no certificates: set paths or enable include-system") +var ErrCATrustsNothing = errors.New("trusts no certificates: set paths or enable include-system") // resolveCA resolves a user-facing *CAConfig into an internal pool config, // shared by root and client CAs. A nil config leaves the base config's pool in @@ -269,7 +441,7 @@ func resolveCA(cc *CAConfig) (caConfig, error) { return caConfig{}, nil // leave the base config's pool } if !cc.hasTrustAnchors() { - return caConfig{}, errCATrustsNothing + return caConfig{}, ErrCATrustsNothing } return cc.customCAConfig(), nil } @@ -281,134 +453,242 @@ func newTLSConfigManager(opts ...TLSConfigManagerOpt) (*TLSConfigManager, error) o(&c) } - // Create and setup base tls.Config - var tlsConfig *tls.Config - var certLoader *TLSCertLoader - if c.useTLS { - // Create / clone TLS configuration as necessary. - tlsConfig = c.baseConfig.Clone() // nil configs are clonable. - if tlsConfig == nil { - tlsConfig = new(tls.Config) - } - - // Modify configuration. - tlsConfig.InsecureSkipVerify = c.allowInsecure - - // Override ClientAuth only when it was explicitly configured; otherwise - // leave the base config's value in place. - if c.clientAuth != nil { - tlsConfig.ClientAuth = *c.clientAuth - } + // Check for configuration errors. + if !c.role.IsValid() { + return nil, fmt.Errorf("newTLSConfigManager: %w", ErrNoRole) + } + if c.monitor == nil { + return nil, fmt.Errorf("newTLSConfigManager: %w", ErrNoCertificateMonitor) + } - // Resolve the user-facing *CAConfig values into internal pool configs, - // applying the not-configured defaults and rejecting configurations - // that trust no certificates. See CAConfig for the nil/non-nil semantics. - rootCAConfig, err := resolveCA(c.rootCA) + // Create certificate loaders. + var serverCertLoader *TLSCertLoader + if c.role.IsServerRole() { + cl, err := NewTLSCertLoader( + ServerOnlyRole, + c.monitor, + append(c.commonCertLoaderOpts(), WithCertLoaderUsage(c.usage+".server"))...) if err != nil { - return nil, fmt.Errorf("root CA configuration %w", err) + return nil, fmt.Errorf("error creating server TLS certificate loader: %w", err) } - clientCAConfig, err := resolveCA(c.clientCA) + serverCertLoader = cl + } + + var clientCertLoader *TLSCertLoader + if c.role.IsClientRole() { + cl, err := NewTLSCertLoader( + ClientOnlyRole, + c.monitor, + append(c.commonCertLoaderOpts(), WithCertLoaderUsage(c.usage+".client"))...) if err != nil { - return nil, fmt.Errorf("client CA configuration %w", err) + return nil, fmt.Errorf("error creating client TLS certificate loader: %w", err) } + clientCertLoader = cl + } + + cm := &TLSConfigManager{ + role: c.role, + usage: c.usage, + serverCertLoader: serverCertLoader, + clientCertLoader: clientCertLoader, + } + + if apply, err := cm.prepareConfigure(&c); err != nil { + return nil, fmt.Errorf("error configuring new TLSConfigManager: %w", err) + } else if err := apply(); err != nil { + return nil, fmt.Errorf("error applying configuration to a new TLSConfigManager: %w", err) + } - // Setup CA pools. - setupPool := func(pool **x509.CertPool, c caConfig) error { - if p, err := c.newCertPool(); err != nil { - return err - } else if p != nil { - // Don't overwrite an existing pool from baseConfig with a nil pool. - *pool = p + return cm, nil +} + +// prepareConfigure changes the configuration of cm based on +func (cm *TLSConfigManager) prepareConfigure(c *tlsConfigManagerConfig) (func() error, error) { + // Create and setup base tls.Config + var tlsConfig *tls.Config + + if cm.disabled { + return nil, ErrConfigureDisabledManager + } + if !c.useTLS { + return func() error { + if cm.clientCertLoader != nil { + cm.clientCertLoader.Clear() } + if cm.serverCertLoader != nil { + cm.serverCertLoader.Clear() + } + cm.mu.Lock() + defer cm.mu.Unlock() + cm.config = c + cm.tlsConfig = nil + return nil + }, nil + } + + // Create / clone TLS configuration as necessary. + tlsConfig = c.baseConfig.Clone() // nil configs are clonable. + if tlsConfig == nil { + tlsConfig = new(tls.Config) + } + + // Modify configuration. + tlsConfig.InsecureSkipVerify = c.allowInsecure + + // Override ClientAuth only when it was explicitly configured; otherwise + // leave the base config's value in place. + if c.clientAuth != nil { + tlsConfig.ClientAuth = *c.clientAuth + } + + // Resolve the user-facing *CAConfig values into internal pool configs, + // applying the not-configured defaults and rejecting configurations + // that trust no certificates. See CAConfig for the nil/non-nil semantics. + rootCAConfig, err := resolveCA(c.rootCA) + if err != nil { + return nil, fmt.Errorf("%s: root CA configuration %w", cm.usage, err) + } + clientCAConfig, err := resolveCA(c.clientCA) + if err != nil { + return nil, fmt.Errorf("%s: client CA configuration %w", cm.usage, err) + } + + // Setup CA pools. + setupPool := func(pool **x509.CertPool, c caConfig) error { + if p, err := c.newCertPool(); err != nil { + return err + } else if p != nil { + // Don't overwrite an existing pool from baseConfig with a nil pool. + *pool = p } + return nil + } - if err := setupPool(&tlsConfig.RootCAs, rootCAConfig); err != nil { - return nil, fmt.Errorf("error creating root CA pool: %w", err) + if err := setupPool(&tlsConfig.RootCAs, rootCAConfig); err != nil { + return nil, fmt.Errorf("%s: error creating root CA pool: %w", cm.usage, err) + } + if err := setupPool(&tlsConfig.ClientCAs, clientCAConfig); err != nil { + return nil, fmt.Errorf("%s: error creating client CA pool: %w", cm.usage, err) + } + + // Prepare certificate load for server certificate loader. + var applyServerCert func() error + if cm.serverCertLoader != nil { + if apply, err := cm.serverCertLoader.PrepareLoad(c.serverCertLoaderOpts()...); err != nil { + return nil, fmt.Errorf("%s: error configuring server cert loader: %w", cm.usage, err) + } else { + applyServerCert = apply + } + } + + // Prepare certificate load for server certificate loader. + var applyClientCert func() error + if cm.clientCertLoader != nil { + if apply, err := cm.clientCertLoader.PrepareLoad(c.clientCertLoaderOpts()...); err != nil { + return nil, fmt.Errorf("%s: error configuring client cert loader: %w", cm.usage, err) + } else { + applyClientCert = apply } - if err := setupPool(&tlsConfig.ClientCAs, clientCAConfig); err != nil { - return nil, fmt.Errorf("error creating client CA pool: %w", err) + } + + return func() error { + if applyServerCert != nil { + if err := applyServerCert(); err != nil { + return fmt.Errorf("%s: error applying server certificate: %w", cm.usage, err) + } + cm.serverCertLoader.SetupTLSConfig(tlsConfig) } - // Create TLSCertLoader and configure it. - // Create TLSCertLoader and configure tlsConfig to use it. No loader is created if no cert - // is provided. This is useful for client-only use cases. - if c.certPath != "" || c.keyPath != "" { - if cl, err := NewTLSCertLoader(c.certPath, c.keyPath, c.certLoaderOpts...); err != nil { - return nil, err - } else { - certLoader = cl + if applyClientCert != nil { + if err := applyClientCert(); err != nil { + return fmt.Errorf("%s: error applying client certificate: %w", cm.usage, err) } - certLoader.SetupTLSConfig(tlsConfig) + cm.clientCertLoader.SetupTLSConfig(tlsConfig) } - } - return &TLSConfigManager{ - tlsConfig: tlsConfig, - certLoader: certLoader, + cm.mu.Lock() + defer cm.mu.Unlock() + cm.config = c + cm.tlsConfig = tlsConfig + + return nil }, nil + } -// NewTLSConfigManager returns a TLSConfigManager with the given configuration. If useTLS is true, -// then the certificate is loaded immediately if specified and the tls.Config instantiated. -// If no certPath and no keyPath is provided, then no TLSCertLoader is created. For this case, the returned -// TLSConfigManager can be used for client operations (e.g. Dial), but not for server operations (e.g. Listen). -// The allowInsecure parameter has no effect on server operations. -func NewTLSConfigManager(useTLS bool, baseConfig *tls.Config, certPath, keyPath string, allowInsecure bool, opts ...TLSConfigManagerOpt) (*TLSConfigManager, error) { +// NewServerTLSConfigManager returns a TLSConfigManager for a given set options for server-only +// use. +// +// Previously, many options were required parameters to this function. They are still +// required, but given using With*() parameters. This has the advantaged of allowing +// a single function to convert a TOML configuration to a slice of With* parameters for +// both construction and re-configuration. It does make missing an option a run-time +// error instead of a compile-time error. +// +// The following are require options. It is an error if they are missing. +// - WithUseTLS +// - WithBaseConfig +// +// If WithUseTLS(true) is given, then WithCertificate must also be given. +// +// All options given as direct positional parameters are required and can not be changed +// after construction. +// +// The returned TLSConfigManager can be used for server operations (e.g. Listen), but not for client operations (e.g. Dial). +func NewServerTLSConfigManager(monitor *TLSCertMonitor, opts ...TLSConfigManagerOpt) (*TLSConfigManager, error) { // Values from explicit parameters should override values set in opts. - co := make([]TLSConfigManagerOpt, 0, len(opts)+4) - co = append(co, opts...) - co = append(co, - WithUseTLS(useTLS), - WithBaseConfig(baseConfig), - WithCertificate(certPath, keyPath), - WithAllowInsecure(allowInsecure)) - return newTLSConfigManager(co...) -} - -// NewClientTLSConfigManager creates a TLSConfigManager that is only useful for clients without -// client certificates. TLS is enabled when useTLS is true. Certificate verification is skipped -// when allowInsecure is true. -// This is convenience wrapper for NewTLSConfigManager(useTLS, baseConfig, "", "", allowInsecure). -func NewClientTLSConfigManager(useTLS bool, baseConfig *tls.Config, allowInsecure bool, opts ...TLSConfigManagerOpt) (*TLSConfigManager, error) { - co := make([]TLSConfigManagerOpt, 0, len(opts)+3) - co = append(co, opts...) - co = append(co, - WithUseTLS(useTLS), - WithBaseConfig(baseConfig), - WithAllowInsecure(allowInsecure)) - return newTLSConfigManager(co...) -} - -// NewDisabledTLSConfigManager creates a TLSConfigManager that has TLS disabled. -// This is a convenience function equivalent to NewTLSConfigManager(false, nil, "", "", false). -// In addition to being more concise, NewDisabledTLSConfigManager can not return an error. + opts = append(opts, withMonitor(monitor), withRole(ServerOnlyRole)) + return newTLSConfigManager(opts...) +} + +// NewClientTLSConfigManager creates a TLSConfigManager that can only be used for clients. +// See NewTLSConfigManager for further information on options. +func NewClientTLSConfigManager(monitor *TLSCertMonitor, opts ...TLSConfigManagerOpt) (*TLSConfigManager, error) { + opts = append(opts, withMonitor(monitor), withRole(ClientOnlyRole)) + return newTLSConfigManager(opts...) +} + +// NewClientServerTLSConfigManager creates a config manager that can be used for both +// client and server operations. See NewServerTLSConfig for further information on options. +func NewClientServerTLSConfigManager(monitor *TLSCertMonitor, opts ...TLSConfigManagerOpt) (*TLSConfigManager, error) { + opts = append(opts, withMonitor(monitor), withRole(ServerAndClientRole)) + return newTLSConfigManager(opts...) +} + +// NewDisabledTLSConfigManager creates a TLSConfigManager that has TLS disabled. A disabled +// config manager can not be reconfigured to enable TLS later. It is primarily useful for tests +// that do not require TLS. func NewDisabledTLSConfigManager() *TLSConfigManager { - return &TLSConfigManager{} + return &TLSConfigManager{disabled: true} } // TLSConfig returns a tls.Config for use with dial and listen functions. When TLS is disabled the return is nil. // The returned tls.Config is a clone and does not need to be cloned again. func (cm *TLSConfigManager) TLSConfig() *tls.Config { + cm.mu.RLock() + defer cm.mu.RUnlock() + // Clone returns nil for a nil tlsConfig return cm.tlsConfig.Clone() } -// TLSCertLoader returns the certificate loader for this TLSConfigManager. When no certificate is provided -// the return value is nil. -func (cm *TLSConfigManager) TLSCertLoader() *TLSCertLoader { - return cm.certLoader -} - // UseTLS returns true if this TLSConfigManager is configured to use TLS. It is a convenience wrapper // around TLSConfig. func (cm *TLSConfigManager) UseTLS() bool { + cm.mu.RLock() + defer cm.mu.RUnlock() + // Don't use TLSConfig() to avoid cloning a tlsConfig only to throw it away. return cm.tlsConfig != nil } // Return a net.Listener for network and address based on current configuration. func (cm *TLSConfigManager) Listen(network, address string) (net.Listener, error) { + if !cm.role.IsServerRole() { + return nil, fmt.Errorf("%s: %w", cm.usage, ErrClientListen) + } + if tlsConfig := cm.TLSConfig(); tlsConfig != nil { return tls.Listen(network, address, tlsConfig) } else { @@ -418,6 +698,10 @@ func (cm *TLSConfigManager) Listen(network, address string) (net.Listener, error // Dial a remote for network and addressing using the current configuration. func (cm *TLSConfigManager) Dial(network, address string) (net.Conn, error) { + if !cm.role.IsClientRole() { + return nil, fmt.Errorf("%s: %w", cm.usage, ErrServerDial) + } + if tlsConfig := cm.TLSConfig(); tlsConfig != nil { return tls.Dial(network, address, tlsConfig) } else { @@ -427,6 +711,10 @@ func (cm *TLSConfigManager) Dial(network, address string) (net.Conn, error) { // Dial a remote for network and addressing using the given dialer and current configuration. func (cm *TLSConfigManager) DialWithDialer(dialer *net.Dialer, network, address string) (net.Conn, error) { + if !cm.role.IsClientRole() { + return nil, fmt.Errorf("%s: %w", cm.usage, ErrServerDial) + } + if tlsConfig := cm.TLSConfig(); tlsConfig != nil { return tls.DialWithDialer(dialer, network, address, tlsConfig) } else { @@ -434,24 +722,50 @@ func (cm *TLSConfigManager) DialWithDialer(dialer *net.Dialer, network, address } } -// PrepareCertificateLoad is a wrapper for the TLSCertLoader's PrepareLoad method. If TLS is not -// enabled, then a NOP callback is returned. -func (cm *TLSConfigManager) PrepareCertificateLoad(certPath, keyPath string) (func() error, error) { - if !cm.UseTLS() { - return func() error { return nil }, nil +// copyCurrentConfig creates a copy of the current configuration. +func (cm *TLSConfigManager) copyCurrentConfig() *tlsConfigManagerConfig { + // A shallow copy is sufficient because the options all overwrite fields + // instead modifying a field in-place. + cm.mu.RLock() + defer cm.mu.RUnlock() + c := &tlsConfigManagerConfig{} + *c = *cm.config + return c +} + +// PrepareReconfigure creates an apply function for a new configuration of the configuration manager. +func (cm *TLSConfigManager) PrepareReconfigure(opts ...TLSConfigManagerOpt) (func() error, error) { + // Use the current configuration. If there are no opts that set a field, then the + // current setting will continue to be used. + c := cm.copyCurrentConfig() + for _, o := range opts { + o(c) } - if certLoader := cm.TLSCertLoader(); certLoader != nil { - return certLoader.PrepareLoad(certPath, keyPath) - } else { - return nil, ErrNoCertLoader + // We cannot currently change some options when the server role is used. + if cm.role.IsServerRole() { + if c.useTLS != cm.UseTLS() { + return nil, fmt.Errorf("%s: changing TLS enabled to %t: %w", cm.usage, c.useTLS, ErrNotSupportedServer) + } } + + return cm.prepareConfigure(c) } // Close closes the underlying TLSCertLoader, if present. This is safe to call multiple times. func (cm *TLSConfigManager) Close() error { - if cm.certLoader != nil { - return cm.certLoader.Close() + var allErrs []error + + if cm.serverCertLoader != nil { + if err := cm.serverCertLoader.Close(); err != nil { + allErrs = append(allErrs, fmt.Errorf("%s: error closing server cert loader: %w", cm.usage, err)) + } + } + if cm.clientCertLoader != nil { + if err := cm.clientCertLoader.Close(); err != nil { + allErrs = append(allErrs, fmt.Errorf("%s: error closing client cert loader: %w", cm.usage, err)) + } } - return nil + + return errors.Join(allErrs...) } diff --git a/pkg/tlsconfig/configmanager_test.go b/pkg/tlsconfig/configmanager_test.go index 720bce2a9aa..376a8890653 100644 --- a/pkg/tlsconfig/configmanager_test.go +++ b/pkg/tlsconfig/configmanager_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + th "github.com/influxdata/influxdb/pkg/testing/helper" "github.com/influxdata/influxdb/pkg/testing/selfsigned" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -21,16 +22,21 @@ import ( func TestTLSConfigManager_ConsistentClonedConfigs(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + // Create an initial baseConfig for manager. baseTLSConfig := ss.ClientTLSConfig(t, false, false) require.NotNil(t, baseTLSConfig.RootCAs, "ClientTLSConfig should have set RootCAs") - manager, err := NewTLSConfigManager(true, baseTLSConfig, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(baseTLSConfig), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) require.NotNil(t, manager) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Get TLS config tlsConfig := manager.TLSConfig() @@ -50,38 +56,35 @@ func TestTLSConfigManager_ConsistentClonedConfigs(t *testing.T) { // Clear out the function pointers before calling require.Equal. require.NotNil(t, tlsConfig.GetCertificate) require.NotNil(t, tlsConfig2.GetCertificate) - require.NotNil(t, tlsConfig.GetClientCertificate) - require.NotNil(t, tlsConfig2.GetClientCertificate) + require.Nil(t, tlsConfig.GetClientCertificate) + require.Nil(t, tlsConfig2.GetClientCertificate) tlsConfig.GetCertificate = nil tlsConfig.GetClientCertificate = nil tlsConfig2.GetCertificate = nil tlsConfig2.GetClientCertificate = nil require.Equal(t, tlsConfig, tlsConfig2) - - // TLSCertLoader should also be available - certLoader := manager.TLSCertLoader() - require.NotNil(t, certLoader) - - // Subsequent calls should return the same instance - certLoader2 := manager.TLSCertLoader() - require.Same(t, certLoader, certLoader2) } func TestTLSConfigManager_BaseConfigCloned(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + baseConfig := &tls.Config{ MinVersion: tls.VersionTLS12, MaxVersion: tls.VersionTLS13, ServerName: "test.example.com", } - manager, err := NewTLSConfigManager(true, baseConfig, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(baseConfig), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) require.NotNil(t, manager) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -102,7 +105,14 @@ func TestTLSConfigManager_BaseConfigCloned(t *testing.T) { func TestTLSConfigManager_NilBaseConfig(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) require.NotNil(t, manager) defer func() { @@ -120,43 +130,36 @@ func TestTLSConfigManager_NilBaseConfig(t *testing.T) { func TestTLSConfigManager_CertLoaderCallbacksSet(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) require.NotNil(t, manager) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) // Verify that TLSCertLoader.SetupTLSConfig was called by checking callbacks are set require.NotNil(t, tlsConfig.GetCertificate, "GetCertificate callback should be set") - require.NotNil(t, tlsConfig.GetClientCertificate, "GetClientCertificate callback should be set") -} - -func TestTLSConfigManager_CertLoaderPathsCorrect(t *testing.T) { - ss := selfsigned.NewSelfSignedCert(t) - - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) - require.NoError(t, err) - require.NotNil(t, manager) - defer func() { - require.NoError(t, manager.Close()) - }() - - certLoader := manager.TLSCertLoader() - require.NotNil(t, certLoader) - - // Verify the paths were passed correctly to TLSCertLoader - certPath, keyPath := certLoader.Paths() - require.Equal(t, ss.CertPath, certPath) - require.Equal(t, ss.KeyPath, keyPath) + require.Nil(t, tlsConfig.GetClientCertificate, "GetClientCertificate callback should not be set") } func TestTLSConfigManager_ConstructorError(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + // Use non-existent paths to verify NewTLSConfigLoader returns error - manager, err := NewTLSConfigManager(true, nil, "/nonexistent/cert.pem", "/nonexistent/key.pem", false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil), + WithServerCertificate("/nonexistent/cert.pem", "/nonexistent/key.pem")) require.ErrorContains(t, err, "LoadCertificate: error opening \"/nonexistent/cert.pem\" for reading: open /nonexistent/cert.pem: no such file or directory") require.Nil(t, manager) } @@ -164,23 +167,45 @@ func TestTLSConfigManager_ConstructorError(t *testing.T) { func TestTLSConfigManager_InsecureSkipVerify(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("allowInsecure true", func(t *testing.T) { - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, true) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil), + WithServerCertificate(ss.CertPath, ss.KeyPath), + WithAllowInsecure(true)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.True(t, tlsConfig.InsecureSkipVerify) }) t.Run("allowInsecure false", func(t *testing.T) { - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil), + WithServerCertificate(ss.CertPath, ss.KeyPath), + WithAllowInsecure(false)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() + + tlsConfig := manager.TLSConfig() + require.False(t, tlsConfig.InsecureSkipVerify) + }) + + t.Run("allowInsecure implied false", func(t *testing.T) { + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.False(t, tlsConfig.InsecureSkipVerify) @@ -191,11 +216,13 @@ func TestTLSConfigManager_InsecureSkipVerify(t *testing.T) { InsecureSkipVerify: true, } - manager, err := NewTLSConfigManager(true, baseConfig, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(baseConfig), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.False(t, tlsConfig.InsecureSkipVerify, "allowInsecure should override base config") @@ -206,64 +233,69 @@ func TestTLSConfigManager_MultipleLoadersIndependent(t *testing.T) { ss1 := selfsigned.NewSelfSignedCert(t, selfsigned.WithDNSName("server1.example.com")) ss2 := selfsigned.NewSelfSignedCert(t, selfsigned.WithDNSName("server2.example.com")) - loader1, err := NewTLSConfigManager(true, nil, ss1.CertPath, ss1.KeyPath, false) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + loader1, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil), + WithServerCertificate(ss1.CertPath, ss1.KeyPath)) require.NoError(t, err) - loader2, err := NewTLSConfigManager(true, nil, ss2.CertPath, ss2.KeyPath, false) + defer th.CheckedClose(t, loader1)() + + loader2, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil), + WithServerCertificate(ss2.CertPath, ss2.KeyPath)) require.NoError(t, err) - defer func() { - require.NoError(t, loader1.Close()) - require.NoError(t, loader2.Close()) - }() + defer th.CheckedClose(t, loader2)() tlsConfig1 := loader1.TLSConfig() tlsConfig2 := loader2.TLSConfig() // Configs should be different instances require.NotSame(t, tlsConfig1, tlsConfig2) - - // Cert loaders should be different instances - certLoader1 := loader1.TLSCertLoader() - certLoader2 := loader2.TLSCertLoader() - require.NotSame(t, certLoader1, certLoader2) - - // Each manager should have its own certificate paths - cp1, kp1 := certLoader1.Paths() - cp2, kp2 := certLoader2.Paths() - require.Equal(t, ss1.CertPath, cp1) - require.Equal(t, ss1.KeyPath, kp1) - require.Equal(t, ss2.CertPath, cp2) - require.Equal(t, ss2.KeyPath, kp2) } func TestTLSConfigManager_UseTLSFalse(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("returns nil config and no error", func(t *testing.T) { - manager, err := NewTLSConfigManager(false, nil, "/any/cert.pem", "/any/key.pem", false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(false), + WithBaseConfig(nil), + WithServerCertificate("/any/cert.pem", "/any/key.pem")) require.NoError(t, err) require.NotNil(t, manager) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.Nil(t, tlsConfig) require.False(t, manager.UseTLS()) }) - t.Run("returns nil cert manager and no error", func(t *testing.T) { - manager, err := NewTLSConfigManager(false, nil, "/any/cert.pem", "/any/key.pem", false) + t.Run("no error on bad cert when disabled", func(t *testing.T) { + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(false), + WithBaseConfig(nil), + WithServerCertificate("/any/cert.pem", "/any/key.pem")) require.NoError(t, err) require.NotNil(t, manager) - defer func() { - require.NoError(t, manager.Close()) - }() - - certLoader := manager.TLSCertLoader() - require.Nil(t, certLoader) + defer th.CheckedClose(t, manager)() }) t.Run("ignores invalid paths when disabled", func(t *testing.T) { // Even with nonexistent paths, useTLS=false should not error - manager, err := NewTLSConfigManager(false, nil, "/nonexistent/cert.pem", "/nonexistent/key.pem", false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(false), + WithBaseConfig(nil), + WithServerCertificate("/nonexistent/cert.pem", "/nonexistent/key.pem")) require.NoError(t, err) require.NotNil(t, manager) defer func() { @@ -272,9 +304,6 @@ func TestTLSConfigManager_UseTLSFalse(t *testing.T) { tlsConfig := manager.TLSConfig() require.Nil(t, tlsConfig) - - certLoader := manager.TLSCertLoader() - require.Nil(t, certLoader) }) t.Run("ignores base config when disabled", func(t *testing.T) { @@ -283,127 +312,104 @@ func TestTLSConfigManager_UseTLSFalse(t *testing.T) { ServerName: "test.example.com", } - manager, err := NewTLSConfigManager(false, baseConfig, "/any/cert.pem", "/any/key.pem", false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(false), + WithBaseConfig(baseConfig), + WithServerCertificate("/any/cert.pem", "/any/key.pem")) require.NoError(t, err) require.NotNil(t, manager) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.Nil(t, tlsConfig, "should return nil config even with base config provided") }) t.Run("close works when disabled", func(t *testing.T) { - manager, err := NewTLSConfigManager(false, nil, "/any/cert.pem", "/any/key.pem", false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(false), + WithBaseConfig(nil), + WithServerCertificate("/any/cert.pem", "/any/key.pem")) require.NoError(t, err) // Close should succeed require.NoError(t, manager.Close()) require.NoError(t, manager.Close()) // idempotent }) - - t.Run("explicit parameters override opts", func(t *testing.T) { - manager, err := NewTLSConfigManager(false, nil, "", "", false, WithUseTLS(true)) - require.NoError(t, err) - require.NotNil(t, manager) - require.False(t, manager.UseTLS()) - }) } func TestTLSConfigManager_UseTLSWithoutCert(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("constructor succeeds with empty paths", func(t *testing.T) { - manager, err := NewTLSConfigManager(true, nil, "", "", true) + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil), + WithServerCertificate("", "")) require.NoError(t, err) require.NotNil(t, manager) require.NoError(t, manager.Close()) }) - t.Run("returns non-nil TLSConfig", func(t *testing.T) { - manager, err := NewTLSConfigManager(true, nil, "", "", true) - require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() - - tlsConfig := manager.TLSConfig() - require.NotNil(t, tlsConfig) - require.True(t, manager.UseTLS()) - }) - - t.Run("returns nil TLSCertLoader", func(t *testing.T) { - manager, err := NewTLSConfigManager(true, nil, "", "", true) - require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() - - certLoader := manager.TLSCertLoader() - require.Nil(t, certLoader) - }) - - t.Run("PrepareCertificateLoad returns ErrNoCertLoader", func(t *testing.T) { - manager, err := NewTLSConfigManager(true, nil, "", "", true) + t.Run("constructor succeeds with implied empty paths", func(t *testing.T) { + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil)) + defer th.CheckedClose(t, manager)() require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() - - callback, err := manager.PrepareCertificateLoad("/any/cert.pem", "/any/key.pem") - require.ErrorIs(t, err, ErrNoCertLoader) - require.Nil(t, callback) - }) - - t.Run("Listen fails without certificate", func(t *testing.T) { - manager, err := NewTLSConfigManager(true, nil, "", "", true) - require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() - - _, err = manager.Listen("tcp", "127.0.0.1:0") - require.ErrorContains(t, err, "tls: neither Certificates, GetCertificate, nor GetConfigForClient set in Config") + require.NotNil(t, manager) }) - t.Run("respects base config", func(t *testing.T) { - baseConfig := &tls.Config{ - MinVersion: tls.VersionTLS12, - ServerName: "test.example.com", - } - - manager, err := NewTLSConfigManager(true, baseConfig, "", "", false) + t.Run("returns non-nil TLSConfig", func(t *testing.T) { + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(nil), + WithServerCertificate("", ""), + WithAllowInsecure(true)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) - require.Equal(t, uint16(tls.VersionTLS12), tlsConfig.MinVersion) - require.Equal(t, "test.example.com", tlsConfig.ServerName) + require.True(t, manager.UseTLS()) }) - t.Run("close works without cert manager", func(t *testing.T) { - manager, err := NewTLSConfigManager(true, nil, "", "", true) - require.NoError(t, err) - - require.NoError(t, manager.Close()) - require.NoError(t, manager.Close()) // idempotent + t.Run("server constructor fails without certificate", func(t *testing.T) { + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithAllowInsecure(true)) + require.ErrorIs(t, err, ErrCertificateEmpty) + require.Nil(t, manager) }) } func TestTLSConfigManager_Close(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("close after construction", func(t *testing.T) { - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) require.NoError(t, manager.Close()) }) t.Run("close is idempotent", func(t *testing.T) { - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) require.NoError(t, manager.Close()) @@ -412,15 +418,20 @@ func TestTLSConfigManager_Close(t *testing.T) { } func TestTLSConfigManager_PrepareCertificateLoad(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("useTLS false returns NOP callback", func(t *testing.T) { - manager, err := NewTLSConfigManager(false, nil, "/any/cert.pem", "/any/key.pem", false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(false), + WithServerCertificate("/any/cert.pem", "/any/key.pem")) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Should return a NOP callback even with invalid paths - callback, err := manager.PrepareCertificateLoad("/nonexistent/cert.pem", "/nonexistent/key.pem") + callback, err := manager.PrepareReconfigure( + WithServerCertificate("/nonexistent/cert.pem", "/nonexistent/key.pem")) require.NoError(t, err) require.NotNil(t, callback) @@ -431,14 +442,16 @@ func TestTLSConfigManager_PrepareCertificateLoad(t *testing.T) { t.Run("useTLS true with valid paths returns callback", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Prepare loading the same cert (valid paths) - callback, err := manager.PrepareCertificateLoad(ss.CertPath, ss.KeyPath) + callback, err := manager.PrepareReconfigure( + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) require.NotNil(t, callback) @@ -449,14 +462,16 @@ func TestTLSConfigManager_PrepareCertificateLoad(t *testing.T) { t.Run("useTLS true with invalid paths returns error", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Prepare loading with nonexistent paths should fail - callback, err := manager.PrepareCertificateLoad("/nonexistent/cert.pem", "/nonexistent/key.pem") + callback, err := manager.PrepareReconfigure( + WithServerCertificate("/nonexistent/cert.pem", "/nonexistent/key.pem")) require.ErrorContains(t, err, "LoadCertificate: error opening \"/nonexistent/cert.pem\" for reading: open /nonexistent/cert.pem: no such file or directory") require.Nil(t, callback) }) @@ -465,31 +480,25 @@ func TestTLSConfigManager_PrepareCertificateLoad(t *testing.T) { ss1 := selfsigned.NewSelfSignedCert(t, selfsigned.WithDNSName("server1.example.com")) ss2 := selfsigned.NewSelfSignedCert(t, selfsigned.WithDNSName("server2.example.com")) - manager, err := NewTLSConfigManager(true, nil, ss1.CertPath, ss1.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss1.CertPath, ss1.KeyPath)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() - - // Verify initial certificate paths - certLoader := manager.TLSCertLoader() - certPath, keyPath := certLoader.Paths() - require.Equal(t, ss1.CertPath, certPath) - require.Equal(t, ss1.KeyPath, keyPath) + defer th.CheckedClose(t, manager)() // Prepare and execute loading a different certificate - callback, err := manager.PrepareCertificateLoad(ss2.CertPath, ss2.KeyPath) + callback, err := manager.PrepareReconfigure( + WithServerCertificate(ss2.CertPath, ss2.KeyPath)) require.NoError(t, err) require.NoError(t, callback()) - - // Verify certificate paths have been updated - certPath, keyPath = certLoader.Paths() - require.Equal(t, ss2.CertPath, certPath) - require.Equal(t, ss2.KeyPath, keyPath) }) } func TestTLSConfigManager_Listen(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + testListenerConnection := func(t *testing.T, listener net.Listener, dial func(addr string) (net.Conn, error)) { t.Helper() @@ -527,17 +536,16 @@ func TestTLSConfigManager_Listen(t *testing.T) { } t.Run("useTLS false returns plain TCP listener", func(t *testing.T) { - manager, err := NewTLSConfigManager(false, nil, "/any/cert.pem", "/any/key.pem", false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(false), + WithServerCertificate("/any/cert.pem", "/any/key.pem")) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() listener, err := manager.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) - defer func() { - require.NoError(t, listener.Close()) - }() + defer th.CheckedClose(t, listener)() testListenerConnection(t, listener, func(addr string) (net.Conn, error) { return net.Dial("tcp", addr) @@ -547,7 +555,10 @@ func TestTLSConfigManager_Listen(t *testing.T) { t.Run("useTLS true returns TLS listener", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) defer func() { require.NoError(t, manager.Close()) @@ -567,7 +578,10 @@ func TestTLSConfigManager_Listen(t *testing.T) { }) t.Run("invalid address returns error", func(t *testing.T) { - manager, err := NewTLSConfigManager(false, nil, "/any/cert.pem", "/any/key.pem", false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(false), + WithServerCertificate("/any/cert.pem", "/any/key.pem")) require.NoError(t, err) defer func() { require.NoError(t, manager.Close()) @@ -579,6 +593,9 @@ func TestTLSConfigManager_Listen(t *testing.T) { } func TestTLSConfigManager_Dial(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + testDialConnection := func(t *testing.T, listener net.Listener, dial func(addr string) (net.Conn, error)) { t.Helper() @@ -607,27 +624,24 @@ func TestTLSConfigManager_Dial(t *testing.T) { require.NoError(t, err) _, err = conn.Write(testData) require.NoError(t, err) - defer func() { - require.NoError(t, conn.Close()) - }() + defer th.CheckedClose(t, conn)() require.NoError(t, <-serverResult) require.Equal(t, testData, <-serverData) } t.Run("useTLS false dials plain TCP", func(t *testing.T) { - manager, err := NewTLSConfigManager(false, nil, "/any/cert.pem", "/any/key.pem", false) + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(false), + WithServerCertificate("/any/cert.pem", "/any/key.pem")) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Create plain TCP server listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) - defer func() { - require.NoError(t, listener.Close()) - }() + defer th.CheckedClose(t, listener)() testDialConnection(t, listener, func(addr string) (net.Conn, error) { return manager.Dial("tcp", addr) @@ -637,11 +651,13 @@ func TestTLSConfigManager_Dial(t *testing.T) { t.Run("useTLS true dials TLS", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, true) + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), + WithAllowInsecure(true)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Create TLS server cert, err := tls.LoadX509KeyPair(ss.CertPath, ss.KeyPath) @@ -650,9 +666,7 @@ func TestTLSConfigManager_Dial(t *testing.T) { Certificates: []tls.Certificate{cert}, }) require.NoError(t, err) - defer func() { - require.NoError(t, listener.Close()) - }() + defer th.CheckedClose(t, listener)() testDialConnection(t, listener, func(addr string) (net.Conn, error) { return manager.Dial("tcp", addr) @@ -663,11 +677,12 @@ func TestTLSConfigManager_Dial(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) // Create manager without client cert (client-only mode) - manager, err := NewTLSConfigManager(true, nil, "", "", true) + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithAllowInsecure(true)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Create TLS server cert, err := tls.LoadX509KeyPair(ss.CertPath, ss.KeyPath) @@ -676,9 +691,7 @@ func TestTLSConfigManager_Dial(t *testing.T) { Certificates: []tls.Certificate{cert}, }) require.NoError(t, err) - defer func() { - require.NoError(t, listener.Close()) - }() + defer th.CheckedClose(t, listener)() testDialConnection(t, listener, func(addr string) (net.Conn, error) { return manager.Dial("tcp", addr) @@ -686,85 +699,102 @@ func TestTLSConfigManager_Dial(t *testing.T) { }) t.Run("invalid address returns error", func(t *testing.T) { - manager, err := NewTLSConfigManager(false, nil, "/any/cert.pem", "/any/key.pem", false) + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(false), + WithServerCertificate("/any/cert.pem", "/any/key.pem")) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() - _, err = manager.Dial("tcp", "invalid:address:format") + dialer, err := manager.Dial("tcp", "invalid:address:format") require.ErrorContains(t, err, "address invalid:address:format: too many colons in address") + require.Nil(t, dialer) }) } func TestNewDisabledTLSConfigManager(t *testing.T) { - // NewDisabledTLSConfigManager should be equivalent to NewTLSConfigManager(false, nil, "", "", false). - // The functional behavior of a disabled manager is already tested in TestTLSConfigManager_UseTLSFalse, - // so we just verify the two constructors produce equivalent managers. + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + disabled := NewDisabledTLSConfigManager() - require.NotNil(t, disabled) - explicit, err := NewTLSConfigManager(false, nil, "", "", false) - require.NoError(t, err) - require.NotNil(t, explicit) + defer th.CheckedClose(t, disabled)() - require.Equal(t, explicit.TLSConfig(), disabled.TLSConfig()) - require.Equal(t, explicit.TLSCertLoader(), disabled.TLSCertLoader()) + require.NotNil(t, disabled) require.False(t, disabled.UseTLS()) - - require.NoError(t, disabled.Close()) - require.NoError(t, explicit.Close()) + require.Nil(t, disabled.TLSConfig()) } func TestNewClientTLSConfigManager(t *testing.T) { - // NewClientTLSConfigManager should be equivalent to NewTLSConfigManager(useTLS, baseConfig, "", "", allowInsecure). - // The functional behavior is already tested in other tests (TestTLSConfigManager_UseTLSWithoutCert, etc.), - // so we just verify the two constructors produce equivalent managers for various parameter combinations. + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + ss := selfsigned.NewSelfSignedCert(t) t.Run("TLS disabled", func(t *testing.T) { - client, err := NewClientTLSConfigManager(false, nil, false) + client, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(false)) require.NoError(t, err) require.NotNil(t, client) - explicit, err := NewTLSConfigManager(false, nil, "", "", false) + defer th.CheckedClose(t, client)() + + server, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(false)) require.NoError(t, err) - require.NotNil(t, explicit) + require.NotNil(t, server) + defer th.CheckedClose(t, server)() - require.Equal(t, explicit.TLSConfig(), client.TLSConfig()) - require.Equal(t, explicit.TLSCertLoader(), client.TLSCertLoader()) + require.Equal(t, server.TLSConfig(), client.TLSConfig()) require.False(t, client.UseTLS()) require.NoError(t, client.Close()) - require.NoError(t, explicit.Close()) + require.NoError(t, server.Close()) }) t.Run("TLS enabled allowInsecure false", func(t *testing.T) { - client, err := NewClientTLSConfigManager(true, nil, false) + client, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true)) require.NoError(t, err) require.NotNil(t, client) - explicit, err := NewTLSConfigManager(true, nil, "", "", false) + defer th.CheckedClose(t, client)() + + server, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), + ) require.NoError(t, err) - require.NotNil(t, explicit) + require.NotNil(t, server) + defer th.CheckedClose(t, server)() - require.Equal(t, explicit.TLSConfig().InsecureSkipVerify, client.TLSConfig().InsecureSkipVerify) - require.Equal(t, explicit.TLSCertLoader(), client.TLSCertLoader()) + require.Equal(t, server.TLSConfig().InsecureSkipVerify, client.TLSConfig().InsecureSkipVerify) require.True(t, client.UseTLS()) require.NoError(t, client.Close()) - require.NoError(t, explicit.Close()) + require.NoError(t, server.Close()) }) t.Run("TLS enabled allowInsecure true", func(t *testing.T) { - client, err := NewClientTLSConfigManager(true, nil, true) + client, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithAllowInsecure(true)) require.NoError(t, err) require.NotNil(t, client) - explicit, err := NewTLSConfigManager(true, nil, "", "", true) - require.NoError(t, err) - require.NotNil(t, explicit) + defer th.CheckedClose(t, client)() - require.Equal(t, explicit.TLSConfig().InsecureSkipVerify, client.TLSConfig().InsecureSkipVerify) - require.Equal(t, explicit.TLSCertLoader(), client.TLSCertLoader()) + server, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithAllowInsecure(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + require.NotNil(t, server) + defer th.CheckedClose(t, server)() - require.NoError(t, client.Close()) - require.NoError(t, explicit.Close()) + require.Equal(t, server.TLSConfig().InsecureSkipVerify, client.TLSConfig().InsecureSkipVerify) }) t.Run("with base config", func(t *testing.T) { @@ -774,22 +804,28 @@ func TestNewClientTLSConfigManager(t *testing.T) { ServerName: "test.example.com", } - client, err := NewClientTLSConfigManager(true, baseConfig, false) + client, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(baseConfig)) require.NoError(t, err) require.NotNil(t, client) - explicit, err := NewTLSConfigManager(true, baseConfig, "", "", false) + defer th.CheckedClose(t, client)() + + server, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(baseConfig), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) - require.NotNil(t, explicit) + require.NotNil(t, server) + defer th.CheckedClose(t, server)() // Verify base config values are preserved in both - require.Equal(t, explicit.TLSConfig().MinVersion, client.TLSConfig().MinVersion) - require.Equal(t, explicit.TLSConfig().MaxVersion, client.TLSConfig().MaxVersion) - require.Equal(t, explicit.TLSConfig().ServerName, client.TLSConfig().ServerName) - require.Equal(t, explicit.TLSConfig().InsecureSkipVerify, client.TLSConfig().InsecureSkipVerify) - require.Equal(t, explicit.TLSCertLoader(), client.TLSCertLoader()) - - require.NoError(t, client.Close()) - require.NoError(t, explicit.Close()) + require.Equal(t, server.TLSConfig().MinVersion, client.TLSConfig().MinVersion) + require.Equal(t, server.TLSConfig().MaxVersion, client.TLSConfig().MaxVersion) + require.Equal(t, server.TLSConfig().ServerName, client.TLSConfig().ServerName) + require.Equal(t, server.TLSConfig().InsecureSkipVerify, client.TLSConfig().InsecureSkipVerify) }) t.Run("base config is cloned", func(t *testing.T) { @@ -797,9 +833,13 @@ func TestNewClientTLSConfigManager(t *testing.T) { ServerName: "original.example.com", } - client, err := NewClientTLSConfigManager(true, baseConfig, false) + client, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(baseConfig)) require.NoError(t, err) require.NotNil(t, client) + defer th.CheckedClose(t, client)() // Verify config is cloned (not same instance) require.NotSame(t, baseConfig, client.TLSConfig()) @@ -810,13 +850,6 @@ func TestNewClientTLSConfigManager(t *testing.T) { require.NoError(t, client.Close()) }) - - t.Run("explicit parameters override opts", func(t *testing.T) { - client, err := NewClientTLSConfigManager(false, nil, false, WithUseTLS(true)) - require.NoError(t, err) - require.NotNil(t, client) - require.False(t, client.UseTLS()) - }) } func simpleEchoServer(serverDone chan error, listener net.Listener, bufSize int) (rErr error) { @@ -844,6 +877,9 @@ func simpleEchoServer(serverDone chan error, listener net.Listener, bufSize int) } func TestTLSConfigManager_WithRootCAFiles(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("sets RootCAs from file", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) @@ -853,12 +889,13 @@ func TestTLSConfigManager_WithRootCAFiles(t *testing.T) { require.NoError(t, err) require.True(t, expectedPool.AppendCertsFromPEM(caCert)) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithRootCA(&CAConfig{Paths: []string{ss.CACertPath}})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -870,26 +907,26 @@ func TestTLSConfigManager_WithRootCAFiles(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) // Server manager with certificate - serverManager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + serverManager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) - defer func() { - require.NoError(t, serverManager.Close()) - }() + defer th.CheckedClose(t, serverManager)() // Client manager trusting the CA that signed the server certificate - clientManager, err := NewClientTLSConfigManager(true, nil, false, + clientManager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), WithRootCA(&CAConfig{Paths: []string{ss.CACertPath}})) require.NoError(t, err) - defer func() { - require.NoError(t, clientManager.Close()) - }() + defer th.CheckedClose(t, clientManager)() // Start server listener, err := serverManager.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) - defer func() { - require.NoError(t, listener.Close()) - }() + require.NotNil(t, listener) + defer th.CheckedClose(t, listener)() // Server accepts and echoes testData := []byte("hello") @@ -899,9 +936,8 @@ func TestTLSConfigManager_WithRootCAFiles(t *testing.T) { // Client connects with CA-trusted connection conn, err := clientManager.Dial("tcp", listener.Addr().String()) require.NoError(t, err) - defer func() { - require.NoError(t, conn.Close()) - }() + require.NotNil(t, conn) + defer th.CheckedClose(t, conn)() n, err := conn.Write(testData) require.NoError(t, err) @@ -923,26 +959,25 @@ func TestTLSConfigManager_WithRootCAFiles(t *testing.T) { otherSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("other", "Other CA")) // Server manager - serverManager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + serverManager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) - defer func() { - require.NoError(t, serverManager.Close()) - }() + defer th.CheckedClose(t, serverManager)() // Client manager trusting only an unrelated CA, so it will not trust the server. - clientManager, err := NewClientTLSConfigManager(true, nil, false, + clientManager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), WithRootCA(&CAConfig{Paths: []string{otherSS.CACertPath}})) require.NoError(t, err) - defer func() { - require.NoError(t, clientManager.Close()) - }() + defer th.CheckedClose(t, clientManager)() // Start server listener, err := serverManager.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) - defer func() { - require.NoError(t, listener.Close()) - }() + defer th.CheckedClose(t, listener)() // Server accepts and waits for handshake to complete (will fail due to client rejecting cert) testData := []byte("test") @@ -972,12 +1007,12 @@ func TestTLSConfigManager_WithRootCAFiles(t *testing.T) { require.NoError(t, err) require.True(t, expectedPool.AppendCertsFromPEM(caCert2)) - manager, err := NewClientTLSConfigManager(true, nil, false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), WithRootCA(&CAConfig{Paths: []string{ss1.CACertPath, ss2.CACertPath}})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -986,10 +1021,13 @@ func TestTLSConfigManager_WithRootCAFiles(t *testing.T) { }) t.Run("error on nonexistent file", func(t *testing.T) { - _, err := NewClientTLSConfigManager(true, nil, false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), WithRootCA(&CAConfig{Paths: []string{"/nonexistent/ca.pem"}})) require.ErrorIs(t, err, os.ErrNotExist) require.ErrorContains(t, err, "error creating root CA pool: error reading file \"/nonexistent/ca.pem\" for CA store: open /nonexistent/ca.pem: no such file or directory") + require.Nil(t, manager) }) t.Run("error on invalid PEM file", func(t *testing.T) { @@ -998,7 +1036,9 @@ func TestTLSConfigManager_WithRootCAFiles(t *testing.T) { tmpFile := path.Join(tmpDir, "invalid.pem") require.NoError(t, os.WriteFile(tmpFile, []byte("not a valid PEM file"), 0644)) - manager, err := NewClientTLSConfigManager(true, nil, false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), WithRootCA(&CAConfig{Paths: []string{tmpFile}})) require.ErrorContains(t, err, "error creating root CA pool: error adding certificates from \""+tmpFile+"\" to CA store: no valid certificates found") require.Nil(t, manager) @@ -1006,17 +1046,20 @@ func TestTLSConfigManager_WithRootCAFiles(t *testing.T) { } func TestTLSConfigManager_WithRootCAIncludeSystem(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("includes system CA pool", func(t *testing.T) { // Get expected system pool expectedPool, err := x509.SystemCertPool() require.NoError(t, err) - manager, err := NewClientTLSConfigManager(true, nil, false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), WithRootCA(&CAConfig{IncludeSystem: true})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1027,20 +1070,23 @@ func TestTLSConfigManager_WithRootCAIncludeSystem(t *testing.T) { t.Run("trusts no certificates is an error", func(t *testing.T) { // A non-nil root CA config that neither lists paths nor includes the // system pool trusts nothing and is rejected at construction. - manager, err := NewClientTLSConfigManager(true, nil, false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), WithRootCA(&CAConfig{IncludeSystem: false})) - require.EqualError(t, err, "root CA configuration trusts no certificates: "+ - "set paths or enable include-system") + require.ErrorIs(t, err, ErrCATrustsNothing) require.Nil(t, manager) }) t.Run("trusts no certificates is an error even when insecure", func(t *testing.T) { // A non-nil config is validated regardless of allowInsecure, so a // no-trust-anchors config is still rejected at construction. - manager, err := NewClientTLSConfigManager(true, nil, true, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithAllowInsecure(true), WithRootCA(&CAConfig{IncludeSystem: false})) - require.EqualError(t, err, "root CA configuration trusts no certificates: "+ - "set paths or enable include-system") + require.ErrorIs(t, err, ErrCATrustsNothing) require.Nil(t, manager) }) @@ -1054,12 +1100,12 @@ func TestTLSConfigManager_WithRootCAIncludeSystem(t *testing.T) { require.NoError(t, err) require.True(t, expectedPool.AppendCertsFromPEM(caCert)) - manager, err := NewClientTLSConfigManager(true, nil, false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), WithRootCA(&CAConfig{Paths: []string{ss.CACertPath}, IncludeSystem: true})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1069,6 +1115,9 @@ func TestTLSConfigManager_WithRootCAIncludeSystem(t *testing.T) { } func TestTLSConfigManager_WithClientCAFiles(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("sets ClientCAs from file", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) @@ -1078,13 +1127,14 @@ func TestTLSConfigManager_WithClientCAFiles(t *testing.T) { require.NoError(t, err) require.True(t, expectedPool.AppendCertsFromPEM(caCert)) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientAuth(tls.RequireAndVerifyClientCert), WithClientCA(&CAConfig{Paths: []string{ss.CACertPath}})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1096,20 +1146,20 @@ func TestTLSConfigManager_WithClientCAFiles(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) // Server manager that requires client certificates. - serverManager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + serverManager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientCA(&CAConfig{Paths: []string{ss.CACertPath}}), WithClientAuth(tls.RequireAndVerifyClientCert)) require.NoError(t, err) - defer func() { - require.NoError(t, serverManager.Close()) - }() + defer th.CheckedClose(t, serverManager)() // Start server listener, err := serverManager.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) - defer func() { - require.NoError(t, listener.Close()) - }() + require.NotNil(t, listener) + defer th.CheckedClose(t, listener)() // Server accepts and echoes testData := []byte("hello") @@ -1118,16 +1168,18 @@ func TestTLSConfigManager_WithClientCAFiles(t *testing.T) { // Client config with client certificate. Ignore certificate validity from server, we're testing // client certificate functionality here. - clientManager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, true) + clientManager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), + WithAllowInsecure(true)) require.NoError(t, err) - defer func() { - require.NoError(t, clientManager.Close()) - }() + defer th.CheckedClose(t, clientManager)() + conn, err := clientManager.Dial("tcp", listener.Addr().String()) require.NoError(t, err) - defer func() { - require.NoError(t, conn.Close()) - }() + require.NotNil(t, conn) + defer th.CheckedClose(t, conn)() n, err := conn.Write(testData) require.NoError(t, err) @@ -1147,31 +1199,34 @@ func TestTLSConfigManager_WithClientCAFiles(t *testing.T) { otherSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("other", "Other CA")) // Server manager that requires client certificates - serverManager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + serverManager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientAuth(tls.RequireAndVerifyClientCert), WithClientCA(&CAConfig{Paths: []string{ss.CACertPath}})) require.NoError(t, err) - defer func() { - require.NoError(t, serverManager.Close()) - }() + defer th.CheckedClose(t, serverManager)() + serverTLSConfig := serverManager.TLSConfig() require.NotNil(t, serverTLSConfig) require.Equal(t, tls.RequireAndVerifyClientCert, serverTLSConfig.ClientAuth) // Client with certificate signed by different CA. Ignore server certificate validity. We're // checking client certificate functionality here. - clientManager, err := NewTLSConfigManager(true, nil, otherSS.CertPath, otherSS.KeyPath, true) + clientManager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(otherSS.CertPath, otherSS.KeyPath), + WithAllowInsecure(true)) require.NoError(t, err) - defer func() { - require.NoError(t, clientManager.Close()) - }() + defer th.CheckedClose(t, clientManager)() // Start server listener, err := serverManager.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) - defer func() { - require.NoError(t, listener.Close()) - }() + require.NotNil(t, listener) + defer th.CheckedClose(t, listener)() // Server accepts and waits for handshake (will fail due to untrusted client cert) testData := []byte("hello") @@ -1211,13 +1266,14 @@ func TestTLSConfigManager_WithClientCAFiles(t *testing.T) { require.NoError(t, err) require.True(t, expectedPool.AppendCertsFromPEM(caCert2)) - manager, err := NewTLSConfigManager(true, nil, ss1.CertPath, ss1.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss1.CertPath, ss1.KeyPath), WithClientAuth(tls.RequireAndVerifyClientCert), WithClientCA(&CAConfig{Paths: []string{ss1.CACertPath, ss2.CACertPath}})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1228,11 +1284,15 @@ func TestTLSConfigManager_WithClientCAFiles(t *testing.T) { t.Run("error on nonexistent file", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - _, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientAuth(tls.RequireAndVerifyClientCert), WithClientCA(&CAConfig{Paths: []string{"/nonexistent/ca.pem"}})) require.ErrorIs(t, err, os.ErrNotExist) require.ErrorContains(t, err, "error creating client CA pool: error reading file \"/nonexistent/ca.pem\" for CA store: open /nonexistent/ca.pem: no such file or directory") + require.Nil(t, manager) }) t.Run("error on invalid PEM file", func(t *testing.T) { @@ -1243,7 +1303,10 @@ func TestTLSConfigManager_WithClientCAFiles(t *testing.T) { tmpFile := path.Join(tmpDir, "invalid.pem") require.NoError(t, os.WriteFile(tmpFile, []byte("not a valid PEM file"), 0644)) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientAuth(tls.RequireAndVerifyClientCert), WithClientCA(&CAConfig{Paths: []string{tmpFile}})) require.ErrorContains(t, err, "error creating client CA pool: error adding certificates from \""+tmpFile+"\" to CA store: no valid certificates found") @@ -1252,6 +1315,9 @@ func TestTLSConfigManager_WithClientCAFiles(t *testing.T) { } func TestTLSConfigManager_WithClientCAIncludeSystem(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("includes system CA pool", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) @@ -1259,13 +1325,14 @@ func TestTLSConfigManager_WithClientCAIncludeSystem(t *testing.T) { expectedPool, err := x509.SystemCertPool() require.NoError(t, err) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientAuth(tls.RequireAndVerifyClientCert), WithClientCA(&CAConfig{IncludeSystem: true})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1278,11 +1345,13 @@ func TestTLSConfigManager_WithClientCAIncludeSystem(t *testing.T) { // A client CA config that neither lists paths nor includes the system pool // trusts nothing and is rejected at construction. - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientAuth(tls.RequireAndVerifyClientCert), WithClientCA(&CAConfig{IncludeSystem: false})) - require.EqualError(t, err, "client CA configuration trusts no certificates: "+ - "set paths or enable include-system") + require.ErrorIs(t, err, ErrCATrustsNothing) require.Nil(t, manager) }) @@ -1296,13 +1365,14 @@ func TestTLSConfigManager_WithClientCAIncludeSystem(t *testing.T) { require.NoError(t, err) require.True(t, expectedPool.AppendCertsFromPEM(caCert)) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientAuth(tls.RequireAndVerifyClientCert), WithClientCA(&CAConfig{Paths: []string{ss.CACertPath}, IncludeSystem: true})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1318,12 +1388,13 @@ func TestTLSConfigManager_WithClientCAIncludeSystem(t *testing.T) { expectedPool, err := x509.SystemCertPool() require.NoError(t, err) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientCA(&CAConfig{IncludeSystem: true})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1341,12 +1412,13 @@ func TestTLSConfigManager_WithClientCAIncludeSystem(t *testing.T) { require.NoError(t, err) require.True(t, expectedPool.AppendCertsFromPEM(caCert)) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientCA(&CAConfig{Paths: []string{ss.CACertPath}, IncludeSystem: true})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1356,15 +1428,19 @@ func TestTLSConfigManager_WithClientCAIncludeSystem(t *testing.T) { } func TestTLSConfigManager_CAOptionsNotSetByDefault(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + // When no CA options are specified, the TLS config should not have // custom CA pools set (allowing Go's default behavior) ss := selfsigned.NewSelfSignedCert(t) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1376,6 +1452,9 @@ func TestTLSConfigManager_CAOptionsWithBaseConfig(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) anotherSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("another", "Another CA")) + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + // Create a base config with existing RootCAs basePool := x509.NewCertPool() baseCACert, err := os.ReadFile(ss.CACertPath) @@ -1393,12 +1472,13 @@ func TestTLSConfigManager_CAOptionsWithBaseConfig(t *testing.T) { require.True(t, expectedPool.AppendCertsFromPEM(anotherCACert)) // Create manager with different CA file - should override base config - manager, err := NewClientTLSConfigManager(true, baseConfig, false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(baseConfig), WithRootCA(&CAConfig{Paths: []string{anotherSS.CACertPath}})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1409,6 +1489,9 @@ func TestTLSConfigManager_CAOptionsWithBaseConfig(t *testing.T) { } func TestTLSConfigManager_CAResolution(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("root nil defers to base config RootCAs", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) @@ -1420,11 +1503,12 @@ func TestTLSConfigManager_CAResolution(t *testing.T) { require.True(t, basePool.AppendCertsFromPEM(caCert)) baseConfig := &tls.Config{RootCAs: basePool} - manager, err := NewClientTLSConfigManager(true, baseConfig, false) + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(baseConfig)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1433,20 +1517,23 @@ func TestTLSConfigManager_CAResolution(t *testing.T) { }) t.Run("root no-trust-anchors is an error", func(t *testing.T) { - manager, err := NewClientTLSConfigManager(true, nil, false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), WithRootCA(&CAConfig{})) - require.EqualError(t, err, "root CA configuration trusts no certificates: "+ - "set paths or enable include-system") + require.ErrorIs(t, err, ErrCATrustsNothing) require.Nil(t, manager) }) t.Run("root no-trust-anchors is an error even when insecure", func(t *testing.T) { // A non-nil config is validated regardless of allowInsecure, so a // no-trust-anchors config is still rejected at construction. - manager, err := NewClientTLSConfigManager(true, nil, true, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithAllowInsecure(true), WithRootCA(&CAConfig{})) - require.EqualError(t, err, "root CA configuration trusts no certificates: "+ - "set paths or enable include-system") + require.ErrorIs(t, err, ErrCATrustsNothing) require.Nil(t, manager) }) @@ -1461,12 +1548,14 @@ func TestTLSConfigManager_CAResolution(t *testing.T) { require.True(t, basePool.AppendCertsFromPEM(caCert)) baseConfig := &tls.Config{ClientCAs: basePool} - manager, err := NewTLSConfigManager(true, baseConfig, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(baseConfig), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientAuth(tls.RequireAndVerifyClientCert)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1480,10 +1569,12 @@ func TestTLSConfigManager_CAResolution(t *testing.T) { // A non-nil client CA config is validated regardless of whether client // auth is enabled, so a config that trusts nothing is rejected here even // with the default NoClientCert. - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientCA(&CAConfig{})) - require.EqualError(t, err, "client CA configuration trusts no certificates: "+ - "set paths or enable include-system") + require.ErrorIs(t, err, ErrCATrustsNothing) require.Nil(t, manager) }) @@ -1497,12 +1588,13 @@ func TestTLSConfigManager_CAResolution(t *testing.T) { require.NoError(t, err) require.True(t, expectedPool.AppendCertsFromPEM(caCert)) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientCA(&CAConfig{Paths: []string{ss.CACertPath}})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1520,13 +1612,14 @@ func TestTLSConfigManager_CAResolution(t *testing.T) { require.NoError(t, err) require.True(t, expectedPool.AppendCertsFromPEM(caCert)) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithClientAuth(tls.RequireAndVerifyClientCert), WithClientCA(&CAConfig{Paths: []string{ss.CACertPath}})) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) @@ -1536,6 +1629,9 @@ func TestTLSConfigManager_CAResolution(t *testing.T) { } func TestTLSConfigManager_ClientAuthOverride(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + // Base config with a non-zero ClientAuth, so we can tell whether an option // overrode it (including overriding it back to the zero value). base := func() *tls.Config { @@ -1543,51 +1639,68 @@ func TestTLSConfigManager_ClientAuthOverride(t *testing.T) { } t.Run("no option leaves base ClientAuth", func(t *testing.T) { - manager, err := NewClientTLSConfigManager(true, base(), false) + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(base())) require.NoError(t, err) - defer func() { require.NoError(t, manager.Close()) }() + defer th.CheckedClose(t, manager)() require.Equal(t, tls.RequireAndVerifyClientCert, manager.TLSConfig().ClientAuth) }) t.Run("WithClientAuth overrides base", func(t *testing.T) { - manager, err := NewClientTLSConfigManager(true, base(), false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), WithBaseConfig(base()), WithClientAuth(tls.VerifyClientCertIfGiven)) require.NoError(t, err) - defer func() { require.NoError(t, manager.Close()) }() + defer th.CheckedClose(t, manager)() require.Equal(t, tls.VerifyClientCertIfGiven, manager.TLSConfig().ClientAuth) }) t.Run("WithClientAuth overrides base with zero value", func(t *testing.T) { - manager, err := NewClientTLSConfigManager(true, base(), false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(base()), WithClientAuth(tls.NoClientCert)) require.NoError(t, err) - defer func() { require.NoError(t, manager.Close()) }() + defer th.CheckedClose(t, manager)() require.Equal(t, tls.NoClientCert, manager.TLSConfig().ClientAuth) }) t.Run("WithClientAuthPtr nil leaves base ClientAuth", func(t *testing.T) { - manager, err := NewClientTLSConfigManager(true, base(), false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(base()), WithClientAuthPtr(nil)) require.NoError(t, err) - defer func() { require.NoError(t, manager.Close()) }() + defer th.CheckedClose(t, manager)() require.Equal(t, tls.RequireAndVerifyClientCert, manager.TLSConfig().ClientAuth) }) t.Run("WithClientAuthPtr non-nil overrides base with zero value", func(t *testing.T) { auth := tls.NoClientCert - manager, err := NewClientTLSConfigManager(true, base(), false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(base()), WithClientAuthPtr(&auth)) require.NoError(t, err) - defer func() { require.NoError(t, manager.Close()) }() + defer th.CheckedClose(t, manager)() require.Equal(t, tls.NoClientCert, manager.TLSConfig().ClientAuth) }) t.Run("WithClientAuthPtr non-nil overrides base with non-zero value", func(t *testing.T) { auth := tls.VerifyClientCertIfGiven - manager, err := NewClientTLSConfigManager(true, base(), false, + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithBaseConfig(base()), WithClientAuthPtr(&auth)) require.NoError(t, err) - defer func() { require.NoError(t, manager.Close()) }() + defer th.CheckedClose(t, manager)() require.Equal(t, tls.VerifyClientCertIfGiven, manager.TLSConfig().ClientAuth) }) } @@ -1600,8 +1713,7 @@ const testManagerCheckTime = 333 * time.Millisecond const testManagerCheckCapture = 500 * time.Millisecond func TestTLSConfigManager_WithCertLoaderOptions(t *testing.T) { - // This test verifies that WithLogger, WithCertificateCheckInterval, and WithExpirationAdvanced - // properly pass through to the underlying TLSCertLoader. + // This test verifies that WithLogger is properly passed to the underlying TLSCertLoader. // Create a certificate that expires in 24 hours notBefore := time.Now().UTC().Truncate(time.Minute).Add(-7 * 24 * time.Hour) @@ -1612,24 +1724,22 @@ func TestTLSConfigManager_WithCertLoaderOptions(t *testing.T) { core, logs := observer.New(zapcore.InfoLevel) logger := zap.New(core) + // No logger so logging has to come through manager. + monitor := newTestCertMonitor(t, WithMonitorExpirationAdvanced(2*24*time.Hour), WithMonitorLogger(logger), WithMonitorTriggerDelay(0)) + defer th.CheckedClose(t, monitor)() + // Create TLSConfigManager with all three options: // - WithLogger: to capture log output // - WithCertificateCheckInterval: to set a short check interval // - WithExpirationAdvanced: to set the expiration warning window to 2 days (> 24 hours remaining) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, - WithLogger(logger), - WithCertificateCheckInterval(testManagerCheckTime), - WithExpirationAdvanced(2*24*time.Hour)) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), + WithLogger(logger)) require.NoError(t, err) require.NotNil(t, manager) - defer func() { - require.NoError(t, manager.Close()) - }() - - // Wait for the certificate monitor to start - certLoader := manager.TLSCertLoader() - require.NotNil(t, certLoader) - certLoader.WaitForMonitorStart() + defer th.CheckedClose(t, manager)() // Verify the "Certificate will expire soon" warning is logged checkWarning := func(t *testing.T) { @@ -1646,6 +1756,7 @@ func TestTLSConfigManager_WithCertLoaderOptions(t *testing.T) { require.WithinDuration(t, notAfter, timeExpires, 2*time.Minute, "untilExpires varies more than expected") logs.TakeAll() // dump all logs } + time.Sleep(testWarnWaitTime) checkWarning(t) // Check for warning during monitor (after another check interval) @@ -1655,62 +1766,74 @@ func TestTLSConfigManager_WithCertLoaderOptions(t *testing.T) { } func TestTLSConfigManager_WithIgnoreFilePermissions(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + t.Run("fails without ignore when cert permissions too open", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) require.NoError(t, os.Chmod(ss.CertPath, 0660)) - _, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.ErrorContains(t, err, fmt.Sprintf("LoadCertificate: file permissions are too open: for %q, maximum is 0644 (-rw-r--r--) but found 0660 (-rw-rw----); extra permissions: 0020 (-----w----)", ss.CertPath)) + require.Nil(t, manager) }) t.Run("succeeds with ignore when cert permissions too open", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) require.NoError(t, os.Chmod(ss.CertPath, 0660)) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithIgnoreFilePermissions(true)) require.NoError(t, err) require.NotNil(t, manager) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Verify the manager works correctly tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) - certLoader := manager.TLSCertLoader() - require.NotNil(t, certLoader) }) t.Run("fails without ignore when key permissions too open", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) require.NoError(t, os.Chmod(ss.KeyPath, 0644)) - _, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false) + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) require.ErrorContains(t, err, fmt.Sprintf("LoadCertificate: file permissions are too open: for %q, maximum is 0600 (-rw-------) but found 0644 (-rw-r--r--); extra permissions: 0044 (----r--r--)", ss.KeyPath)) + require.Nil(t, manager) }) t.Run("succeeds with ignore when key permissions too open", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) require.NoError(t, os.Chmod(ss.KeyPath, 0644)) - manager, err := NewTLSConfigManager(true, nil, ss.CertPath, ss.KeyPath, false, + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), WithIgnoreFilePermissions(true)) require.NoError(t, err) require.NotNil(t, manager) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Verify the manager works correctly tlsConfig := manager.TLSConfig() require.NotNil(t, tlsConfig) - certLoader := manager.TLSCertLoader() - require.NotNil(t, certLoader) }) } func TestTLSConfigManager_DialWithDialer(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + testDialWithDialerConnection := func(t *testing.T, listener net.Listener, dial func(dialer *net.Dialer, addr string) (net.Conn, error)) { t.Helper() @@ -1750,27 +1873,24 @@ func TestTLSConfigManager_DialWithDialer(t *testing.T) { _, err = conn.Write(testData) require.NoError(t, err) - defer func() { - require.NoError(t, conn.Close()) - }() + defer th.CheckedClose(t, conn)() require.NoError(t, <-serverResult) require.Equal(t, testData, <-serverData) } t.Run("dialer LocalAddr is used for plain TCP", func(t *testing.T) { - manager, err := NewTLSConfigManager(false, nil, "/any/cert.pem", "/any/key.pem", false) + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(false), + WithServerCertificate("/any/cert.pem", "/any/key.pem")) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Create plain TCP server listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) - defer func() { - require.NoError(t, listener.Close()) - }() + defer th.CheckedClose(t, listener) testDialWithDialerConnection(t, listener, func(dialer *net.Dialer, addr string) (net.Conn, error) { return manager.DialWithDialer(dialer, "tcp", addr) @@ -1780,11 +1900,12 @@ func TestTLSConfigManager_DialWithDialer(t *testing.T) { t.Run("dialer LocalAddr is used for TLS", func(t *testing.T) { ss := selfsigned.NewSelfSignedCert(t) - manager, err := NewTLSConfigManager(true, nil, "", "", true) + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithAllowInsecure(true)) require.NoError(t, err) - defer func() { - require.NoError(t, manager.Close()) - }() + defer th.CheckedClose(t, manager)() // Create TLS server cert, err := tls.LoadX509KeyPair(ss.CertPath, ss.KeyPath) @@ -1793,9 +1914,7 @@ func TestTLSConfigManager_DialWithDialer(t *testing.T) { Certificates: []tls.Certificate{cert}, }) require.NoError(t, err) - defer func() { - require.NoError(t, listener.Close()) - }() + defer th.CheckedClose(t, listener)() testDialWithDialerConnection(t, listener, func(dialer *net.Dialer, addr string) (net.Conn, error) { return manager.DialWithDialer(dialer, "tcp", addr) diff --git a/pkg/tlsconfig/loadcert.go b/pkg/tlsconfig/loadcert.go new file mode 100644 index 00000000000..41b4dc1ea58 --- /dev/null +++ b/pkg/tlsconfig/loadcert.go @@ -0,0 +1,246 @@ +package tlsconfig + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "os" + "time" + + "github.com/influxdata/influxdb/pkg/file" + "go.uber.org/zap" +) + +const ( + // logCertContext is the key for log context with the certificate path. + // Use literals instead of constants in tests to ensure no one changes + // the constant unintentionally. + logCertContext = "cert" + + // logKeyContext is the key for log context with the key path. + // Use literals instead of constants in tests to ensure no one changes + // the constant unintentionally. + logKeyContext = "key" + + // logSerialContext is the key for log context with the certificate serial. + // Use literals instead of constants in tests to ensure no one changes + // the constant unintentionally. + logSerialContext = "serial" +) + +// X509Certificate is a wrapper around an x509.Certificate that adds +// some extra utility methods. +type X509Certificate struct { + *x509.Certificate +} + +// IsPremature determines if an x509 cert is premature (not valid yet). +// Returns true if certificate is premature, false otherwise. +func (xc *X509Certificate) IsPremature() bool { + return time.Now().Before(xc.NotBefore) +} + +// IsExpired determines if an x509 cert is expired. Returns if true if certificate +// is expired, false otherwise. +func (xc *X509Certificate) IsExpired() bool { + return time.Now().After(xc.NotAfter) +} + +// certExpiresSoon determines if an x509 cert is about to expire, based on expirationAdvanced. +// It also returns how long until the cert expires if we are within the expiration warn window. +func (xc *X509Certificate) ExpiresSoon(expirationAdvanced time.Duration) (bool, time.Duration) { + untilExpires := time.Until(xc.NotAfter) + if untilExpires < expirationAdvanced { + return true, untilExpires + } + return false, 0 +} + +// logIssues logs issues with xc. Issues include: +// - expired certificate +// - certificates that are about to expire +// - certificate that is not valid yet +func (xc *X509Certificate) logIssues(log *zap.Logger, expirationAdvanced time.Duration) { + if log == nil || xc == nil { + return + } + + if xc.IsExpired() { + log.Warn("Certificate is expired", zap.Time("NotAfter", xc.NotAfter)) + } else if xc.IsPremature() { + log.Warn("Certificate is not valid yet", zap.Time("NotBefore", xc.NotBefore)) + } else if expiresSoon, timeUntilExpires := xc.ExpiresSoon(expirationAdvanced); expiresSoon { + log.Warn("Certificate will expire soon", zap.Time("NotAfter", xc.NotAfter), zap.Duration("untilExpires", timeUntilExpires)) + } +} + +// LoadedCertificate encapsulates information about a loaded certificate. +type LoadedCertificate struct { + // valid indicates if this object is valid. + valid bool + + // CertPath is the path the certificate was loaded from. + CertificatePath string + + // KeyPath is the path the private key was loaded from. + KeyPath string + + // Certificate is the certificate that was loaded. + Certificate *tls.Certificate + + // Leaf is the parsed x509 certificate of Certificate's leaf certificate. + Leaf *x509.Certificate +} + +func (lc LoadedCertificate) IsValid() bool { + return lc.valid +} + +func (lc LoadedCertificate) IsEmpty() bool { + return !lc.IsValid() || (lc.CertificatePath == "" && lc.KeyPath == "") +} + +func (lc LoadedCertificate) Serial() string { + if lc.Leaf != nil { + return lc.Leaf.SerialNumber.String() + } + return "N/A" +} + +// WithLogContext adds context about lc to log and returns the new logger. +func (lc LoadedCertificate) WithLogContext(log *zap.Logger) *zap.Logger { + return log.With(zap.String(logCertContext, lc.CertificatePath), zap.String(logKeyContext, lc.KeyPath), zap.String(logSerialContext, lc.Serial())) +} + +// loadCertificateConfig is an internal config for LoadCertificate. +type loadCertificateConfig struct { + // ignoreFilePermissions indicates if file permissions should be ignored during load. + ignoreFilePermissions bool +} + +// LoadCertificateOpt are functions to change the behavior of LoadCertificate. +type LoadCertificateOpt func(*loadCertificateConfig) + +// WithLoadCertificateIgnoreFilePermissions instructs LoadCertificate to ignore file permissions +// if ignore is true. +func WithLoadCertificateIgnoreFilePermissions(ignore bool) LoadCertificateOpt { + return func(c *loadCertificateConfig) { + c.ignoreFilePermissions = ignore + } +} + +// LoadCertificate loads a key pair from certPath and keyPath, performing several checks +// along the way. If any checks fail or an error occurs loading the files, then an error is returned. +// If keyPath is empty, then certPath is assumed to contain both the certificate and the private key. +// Only trusted input (standard configuration files) should be used for certPath and keyPath. +func LoadCertificate(certPath, keyPath string, opts ...LoadCertificateOpt) (LoadedCertificate, error) { + fail := func(err error) (LoadedCertificate, error) { return LoadedCertificate{valid: false}, err } + + // Return empty certificate for empty paths. + if certPath == "" && keyPath == "" { + return LoadedCertificate{}, nil + } + + config := loadCertificateConfig{} + for _, o := range opts { + o(&config) + } + + if certPath == "" { + return fail(fmt.Errorf("LoadCertificate: certificate: %w", ErrPathEmpty)) + } + + if keyPath == "" { + // Assume key is combined with certificate. + keyPath = certPath + } + + wipeData := func(d []byte) { + for i := range d { + d[i] = 0 + } + } + + // Load the certificate and private key from their files. + loadFile := func(path string, maxPerms os.FileMode) (rData []byte, rErr error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("LoadCertificate: error opening %q for reading: %w", path, err) + } + defer func() { + if err := f.Close(); err != nil { + wipeData(rData) + rData = nil + rErr = errors.Join(rErr, fmt.Errorf("LoadCertificate: error closing file %q: %w", path, err)) + } + }() + + if !config.ignoreFilePermissions { + if err := file.VerifyFilePermissivenessF(f, maxPerms); err != nil { + // VerifyFilePermissivenessF includes a lot context in its errors. No need to add duplicate here. + return nil, fmt.Errorf("LoadCertificate: %w", err) + } + } + data, err := io.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("LoadCertificate: error data from %q: %w", path, err) + } + return data, nil + } + certData, err := loadFile(certPath, CertMaxPermissions) + defer wipeData(certData) + if err != nil { + return fail(err) + } + + keyData, err := loadFile(keyPath, KeyMaxPermissions) + defer wipeData(keyData) + if err != nil { + return fail(err) + } + + // Create key pair from loaded data + cert, err := tls.X509KeyPair(certData, keyData) + if err != nil { + return fail(fmt.Errorf("error loading x509 key pair (%q / %q): %w", certPath, keyPath, err)) + } + + // Parse the first X509 certificate in cert's chain. + // X509KeyPair() guarantees that cert.Certificate is not empty. + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + // This should be impossible to reach because `tls.X509KeyPair` will fail + // if the leaf certificate can't be parsed. + return fail(fmt.Errorf("error parsing leaf certificate (%q / %q): %w", certPath, keyPath, err)) + } + if leaf == nil { + // This shouldn't happen, but we should be extra careful with TLS certs. + return fail(fmt.Errorf("error parsing leaf certificate (%q / %q): %w", certPath, keyPath, ErrCertificateNil)) + } + + return LoadedCertificate{ + valid: true, + CertificatePath: certPath, + KeyPath: keyPath, + Certificate: &cert, + Leaf: leaf, + }, nil +} + +// GetLeaf returns the loaded leaf certificate, wrapped as a X509Certificate. +func (lc *LoadedCertificate) GetLeaf() (*X509Certificate, error) { + if !lc.IsValid() { + return nil, ErrCertificateInvalid + } + if lc.IsEmpty() { + return nil, ErrCertificateEmpty + } + if lc.Leaf == nil { + return nil, ErrCertificateNil + } + return &X509Certificate{ + Certificate: lc.Leaf, + }, nil +} diff --git a/services/httpd/config_test.go b/services/httpd/config_test.go index 1fc704a4a6d..0f745ef7c31 100644 --- a/services/httpd/config_test.go +++ b/services/httpd/config_test.go @@ -5,7 +5,10 @@ import ( "testing" "github.com/BurntSushi/toml" + th "github.com/influxdata/influxdb/pkg/testing/helper" + "github.com/influxdata/influxdb/pkg/tlsconfig" "github.com/influxdata/influxdb/services/httpd" + "github.com/stretchr/testify/require" ) func TestConfig_Parse(t *testing.T) { @@ -51,8 +54,12 @@ max-body-size = 100 } func TestConfig_WriteTracing(t *testing.T) { + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + defer th.CheckedClose(t, certMonitor)() + c := httpd.Config{WriteTracing: true} - s := httpd.NewService(c) + s := httpd.NewService(c, certMonitor) if !s.Handler.Config.WriteTracing { t.Fatalf("write tracing was not set") } diff --git a/services/httpd/service.go b/services/httpd/service.go index dc22b04598f..2b4f8f0eb68 100644 --- a/services/httpd/service.go +++ b/services/httpd/service.go @@ -61,6 +61,8 @@ type Service struct { // Fields above mu are not protected by mu and should be read-only after being // set in NewService. + certMonitor *tlsconfig.TLSCertMonitor + addr string httpsEnabled bool @@ -122,7 +124,7 @@ type Service struct { } // NewService returns a new instance of Service. -func NewService(c Config) *Service { +func NewService(c Config, certMonitor *tlsconfig.TLSCertMonitor) *Service { handler := NewHandler(c) // Convert the optional client-auth type to a *tls.ClientAuthType so a nil @@ -134,6 +136,7 @@ func NewService(c Config) *Service { } s := &Service{ + certMonitor: certMonitor, addr: c.BindAddress, httpsEnabled: c.HTTPSEnabled, httpsCertificate: c.HTTPSCertificate, @@ -178,7 +181,11 @@ func (s *Service) Open() error { s.Handler.Open() // Open listener. - tm, err := tlsconfig.NewTLSConfigManager(s.httpsEnabled, s.tlsConfig, s.httpsCertificate, s.httpsPrivateKey, false, + tm, err := tlsconfig.NewServerTLSConfigManager( + s.certMonitor, + tlsconfig.WithUseTLS(s.httpsEnabled), + tlsconfig.WithBaseConfig(s.tlsConfig), + tlsconfig.WithServerCertificate(s.httpsCertificate, s.httpsPrivateKey), tlsconfig.WithIgnoreFilePermissions(s.httpsInsecureCertificate), tlsconfig.WithClientAuthPtr(s.httpsClientAuthType), tlsconfig.WithClientCA(s.httpsClientCA), @@ -332,7 +339,8 @@ func (s *Service) PrepareReloadConfig(c Config) (func() error, error) { } // Make sure the specified certificate will load correctly and return an apply function. - if apply, err := s.tlsManager.PrepareCertificateLoad(c.HTTPSCertificate, c.HTTPSPrivateKey); err == nil { + if apply, err := s.tlsManager.PrepareReconfigure( + tlsconfig.WithServerCertificate(c.HTTPSCertificate, c.HTTPSPrivateKey)); err == nil { return apply, nil } else { return nil, fmt.Errorf("httpd: error loading certificate at (%q, %q): %w", c.HTTPSCertificate, c.HTTPSPrivateKey, err) diff --git a/services/httpd/service_test.go b/services/httpd/service_test.go index b7121f288f9..4eaf509d325 100644 --- a/services/httpd/service_test.go +++ b/services/httpd/service_test.go @@ -19,12 +19,16 @@ import ( ) func TestService_VerifyReloadedConfig(t *testing.T) { + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + defer th.CheckedClose(t, certMonitor)() + t.Run("HTTPS disabled, no change", func(t *testing.T) { // Create service with HTTPS disabled config := httpd.Config{ HTTPSEnabled: false, } - s := httpd.NewService(config) + s := httpd.NewService(config, certMonitor) s.WithLogger(zap.NewNop()) // Verify reload with HTTPS still disabled @@ -46,7 +50,7 @@ func TestService_VerifyReloadedConfig(t *testing.T) { HTTPSCertificate: ss.CertPath, HTTPSPrivateKey: ss.KeyPath, } - s := httpd.NewService(config) + s := httpd.NewService(config, certMonitor) s.WithLogger(zap.NewNop()) // Open service to initialize certLoader @@ -67,7 +71,7 @@ func TestService_VerifyReloadedConfig(t *testing.T) { config := httpd.Config{ HTTPSEnabled: false, } - s := httpd.NewService(config) + s := httpd.NewService(config, certMonitor) s.WithLogger(zap.NewNop()) // Create certificates for reload attempt @@ -96,7 +100,7 @@ func TestService_VerifyReloadedConfig(t *testing.T) { HTTPSPrivateKey: ss1.KeyPath, TLS: new(tls.Config), } - s := httpd.NewService(config) + s := httpd.NewService(config, certMonitor) s.WithLogger(zap.NewNop()) // Open service to initialize certLoader @@ -169,7 +173,7 @@ func TestService_VerifyReloadedConfig(t *testing.T) { HTTPSCertificate: ss.CertPath, HTTPSPrivateKey: ss.KeyPath, } - s := httpd.NewService(config) + s := httpd.NewService(config, certMonitor) s.WithLogger(zap.NewNop()) // Open service to initialize certLoader @@ -198,7 +202,7 @@ func TestService_VerifyReloadedConfig(t *testing.T) { HTTPSPrivateKey: ss.KeyPath, TLS: new(tls.Config), } - s := httpd.NewService(config) + s := httpd.NewService(config, certMonitor) s.WithLogger(zap.NewNop()) // Open service to initialize certLoader @@ -265,6 +269,10 @@ func TestService_Open_ClientCertAuth(t *testing.T) { serverSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithDNSName("localhost")) clientSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("client", "Client CA")) + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + defer th.CheckedClose(t, certMonitor)() + authType := toml.TlsClientAuthType(tls.RequireAndVerifyClientCert) s := httpd.NewService(httpd.Config{ BindAddress: "localhost:", @@ -274,7 +282,7 @@ func TestService_Open_ClientCertAuth(t *testing.T) { HTTPSClientAuthType: &authType, HTTPSClientCA: &tlsconfig.CAConfig{Paths: []string{clientSS.CACertPath}}, TLS: new(tls.Config), - }) + }, certMonitor) s.WithLogger(zap.NewNop()) require.NoError(t, s.Open()) defer th.CheckedClose(t, s)() @@ -327,7 +335,12 @@ func TestService_Open_ClientCertAuth(t *testing.T) { // openService creates, opens, and registers cleanup for an httpd.Service. func openService(t *testing.T, config httpd.Config) *httpd.Service { t.Helper() - s := httpd.NewService(config) + + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + s := httpd.NewService(config, certMonitor) s.WithLogger(zap.NewNop()) require.NoError(t, s.Open()) t.Cleanup(th.CheckedClose(t, s)) @@ -370,6 +383,10 @@ func loadCertSerial(t *testing.T, certPath, keyPath string) *big.Int { // TestService_Open_TLSConfigManager verifies that the TLSConfigManager created // in Service.Open is configured properly based on the Service's configuration. func TestService_Open_TLSConfigManager(t *testing.T) { + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + defer th.CheckedClose(t, certMonitor)() + t.Run("HTTPS disabled creates non-TLS listener", func(t *testing.T) { s := openService(t, httpd.Config{ BindAddress: "localhost:", @@ -402,7 +419,7 @@ func TestService_Open_TLSConfigManager(t *testing.T) { HTTPSEnabled: true, HTTPSCertificate: "/nonexistent/cert.pem", HTTPSPrivateKey: "/nonexistent/key.pem", - }) + }, certMonitor) s.WithLogger(zap.NewNop()) err := s.Open() @@ -449,7 +466,7 @@ func TestService_Open_TLSConfigManager(t *testing.T) { HTTPSCertificate: ss.CertPath, HTTPSPrivateKey: ss.KeyPath, HTTPSInsecureCertificate: false, - }) + }, certMonitor) s.WithLogger(zap.NewNop()) err := s.Open() diff --git a/services/opentsdb/service.go b/services/opentsdb/service.go index 64f6928ce55..1ee9aa46621 100644 --- a/services/opentsdb/service.go +++ b/services/opentsdb/service.go @@ -50,7 +50,10 @@ type Service struct { ln net.Listener // main listener httpln *chanListener // http channel-based listener - wg sync.WaitGroup + wg sync.WaitGroup + + certMonitor *tlsconfig.TLSCertMonitor + tls bool tlsManager *tlsconfig.TLSConfigManager tlsConfig *tls.Config @@ -94,7 +97,7 @@ type Service struct { } // NewService returns a new instance of Service. -func NewService(c Config) (*Service, error) { +func NewService(c Config, certMonitor *tlsconfig.TLSCertMonitor) (*Service, error) { // Use defaults where necessary. d := c.WithDefaults() @@ -107,6 +110,7 @@ func NewService(c Config) (*Service, error) { } s := &Service{ + certMonitor: certMonitor, tls: d.TLSEnabled, tlsConfig: d.TLS, cert: d.Certificate, @@ -152,7 +156,11 @@ func (s *Service) Open() error { go func() { defer s.wg.Done(); s.processBatches(s.batcher) }() // Open listener. - cm, err := tlsconfig.NewTLSConfigManager(s.tls, s.tlsConfig, s.cert, s.privateKey, false, + cm, err := tlsconfig.NewServerTLSConfigManager( + s.certMonitor, + tlsconfig.WithUseTLS(s.tls), + tlsconfig.WithBaseConfig(s.tlsConfig), + tlsconfig.WithServerCertificate(s.cert, s.privateKey), tlsconfig.WithIgnoreFilePermissions(s.insecureCert), tlsconfig.WithClientAuthPtr(s.clientAuthType), tlsconfig.WithClientCA(s.clientCA), @@ -239,7 +247,7 @@ func (s *Service) PrepareReloadTLSCertificates() (func() error, error) { return nil, errors.New("opentsdb: no TLS manager available") } - if apply, err := s.tlsManager.PrepareCertificateLoad(s.cert, s.privateKey); err == nil { + if apply, err := s.tlsManager.PrepareReconfigure(tlsconfig.WithServerCertificate(s.cert, s.privateKey)); err == nil { return apply, nil } else { return nil, fmt.Errorf("opentsdb: TLS certificate reload failed (%q, %q): %w", s.cert, s.privateKey, err) diff --git a/services/opentsdb/service_test.go b/services/opentsdb/service_test.go index e2042198530..daac98f6cee 100644 --- a/services/opentsdb/service_test.go +++ b/services/opentsdb/service_test.go @@ -14,9 +14,11 @@ import ( "time" "github.com/davecgh/go-spew/spew" + "github.com/influxdata/influxdb/internal" "github.com/influxdata/influxdb/logger" "github.com/influxdata/influxdb/models" + th "github.com/influxdata/influxdb/pkg/testing/helper" "github.com/influxdata/influxdb/pkg/testing/selfsigned" "github.com/influxdata/influxdb/pkg/tlsconfig" "github.com/influxdata/influxdb/services/meta" @@ -28,7 +30,7 @@ import ( func Test_Service_OpenClose(t *testing.T) { // Let the OS assign a random port since we are only opening and closing the service, // not actually connecting to it. - service := NewTestService("db0", "127.0.0.1:0") + service := NewTestService(t, "db0", "127.0.0.1:0") // Closing a closed service is fine. if err := service.Service.Close(); err != nil { @@ -69,7 +71,7 @@ func TestService_CreatesDatabase(t *testing.T) { t.Parallel() database := "db0" - s := NewTestService(database, "127.0.0.1:0") + s := NewTestService(t, database, "127.0.0.1:0") s.WritePointsFn = func(tsdb.WriteContext, string, string, models.ConsistencyLevel, []models.Point) error { return nil } @@ -145,7 +147,7 @@ func TestService_CreatesDatabase(t *testing.T) { func TestService_Telnet(t *testing.T) { t.Parallel() - s := NewTestService("db0", "127.0.0.1:0") + s := NewTestService(t, "db0", "127.0.0.1:0") if err := s.Service.Open(); err != nil { t.Fatal(err) } @@ -208,7 +210,7 @@ func TestService_Telnet(t *testing.T) { func TestService_HTTP(t *testing.T) { t.Parallel() - s := NewTestService("db0", "127.0.0.1:0") + s := NewTestService(t, "db0", "127.0.0.1:0") if err := s.Service.Open(); err != nil { t.Fatal(err) } @@ -255,26 +257,30 @@ func TestService_HTTP(t *testing.T) { } type TestService struct { + t *testing.T Service *Service MetaClient *internal.MetaClientMock WritePointsFn func(ctx tsdb.WriteContext, database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error + certMonitor *tlsconfig.TLSCertMonitor } // NewTestService returns a new instance of Service. -func NewTestService(database string, bind string) *TestService { +func NewTestService(t *testing.T, database string, bind string) *TestService { + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + defer th.CheckedClose(t, certMonitor)() + s, err := NewService(Config{ BindAddress: bind, Database: database, ConsistencyLevel: "one", - }) - - if err != nil { - panic(err) - } + }, certMonitor) + require.NoError(t, err) service := &TestService{ - Service: s, - MetaClient: &internal.MetaClientMock{}, + Service: s, + MetaClient: &internal.MetaClientMock{}, + certMonitor: certMonitor, } service.MetaClient.CreateDatabaseFn = func(db string) (*meta.DatabaseInfo, error) { @@ -293,6 +299,19 @@ func NewTestService(database string, bind string) *TestService { return service } +func (s *TestService) Close() error { + var allErrs []error + + if err := s.certMonitor.Close(); err != nil { + allErrs = append(allErrs, fmt.Errorf("error closing cert monitor: %w", err)) + } + if err := s.Service.Close(); err != nil { + allErrs = append(allErrs, fmt.Errorf("error closing opentsdb service: %w", err)) + } + + return errors.Join(allErrs...) +} + func (s *TestService) WritePointsPrivileged(ctx tsdb.WriteContext, database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error { return s.WritePointsFn(ctx, database, retentionPolicy, consistencyLevel, points) } @@ -316,6 +335,10 @@ func TestService_ClientCertAuth(t *testing.T) { serverSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithDNSName("localhost")) clientSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("client", "Client CA")) + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + defer th.CheckedClose(t, certMonitor) + authType := toml.TlsClientAuthType(tls.RequireAndVerifyClientCert) s, err := NewService(Config{ BindAddress: "127.0.0.1:0", @@ -327,7 +350,7 @@ func TestService_ClientCertAuth(t *testing.T) { ClientAuthType: &authType, ClientCA: &tlsconfig.CAConfig{Paths: []string{clientSS.CACertPath}}, TLS: new(tls.Config), - }) + }, certMonitor) require.NoError(t, err) // Wire the minimal dependencies so /api/put succeeds once the handshake passes. diff --git a/services/subscriber/mtls_internal_test.go b/services/subscriber/mtls_internal_test.go index 2548213b899..bc64a1ae5e1 100644 --- a/services/subscriber/mtls_internal_test.go +++ b/services/subscriber/mtls_internal_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + th "github.com/influxdata/influxdb/pkg/testing/helper" "github.com/influxdata/influxdb/pkg/testing/selfsigned" "github.com/influxdata/influxdb/pkg/tlsconfig" "github.com/influxdata/influxdb/services/meta" @@ -59,6 +60,10 @@ func TestHTTP_PresentsClientCertificate(t *testing.T) { serverSS := selfsigned.NewSelfSignedCert(t) clientSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("client", "Client CA")) + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + defer th.CheckedClose(t, certMonitor)() + var sawClientCert bool srv := newMTLSServer(t, serverSS, clientSS.CACertPath, &sawClientCert) defer srv.Close() @@ -73,11 +78,15 @@ func TestHTTP_PresentsClientCertificate(t *testing.T) { } newManager := func(withClientCert bool) *tlsconfig.TLSConfigManager { - opts := []tlsconfig.TLSConfigManagerOpt{tlsconfig.WithRootCA(conf.effectiveRootCA())} + opts := []tlsconfig.TLSConfigManagerOpt{ + tlsconfig.WithUseTLS(true), + tlsconfig.WithBaseConfig(conf.TLS), + tlsconfig.WithAllowInsecure(conf.InsecureSkipVerify), + tlsconfig.WithRootCA(conf.effectiveRootCA())} if withClientCert { - opts = append(opts, tlsconfig.WithCertificate(conf.Certificate, conf.PrivateKey)) + opts = append(opts, tlsconfig.WithServerCertificate(conf.Certificate, conf.PrivateKey)) } - cm, err := tlsconfig.NewClientTLSConfigManager(true, conf.TLS, conf.InsecureSkipVerify, opts...) + cm, err := tlsconfig.NewClientTLSConfigManager(certMonitor, opts...) require.NoError(t, err) return cm } @@ -126,8 +135,14 @@ func (stubMetaClient) WaitForDataChanged() chan struct{} { return make(chan stru func TestService_PrepareReloadTLSCertificates(t *testing.T) { open := func(t *testing.T, c Config) *Service { t.Helper() - s := NewService(c) + + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + s := NewService(c, certMonitor) s.MetaClient = stubMetaClient{} + require.NoError(t, s.Open()) t.Cleanup(func() { require.NoError(t, s.Close()) }) return s diff --git a/services/subscriber/service.go b/services/subscriber/service.go index 210949a19c0..0e5485bb747 100644 --- a/services/subscriber/service.go +++ b/services/subscriber/service.go @@ -121,6 +121,9 @@ type Service struct { conf Config subs map[subEntry]*chanWriter + // certMonitor is the TLS certificate monitor to use for tlsManager. + certMonitor *tlsconfig.TLSCertMonitor + // tlsManager builds the client TLS configuration for HTTPS writers and owns // the client certificate loader so rotated certificates can be reloaded. tlsManager *tlsconfig.TLSConfigManager @@ -130,13 +133,14 @@ type Service struct { } // NewService returns a subscriber service with given settings -func NewService(c Config) *Service { +func NewService(c Config, certMonitor *tlsconfig.TLSCertMonitor) *Service { stats := &Statistics{} s := &Service{ - Logger: zap.NewNop(), - stats: stats, - conf: c, - router: newSubscriptionRouter(stats), + Logger: zap.NewNop(), + stats: stats, + conf: c, + router: newSubscriptionRouter(stats), + certMonitor: certMonitor, } s.NewPointsWriter = s.newPointsWriter return s @@ -159,8 +163,12 @@ func (s *Service) Open() error { // Build the client TLS manager once for all HTTPS writers. It resolves // the root CAs (RootCA block plus the legacy ca-certs) and any client // certificate, and owns the certificate loader used for reloads. - cm, err := tlsconfig.NewClientTLSConfigManager(true, s.conf.TLS, s.conf.InsecureSkipVerify, - tlsconfig.WithCertificate(s.conf.Certificate, s.conf.PrivateKey), + cm, err := tlsconfig.NewClientTLSConfigManager( + s.certMonitor, + tlsconfig.WithUseTLS(true), + tlsconfig.WithBaseConfig(s.conf.TLS), + tlsconfig.WithAllowInsecure(s.conf.InsecureSkipVerify), + tlsconfig.WithClientCertificate(s.conf.Certificate, s.conf.PrivateKey), tlsconfig.WithRootCA(s.conf.effectiveRootCA()), tlsconfig.WithIgnoreFilePermissions(s.conf.InsecureCertificate), tlsconfig.WithLogger(s.Logger)) @@ -249,10 +257,10 @@ func (s *Service) PrepareReloadTLSCertificates() (func() error, error) { s.mu.Lock() defer s.mu.Unlock() - if s.tlsManager == nil || s.tlsManager.TLSCertLoader() == nil { + if s.tlsManager == nil { return func() error { return nil }, nil } - apply, err := s.tlsManager.PrepareCertificateLoad(s.conf.Certificate, s.conf.PrivateKey) + apply, err := s.tlsManager.PrepareReconfigure(tlsconfig.WithServerCertificate(s.conf.Certificate, s.conf.PrivateKey)) if err != nil { return nil, fmt.Errorf("subscriber: TLS certificate reload failed (%q, %q): %w", s.conf.Certificate, s.conf.PrivateKey, err) } diff --git a/services/subscriber/service_test.go b/services/subscriber/service_test.go index cea84cb5f1f..f91180c572b 100644 --- a/services/subscriber/service_test.go +++ b/services/subscriber/service_test.go @@ -13,6 +13,8 @@ import ( "github.com/influxdata/influxdb/coordinator" "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/pkg/testing/assert" + th "github.com/influxdata/influxdb/pkg/testing/helper" + "github.com/influxdata/influxdb/pkg/tlsconfig" "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxdb/services/subscriber" "github.com/stretchr/testify/require" @@ -42,7 +44,17 @@ func (s Subscription) WritePointsContext(_ context.Context, request subscriber.W return s.WritePointsFn(request) } +func newCertMonitor(t *testing.T) *tlsconfig.TLSCertMonitor { + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + return certMonitor +} + func TestService_IgnoreNonMatch(t *testing.T) { + certMonitor := newCertMonitor(t) + dataChanged := make(chan struct{}) ms := MetaClient{} ms.WaitForDataChangedFn = func() chan struct{} { @@ -76,7 +88,7 @@ func TestService_IgnoreNonMatch(t *testing.T) { return sub, nil } - s := subscriber.NewService(subscriber.NewConfig()) + s := subscriber.NewService(subscriber.NewConfig(), certMonitor) s.MetaClient = ms s.NewPointsWriter = newPointsWriter require.NoError(t, s.Open()) @@ -120,6 +132,7 @@ func TestService_IgnoreNonMatch(t *testing.T) { } func TestService_ModeALL(t *testing.T) { + certMonitor := newCertMonitor(t) dataChanged := make(chan struct{}) ms := MetaClient{} ms.WaitForDataChangedFn = func() chan struct{} { @@ -153,7 +166,7 @@ func TestService_ModeALL(t *testing.T) { return sub, nil } - s := subscriber.NewService(subscriber.NewConfig()) + s := subscriber.NewService(subscriber.NewConfig(), certMonitor) s.MetaClient = ms s.NewPointsWriter = newPointsWriter require.NoError(t, s.Open()) @@ -199,6 +212,7 @@ func TestService_ModeALL(t *testing.T) { } func TestService_ModeANY(t *testing.T) { + certMonitor := newCertMonitor(t) dataChanged := make(chan struct{}) ms := MetaClient{} ms.WaitForDataChangedFn = func() chan struct{} { @@ -232,7 +246,7 @@ func TestService_ModeANY(t *testing.T) { return sub, nil } - s := subscriber.NewService(subscriber.NewConfig()) + s := subscriber.NewService(subscriber.NewConfig(), certMonitor) s.MetaClient = ms s.NewPointsWriter = newPointsWriter require.NoError(t, s.Open()) @@ -282,6 +296,7 @@ func TestService_ModeANY(t *testing.T) { } func TestService_Multiple(t *testing.T) { + certMonitor := newCertMonitor(t) dataChanged := make(chan struct{}) ms := MetaClient{} ms.WaitForDataChangedFn = func() chan struct{} { @@ -321,7 +336,7 @@ func TestService_Multiple(t *testing.T) { return sub, nil } - s := subscriber.NewService(subscriber.NewConfig()) + s := subscriber.NewService(subscriber.NewConfig(), certMonitor) s.MetaClient = ms s.NewPointsWriter = newPointsWriter require.NoError(t, s.Open()) @@ -400,6 +415,7 @@ func TestService_Multiple(t *testing.T) { } func TestService_WaitForDataChanged(t *testing.T) { + certMonitor := newCertMonitor(t) dataChanged := make(chan struct{}, 1) defer close(dataChanged) ms := MetaClient{} @@ -445,7 +461,7 @@ func TestService_WaitForDataChanged(t *testing.T) { return currentDatabaseInfo } - s := subscriber.NewService(subscriber.NewConfig()) + s := subscriber.NewService(subscriber.NewConfig(), certMonitor) s.MetaClient = ms // Explicitly closed below for testing require.NoError(t, s.Open()) @@ -528,6 +544,7 @@ func TestService_WaitForDataChanged(t *testing.T) { } func TestService_BadUTF8(t *testing.T) { + certMonitor := newCertMonitor(t) dataChanged := make(chan struct{}) ms := MetaClient{} ms.WaitForDataChangedFn = func() chan struct{} { @@ -561,7 +578,7 @@ func TestService_BadUTF8(t *testing.T) { return sub, nil } - s := subscriber.NewService(subscriber.NewConfig()) + s := subscriber.NewService(subscriber.NewConfig(), certMonitor) s.MetaClient = ms s.NewPointsWriter = newPointsWriter require.NoError(t, s.Open()) From 0a1fd24ab964a39c9d8459499b58263cd49148e1 Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Tue, 14 Jul 2026 23:33:17 -0500 Subject: [PATCH 02/18] chore: code cleanups and comment fixes --- pkg/tlsconfig/certconfig.go | 35 +--------------------------------- pkg/tlsconfig/certmonitor.go | 28 ++++++++++++++++----------- pkg/tlsconfig/configmanager.go | 15 +++++++++++---- 3 files changed, 29 insertions(+), 49 deletions(-) diff --git a/pkg/tlsconfig/certconfig.go b/pkg/tlsconfig/certconfig.go index 516d4ed670e..4d1ca5dc7a6 100644 --- a/pkg/tlsconfig/certconfig.go +++ b/pkg/tlsconfig/certconfig.go @@ -58,9 +58,6 @@ type TLSCertLoader struct { // be set at construction time using WIthCertLoaderUsage. usage string - // closeCh is used to trigger closing the monitor. - closeCh chan struct{} - // mu protects all members below. All fields below can be set at construction // time or with Reconfigure. mu sync.RWMutex @@ -70,9 +67,6 @@ type TLSCertLoader struct { // config is the current configuration of the loader. config *tlsCertLoaderConfig - - // ignorePermissions is true if file permission checks should be bypassed. - ignorePermissions bool } // tlsCertLoaderConfig holds configuration data for TLSCertLoader. It is the actual @@ -91,10 +85,6 @@ type tlsCertLoaderConfig struct { // It is during a reconfiguration if ignoreFilePermissionsSet is false. ignoreFilePermissions bool - // ignoreFilePermissionsSet indicates if ignoreFilePermissions was set or is - // simply the default value. - ignoreFilePermissionsSet bool - // logger is the logger to use for logging. It is ignored during reconfiguration // if loggerSet is false. logger *zap.Logger @@ -134,7 +124,6 @@ func WithCertLoaderUsage(usage string) TLSCertLoaderOpt { func WithCertLoaderIgnoreFilePermissions(ignore bool) TLSCertLoaderOpt { return func(c *tlsCertLoaderConfig) { c.ignoreFilePermissions = ignore - c.ignoreFilePermissionsSet = true } } @@ -197,20 +186,6 @@ func (cl *TLSCertLoader) setLogger(logger *zap.Logger) { cl.logger = cl.logger.With(zap.String(logUsageContext, cl.usage)) } -// ignoreFilePermissions indicates if file permissions should be ignored on load. -func (cl *TLSCertLoader) ignoreFilePermissions() bool { - cl.mu.RLock() - defer cl.mu.RUnlock() - return cl.config.ignoreFilePermissions -} - -// SetIgnoreFilePermissions sets if file permissions should be ignored on load. -func (cl *TLSCertLoader) SetIgnoreFilePermissions(ignore bool) { - cl.mu.Lock() - defer cl.mu.Unlock() - cl.ignorePermissions = ignore -} - // Usage is the descriptive usage set using WithCertLoaderUsage. func (cl *TLSCertLoader) Usage() string { return cl.usage @@ -304,10 +279,6 @@ func (cl *TLSCertLoader) Leaf() *x509.Certificate { return cl.cert.Leaf } -func (cl *TLSCertLoader) loadCertificate(certPath, keyPath string) (LoadedCertificate, error) { - return LoadCertificate(certPath, keyPath, WithLoadCertificateIgnoreFilePermissions(cl.ignoreFilePermissions())) -} - // Load loads the certificate at the given certificate path and private keyfile path. // Only trusted input (standard configuration files) should be used for certPath and keyPath. func (cl *TLSCertLoader) Load(certPath, keyPath string) error { @@ -361,11 +332,7 @@ func (cl *TLSCertLoader) PrepareLoad(opts ...TLSCertLoaderOpt) (func() error, er } } - ignoreFilePermissions := cl.ignoreFilePermissions() - if c.ignoreFilePermissionsSet { - ignoreFilePermissions = c.ignoreFilePermissions - } - loadedCert, err := LoadCertificate(c.certPath, c.keyPath, WithLoadCertificateIgnoreFilePermissions(ignoreFilePermissions)) + loadedCert, err := LoadCertificate(c.certPath, c.keyPath, WithLoadCertificateIgnoreFilePermissions(c.ignoreFilePermissions)) if err != nil { logLoadError(err) return nil, err diff --git a/pkg/tlsconfig/certmonitor.go b/pkg/tlsconfig/certmonitor.go index 9bc17e47250..cac60243d05 100644 --- a/pkg/tlsconfig/certmonitor.go +++ b/pkg/tlsconfig/certmonitor.go @@ -14,10 +14,13 @@ const ( DefaultTriggerDelay = 5 * time.Second ) -// certMonitor is an internal class that implements periodic certificate -// monitoring. It avoids logging about a single certificate / key pair +// TLSCertMonitor implements periodic certificate monitoring. +// +// It avoids logging about a single certificate / key pair +// // multiple times, as well as the number of goroutines required to monitor -// certificates. +// certificates. There should be a single TLSCertMonitor in an application +// that is shared amongst all TLSConfigManager objects. type TLSCertMonitor struct { // log is the logger to use. log *zap.Logger @@ -82,7 +85,7 @@ type TLSCertMonitor struct { // TLSCertMonitorOpt is an option for NewTLSCertMonitor. type TLSCertMonitorOpt func(*TLSCertMonitor) -// WithMonitorCheckInternal sets the initial check interval for the monitor. +// WithMonitorCheckInterval sets the initial check interval for the monitor. // It can be changed later with SetCheckInterval. func WithMonitorCheckInterval(d time.Duration) TLSCertMonitorOpt { return func(m *TLSCertMonitor) { @@ -91,7 +94,7 @@ func WithMonitorCheckInterval(d time.Duration) TLSCertMonitorOpt { } // WithMonitorExpirationWarn sets the initial expiration warn time for the -// monitor. It can be changed later with SetExpirationWarn. +// monitor. It can be changed later with SetExpirationAdvanced. func WithMonitorExpirationAdvanced(d time.Duration) TLSCertMonitorOpt { return func(m *TLSCertMonitor) { m.certExpirationAdvanced = d @@ -181,7 +184,8 @@ func (m *TLSCertMonitor) Close() error { } // WaitForMonitorStop waits for the certificate monitor goroutine to stop. -// This is mainly useful for tests to avoid race conditions. +// This is mainly useful for tests to avoid race conditions. This will block +// forever if Open is not called before Close. func (m *TLSCertMonitor) WaitForMonitorStop() { m.monitorStopWg.Wait() } @@ -230,9 +234,10 @@ func (m *TLSCertMonitor) resetCheckInterval(d time.Duration) { func (m *TLSCertMonitor) SetCheckInterval(checkInterval time.Duration) { m.mu.Lock() - defer m.mu.Unlock() m.certCheckInterval = checkInterval - m.resetCheckInterval(m.certCheckInterval) + m.mu.Unlock() + + m.resetCheckInterval(checkInterval) } func (m *TLSCertMonitor) checkInterval() time.Duration { @@ -243,8 +248,9 @@ func (m *TLSCertMonitor) checkInterval() time.Duration { func (m *TLSCertMonitor) SetExpirationAdvanced(expirationAdvanced time.Duration) { m.mu.Lock() - defer m.mu.Unlock() m.certExpirationAdvanced = expirationAdvanced + m.mu.Unlock() + m.resetTrigger(m.triggerDelayInterval) } @@ -268,9 +274,9 @@ func (m *TLSCertMonitor) SetTriggerDelay(d time.Duration) { // have certificates have been reloaded. func (m *TLSCertMonitor) QueueWarnIssues(cl *TLSCertLoader) { m.mu.Lock() - defer m.mu.Unlock() - m.queuedCertLoaders = append(m.queuedCertLoaders, cl) + m.mu.Unlock() + m.resetTrigger(m.triggerDelayInterval) } diff --git a/pkg/tlsconfig/configmanager.go b/pkg/tlsconfig/configmanager.go index 16f103fa469..fe3c29d3361 100644 --- a/pkg/tlsconfig/configmanager.go +++ b/pkg/tlsconfig/configmanager.go @@ -97,7 +97,7 @@ func (r Role) IsClientRole() bool { // TLSConfigManager will manage a TLS configuration and make sure that only one instance of its tls.Config exists. // Different TLSConfigManager objects will have different configurations, even if they are instantiated in exactly -// the same way. No struct member is modified once the NewTLSConfigManager constructor is finished. +// the same way. type TLSConfigManager struct { // Fields above mu are not protected by mu and can only be set at construction time. @@ -233,7 +233,7 @@ type tlsConfigManagerConfig struct { // baseConfig is the *tls.Config to use as the basis for the manager's *tls.Config. baseConfig *tls.Config - // serverCertPath is the path to the server certificate. It is also used a s fallback for + // serverCertPath is the path to the server certificate. It is also used as fallback for // the client certificate. serverCertPath string @@ -286,6 +286,8 @@ func (c *tlsConfigManagerConfig) serverCertLoaderOpts() []TLSCertLoaderOpt { } // clientCertLoaderOpts returns the options needed for the client TLSCertLoader. +// If the client certificate is not set, then the server certificate will +// be used as a fallback. func (c *tlsConfigManagerConfig) clientCertLoaderOpts() []TLSCertLoaderOpt { certPath := c.clientCertPath keyPath := c.clientKeyPath @@ -325,8 +327,9 @@ func WithServerCertificate(certPath, keyPath string) TLSConfigManagerOpt { } } -// WithServerCertificate sets the config manager's client certificate and private -// key path. These will also be used as fallbacks for a client if no client +// WithClientCertificate sets the config manager's client certificate and private +// key path. If no client certificate is set, then the server certificate will +// be used as a fallback. func WithClientCertificate(certPath, keyPath string) TLSConfigManagerOpt { return func(cp *tlsConfigManagerConfig) { cp.clientCertPath = certPath @@ -735,6 +738,10 @@ func (cm *TLSConfigManager) copyCurrentConfig() *tlsConfigManagerConfig { // PrepareReconfigure creates an apply function for a new configuration of the configuration manager. func (cm *TLSConfigManager) PrepareReconfigure(opts ...TLSConfigManagerOpt) (func() error, error) { + if cm.disabled { + return nil, ErrConfigureDisabledManager + } + // Use the current configuration. If there are no opts that set a field, then the // current setting will continue to be used. c := cm.copyCurrentConfig() From 02cab0b5340c9f4c935902e1a9eabe4d8b010cbe Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 00:00:59 -0500 Subject: [PATCH 03/18] chore: add tests and improve existing tests --- pkg/tlsconfig/certconfig_test.go | 156 ++++++++++ pkg/tlsconfig/certmonitor_test.go | 437 ++++++++++++++++++++++++++++ pkg/tlsconfig/clientcert_test.go | 389 +++++++++++++++++++++++++ pkg/tlsconfig/configmanager_test.go | 214 +++++++++++++- pkg/tlsconfig/loadcert_test.go | 301 +++++++++++++++++++ pkg/tlsconfig/tls_config_test.go | 168 +++++++++++ 6 files changed, 1661 insertions(+), 4 deletions(-) create mode 100644 pkg/tlsconfig/certmonitor_test.go create mode 100644 pkg/tlsconfig/clientcert_test.go create mode 100644 pkg/tlsconfig/loadcert_test.go create mode 100644 pkg/tlsconfig/tls_config_test.go diff --git a/pkg/tlsconfig/certconfig_test.go b/pkg/tlsconfig/certconfig_test.go index bc106b3faf7..709ab890c03 100644 --- a/pkg/tlsconfig/certconfig_test.go +++ b/pkg/tlsconfig/certconfig_test.go @@ -729,3 +729,159 @@ func TestTLSCertLoader_SetupTLSConfig(t *testing.T) { require.Nil(t, tlsConfig.GetClientCertificate) }) } + +func TestTLSCertLoader_ConstructorValidation(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + t.Run("combined role is rejected", func(t *testing.T) { + // A cert loader holds one certificate, so it serves one role. A manager + // that acts as both uses two loaders. + cl, err := NewTLSCertLoader( + ServerAndClientRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) + require.ErrorIs(t, err, ErrSingleRoleRequired) + require.Nil(t, cl) + }) + + t.Run("invalid role is rejected", func(t *testing.T) { + cl, err := NewTLSCertLoader( + InvalidRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) + require.ErrorIs(t, err, ErrSingleRoleRequired) + require.Nil(t, cl) + }) + + t.Run("missing monitor is rejected", func(t *testing.T) { + cl, err := NewTLSCertLoader( + ServerOnlyRole, + nil, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) + require.ErrorIs(t, err, ErrNoCertificateMonitor) + require.Nil(t, cl) + }) + + t.Run("error names the usage", func(t *testing.T) { + cl, err := NewTLSCertLoader( + ServerAndClientRole, + monitor, + WithCertLoaderUsage("httpd.server"), + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) + require.ErrorContains(t, err, `usage="httpd.server"`) + require.Nil(t, cl) + }) +} + +func TestTLSCertLoader_GetCertificateWithoutCertificate(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + // Only a client loader can exist without a certificate; a server loader + // with an empty certificate is refused at construction. + cl, err := NewTLSCertLoader(ClientOnlyRole, monitor) + require.NoError(t, err) + require.NotNil(t, cl) + defer th.CheckedClose(t, cl)() + + require.Nil(t, cl.Certificate()) + require.Nil(t, cl.Leaf()) + require.True(t, cl.LoadedCertificate().IsEmpty()) + + cert, err := cl.GetCertificate(&tls.ClientHelloInfo{}) + require.ErrorIs(t, err, ErrCertificateNil) + require.Nil(t, cert) + + // GetClientCertificate must honor its contract and return a non-nil, + // empty certificate alongside the error. + clientCert, err := cl.GetClientCertificate(&tls.CertificateRequestInfo{ + SignatureSchemes: []tls.SignatureScheme{tls.PKCS1WithSHA256}, + }) + require.ErrorIs(t, err, ErrCertificateNil) + require.NotNil(t, clientCert) + require.Empty(t, clientCert.Certificate) + + // A loader with no certificate leaves the tls.Config callbacks unset so the + // config falls back to whatever the base config specified. + tlsConfig := &tls.Config{} + cl.SetupTLSConfig(tlsConfig) + require.Nil(t, tlsConfig.GetClientCertificate) + require.Nil(t, tlsConfig.GetCertificate) +} + +func TestTLSCertLoader_EmptyCertificateRejectedForServer(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + ss := selfsigned.NewSelfSignedCert(t) + + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, cl)() + + // Clearing the paths would leave the server with no certificate to present. + apply, err := cl.PrepareLoad(WithCertLoaderCertificate("", "")) + require.ErrorIs(t, err, ErrCertificateEmpty) + require.Nil(t, apply) + + // The previously loaded certificate is still in place. + certPath, _ := cl.Paths() + require.Equal(t, ss.CertPath, certPath) +} + +func TestTLSCertLoader_Clear(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + ss := selfsigned.NewSelfSignedCert(t) + + cl, err := NewTLSCertLoader( + ClientOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, cl)() + require.False(t, cl.LoadedCertificate().IsEmpty()) + + cl.Clear() + + require.True(t, cl.LoadedCertificate().IsEmpty()) + require.Nil(t, cl.Certificate()) + certPath, keyPath := cl.Paths() + require.Empty(t, certPath) + require.Empty(t, keyPath) + + // A cleared loader can be reloaded. + require.NoError(t, cl.Load(ss.CertPath, ss.KeyPath)) + certPath, keyPath = cl.Paths() + require.Equal(t, ss.CertPath, certPath) + require.Equal(t, ss.KeyPath, keyPath) +} + +func TestRole_Predicates(t *testing.T) { + tests := []struct { + name string + role Role + valid, single, serverRole, clientRole bool + }{ + {name: "invalid", role: InvalidRole}, + {name: "server only", role: ServerOnlyRole, valid: true, single: true, serverRole: true}, + {name: "client only", role: ClientOnlyRole, valid: true, single: true, clientRole: true}, + {name: "server and client", role: ServerAndClientRole, valid: true, serverRole: true, clientRole: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.valid, tt.role.IsValid()) + require.Equal(t, tt.single, tt.role.IsSingleRole()) + require.Equal(t, tt.serverRole, tt.role.IsServerRole()) + require.Equal(t, tt.clientRole, tt.role.IsClientRole()) + }) + } +} diff --git a/pkg/tlsconfig/certmonitor_test.go b/pkg/tlsconfig/certmonitor_test.go new file mode 100644 index 00000000000..70fc5293b2e --- /dev/null +++ b/pkg/tlsconfig/certmonitor_test.go @@ -0,0 +1,437 @@ +package tlsconfig + +import ( + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + th "github.com/influxdata/influxdb/pkg/testing/helper" + "github.com/influxdata/influxdb/pkg/testing/selfsigned" +) + +// newExpiredCert returns a self-signed certificate that expired a week ago. +func newExpiredCert(t *testing.T) *selfsigned.Cert { + t.Helper() + notAfter := time.Now().UTC().Truncate(time.Hour).Add(-7 * 24 * time.Hour) + return selfsigned.NewSelfSignedCert(t, + selfsigned.WithNotBefore(notAfter.Add(-7*24*time.Hour)), + selfsigned.WithNotAfter(notAfter)) +} + +// newQuietCertLoader creates a cert loader whose own logging is discarded, so +// only the monitor's log output is observed. +func newQuietCertLoader(t *testing.T, monitor *TLSCertMonitor, certPath, keyPath, usage string) *TLSCertLoader { + t.Helper() + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(certPath, keyPath), + WithCertLoaderLogger(zap.NewNop()), + WithCertLoaderUsage(usage)) + require.NoError(t, err) + require.NotNil(t, cl) + return cl +} + +// takeMessages drains every observed entry and returns those matching msg. +// It drains rather than filters so that each call only reports entries logged +// since the previous call: ObservedLogs.FilterMessage returns a copy, and +// calling TakeAll on that copy leaves the original entries in place. +func takeMessages(logs *observer.ObservedLogs, msg string) []observer.LoggedEntry { + var matched []observer.LoggedEntry + for _, e := range logs.TakeAll() { + if e.Message == msg { + matched = append(matched, e) + } + } + return matched +} + +func takeExpiredWarnings(logs *observer.ObservedLogs) []observer.LoggedEntry { + return takeMessages(logs, "Certificate is expired") +} + +func TestTLSCertMonitor_Defaults(t *testing.T) { + t.Run("unset options use defaults", func(t *testing.T) { + m := NewTLSCertMonitor() + require.NotNil(t, m) + + require.Equal(t, DefaultCertificateCheckTime, m.checkInterval()) + require.Equal(t, DefaultExpirationWarnTime, m.expirationAdvanced()) + require.Equal(t, DefaultTriggerDelay, m.triggerDelayInterval) + require.NotNil(t, m.logger(), "a monitor must always have a usable logger") + }) + + t.Run("non-positive durations fall back to defaults", func(t *testing.T) { + m := NewTLSCertMonitor( + WithMonitorCheckInterval(0), + WithMonitorExpirationAdvanced(-time.Second)) + require.NotNil(t, m) + + require.Equal(t, DefaultCertificateCheckTime, m.checkInterval()) + require.Equal(t, DefaultExpirationWarnTime, m.expirationAdvanced()) + }) + + t.Run("options are honored", func(t *testing.T) { + m := NewTLSCertMonitor( + WithMonitorCheckInterval(time.Minute), + WithMonitorExpirationAdvanced(time.Hour), + WithMonitorTriggerDelay(time.Second)) + require.NotNil(t, m) + + require.Equal(t, time.Minute, m.checkInterval()) + require.Equal(t, time.Hour, m.expirationAdvanced()) + require.Equal(t, time.Second, m.triggerDelayInterval) + }) +} + +func TestTLSCertMonitor_OpenCloseIdempotent(t *testing.T) { + m := NewTLSCertMonitor(WithMonitorCheckInterval(time.Hour)) + require.NotNil(t, m) + + // Open starts exactly one goroutine no matter how many times it is called. + require.NoError(t, m.Open()) + require.NoError(t, m.Open()) + m.WaitForMonitorStart() + + require.NoError(t, m.Close()) + require.NoError(t, m.Close()) + m.WaitForMonitorStop() +} + +func TestTLSCertMonitor_SetLogger(t *testing.T) { + ss := newExpiredCert(t) + + coreA, logsA := observer.New(zapcore.InfoLevel) + coreB, logsB := observer.New(zapcore.InfoLevel) + + // A long check interval keeps the periodic check from firing so the test + // controls exactly when certificates are checked. + monitor := newTestCertMonitor(t, + WithMonitorLogger(zap.New(coreA)), + WithMonitorCheckInterval(time.Hour), + WithMonitorTriggerDelay(time.Millisecond)) + defer th.CheckedClose(t, monitor)() + + cl := newQuietCertLoader(t, monitor, ss.CertPath, ss.KeyPath, "httpd.server") + defer th.CheckedClose(t, cl)() + + // The load queued at construction warns through the original logger. + time.Sleep(testWarnWaitTime) + require.Len(t, takeExpiredWarnings(logsA), 1) + require.Empty(t, takeExpiredWarnings(logsB)) + + monitor.SetLogger(zap.New(coreB)) + + // A subsequent check must warn through the new logger only. + require.NoError(t, cl.Load(ss.CertPath, ss.KeyPath)) + time.Sleep(testWarnWaitTime) + + require.Len(t, takeExpiredWarnings(logsB), 1, "warnings should go to the logger set by SetLogger") + require.Empty(t, takeExpiredWarnings(logsA), "the replaced logger should no longer receive warnings") +} + +func TestTLSCertMonitor_SetCheckInterval(t *testing.T) { + ss := newExpiredCert(t) + + core, logs := observer.New(zapcore.InfoLevel) + + // Both the periodic check and the queued trigger are effectively disabled, + // so nothing is checked until SetCheckInterval shortens the interval. + monitor := newTestCertMonitor(t, + WithMonitorLogger(zap.New(core)), + WithMonitorCheckInterval(time.Hour), + WithMonitorTriggerDelay(time.Hour)) + defer th.CheckedClose(t, monitor)() + + cl := newQuietCertLoader(t, monitor, ss.CertPath, ss.KeyPath, "httpd.server") + defer th.CheckedClose(t, cl)() + + time.Sleep(testWarnWaitTime) + require.Empty(t, takeExpiredWarnings(logs), "no check should have run yet") + + monitor.SetCheckInterval(testCheckTime) + require.Equal(t, testCheckTime, monitor.checkInterval()) + + time.Sleep(testCheckCapture) + require.NotEmpty(t, takeExpiredWarnings(logs), "shortened check interval should trigger a periodic check") +} + +func TestTLSCertMonitor_SetExpirationAdvanced(t *testing.T) { + // A certificate that expires in a day: inside a 48 hour warn window but + // outside a one hour window. + notAfter := time.Now().UTC().Truncate(time.Hour).Add(24 * time.Hour) + ss := selfsigned.NewSelfSignedCert(t, + selfsigned.WithNotBefore(time.Now().UTC().Add(-24*time.Hour)), + selfsigned.WithNotAfter(notAfter)) + + core, logs := observer.New(zapcore.InfoLevel) + + monitor := newTestCertMonitor(t, + WithMonitorLogger(zap.New(core)), + WithMonitorCheckInterval(time.Hour), + WithMonitorExpirationAdvanced(time.Hour), + WithMonitorTriggerDelay(time.Millisecond)) + defer th.CheckedClose(t, monitor)() + + cl := newQuietCertLoader(t, monitor, ss.CertPath, ss.KeyPath, "httpd.server") + defer th.CheckedClose(t, cl)() + + time.Sleep(testWarnWaitTime) + require.Empty(t, takeMessages(logs, "Certificate will expire soon"), + "certificate expires outside the one hour warn window") + + // Widening the warn window re-checks the registered certificates, so the + // same certificate now warns without needing a reload. + monitor.SetExpirationAdvanced(48 * time.Hour) + require.Equal(t, 48*time.Hour, monitor.expirationAdvanced()) + + time.Sleep(testWarnWaitTime) + warnings := takeMessages(logs, "Certificate will expire soon") + require.Len(t, warnings, 1, "widened warn window should trigger a re-check") + require.Equal(t, ss.CertPath, warnings[0].ContextMap()["cert"]) + require.Equal(t, notAfter, warnings[0].ContextMap()["NotAfter"]) +} + +func TestTLSCertMonitor_SetTriggerDelay(t *testing.T) { + ss := newExpiredCert(t) + + core, logs := observer.New(zapcore.InfoLevel) + + monitor := newTestCertMonitor(t, + WithMonitorLogger(zap.New(core)), + WithMonitorCheckInterval(time.Hour), + WithMonitorTriggerDelay(time.Hour)) + defer th.CheckedClose(t, monitor)() + + cl := newQuietCertLoader(t, monitor, ss.CertPath, ss.KeyPath, "httpd.server") + defer th.CheckedClose(t, cl)() + + time.Sleep(testWarnWaitTime) + require.Empty(t, takeExpiredWarnings(logs), "the long trigger delay should defer the queued check") + + // SetTriggerDelay only affects the next queued trigger; it does not reset a + // trigger that is already pending. + monitor.SetTriggerDelay(time.Millisecond) + time.Sleep(testWarnWaitTime) + require.Empty(t, takeExpiredWarnings(logs), "SetTriggerDelay should not fire an already pending trigger") + + // Queueing a new warning picks up the shortened delay. + require.NoError(t, cl.Load(ss.CertPath, ss.KeyPath)) + time.Sleep(testWarnWaitTime) + require.NotEmpty(t, takeExpiredWarnings(logs), "next queued trigger should use the new delay") +} + +func TestTLSCertMonitor_DuplicateCertificatesGrouped(t *testing.T) { + ss := newExpiredCert(t) + + core, logs := observer.New(zapcore.InfoLevel) + + monitor := newTestCertMonitor(t, + WithMonitorLogger(zap.New(core)), + WithMonitorCheckInterval(time.Hour), + WithMonitorTriggerDelay(time.Hour)) + defer th.CheckedClose(t, monitor)() + + // Two services sharing one certificate should produce one warning listing + // both usages, rather than one warning per service. + cl1 := newQuietCertLoader(t, monitor, ss.CertPath, ss.KeyPath, "opentsdb.server") + defer th.CheckedClose(t, cl1)() + cl2 := newQuietCertLoader(t, monitor, ss.CertPath, ss.KeyPath, "httpd.server") + defer th.CheckedClose(t, cl2)() + + monitor.SetCheckInterval(testCheckTime) + time.Sleep(testCheckCapture) + + warnings := takeExpiredWarnings(logs) + require.Len(t, warnings, 1, "a shared certificate should only be reported once") + require.Equal(t, ss.CertPath, warnings[0].ContextMap()["cert"]) + require.Equal(t, []any{"httpd.server", "opentsdb.server"}, warnings[0].ContextMap()["usages"], + "usages should be merged and sorted") +} + +func TestTLSCertMonitor_DistinctCertificatesLoggedSeparately(t *testing.T) { + ss1 := newExpiredCert(t) + ss2 := newExpiredCert(t) + + core, logs := observer.New(zapcore.InfoLevel) + + monitor := newTestCertMonitor(t, + WithMonitorLogger(zap.New(core)), + WithMonitorCheckInterval(time.Hour), + WithMonitorTriggerDelay(time.Hour)) + defer th.CheckedClose(t, monitor)() + + cl1 := newQuietCertLoader(t, monitor, ss1.CertPath, ss1.KeyPath, "httpd.server") + defer th.CheckedClose(t, cl1)() + cl2 := newQuietCertLoader(t, monitor, ss2.CertPath, ss2.KeyPath, "opentsdb.server") + defer th.CheckedClose(t, cl2)() + + monitor.SetCheckInterval(testCheckTime) + time.Sleep(testCheckCapture) + + warnings := takeExpiredWarnings(logs) + require.Len(t, warnings, 2, "distinct certificates must each be reported") + + byCert := make(map[string][]any, len(warnings)) + for _, w := range warnings { + byCert[w.ContextMap()["cert"].(string)] = w.ContextMap()["usages"].([]any) + } + require.Equal(t, []any{"httpd.server"}, byCert[ss1.CertPath]) + require.Equal(t, []any{"opentsdb.server"}, byCert[ss2.CertPath]) +} + +func TestTLSCertMonitor_UnregisterStopsMonitoring(t *testing.T) { + ss := newExpiredCert(t) + + core, logs := observer.New(zapcore.InfoLevel) + + monitor := newTestCertMonitor(t, + WithMonitorLogger(zap.New(core)), + WithMonitorCheckInterval(time.Hour), + WithMonitorTriggerDelay(time.Hour)) + defer th.CheckedClose(t, monitor)() + + cl := newQuietCertLoader(t, monitor, ss.CertPath, ss.KeyPath, "httpd.server") + require.Len(t, takeMessages(logs, "Registered certificate loader"), 1) + + // Closing the loader unregisters it, so its certificate is no longer checked. + require.NoError(t, cl.Close()) + unregistered := takeMessages(logs, "Unregistered certificate loader") + require.Len(t, unregistered, 1) + require.Equal(t, "httpd.server", unregistered[0].ContextMap()["usage"]) + + // Close is safe to call more than once and only unregisters once. + require.NoError(t, cl.Close()) + require.Empty(t, takeMessages(logs, "Unregistered certificate loader"), + "a second Close should not log another unregistration") + + monitor.SetCheckInterval(testCheckTime) + time.Sleep(testCheckCapture) + require.Empty(t, takeExpiredWarnings(logs), "an unregistered certificate should not be checked") +} + +func TestTLSCertMonitor_ClosedMonitorDoesNotBlock(t *testing.T) { + ss := newExpiredCert(t) + + monitor := newTestCertMonitor(t, WithMonitorCheckInterval(time.Hour)) + cl := newQuietCertLoader(t, monitor, ss.CertPath, ss.KeyPath, "httpd.server") + defer th.CheckedClose(t, cl)() + + require.NoError(t, monitor.Close()) + monitor.WaitForMonitorStop() + + // Once the monitor goroutine has stopped nothing drains the reset channels. + // These calls must not block, and loads must keep working. The counts here + // exceed the reset channel depth to catch a regression that only appears + // once the buffer fills. + done := make(chan struct{}) + go func() { + defer close(done) + for range 50 { + monitor.SetCheckInterval(time.Minute) + monitor.SetExpirationAdvanced(time.Hour) + require.NoError(t, cl.Load(ss.CertPath, ss.KeyPath)) + } + }() + + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("monitor calls blocked after the monitor was closed") + } +} + +func TestTLSCertMonitor_LogIssuesGuards(t *testing.T) { + monitor := newTestCertMonitor(t, WithMonitorCheckInterval(time.Hour)) + defer th.CheckedClose(t, monitor)() + + core, logs := observer.New(zapcore.DebugLevel) + log := zap.New(core) + + t.Run("nil logger does not panic", func(t *testing.T) { + require.NotPanics(t, func() { + monitor.logIssues(nil, LoadedCertificate{valid: true, CertificatePath: "cert.pem"}) + }) + }) + + t.Run("empty certificate reports nothing", func(t *testing.T) { + monitor.logIssues(log, LoadedCertificate{}) + require.Zero(t, logs.Len(), "an empty certificate has no issues to report") + }) + + t.Run("unusable leaf is reported", func(t *testing.T) { + // A certificate with no leaf cannot come out of LoadCertificate. If one + // ever appears, the monitor should say so rather than skip it silently. + monitor.logIssues(log, LoadedCertificate{valid: true, CertificatePath: "cert.pem"}) + + entries := takeMessages(logs, "error logging certificate issues") + require.Len(t, entries, 1) + require.Equal(t, zap.ErrorLevel, entries[0].Level) + }) +} + +// TestTLSCertMonitor_RotatedCertificateAtSamePath covers two loaders reading the +// same certificate path where one loaded before the file was replaced. They are +// grouped by serial as well as path, so both the stale and the current +// certificate are reported. +func TestTLSCertMonitor_RotatedCertificateAtSamePath(t *testing.T) { + copyFile := func(t *testing.T, src, dst string) { + t.Helper() + data, err := os.ReadFile(src) + require.NoError(t, err) + require.NoError(t, os.WriteFile(dst, data, 0600)) + } + + dir := t.TempDir() + certPath := path.Join(dir, "cert.pem") + keyPath := path.Join(dir, "key.pem") + + core, logs := observer.New(zapcore.InfoLevel) + + monitor := newTestCertMonitor(t, + WithMonitorLogger(zap.New(core)), + WithMonitorCheckInterval(time.Hour), + WithMonitorTriggerDelay(time.Hour)) + defer th.CheckedClose(t, monitor)() + + original := newExpiredCert(t) + copyFile(t, original.CertPath, certPath) + copyFile(t, original.KeyPath, keyPath) + + cl1 := newQuietCertLoader(t, monitor, certPath, keyPath, "httpd.server") + defer th.CheckedClose(t, cl1)() + + // Replace the files on disk. cl1 keeps serving the certificate it already + // loaded; cl2 picks up the replacement. + rotated := newExpiredCert(t) + copyFile(t, rotated.CertPath, certPath) + copyFile(t, rotated.KeyPath, keyPath) + + cl2 := newQuietCertLoader(t, monitor, certPath, keyPath, "opentsdb.server") + defer th.CheckedClose(t, cl2)() + + originalSerial := cl1.Leaf().SerialNumber.String() + rotatedSerial := cl2.Leaf().SerialNumber.String() + require.NotEqual(t, originalSerial, rotatedSerial, "the rotated certificate should be a different one") + + monitor.SetCheckInterval(testCheckTime) + time.Sleep(testCheckCapture) + + warnings := takeExpiredWarnings(logs) + require.Len(t, warnings, 2, "certificates sharing a path but not a serial are reported separately") + + bySerial := make(map[string][]any, len(warnings)) + for _, w := range warnings { + require.Equal(t, certPath, w.ContextMap()["cert"]) + bySerial[w.ContextMap()["serial"].(string)] = w.ContextMap()["usages"].([]any) + } + require.Equal(t, []any{"httpd.server"}, bySerial[originalSerial]) + require.Equal(t, []any{"opentsdb.server"}, bySerial[rotatedSerial]) +} diff --git a/pkg/tlsconfig/clientcert_test.go b/pkg/tlsconfig/clientcert_test.go new file mode 100644 index 00000000000..252f9f36e26 --- /dev/null +++ b/pkg/tlsconfig/clientcert_test.go @@ -0,0 +1,389 @@ +package tlsconfig + +// Tests for TLSConfigManager's support for a client certificate that is +// separate from the server certificate, including the fallback to the server +// certificate when no client certificate is configured. + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "os" + "path" + "testing" + + "github.com/stretchr/testify/require" + + th "github.com/influxdata/influxdb/pkg/testing/helper" + "github.com/influxdata/influxdb/pkg/testing/selfsigned" +) + +// acceptableCAs builds the AcceptableCAs list a server trusting ss's CA would +// send in a certificate request. +func acceptableCAs(t *testing.T, ss *selfsigned.Cert) [][]byte { + t.Helper() + caPEM, err := os.ReadFile(ss.CACertPath) + require.NoError(t, err) + + block, _ := pem.Decode(caPEM) + require.NotNil(t, block) + parsed, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + + return [][]byte{parsed.RawSubject} +} + +// certRequestFrom builds a CertificateRequestInfo for a server that trusts ss's +// CA. The selfsigned package issues RSA certificates. +func certRequestFrom(t *testing.T, ss *selfsigned.Cert) *tls.CertificateRequestInfo { + t.Helper() + return &tls.CertificateRequestInfo{ + SignatureSchemes: []tls.SignatureScheme{tls.PKCS1WithSHA256}, + AcceptableCAs: acceptableCAs(t, ss), + } +} + +// leafSerial returns the serial of the certificate a loader currently holds. +func leafSerial(t *testing.T, cl *TLSCertLoader) string { + t.Helper() + require.NotNil(t, cl) + leaf := cl.Leaf() + require.NotNil(t, leaf) + return leaf.SerialNumber.String() +} + +func TestTLSConfigManager_WithClientCertificate_Paths(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + t.Run("separate client and server certificates", func(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t) + + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCertificate(clientSS.CertPath, clientSS.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + serverCert, serverKey := manager.serverCertLoader.Paths() + require.Equal(t, serverSS.CertPath, serverCert) + require.Equal(t, serverSS.KeyPath, serverKey) + + clientCert, clientKey := manager.clientCertLoader.Paths() + require.Equal(t, clientSS.CertPath, clientCert) + require.Equal(t, clientSS.KeyPath, clientKey) + + // The loaders must hold genuinely different certificates, not two + // handles onto the same one. + require.NotEqual(t, leafSerial(t, manager.serverCertLoader), leafSerial(t, manager.clientCertLoader)) + }) + + t.Run("client falls back to server certificate", func(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + clientCert, clientKey := manager.clientCertLoader.Paths() + require.Equal(t, serverSS.CertPath, clientCert, "client should fall back to the server certificate") + require.Equal(t, serverSS.KeyPath, clientKey) + require.Equal(t, leafSerial(t, manager.serverCertLoader), leafSerial(t, manager.clientCertLoader)) + }) + + t.Run("client-only manager uses client certificate", func(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t) + + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCertificate(clientSS.CertPath, clientSS.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + require.Nil(t, manager.serverCertLoader, "a client-only manager has no server cert loader") + + clientCert, clientKey := manager.clientCertLoader.Paths() + require.Equal(t, clientSS.CertPath, clientCert) + require.Equal(t, clientSS.KeyPath, clientKey) + }) + + t.Run("server-only manager ignores client certificate", func(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t) + + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCertificate(clientSS.CertPath, clientSS.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + require.Nil(t, manager.clientCertLoader, "a server-only manager has no client cert loader") + + serverCert, _ := manager.serverCertLoader.Paths() + require.Equal(t, serverSS.CertPath, serverCert) + }) + + t.Run("client certificate without key uses a combined file", func(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + combinedSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCombinedFile()) + + // Setting only the client certificate path suppresses the fallback + // entirely: the key is read from the certificate file rather than + // borrowed from the server configuration. + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithIgnoreFilePermissions(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCertificate(combinedSS.CertPath, "")) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + clientCert, clientKey := manager.clientCertLoader.Paths() + require.Equal(t, combinedSS.CertPath, clientCert) + require.Equal(t, combinedSS.CertPath, clientKey, "key should come from the combined certificate file") + }) + + t.Run("client key without certificate is an error", func(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t) + + // A client key with no client certificate does not fall back to the + // server pair; there is no certificate to pair the key with. + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCertificate("", clientSS.KeyPath)) + require.ErrorIs(t, err, ErrPathEmpty) + require.Nil(t, manager) + }) + + t.Run("unloadable client certificate is an error", func(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + missing := path.Join(t.TempDir(), "absent.pem") + + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCertificate(missing, missing)) + require.ErrorContains(t, err, "error configuring client cert loader") + require.ErrorContains(t, err, "no such file or directory") + require.Nil(t, manager) + }) +} + +func TestTLSConfigManager_WithClientCertificate_TLSConfigCallbacks(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t) + + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCertificate(clientSS.CertPath, clientSS.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + tlsConfig := manager.TLSConfig() + require.NotNil(t, tlsConfig) + require.NotNil(t, tlsConfig.GetCertificate) + require.NotNil(t, tlsConfig.GetClientCertificate) + + t.Run("GetCertificate returns the server certificate", func(t *testing.T) { + cert, err := tlsConfig.GetCertificate(&tls.ClientHelloInfo{}) + require.NoError(t, err) + require.NotNil(t, cert) + require.Equal(t, manager.serverCertLoader.Certificate(), cert) + }) + + t.Run("GetClientCertificate returns the client certificate", func(t *testing.T) { + cert, err := tlsConfig.GetClientCertificate(certRequestFrom(t, clientSS)) + require.NoError(t, err) + require.NotNil(t, cert) + require.Equal(t, manager.clientCertLoader.Certificate(), cert, + "the client certificate, not the server certificate, must be offered") + }) + + t.Run("GetClientCertificate withholds a certificate the server won't accept", func(t *testing.T) { + otherSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("other", "Other CA")) + + // Go's behavior when no configured certificate matches the request is + // to send an empty certificate rather than fail the handshake locally. + cert, err := tlsConfig.GetClientCertificate(certRequestFrom(t, otherSS)) + require.NoError(t, err) + require.NotNil(t, cert) + require.Empty(t, cert.Certificate) + }) +} + +// TestTLSConfigManager_ClientCertificateMTLS exercises a real handshake where +// the server trusts only the CA that issued the separate client certificate. +// The handshake can therefore only succeed if the client presents its client +// certificate rather than its server certificate. +func TestTLSConfigManager_ClientCertificateMTLS(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + newServer := func(t *testing.T, serverSS, clientCASS *selfsigned.Cert) *TLSConfigManager { + t.Helper() + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCA(&CAConfig{Paths: []string{clientCASS.CACertPath}}), + WithClientAuth(tls.RequireAndVerifyClientCert)) + require.NoError(t, err) + return manager + } + + t.Run("separate client certificate is accepted", func(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("client_org", "Client CA")) + + // The server trusts only the client CA, so the server certificate would + // not be accepted if it were offered instead. + serverManager := newServer(t, serverSS, clientSS) + defer th.CheckedClose(t, serverManager)() + + listener, err := serverManager.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + testData := []byte("hello") + serverDone := make(chan error, 1) + go simpleEchoServer(serverDone, listener, len(testData)) + + clientManager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCertificate(clientSS.CertPath, clientSS.KeyPath), + WithAllowInsecure(true)) + require.NoError(t, err) + defer th.CheckedClose(t, clientManager)() + + conn, err := clientManager.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer th.CheckedClose(t, conn)() + + n, err := conn.Write(testData) + require.NoError(t, err) + require.Equal(t, len(testData), n) + + buf := make([]byte, len(testData)) + n, err = conn.Read(buf) + require.NoError(t, err) + require.Equal(t, len(testData), n) + require.Equal(t, testData, buf) + + require.NoError(t, <-serverDone) + }) + + t.Run("fallback server certificate is rejected", func(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("client_org", "Client CA")) + + serverManager := newServer(t, serverSS, clientSS) + defer th.CheckedClose(t, serverManager)() + + listener, err := serverManager.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + testData := []byte("hello") + serverDone := make(chan error, 1) + go simpleEchoServer(serverDone, listener, len(testData)) + + // Without a client certificate the manager falls back to the server + // certificate, which this server does not trust. + clientManager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithAllowInsecure(true)) + require.NoError(t, err) + defer th.CheckedClose(t, clientManager)() + + conn, dialErr := clientManager.Dial("tcp", listener.Addr().String()) + if dialErr == nil { + defer func() { + require.NoError(t, conn.Close()) + }() + buf := make([]byte, 1) + _, dialErr = conn.Read(buf) + } + require.ErrorContains(t, dialErr, "remote error: tls: certificate required") + + serverErr := <-serverDone + require.ErrorContains(t, serverErr, "tls: client didn't provide a certificate") + }) +} + +// TestTLSConfigManager_ReconfigureClientCertificate covers reconfiguring the +// client certificate independently of the server certificate. +func TestTLSConfigManager_ReconfigureClientCertificate(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t) + newClientSS := selfsigned.NewSelfSignedCert(t) + + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCertificate(clientSS.CertPath, clientSS.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + serverSerial := leafSerial(t, manager.serverCertLoader) + + apply, err := manager.PrepareReconfigure( + WithClientCertificate(newClientSS.CertPath, newClientSS.KeyPath)) + require.NoError(t, err) + require.NotNil(t, apply) + + // PrepareReconfigure only validates; nothing changes until apply runs. + clientCert, _ := manager.clientCertLoader.Paths() + require.Equal(t, clientSS.CertPath, clientCert, "PrepareReconfigure must not swap the certificate") + + require.NoError(t, apply()) + + clientCert, clientKey := manager.clientCertLoader.Paths() + require.Equal(t, newClientSS.CertPath, clientCert) + require.Equal(t, newClientSS.KeyPath, clientKey) + + // The server certificate is left untouched by a client-only reconfigure. + serverCert, _ := manager.serverCertLoader.Paths() + require.Equal(t, serverSS.CertPath, serverCert) + require.Equal(t, serverSerial, leafSerial(t, manager.serverCertLoader)) + + t.Run("failed client reconfigure keeps the previous certificate", func(t *testing.T) { + missing := path.Join(t.TempDir(), "absent.pem") + + apply, err := manager.PrepareReconfigure(WithClientCertificate(missing, missing)) + require.ErrorContains(t, err, "no such file or directory") + require.Nil(t, apply) + + clientCert, _ := manager.clientCertLoader.Paths() + require.Equal(t, newClientSS.CertPath, clientCert, + "a certificate that fails to load must not disturb the active one") + }) +} diff --git a/pkg/tlsconfig/configmanager_test.go b/pkg/tlsconfig/configmanager_test.go index 376a8890653..14878c5edd4 100644 --- a/pkg/tlsconfig/configmanager_test.go +++ b/pkg/tlsconfig/configmanager_test.go @@ -1705,11 +1705,8 @@ func TestTLSConfigManager_ClientAuthOverride(t *testing.T) { }) } -// testManagerCheckTime is the TLS certificate check time in logging tests. -const testManagerCheckTime = 333 * time.Millisecond - // testManagerCheckCapture time is how long to capture logs during logging tests. To prevent flaky tests, -// it should be more than testManagerCheckTime, but less than 2 * testManagerCheckTime. +// it should be more than testCheckTime, but less than 2 * testCheckTime. const testManagerCheckCapture = 500 * time.Millisecond func TestTLSConfigManager_WithCertLoaderOptions(t *testing.T) { @@ -1921,3 +1918,212 @@ func TestTLSConfigManager_DialWithDialer(t *testing.T) { }) }) } + +func TestTLSConfigManager_WithUsage(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + ss := selfsigned.NewSelfSignedCert(t) + + t.Run("usage names the cert loaders by role", func(t *testing.T) { + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithUsage("subscriber"), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + require.Equal(t, "subscriber.server", manager.serverCertLoader.Usage()) + require.Equal(t, "subscriber.client", manager.clientCertLoader.Usage()) + }) + + t.Run("usage prefixes manager errors", func(t *testing.T) { + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithUsage("httpd"), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + // A server manager cannot dial; the message should say which manager. + _, err = manager.Dial("tcp", "127.0.0.1:0") + require.ErrorContains(t, err, "httpd: ") + }) + + t.Run("unset usage still names the loaders", func(t *testing.T) { + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + require.Equal(t, ".server", manager.serverCertLoader.Usage()) + require.Equal(t, ".client", manager.clientCertLoader.Usage()) + }) +} + +func TestTLSConfigManager_RoleRestrictions(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + ss := selfsigned.NewSelfSignedCert(t) + + t.Run("client manager cannot Listen", func(t *testing.T) { + manager, err := NewClientTLSConfigManager(monitor, WithUseTLS(true)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + listener, err := manager.Listen("tcp", "127.0.0.1:0") + require.ErrorIs(t, err, ErrClientListen) + require.Nil(t, listener) + }) + + t.Run("server manager cannot Dial", func(t *testing.T) { + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + conn, err := manager.Dial("tcp", "127.0.0.1:0") + require.ErrorIs(t, err, ErrServerDial) + require.Nil(t, conn) + + conn, err = manager.DialWithDialer(&net.Dialer{}, "tcp", "127.0.0.1:0") + require.ErrorIs(t, err, ErrServerDial) + require.Nil(t, conn) + }) + + t.Run("client and server manager can do both", func(t *testing.T) { + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath), + WithAllowInsecure(true)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + listener, err := manager.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + // The same manager serves and dials its own listener. + testData := []byte("hello") + serverDone := make(chan error, 1) + go simpleEchoServer(serverDone, listener, len(testData)) + + conn, err := manager.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer th.CheckedClose(t, conn)() + + _, err = conn.Write(testData) + require.NoError(t, err) + + buf := make([]byte, len(testData)) + _, err = conn.Read(buf) + require.NoError(t, err) + require.Equal(t, testData, buf) + + require.NoError(t, <-serverDone) + }) +} + +func TestTLSConfigManager_ConstructorValidation(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + t.Run("missing monitor", func(t *testing.T) { + manager, err := NewServerTLSConfigManager(nil, WithUseTLS(false)) + require.ErrorIs(t, err, ErrNoCertificateMonitor) + require.Nil(t, manager) + }) + + t.Run("missing role", func(t *testing.T) { + // The exported constructors always supply a role, so this guards + // against misuse of the internal constructor. + manager, err := newTLSConfigManager(withMonitor(monitor), WithUseTLS(false)) + require.ErrorIs(t, err, ErrNoRole) + require.Nil(t, manager) + }) +} + +func TestTLSConfigManager_PrepareReconfigureDisabled(t *testing.T) { + disabled := NewDisabledTLSConfigManager() + defer th.CheckedClose(t, disabled)() + + // A disabled manager has no configuration to copy. Reconfiguring it must + // report that clearly instead of panicking. + require.NotPanics(t, func() { + apply, err := disabled.PrepareReconfigure(WithUseTLS(true)) + require.ErrorIs(t, err, ErrConfigureDisabledManager) + require.Nil(t, apply) + }) + + require.False(t, disabled.UseTLS(), "a disabled manager stays disabled") +} + +func TestTLSConfigManager_PrepareReconfigureUseTLS(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + ss := selfsigned.NewSelfSignedCert(t) + + t.Run("server cannot toggle TLS", func(t *testing.T) { + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + // A listener is already bound with the current configuration, so + // turning TLS off underneath it is refused. + apply, err := manager.PrepareReconfigure(WithUseTLS(false)) + require.ErrorIs(t, err, ErrNotSupportedServer) + require.Nil(t, apply) + require.True(t, manager.UseTLS(), "a refused reconfigure must not change the manager") + }) + + t.Run("server reconfigure without a TLS change is allowed", func(t *testing.T) { + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + newSS := selfsigned.NewSelfSignedCert(t) + apply, err := manager.PrepareReconfigure(WithServerCertificate(newSS.CertPath, newSS.KeyPath)) + require.NoError(t, err) + require.NoError(t, apply()) + + certPath, _ := manager.serverCertLoader.Paths() + require.Equal(t, newSS.CertPath, certPath) + }) + + t.Run("client can turn TLS off", func(t *testing.T) { + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + require.True(t, manager.UseTLS()) + + apply, err := manager.PrepareReconfigure(WithUseTLS(false)) + require.NoError(t, err) + require.NoError(t, apply()) + + require.False(t, manager.UseTLS()) + require.Nil(t, manager.TLSConfig()) + + // Disabling TLS clears the loaded certificate. + certPath, keyPath := manager.clientCertLoader.Paths() + require.Empty(t, certPath) + require.Empty(t, keyPath) + }) +} diff --git a/pkg/tlsconfig/loadcert_test.go b/pkg/tlsconfig/loadcert_test.go new file mode 100644 index 00000000000..5a9cdecfa38 --- /dev/null +++ b/pkg/tlsconfig/loadcert_test.go @@ -0,0 +1,301 @@ +package tlsconfig + +import ( + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/influxdata/influxdb/pkg/testing/selfsigned" +) + +func TestLoadCertificate_EmptyPaths(t *testing.T) { + t.Run("both paths empty loads nothing", func(t *testing.T) { + // An entirely unconfigured certificate is not an error: it is how a + // client-only manager with no client certificate is expressed. + lc, err := LoadCertificate("", "") + require.NoError(t, err) + require.True(t, lc.IsEmpty()) + require.False(t, lc.IsValid()) + }) + + t.Run("key without certificate is an error", func(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + lc, err := LoadCertificate("", ss.KeyPath) + require.ErrorIs(t, err, ErrPathEmpty) + require.ErrorContains(t, err, "LoadCertificate: certificate:") + require.False(t, lc.IsValid()) + }) +} + +func TestLoadCertificate_KeyDefaultsToCertPath(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t, selfsigned.WithCombinedFile()) + + // An empty keyPath means the key lives in the certificate file. The loaded + // certificate should report the certificate path as the key path. + lc, err := LoadCertificate(ss.CertPath, "", WithLoadCertificateIgnoreFilePermissions(true)) + require.NoError(t, err) + require.True(t, lc.IsValid()) + require.Equal(t, ss.CertPath, lc.CertificatePath) + require.Equal(t, ss.CertPath, lc.KeyPath) +} + +func TestLoadedCertificate_GetLeaf(t *testing.T) { + t.Run("invalid certificate", func(t *testing.T) { + lc := LoadedCertificate{} + leaf, err := lc.GetLeaf() + require.ErrorIs(t, err, ErrCertificateInvalid) + require.Nil(t, leaf) + }) + + t.Run("valid but empty certificate", func(t *testing.T) { + // valid with no paths is only reachable inside the package, but GetLeaf + // must still report it rather than returning a nil leaf. + lc := LoadedCertificate{valid: true} + leaf, err := lc.GetLeaf() + require.ErrorIs(t, err, ErrCertificateEmpty) + require.Nil(t, leaf) + }) + + t.Run("certificate with nil leaf", func(t *testing.T) { + lc := LoadedCertificate{valid: true, CertificatePath: "cert.pem", KeyPath: "key.pem"} + leaf, err := lc.GetLeaf() + require.ErrorIs(t, err, ErrCertificateNil) + require.Nil(t, leaf) + }) + + t.Run("loaded certificate", func(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + lc, err := LoadCertificate(ss.CertPath, ss.KeyPath) + require.NoError(t, err) + + leaf, err := lc.GetLeaf() + require.NoError(t, err) + require.NotNil(t, leaf) + require.NotNil(t, leaf.Certificate) + require.Equal(t, lc.Leaf.SerialNumber, leaf.SerialNumber) + }) +} + +func TestLoadedCertificate_Serial(t *testing.T) { + t.Run("no leaf reports N/A", func(t *testing.T) { + lc := LoadedCertificate{valid: true, CertificatePath: "cert.pem"} + require.Equal(t, "N/A", lc.Serial()) + }) + + t.Run("loaded certificate reports leaf serial", func(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + lc, err := LoadCertificate(ss.CertPath, ss.KeyPath) + require.NoError(t, err) + require.Equal(t, lc.Leaf.SerialNumber.String(), lc.Serial()) + }) +} + +func TestLoadedCertificate_IsEmpty(t *testing.T) { + tests := []struct { + name string + lc LoadedCertificate + isEmpty bool + isValid bool + }{ + { + name: "zero value", + lc: LoadedCertificate{}, + isEmpty: true, + isValid: false, + }, + { + name: "invalid with paths is still empty", + lc: LoadedCertificate{CertificatePath: "cert.pem", KeyPath: "key.pem"}, + isEmpty: true, + isValid: false, + }, + { + name: "valid without paths is empty", + lc: LoadedCertificate{valid: true}, + isEmpty: true, + isValid: true, + }, + { + name: "valid with cert path only", + lc: LoadedCertificate{valid: true, CertificatePath: "cert.pem"}, + isEmpty: false, + isValid: true, + }, + { + name: "valid with both paths", + lc: LoadedCertificate{valid: true, CertificatePath: "cert.pem", KeyPath: "key.pem"}, + isEmpty: false, + isValid: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.isEmpty, tt.lc.IsEmpty()) + require.Equal(t, tt.isValid, tt.lc.IsValid()) + }) + } +} + +func TestLoadedCertificate_WithLogContext(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + lc, err := LoadCertificate(ss.CertPath, ss.KeyPath) + require.NoError(t, err) + + core, logs := observer.New(zapcore.InfoLevel) + lc.WithLogContext(zap.New(core)).Info("test message") + + entries := logs.FilterMessage("test message").TakeAll() + require.Len(t, entries, 1) + require.Equal(t, ss.CertPath, entries[0].ContextMap()["cert"]) + require.Equal(t, ss.KeyPath, entries[0].ContextMap()["key"]) + require.Equal(t, lc.Serial(), entries[0].ContextMap()["serial"]) +} + +func TestX509Certificate_ExpiresSoon(t *testing.T) { + loadLeaf := func(t *testing.T, notBefore, notAfter time.Time) *X509Certificate { + t.Helper() + ss := selfsigned.NewSelfSignedCert(t, + selfsigned.WithNotBefore(notBefore), + selfsigned.WithNotAfter(notAfter)) + lc, err := LoadCertificate(ss.CertPath, ss.KeyPath) + require.NoError(t, err) + leaf, err := lc.GetLeaf() + require.NoError(t, err) + return leaf + } + + t.Run("outside warn window", func(t *testing.T) { + leaf := loadLeaf(t, time.Now().Add(-24*time.Hour), time.Now().Add(30*24*time.Hour)) + + expiresSoon, untilExpires := leaf.ExpiresSoon(5 * 24 * time.Hour) + require.False(t, expiresSoon) + require.Zero(t, untilExpires, "untilExpires is only meaningful inside the warn window") + }) + + t.Run("inside warn window", func(t *testing.T) { + notAfter := time.Now().Add(24 * time.Hour) + leaf := loadLeaf(t, time.Now().Add(-24*time.Hour), notAfter) + + expiresSoon, untilExpires := leaf.ExpiresSoon(5 * 24 * time.Hour) + require.True(t, expiresSoon) + require.Positive(t, untilExpires) + require.WithinDuration(t, notAfter, time.Now().Add(untilExpires), time.Minute) + }) + + t.Run("already expired is inside any window", func(t *testing.T) { + leaf := loadLeaf(t, time.Now().Add(-48*time.Hour), time.Now().Add(-24*time.Hour)) + + expiresSoon, untilExpires := leaf.ExpiresSoon(time.Hour) + require.True(t, expiresSoon) + require.Negative(t, untilExpires, "an expired certificate reports a negative time to expiration") + }) +} + +func TestX509Certificate_IsExpiredIsPremature(t *testing.T) { + loadLeaf := func(t *testing.T, notBefore, notAfter time.Time) *X509Certificate { + t.Helper() + ss := selfsigned.NewSelfSignedCert(t, + selfsigned.WithNotBefore(notBefore), + selfsigned.WithNotAfter(notAfter)) + lc, err := LoadCertificate(ss.CertPath, ss.KeyPath) + require.NoError(t, err) + leaf, err := lc.GetLeaf() + require.NoError(t, err) + return leaf + } + + t.Run("current certificate", func(t *testing.T) { + leaf := loadLeaf(t, time.Now().Add(-24*time.Hour), time.Now().Add(24*time.Hour)) + require.False(t, leaf.IsExpired()) + require.False(t, leaf.IsPremature()) + }) + + t.Run("expired certificate", func(t *testing.T) { + leaf := loadLeaf(t, time.Now().Add(-48*time.Hour), time.Now().Add(-24*time.Hour)) + require.True(t, leaf.IsExpired()) + require.False(t, leaf.IsPremature()) + }) + + t.Run("premature certificate", func(t *testing.T) { + leaf := loadLeaf(t, time.Now().Add(24*time.Hour), time.Now().Add(48*time.Hour)) + require.False(t, leaf.IsExpired()) + require.True(t, leaf.IsPremature()) + }) +} + +func TestX509Certificate_LogIssues(t *testing.T) { + loadLeaf := func(t *testing.T, notBefore, notAfter time.Time) *X509Certificate { + t.Helper() + ss := selfsigned.NewSelfSignedCert(t, + selfsigned.WithNotBefore(notBefore), + selfsigned.WithNotAfter(notAfter)) + lc, err := LoadCertificate(ss.CertPath, ss.KeyPath) + require.NoError(t, err) + leaf, err := lc.GetLeaf() + require.NoError(t, err) + return leaf + } + + t.Run("healthy certificate logs nothing", func(t *testing.T) { + leaf := loadLeaf(t, time.Now().Add(-24*time.Hour), time.Now().Add(30*24*time.Hour)) + + core, logs := observer.New(zapcore.DebugLevel) + leaf.logIssues(zap.New(core), 5*24*time.Hour) + require.Zero(t, logs.Len(), "a healthy certificate should not produce log noise") + }) + + t.Run("nil logger does not panic", func(t *testing.T) { + leaf := loadLeaf(t, time.Now().Add(-48*time.Hour), time.Now().Add(-24*time.Hour)) + require.NotPanics(t, func() { leaf.logIssues(nil, time.Hour) }) + }) + + t.Run("nil certificate does not panic", func(t *testing.T) { + core, _ := observer.New(zapcore.DebugLevel) + var xc *X509Certificate + require.NotPanics(t, func() { xc.logIssues(zap.New(core), time.Hour) }) + }) + + t.Run("expired outranks expires soon", func(t *testing.T) { + leaf := loadLeaf(t, time.Now().Add(-48*time.Hour), time.Now().Add(-24*time.Hour)) + + core, logs := observer.New(zapcore.DebugLevel) + // An expired certificate is also inside the warn window; only the more + // severe "expired" message should be reported. + leaf.logIssues(zap.New(core), 30*24*time.Hour) + + entries := logs.TakeAll() + require.Len(t, entries, 1, "expired certificate should log exactly one issue") + require.Equal(t, "Certificate is expired", entries[0].Message) + }) +} + +func TestLoadCertificate_MalformedFiles(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + t.Run("garbage certificate file", func(t *testing.T) { + dir := t.TempDir() + certPath := path.Join(dir, "cert.pem") + require.NoError(t, os.WriteFile(certPath, []byte("not a certificate"), 0600)) + + lc, err := LoadCertificate(certPath, ss.KeyPath) + require.ErrorContains(t, err, "error loading x509 key pair") + require.False(t, lc.IsValid()) + }) + + t.Run("missing key file", func(t *testing.T) { + lc, err := LoadCertificate(ss.CertPath, path.Join(t.TempDir(), "absent.pem")) + require.ErrorContains(t, err, "LoadCertificate: error opening") + require.False(t, lc.IsValid()) + }) +} diff --git a/pkg/tlsconfig/tls_config_test.go b/pkg/tlsconfig/tls_config_test.go new file mode 100644 index 00000000000..40c9e1585eb --- /dev/null +++ b/pkg/tlsconfig/tls_config_test.go @@ -0,0 +1,168 @@ +package tlsconfig + +import ( + "crypto/tls" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConfig_NewConfig(t *testing.T) { + c := NewConfig() + require.Empty(t, c.Ciphers) + require.Empty(t, c.MinVersion) + require.Empty(t, c.MaxVersion) + + // An unset Config is valid and parses to a nil *tls.Config so callers can + // distinguish "nothing configured" from "configured". + require.NoError(t, c.Validate()) + parsed, err := c.Parse() + require.NoError(t, err) + require.Nil(t, parsed) +} + +func TestConfig_Parse(t *testing.T) { + tests := []struct { + name string + config Config + expected *tls.Config + }{ + { + name: "empty config parses to nil", + config: Config{}, + expected: nil, + }, + { + name: "single cipher", + config: Config{Ciphers: []string{"TLS_AES_128_GCM_SHA256"}}, + expected: &tls.Config{ + CipherSuites: []uint16{tls.TLS_AES_128_GCM_SHA256}, + }, + }, + { + name: "multiple ciphers preserve order", + config: Config{Ciphers: []string{ + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_AES_256_GCM_SHA384", + }}, + expected: &tls.Config{ + CipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + }, + }, + }, + { + name: "cipher names are case insensitive", + config: Config{Ciphers: []string{"tls_aes_128_gcm_sha256"}}, + expected: &tls.Config{ + CipherSuites: []uint16{tls.TLS_AES_128_GCM_SHA256}, + }, + }, + { + name: "min version only", + config: Config{MinVersion: "TLS1.2"}, + expected: &tls.Config{MinVersion: tls.VersionTLS12}, + }, + { + name: "max version only", + config: Config{MaxVersion: "TLS1.3"}, + expected: &tls.Config{MaxVersion: tls.VersionTLS13}, + }, + { + name: "version names are case insensitive", + config: Config{MinVersion: "tls1.3"}, + expected: &tls.Config{MinVersion: tls.VersionTLS13}, + }, + { + name: "numeric version aliases", + config: Config{MinVersion: "1.0", MaxVersion: "1.3"}, + expected: &tls.Config{MinVersion: tls.VersionTLS10, MaxVersion: tls.VersionTLS13}, + }, + { + name: "ciphers and versions together", + config: Config{ + Ciphers: []string{"TLS_AES_128_GCM_SHA256"}, + MinVersion: "TLS1.2", + MaxVersion: "TLS1.3", + }, + expected: &tls.Config{ + CipherSuites: []uint16{tls.TLS_AES_128_GCM_SHA256}, + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS13, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parsed, err := tt.config.Parse() + require.NoError(t, err) + require.Equal(t, tt.expected, parsed) + + // Anything that parses cleanly must also validate cleanly. + require.NoError(t, tt.config.Validate()) + }) + } +} + +func TestConfig_ParseErrors(t *testing.T) { + tests := []struct { + name string + config Config + errMsg string + }{ + { + name: "unknown cipher", + config: Config{Ciphers: []string{"TLS_NOT_A_REAL_CIPHER"}}, + errMsg: `unknown cipher suite: "TLS_NOT_A_REAL_CIPHER". available ciphers: `, + }, + { + name: "unknown cipher among valid ciphers", + config: Config{Ciphers: []string{"TLS_AES_128_GCM_SHA256", "BOGUS"}}, + errMsg: `unknown cipher suite: "BOGUS". available ciphers: `, + }, + { + name: "unknown min version", + config: Config{MinVersion: "TLS9.9"}, + errMsg: `unknown tls version: "TLS9.9". available versions: `, + }, + { + name: "unknown max version", + config: Config{MaxVersion: "nope"}, + errMsg: `unknown tls version: "nope". available versions: `, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parsed, err := tt.config.Parse() + require.ErrorContains(t, err, tt.errMsg) + require.Nil(t, parsed) + + // Validate is a thin wrapper around Parse and must report the same error. + require.ErrorContains(t, tt.config.Validate(), tt.errMsg) + }) + } +} + +func TestConfig_UnknownCipherListsAvailable(t *testing.T) { + err := unknownCipher("bogus") + require.ErrorContains(t, err, `unknown cipher suite: "bogus"`) + + // The advice is only useful if it names ciphers the user can actually pick. + require.ErrorContains(t, err, "TLS_AES_128_GCM_SHA256") + require.ErrorContains(t, err, "TLS_RSA_WITH_AES_128_CBC_SHA") +} + +func TestConfig_UnknownVersionListsAvailable(t *testing.T) { + err := unknownVersion("bogus") + require.ErrorContains(t, err, `unknown tls version: "bogus"`) + require.ErrorContains(t, err, "TLS1.0") + require.ErrorContains(t, err, "TLS1.3") + + // The numeric aliases ("1.0", "1.3") are deliberately omitted from the + // suggestions to avoid a confusing, duplicated list. + require.NotContains(t, err.Error(), " 1.0") + require.NotContains(t, err.Error(), " 1.3") +} From f587913819fa89693cb2cc7f39807f86ed3f4e1a Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 09:34:28 -0500 Subject: [PATCH 04/18] fix: subscriber service properly reloads TLS configuration Fix subscriber service so it now does a full TLS configuration reload. Previously the certificate was not being propagated to existing subscriber clients. Also added doing a full TLS configuration reload for all configs, including CAs. --- client/v2/client.go | 17 +- cmd/influxd/run/server.go | 2 +- pkg/tlsconfig/configmanager.go | 29 ++++ pkg/tlsconfig/configmanager_test.go | 179 ++++++++++++++++++++++ services/subscriber/config.go | 12 ++ services/subscriber/http.go | 33 ++-- services/subscriber/mtls_internal_test.go | 11 +- services/subscriber/service.go | 43 +++--- 8 files changed, 289 insertions(+), 37 deletions(-) diff --git a/client/v2/client.go b/client/v2/client.go index bf6467abb69..b24cc02de98 100644 --- a/client/v2/client.go +++ b/client/v2/client.go @@ -54,6 +54,18 @@ type HTTPConfig struct { // DialContext specifies the dial function for creating unencrypted TCP connections. // If DialContext is nil then the transport dials using package net. DialContext func(ctx context.Context, network, addr string) (net.Conn, error) + + // DialTLSContext specifies the dial function for creating TLS connections for + // non-proxied HTTPS requests. The returned connection is assumed to have + // already completed its TLS handshake. It allows the TLS configuration to be + // resolved per connection instead of being fixed when the client is created. + // + // When DialTLSContext is set, TLSConfig is only used for proxied requests, + // which tunnel through the proxy and so do not use this function. Set both to + // keep proxied requests working. + // + // If DialTLSContext is nil then DialContext and TLSConfig are used. + DialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error) } // BatchPointsConfig is the config data needed to create an instance of the BatchPoints struct. @@ -125,8 +137,9 @@ func NewHTTPClient(conf HTTPConfig) (HTTPClient, error) { TLSClientConfig: &tls.Config{ InsecureSkipVerify: conf.InsecureSkipVerify, }, - Proxy: conf.Proxy, - DialContext: conf.DialContext, + Proxy: conf.Proxy, + DialContext: conf.DialContext, + DialTLSContext: conf.DialTLSContext, } if conf.TLSConfig != nil { // A supplied TLSConfig is used as-is, including its own diff --git a/cmd/influxd/run/server.go b/cmd/influxd/run/server.go index 298b9f429db..c779fce55fa 100644 --- a/cmd/influxd/run/server.go +++ b/cmd/influxd/run/server.go @@ -633,7 +633,7 @@ func (s *Server) ApplyReloadedConfig(config *Config, log *zap.Logger) error { // Reload the subscriber's client TLS certificate at its currently // configured path so rotated certificates are picked up. if s.Subscriber != nil { - if af, err := s.Subscriber.PrepareReloadTLSCertificates(); err == nil { + if af, err := s.Subscriber.PrepareReloadTLSCertificates(config.Subscriber); err == nil { applyFuncs = append(applyFuncs, af) } else { log.Error("error reloading subscriber service TLS certificate, no new configuration applied", zap.Error(err)) diff --git a/pkg/tlsconfig/configmanager.go b/pkg/tlsconfig/configmanager.go index fe3c29d3361..7fe4132f1c2 100644 --- a/pkg/tlsconfig/configmanager.go +++ b/pkg/tlsconfig/configmanager.go @@ -1,6 +1,7 @@ package tlsconfig import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -712,6 +713,34 @@ func (cm *TLSConfigManager) Dial(network, address string) (net.Conn, error) { } } +// DialContext dials a remote for network and address using the current +// configuration, honoring ctx for cancellation. +// +// The configuration is resolved on each call, so a manager reconfigured through +// PrepareReconfigure takes effect on the next connection without the caller +// rebuilding anything. This makes it suitable for an http.Transport's +// DialTLSContext, where a *tls.Config handed over once would otherwise freeze +// the settings in place. +// +// No dial timeout is imposed: ctx is the only bound, matching Dial. Callers that +// need one should pass a ctx carrying it. Note that an http.Client.Timeout is +// delivered as a cancellation of ctx rather than as a ctx deadline, and is +// honored either way. +func (cm *TLSConfigManager) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + if !cm.role.IsClientRole() { + return nil, fmt.Errorf("%s: %w", cm.usage, ErrServerDial) + } + + if tlsConfig := cm.TLSConfig(); tlsConfig != nil { + // tls.Dialer fills in ServerName from address when the config does not + // set one, so peer verification still works. + dialer := &tls.Dialer{NetDialer: new(net.Dialer), Config: tlsConfig} + return dialer.DialContext(ctx, network, address) + } else { + return (&net.Dialer{}).DialContext(ctx, network, address) + } +} + // Dial a remote for network and addressing using the given dialer and current configuration. func (cm *TLSConfigManager) DialWithDialer(dialer *net.Dialer, network, address string) (net.Conn, error) { if !cm.role.IsClientRole() { diff --git a/pkg/tlsconfig/configmanager_test.go b/pkg/tlsconfig/configmanager_test.go index 14878c5edd4..bb5492fc73c 100644 --- a/pkg/tlsconfig/configmanager_test.go +++ b/pkg/tlsconfig/configmanager_test.go @@ -1,6 +1,7 @@ package tlsconfig import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -2127,3 +2128,181 @@ func TestTLSConfigManager_PrepareReconfigureUseTLS(t *testing.T) { require.Empty(t, keyPath) }) } + +func TestTLSConfigManager_DialContext(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + t.Run("useTLS false dials plain TCP", func(t *testing.T) { + manager, err := NewClientTLSConfigManager(monitor, WithUseTLS(false)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + testData := []byte("hello") + serverDone := make(chan error, 1) + go simpleEchoServer(serverDone, listener, len(testData)) + + conn, err := manager.DialContext(context.Background(), "tcp", listener.Addr().String()) + require.NoError(t, err) + defer th.CheckedClose(t, conn)() + + _, err = conn.Write(testData) + require.NoError(t, err) + + buf := make([]byte, len(testData)) + _, err = conn.Read(buf) + require.NoError(t, err) + require.Equal(t, testData, buf) + require.NoError(t, <-serverDone) + }) + + t.Run("useTLS true dials TLS", func(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + serverManager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, serverManager)() + + listener, err := serverManager.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + testData := []byte("hello") + serverDone := make(chan error, 1) + go simpleEchoServer(serverDone, listener, len(testData)) + + clientManager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithRootCA(&CAConfig{Paths: []string{ss.CACertPath}})) + require.NoError(t, err) + defer th.CheckedClose(t, clientManager)() + + conn, err := clientManager.DialContext(context.Background(), "tcp", listener.Addr().String()) + require.NoError(t, err) + defer th.CheckedClose(t, conn)() + + // ServerName is inferred from the address, so the peer is verified + // against the root CAs rather than skipped. + require.True(t, conn.(*tls.Conn).ConnectionState().HandshakeComplete) + + _, err = conn.Write(testData) + require.NoError(t, err) + + buf := make([]byte, len(testData)) + _, err = conn.Read(buf) + require.NoError(t, err) + require.Equal(t, testData, buf) + require.NoError(t, <-serverDone) + }) + + t.Run("server manager cannot DialContext", func(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + conn, err := manager.DialContext(context.Background(), "tcp", "127.0.0.1:0") + require.ErrorIs(t, err, ErrServerDial) + require.Nil(t, conn) + }) + + t.Run("cancellation aborts the handshake", func(t *testing.T) { + // A peer that accepts the connection but never speaks TLS: without + // honoring ctx the handshake would hang here. + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + accepted := make(chan net.Conn, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + close(accepted) + return + } + accepted <- conn + }() + t.Cleanup(func() { + if conn, ok := <-accepted; ok && conn != nil { + conn.Close() + } + }) + + manager, err := NewClientTLSConfigManager(monitor, WithUseTLS(true), WithAllowInsecure(true)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + // An http.Client.Timeout reaches a dialer as a cancellation with no + // deadline attached, so that is what is reproduced here. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, hasDeadline := ctx.Deadline() + require.False(t, hasDeadline, "the timeout arrives as a cancellation, not a deadline") + time.AfterFunc(50*time.Millisecond, cancel) + + start := time.Now() + conn, err := manager.DialContext(ctx, "tcp", listener.Addr().String()) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, conn) + require.Less(t, time.Since(start), 10*time.Second, "the handshake should be abandoned when ctx is cancelled") + }) + + t.Run("resolves the configuration per connection", func(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + otherSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("other", "Other CA")) + + cert, err := tls.LoadX509KeyPair(ss.CertPath, ss.KeyPath) + require.NoError(t, err) + listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}}) + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { + buf := make([]byte, 1) + conn.Read(buf) + conn.Close() + }() + } + }() + + // Trust the wrong CA to begin with. + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithRootCA(&CAConfig{Paths: []string{otherSS.CACertPath}})) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + conn, err := manager.DialContext(context.Background(), "tcp", listener.Addr().String()) + require.ErrorContains(t, err, "certificate signed by unknown authority") + require.Nil(t, conn) + + // Reconfiguring is enough: the next dial resolves the new roots without + // the caller rebuilding anything. + apply, err := manager.PrepareReconfigure(WithRootCA(&CAConfig{Paths: []string{ss.CACertPath}})) + require.NoError(t, err) + require.NoError(t, apply()) + + conn, err = manager.DialContext(context.Background(), "tcp", listener.Addr().String()) + require.NoError(t, err) + require.NoError(t, conn.Close()) + }) +} diff --git a/services/subscriber/config.go b/services/subscriber/config.go index 85a5f4cf997..53660e50219 100644 --- a/services/subscriber/config.go +++ b/services/subscriber/config.go @@ -126,6 +126,18 @@ func (c Config) effectiveRootCA() *tlsconfig.CAConfig { return cc } +// TLSManagerOpts returns the list of TLS manager options specified by c. +func (c Config) TLSManagerOpts() []tlsconfig.TLSConfigManagerOpt { + return []tlsconfig.TLSConfigManagerOpt{ + tlsconfig.WithUseTLS(true), + tlsconfig.WithBaseConfig(c.TLS), + tlsconfig.WithAllowInsecure(c.InsecureSkipVerify), + tlsconfig.WithClientCertificate(c.Certificate, c.PrivateKey), + tlsconfig.WithRootCA(c.effectiveRootCA()), + tlsconfig.WithIgnoreFilePermissions(c.InsecureCertificate), + } +} + func fileExists(fileName string) bool { info, err := os.Stat(fileName) return err == nil && !info.IsDir() diff --git a/services/subscriber/http.go b/services/subscriber/http.go index 1f3ca115b68..8b7999f867e 100644 --- a/services/subscriber/http.go +++ b/services/subscriber/http.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/tls" + "net" "time" "github.com/influxdata/influxdb/client/v2" @@ -17,20 +18,32 @@ type HTTP struct { // NewHTTP returns a new HTTP points writer with default options. func NewHTTP(addr string, timeout time.Duration) (*HTTP, error) { - return NewHTTPS(addr, timeout, nil) + return NewHTTPS(addr, timeout, nil, nil) } // NewHTTPS returns a new HTTPS points writer with default options and HTTPS -// configured. tlsConfig is the fully-resolved client TLS configuration (root -// CAs, any client certificate, and InsecureSkipVerify) built by the service via -// tlsconfig.TLSConfigManager; it may be nil to use Go's defaults. When it -// carries a manager-backed GetClientCertificate, rotated client certificates -// are picked up automatically on new connections. -func NewHTTPS(addr string, timeout time.Duration, tlsConfig *tls.Config) (*HTTP, error) { +// configured. +// +// dialTLSContext dials the writer's TLS connections, normally +// tlsconfig.TLSConfigManager.DialContext, which resolves the TLS configuration +// on each connection. That is what allows a reloaded configuration to reach a +// writer that already exists: the writer keeps dialing through the manager +// rather than through a configuration captured when it was built. +// +// tlsConfig is the fully-resolved client TLS configuration (root CAs, any client +// certificate, and InsecureSkipVerify). It is only consulted for proxied +// requests, which tunnel through the proxy instead of using dialTLSContext, and +// is therefore a snapshot that a reload does not update. Both may be nil to use +// Go's defaults. +// +// timeout bounds the whole request, including the connection and TLS handshake, +// so no separate handshake timeout is needed. +func NewHTTPS(addr string, timeout time.Duration, tlsConfig *tls.Config, dialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error)) (*HTTP, error) { conf := client.HTTPConfig{ - Addr: addr, - Timeout: timeout, - TLSConfig: tlsConfig, + Addr: addr, + Timeout: timeout, + TLSConfig: tlsConfig, + DialTLSContext: dialTLSContext, } c, err := client.NewHTTPClient(conf) diff --git a/services/subscriber/mtls_internal_test.go b/services/subscriber/mtls_internal_test.go index bc64a1ae5e1..c1c9604b9fc 100644 --- a/services/subscriber/mtls_internal_test.go +++ b/services/subscriber/mtls_internal_test.go @@ -95,7 +95,7 @@ func TestHTTP_PresentsClientCertificate(t *testing.T) { cm := newManager(true) defer cm.Close() - w, err := NewHTTPS(srv.URL, time.Duration(conf.HTTPTimeout), cm.TLSConfig()) + w, err := NewHTTPS(srv.URL, time.Duration(conf.HTTPTimeout), cm.TLSConfig(), cm.DialContext) require.NoError(t, err) _, err = w.WritePointsContext(context.Background(), WriteRequest{ @@ -111,7 +111,7 @@ func TestHTTP_PresentsClientCertificate(t *testing.T) { cm := newManager(false) defer cm.Close() - w, err := NewHTTPS(srv.URL, time.Duration(conf.HTTPTimeout), cm.TLSConfig()) + w, err := NewHTTPS(srv.URL, time.Duration(conf.HTTPTimeout), cm.TLSConfig(), cm.DialContext) require.NoError(t, err) _, err = w.WritePointsContext(context.Background(), WriteRequest{ @@ -149,8 +149,9 @@ func TestService_PrepareReloadTLSCertificates(t *testing.T) { } t.Run("no client certificate is a no-op", func(t *testing.T) { - s := open(t, NewConfig()) - apply, err := s.PrepareReloadTLSCertificates() + conf := NewConfig() + s := open(t, conf) + apply, err := s.PrepareReloadTLSCertificates(conf) require.NoError(t, err) require.NotNil(t, apply) require.NoError(t, apply()) @@ -163,7 +164,7 @@ func TestService_PrepareReloadTLSCertificates(t *testing.T) { c.PrivateKey = clientSS.KeyPath s := open(t, c) - apply, err := s.PrepareReloadTLSCertificates() + apply, err := s.PrepareReloadTLSCertificates(c) require.NoError(t, err) require.NotNil(t, apply) require.NoError(t, apply()) diff --git a/services/subscriber/service.go b/services/subscriber/service.go index 0e5485bb747..49eb7e9119f 100644 --- a/services/subscriber/service.go +++ b/services/subscriber/service.go @@ -7,6 +7,7 @@ import ( "crypto/tls" "errors" "fmt" + "net" "net/url" "sync" "sync/atomic" @@ -165,13 +166,9 @@ func (s *Service) Open() error { // certificate, and owns the certificate loader used for reloads. cm, err := tlsconfig.NewClientTLSConfigManager( s.certMonitor, - tlsconfig.WithUseTLS(true), - tlsconfig.WithBaseConfig(s.conf.TLS), - tlsconfig.WithAllowInsecure(s.conf.InsecureSkipVerify), - tlsconfig.WithClientCertificate(s.conf.Certificate, s.conf.PrivateKey), - tlsconfig.WithRootCA(s.conf.effectiveRootCA()), - tlsconfig.WithIgnoreFilePermissions(s.conf.InsecureCertificate), - tlsconfig.WithLogger(s.Logger)) + append( + s.conf.TLSManagerOpts(), + tlsconfig.WithLogger(s.Logger))...) if err != nil { return fmt.Errorf("subscriber: error creating TLS manager: %w", err) } @@ -248,21 +245,26 @@ func (s *Service) Close() error { return nil } -// PrepareReloadTLSCertificates verifies that the configured client TLS -// certificate can be reloaded from its current path. On success it returns a -// function that applies the reloaded certificate; the new certificate is then -// presented on subsequent HTTPS connections. If no client certificate is -// configured there is nothing to reload and the returned function is a no-op. -func (s *Service) PrepareReloadTLSCertificates() (func() error, error) { +// PrepareReloadTLSCertificates verifies that the client TLS settings in conf can +// be loaded. On success it returns a function that applies them; subsequent +// HTTPS connections then use the new settings, including from writers that +// already exist, because writers dial through the TLS manager. Connections that +// are already established are left alone. +// +// A service with TLS disabled has no TLS manager, so there is nothing to reload +// and the returned function is a no-op. +func (s *Service) PrepareReloadTLSCertificates(conf Config) (func() error, error) { s.mu.Lock() defer s.mu.Unlock() if s.tlsManager == nil { return func() error { return nil }, nil } - apply, err := s.tlsManager.PrepareReconfigure(tlsconfig.WithServerCertificate(s.conf.Certificate, s.conf.PrivateKey)) + apply, err := s.tlsManager.PrepareReconfigure(conf.TLSManagerOpts()...) if err != nil { - return nil, fmt.Errorf("subscriber: TLS certificate reload failed (%q, %q): %w", s.conf.Certificate, s.conf.PrivateKey, err) + // Report the paths being loaded, not the ones currently in use: the + // whole point of the message is to name what failed. + return nil, fmt.Errorf("subscriber: TLS certificate reload failed (%q, %q): %w", conf.Certificate, conf.PrivateKey, err) } return apply, nil } @@ -455,14 +457,17 @@ func (s *Service) newPointsWriter(u url.URL) (PointsWriter, error) { if s.conf.InsecureSkipVerify { s.Logger.Warn("'insecure-skip-verify' is true. This will skip all certificate verifications.") } - // Each writer gets its own clone of the manager's config; the clone - // shares the certificate loader via GetClientCertificate, so a reloaded - // client certificate is picked up on new connections. + // Writers dial through the manager, which resolves the TLS + // configuration on each connection, so a reloaded configuration reaches + // writers that already exist. The config snapshot is only a fallback for + // proxied requests, which do not dial through the manager. var tlsConfig *tls.Config + var dialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error) if s.tlsManager != nil { tlsConfig = s.tlsManager.TLSConfig() + dialTLSContext = s.tlsManager.DialContext } - return NewHTTPS(u.String(), time.Duration(s.conf.HTTPTimeout), tlsConfig) + return NewHTTPS(u.String(), time.Duration(s.conf.HTTPTimeout), tlsConfig, dialTLSContext) default: return nil, fmt.Errorf("unknown destination scheme %s", u.Scheme) } From cc846f568113dfb469fe4f38ed12aa70b8fbfdca Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 10:59:05 -0500 Subject: [PATCH 05/18] chore: code cleanup Remove unused and misleading *tls.Config parameter from NewHTTPS. --- services/subscriber/http.go | 23 +- services/subscriber/mtls_internal_test.go | 4 +- services/subscriber/service.go | 8 +- .../subscriber/tlsreload_internal_test.go | 407 ++++++++++++++++++ 4 files changed, 419 insertions(+), 23 deletions(-) create mode 100644 services/subscriber/tlsreload_internal_test.go diff --git a/services/subscriber/http.go b/services/subscriber/http.go index 8b7999f867e..00170808fd8 100644 --- a/services/subscriber/http.go +++ b/services/subscriber/http.go @@ -3,7 +3,6 @@ package subscriber import ( "bytes" "context" - "crypto/tls" "net" "time" @@ -18,31 +17,25 @@ type HTTP struct { // NewHTTP returns a new HTTP points writer with default options. func NewHTTP(addr string, timeout time.Duration) (*HTTP, error) { - return NewHTTPS(addr, timeout, nil, nil) + return NewHTTPS(addr, timeout, nil) } // NewHTTPS returns a new HTTPS points writer with default options and HTTPS // configured. // -// dialTLSContext dials the writer's TLS connections, normally -// tlsconfig.TLSConfigManager.DialContext, which resolves the TLS configuration -// on each connection. That is what allows a reloaded configuration to reach a -// writer that already exists: the writer keeps dialing through the manager -// rather than through a configuration captured when it was built. -// -// tlsConfig is the fully-resolved client TLS configuration (root CAs, any client -// certificate, and InsecureSkipVerify). It is only consulted for proxied -// requests, which tunnel through the proxy instead of using dialTLSContext, and -// is therefore a snapshot that a reload does not update. Both may be nil to use -// Go's defaults. +// dialTLSContext dials the writer's TLS connections and is the only source of +// its TLS configuration. It is normally tlsconfig.TLSConfigManager.DialContext, +// which resolves the configuration on each connection. That is what allows a +// reloaded configuration to reach a writer that already exists: the writer keeps +// dialing through the manager rather than through a configuration captured when +// it was built. It may be nil to use Go's defaults, as NewHTTP does. // // timeout bounds the whole request, including the connection and TLS handshake, // so no separate handshake timeout is needed. -func NewHTTPS(addr string, timeout time.Duration, tlsConfig *tls.Config, dialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error)) (*HTTP, error) { +func NewHTTPS(addr string, timeout time.Duration, dialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error)) (*HTTP, error) { conf := client.HTTPConfig{ Addr: addr, Timeout: timeout, - TLSConfig: tlsConfig, DialTLSContext: dialTLSContext, } diff --git a/services/subscriber/mtls_internal_test.go b/services/subscriber/mtls_internal_test.go index c1c9604b9fc..55ad18d2cc2 100644 --- a/services/subscriber/mtls_internal_test.go +++ b/services/subscriber/mtls_internal_test.go @@ -95,7 +95,7 @@ func TestHTTP_PresentsClientCertificate(t *testing.T) { cm := newManager(true) defer cm.Close() - w, err := NewHTTPS(srv.URL, time.Duration(conf.HTTPTimeout), cm.TLSConfig(), cm.DialContext) + w, err := NewHTTPS(srv.URL, time.Duration(conf.HTTPTimeout), cm.DialContext) require.NoError(t, err) _, err = w.WritePointsContext(context.Background(), WriteRequest{ @@ -111,7 +111,7 @@ func TestHTTP_PresentsClientCertificate(t *testing.T) { cm := newManager(false) defer cm.Close() - w, err := NewHTTPS(srv.URL, time.Duration(conf.HTTPTimeout), cm.TLSConfig(), cm.DialContext) + w, err := NewHTTPS(srv.URL, time.Duration(conf.HTTPTimeout), cm.DialContext) require.NoError(t, err) _, err = w.WritePointsContext(context.Background(), WriteRequest{ diff --git a/services/subscriber/service.go b/services/subscriber/service.go index 49eb7e9119f..45fbddf93c6 100644 --- a/services/subscriber/service.go +++ b/services/subscriber/service.go @@ -4,7 +4,6 @@ package subscriber // import "github.com/influxdata/influxdb/services/subscriber import ( "context" - "crypto/tls" "errors" "fmt" "net" @@ -459,15 +458,12 @@ func (s *Service) newPointsWriter(u url.URL) (PointsWriter, error) { } // Writers dial through the manager, which resolves the TLS // configuration on each connection, so a reloaded configuration reaches - // writers that already exist. The config snapshot is only a fallback for - // proxied requests, which do not dial through the manager. - var tlsConfig *tls.Config + // writers that already exist. var dialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error) if s.tlsManager != nil { - tlsConfig = s.tlsManager.TLSConfig() dialTLSContext = s.tlsManager.DialContext } - return NewHTTPS(u.String(), time.Duration(s.conf.HTTPTimeout), tlsConfig, dialTLSContext) + return NewHTTPS(u.String(), time.Duration(s.conf.HTTPTimeout), dialTLSContext) default: return nil, fmt.Errorf("unknown destination scheme %s", u.Scheme) } diff --git a/services/subscriber/tlsreload_internal_test.go b/services/subscriber/tlsreload_internal_test.go new file mode 100644 index 00000000000..954846075f9 --- /dev/null +++ b/services/subscriber/tlsreload_internal_test.go @@ -0,0 +1,407 @@ +package subscriber + +// Tests for reloading the subscriber's client TLS configuration. Open and +// PrepareReloadTLSCertificates both build their options from +// Config.TLSManagerOpts, so a reload can change any TLS setting, not just the +// certificate. + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path" + "sync" + "testing" + "time" + + th "github.com/influxdata/influxdb/pkg/testing/helper" + "github.com/influxdata/influxdb/pkg/testing/selfsigned" + "github.com/influxdata/influxdb/pkg/tlsconfig" + "github.com/influxdata/influxdb/toml" + "github.com/stretchr/testify/require" +) + +// certSerial returns the serial of ss's leaf certificate. +func certSerial(t *testing.T, ss *selfsigned.Cert) string { + t.Helper() + lc, err := tlsconfig.LoadCertificate(ss.CertPath, ss.KeyPath) + require.NoError(t, err) + return lc.Serial() +} + +// presentedClientSerial returns the serial of the certificate cm's TLS config +// would offer to a server asking for a client certificate. +func presentedClientSerial(t *testing.T, cm *tlsconfig.TLSConfigManager) string { + t.Helper() + + tlsConfig := cm.TLSConfig() + require.NotNil(t, tlsConfig) + require.NotNil(t, tlsConfig.GetClientCertificate) + + cert, err := tlsConfig.GetClientCertificate(&tls.CertificateRequestInfo{ + SignatureSchemes: []tls.SignatureScheme{tls.PKCS1WithSHA256}, + }) + require.NoError(t, err) + require.NotEmpty(t, cert.Certificate) + + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + require.NoError(t, err) + return leaf.SerialNumber.String() +} + +// recordingMTLSServer starts an HTTPS server that requires a client certificate +// signed by any of clientCAs and records the serial most recently presented. +func recordingMTLSServer(t *testing.T, serverCert *selfsigned.Cert, clientCAs ...*selfsigned.Cert) (srv *httptest.Server, lastSerial func() string) { + t.Helper() + + pool := x509.NewCertPool() + for _, ca := range clientCAs { + caPEM, err := os.ReadFile(ca.CACertPath) + require.NoError(t, err) + require.True(t, pool.AppendCertsFromPEM(caPEM)) + } + + cert, err := tls.LoadX509KeyPair(serverCert.CertPath, serverCert.KeyPath) + require.NoError(t, err) + + var mu sync.Mutex + var serial string + srv = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 { + serial = r.TLS.PeerCertificates[0].SerialNumber.String() + } + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + srv.TLS = &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: pool, + } + srv.StartTLS() + t.Cleanup(srv.Close) + + return srv, func() string { + mu.Lock() + defer mu.Unlock() + return serial + } +} + +// openService opens a subscriber Service with c and its own certificate monitor. +func openService(t *testing.T, c Config) *Service { + t.Helper() + + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + s := NewService(c, certMonitor) + s.MetaClient = stubMetaClient{} + require.NoError(t, s.Open()) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + return s +} + +func writePoint(t *testing.T, w PointsWriter) { + t.Helper() + _, err := w.WritePointsContext(context.Background(), WriteRequest{ + Database: "db0", + lineProtocol: []byte("cpu value=1\n"), + }) + require.NoError(t, err) +} + +func TestConfig_TLSManagerOpts(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + c := NewConfig() + c.CaCerts = ss.CACertPath + c.Certificate = ss.CertPath + c.PrivateKey = ss.KeyPath + c.InsecureSkipVerify = true + + // Open and the reload hook share these options, so building a manager from + // them must reproduce the configured settings. + cm, err := tlsconfig.NewClientTLSConfigManager(certMonitor, c.TLSManagerOpts()...) + require.NoError(t, err) + defer th.CheckedClose(t, cm)() + + tlsConfig := cm.TLSConfig() + require.NotNil(t, tlsConfig) + require.True(t, tlsConfig.InsecureSkipVerify) + require.NotNil(t, tlsConfig.RootCAs, "ca-certs should resolve into the root CA pool") + + // The certificate is offered as a client certificate, not a server one. + require.NotNil(t, tlsConfig.GetClientCertificate) + require.Nil(t, tlsConfig.GetCertificate) + + cert, err := tlsConfig.GetClientCertificate(&tls.CertificateRequestInfo{ + SignatureSchemes: []tls.SignatureScheme{tls.PKCS1WithSHA256}, + }) + require.NoError(t, err) + require.NotEmpty(t, cert.Certificate) +} + +func TestService_ReloadTLSConfig_ReconfiguresEverything(t *testing.T) { + ssA := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("a", "CA A")) + ssB := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("b", "CA B")) + + confA := NewConfig() + confA.CaCerts = ssA.CACertPath + confA.Certificate = ssA.CertPath + confA.PrivateKey = ssA.KeyPath + + s := openService(t, confA) + + before := s.tlsManager.TLSConfig() + require.NotNil(t, before) + require.False(t, before.InsecureSkipVerify) + + // Reload with a different CA, a different client certificate, and insecure + // verification enabled. + confB := NewConfig() + confB.CaCerts = ssB.CACertPath + confB.Certificate = ssB.CertPath + confB.PrivateKey = ssB.KeyPath + confB.InsecureSkipVerify = true + + apply, err := s.PrepareReloadTLSCertificates(confB) + require.NoError(t, err) + require.NotNil(t, apply) + + // Preparing must not change anything on its own. + require.False(t, s.tlsManager.TLSConfig().InsecureSkipVerify, "PrepareReload must only validate") + + require.NoError(t, apply()) + + after := s.tlsManager.TLSConfig() + require.True(t, after.InsecureSkipVerify, "allow-insecure should be reconfigured") + require.NotNil(t, after.RootCAs) + require.False(t, before.RootCAs.Equal(after.RootCAs), "root CAs should be rebuilt from the new config") + + require.Equal(t, certSerial(t, ssB), presentedClientSerial(t, s.tlsManager), + "the reloaded client certificate should be the one offered to servers") +} + +func TestService_ReloadTLSConfig_NewWriterUsesReloadedConfig(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientA := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("a", "Client CA A")) + clientB := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("b", "Client CA B")) + + srv, lastSerial := recordingMTLSServer(t, serverSS, clientA, clientB) + + conf := NewConfig() + conf.HTTPTimeout = toml.Duration(5 * time.Second) + conf.CaCerts = serverSS.CACertPath + conf.Certificate = clientA.CertPath + conf.PrivateKey = clientA.KeyPath + + s := openService(t, conf) + + u, err := url.Parse(srv.URL) + require.NoError(t, err) + + w, err := s.newPointsWriter(*u) + require.NoError(t, err) + writePoint(t, w) + require.Equal(t, certSerial(t, clientA), lastSerial(), "the configured client certificate should be presented") + + // Reload with a different client certificate. + confB := conf + confB.Certificate = clientB.CertPath + confB.PrivateKey = clientB.KeyPath + + apply, err := s.PrepareReloadTLSCertificates(confB) + require.NoError(t, err) + require.NoError(t, apply()) + + // A writer created after the reload handshakes fresh and presents the new + // certificate. This is the path subscriptions take when they are recreated. + w2, err := s.newPointsWriter(*u) + require.NoError(t, err) + writePoint(t, w2) + require.Equal(t, certSerial(t, clientB), lastSerial(), "a new writer should present the reloaded certificate") +} + +// TestService_ReloadTLSConfig_EstablishedConnectionUndisturbed covers the two +// halves of a reload's effect on a writer that already exists. A connection that +// is already up is working and stays on the certificate it handshook with: a +// reload must not disturb it. Once that connection is replaced, the writer dials +// through the manager again and the new connection uses the reloaded settings. +func TestService_ReloadTLSConfig_EstablishedConnectionUndisturbed(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientA := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("a", "Client CA A")) + clientB := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("b", "Client CA B")) + + srv, lastSerial := recordingMTLSServer(t, serverSS, clientA, clientB) + + conf := NewConfig() + conf.HTTPTimeout = toml.Duration(5 * time.Second) + conf.CaCerts = serverSS.CACertPath + conf.Certificate = clientA.CertPath + conf.PrivateKey = clientA.KeyPath + + s := openService(t, conf) + + u, err := url.Parse(srv.URL) + require.NoError(t, err) + + w, err := s.newPointsWriter(*u) + require.NoError(t, err) + writePoint(t, w) + require.Equal(t, certSerial(t, clientA), lastSerial()) + + confB := conf + confB.Certificate = clientB.CertPath + confB.PrivateKey = clientB.KeyPath + + apply, err := s.PrepareReloadTLSCertificates(confB) + require.NoError(t, err) + require.NoError(t, apply()) + + // Same writer, connection reused: no handshake, so the working connection + // carries on with the certificate it was built with. + writePoint(t, w) + require.Equal(t, certSerial(t, clientA), lastSerial(), + "a reload must not disturb an established connection") + + // Drop the pooled connection from the client side so the next write has to + // dial again. Closing it from the server side instead would race: the client + // can pick the already-dead connection for a write, which is not idempotent + // and so is not always retried. + require.NoError(t, w.(*HTTP).c.Close()) + + // The writer dials through the manager, so its next connection picks up the + // reloaded certificate without the writer being rebuilt. + writePoint(t, w) + require.Equal(t, certSerial(t, clientB), lastSerial(), + "a reconnect should present the reloaded certificate") +} + +// TestService_ReloadTLSConfig_ExistingWriterHonorsReloadedRootCA covers a +// setting that has no per-connection callback of its own. Because writers dial +// through the manager, root CAs are resolved per connection like everything +// else, so a writer that already exists picks up a reloaded pool. Nothing here +// forces a reconnect: the first handshake fails, so no connection is pooled and +// the next write dials again on its own. +func TestService_ReloadTLSConfig_ExistingWriterHonorsReloadedRootCA(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + otherSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("other", "Other CA")) + + cert, err := tls.LoadX509KeyPair(serverSS.CertPath, serverSS.KeyPath) + require.NoError(t, err) + + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + srv.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + srv.StartTLS() + t.Cleanup(srv.Close) + + // Trust the wrong CA to begin with, so the endpoint cannot be verified. + conf := NewConfig() + conf.HTTPTimeout = toml.Duration(5 * time.Second) + conf.CaCerts = otherSS.CACertPath + + s := openService(t, conf) + + u, err := url.Parse(srv.URL) + require.NoError(t, err) + + w, err := s.newPointsWriter(*u) + require.NoError(t, err) + + _, err = w.WritePointsContext(context.Background(), WriteRequest{ + Database: "db0", + lineProtocol: []byte("cpu value=1\n"), + }) + require.ErrorContains(t, err, "certificate signed by unknown authority") + + // Reload trusting the endpoint's CA. + good := conf + good.CaCerts = serverSS.CACertPath + + apply, err := s.PrepareReloadTLSCertificates(good) + require.NoError(t, err) + require.NoError(t, apply()) + + // The same writer now verifies the endpoint against the reloaded pool. + writePoint(t, w) +} + +func TestService_ReloadTLSConfig_Failures(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + conf := NewConfig() + conf.Certificate = ss.CertPath + conf.PrivateKey = ss.KeyPath + + t.Run("unloadable certificate keeps the previous configuration", func(t *testing.T) { + s := openService(t, conf) + + bad := conf + bad.Certificate = path.Join(t.TempDir(), "absent.pem") + bad.PrivateKey = bad.Certificate + + apply, err := s.PrepareReloadTLSCertificates(bad) + require.ErrorContains(t, err, "subscriber: TLS certificate reload failed") + require.Nil(t, apply) + + // The message must name the path that failed to load, not the one still + // in use, or it sends the reader to the wrong file. + require.ErrorContains(t, err, bad.Certificate) + require.NotContains(t, err.Error(), ss.CertPath) + + // The active certificate is untouched, so the service keeps working. + require.Equal(t, certSerial(t, ss), presentedClientSerial(t, s.tlsManager)) + }) + + t.Run("unusable root CA keeps the previous configuration", func(t *testing.T) { + s := openService(t, conf) + before := s.tlsManager.TLSConfig() + + bad := conf + bad.CaCerts = path.Join(t.TempDir(), "absent-ca.pem") + + apply, err := s.PrepareReloadTLSCertificates(bad) + require.Error(t, err) + require.Nil(t, apply) + + require.Equal(t, before.RootCAs, s.tlsManager.TLSConfig().RootCAs) + }) +} + +func TestService_ReloadTLSConfig_DisabledService(t *testing.T) { + conf := NewConfig() + conf.Enabled = false + + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + s := NewService(conf, certMonitor) + s.MetaClient = stubMetaClient{} + + // Open on a disabled service returns immediately without starting anything, + // so there is nothing to Close afterwards. + require.NoError(t, s.Open()) + + // A disabled service never built a TLS manager, so a reload has nothing to + // do and must not fail the whole configuration reload. + require.Nil(t, s.tlsManager) + + apply, err := s.PrepareReloadTLSCertificates(NewConfig()) + require.NoError(t, err) + require.NotNil(t, apply) + require.NoError(t, apply()) +} From 24e28e520651266c28a6100ec448e8fdb33b8da4 Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 11:59:11 -0500 Subject: [PATCH 06/18] feat: httpd service supports full TLS config reloading - Add support for full httpd TLS configuration reloading. - Add usage to production TLS managers for improved logging. - Fix some existing test issues. --- pkg/tlsconfig/configmanager.go | 30 ++- pkg/tlsconfig/configmanager_test.go | 131 ++++++++++ services/httpd/config.go | 25 ++ services/httpd/service.go | 86 ++---- services/httpd/service_test.go | 3 +- services/httpd/tlsreload_test.go | 247 ++++++++++++++++++ services/opentsdb/service.go | 5 +- services/opentsdb/service_test.go | 44 +++- services/subscriber/config.go | 1 + .../subscriber/tlsreload_internal_test.go | 28 ++ 10 files changed, 533 insertions(+), 67 deletions(-) create mode 100644 services/httpd/tlsreload_test.go diff --git a/pkg/tlsconfig/configmanager.go b/pkg/tlsconfig/configmanager.go index 7fe4132f1c2..384414a6696 100644 --- a/pkg/tlsconfig/configmanager.go +++ b/pkg/tlsconfig/configmanager.go @@ -42,6 +42,10 @@ var ( // with a valid Role. It is generally due to a misuse of an internal API. ErrNoRole = errors.New("no role specified for TLS certificate") + // ErrNoTLSConfig indicates that a TLS connection was attempted against a manager + // that has no TLS configuration to serve it. + ErrNoTLSConfig = errors.New("no TLS configuration available") + // ErrNotSupportedServer indicates that an operation is not supported by a server role // config manager. ErrNotSupportedServer = errors.New("operation not supported by server role TLS manager") @@ -688,16 +692,36 @@ func (cm *TLSConfigManager) UseTLS() bool { } // Return a net.Listener for network and address based on current configuration. +// +// When TLS is enabled the listener resolves its configuration on each +// connection, so a manager reconfigured through PrepareReconfigure takes effect +// on the next connection without the listener being rebound. Connections that +// are already established keep the configuration they handshook with. Whether +// the listener is TLS at all is fixed here, when the socket is bound, which is +// why PrepareReconfigure refuses to change useTLS for a server. func (cm *TLSConfigManager) Listen(network, address string) (net.Listener, error) { if !cm.role.IsServerRole() { return nil, fmt.Errorf("%s: %w", cm.usage, ErrClientListen) } - if tlsConfig := cm.TLSConfig(); tlsConfig != nil { - return tls.Listen(network, address, tlsConfig) - } else { + if !cm.UseTLS() { return net.Listen(network, address) } + + // This config carries only the callback; the configuration it returns is + // what serves each connection. Session ticket keys are read from this outer + // config rather than from the returned one, so they stay stable across + // connections and session resumption is unaffected. + return tls.Listen(network, address, &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + if tlsConfig := cm.TLSConfig(); tlsConfig != nil { + return tlsConfig, nil + } + // Unreachable while a server cannot turn useTLS off, but failing + // the handshake beats serving a connection with no configuration. + return nil, fmt.Errorf("%s: %w", cm.usage, ErrNoTLSConfig) + }, + }) } // Dial a remote for network and addressing using the current configuration. diff --git a/pkg/tlsconfig/configmanager_test.go b/pkg/tlsconfig/configmanager_test.go index bb5492fc73c..9696466196a 100644 --- a/pkg/tlsconfig/configmanager_test.go +++ b/pkg/tlsconfig/configmanager_test.go @@ -2306,3 +2306,134 @@ func TestTLSConfigManager_DialContext(t *testing.T) { require.NoError(t, conn.Close()) }) } + +// TestTLSConfigManager_ListenResolvesConfigPerConnection covers the listener +// resolving its configuration on each connection rather than freezing it when +// the socket is bound. +func TestTLSConfigManager_ListenResolvesConfigPerConnection(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + ss := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("c", "Client CA")) + + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + listener, err := manager.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + // Report the server's view of each handshake: that is what enforces the + // client auth policy. + serverErr := make(chan error, 8) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { + serverErr <- conn.(*tls.Conn).Handshake() + conn.Close() + }() + } + }() + + dial := func(t *testing.T, cfg *tls.Config) { + t.Helper() + cfg.InsecureSkipVerify = true + conn, err := tls.Dial("tcp", listener.Addr().String(), cfg) + if err == nil { + conn.Handshake() + conn.Close() + } + } + + dial(t, &tls.Config{}) + require.NoError(t, <-serverErr, "no client certificate is required initially") + + // Require client certificates. The listener is not rebound. + apply, err := manager.PrepareReconfigure( + WithClientAuth(tls.RequireAndVerifyClientCert), + WithClientCA(&CAConfig{Paths: []string{clientSS.CACertPath}})) + require.NoError(t, err) + + dial(t, &tls.Config{}) + require.NoError(t, <-serverErr, "PrepareReconfigure must not change the listener") + + require.NoError(t, apply()) + + dial(t, &tls.Config{}) + require.Error(t, <-serverErr, "the reconfigured client auth policy should be enforced") + + clientCert, err := tls.LoadX509KeyPair(clientSS.CertPath, clientSS.KeyPath) + require.NoError(t, err) + dial(t, &tls.Config{Certificates: []tls.Certificate{clientCert}}) + require.NoError(t, <-serverErr, "the reconfigured client CA should verify the client") +} + +// TestTLSConfigManager_ListenSessionResumption guards the session ticket keys. +// The listener resolves a fresh *tls.Config per connection; ticket keys are +// taken from the listener's own config, so they stay stable and resumption +// keeps working. Fresh per-connection keys would silently break it. +func TestTLSConfigManager_ListenSessionResumption(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + ss := selfsigned.NewSelfSignedCert(t) + + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + listener, err := manager.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + if err := conn.(*tls.Conn).Handshake(); err != nil { + return + } + conn.Write([]byte("x")) + buf := make([]byte, 1) + conn.Read(buf) + }() + } + }() + + // One cache across dials, as a real client would have. + clientConfig := &tls.Config{ + InsecureSkipVerify: true, + ClientSessionCache: tls.NewLRUClientSessionCache(8), + } + + var resumed []bool + for range 3 { + conn, err := tls.Dial("tcp", listener.Addr().String(), clientConfig) + require.NoError(t, err) + resumed = append(resumed, conn.ConnectionState().DidResume) + + // Reading processes the session ticket, caching it for the next dial. + buf := make([]byte, 1) + _, err = conn.Read(buf) + require.NoError(t, err) + require.NoError(t, conn.Close()) + } + + require.Contains(t, resumed, true, "per-connection configs must not break session resumption") +} diff --git a/services/httpd/config.go b/services/httpd/config.go index f9a0009e007..2feca99a4a4 100644 --- a/services/httpd/config.go +++ b/services/httpd/config.go @@ -215,3 +215,28 @@ func (filters StatusFilters) Match(statusCode int) bool { } return false } + +// TLSManagerOpts returns the list of TLS manager options specified by c. +// +// Open and PrepareReloadConfig both build the manager from these, so a reload +// applies every TLS setting rather than only the certificate, and the two paths +// cannot drift apart. +func (c Config) TLSManagerOpts() []tlsconfig.TLSConfigManagerOpt { + // Convert the optional client-auth type to a *tls.ClientAuthType so a nil + // (unset) config leaves the base TLS config's ClientAuth in place. + var clientAuthType *tls.ClientAuthType + if c.HTTPSClientAuthType != nil { + auth := tls.ClientAuthType(*c.HTTPSClientAuthType) + clientAuthType = &auth + } + + return []tlsconfig.TLSConfigManagerOpt{ + tlsconfig.WithUsage("httpd"), + tlsconfig.WithUseTLS(c.HTTPSEnabled), + tlsconfig.WithBaseConfig(c.TLS), + tlsconfig.WithServerCertificate(c.HTTPSCertificate, c.HTTPSPrivateKey), + tlsconfig.WithIgnoreFilePermissions(c.HTTPSInsecureCertificate), + tlsconfig.WithClientAuthPtr(clientAuthType), + tlsconfig.WithClientCA(c.HTTPSClientCA), + } +} diff --git a/services/httpd/service.go b/services/httpd/service.go index 2b4f8f0eb68..e7b395ff10c 100644 --- a/services/httpd/service.go +++ b/services/httpd/service.go @@ -3,7 +3,6 @@ package httpd // import "github.com/influxdata/influxdb/services/httpd" import ( "context" - "crypto/tls" "errors" "fmt" "net" @@ -63,6 +62,11 @@ type Service struct { certMonitor *tlsconfig.TLSCertMonitor + // conf is the configuration the service was created with. It is the single + // source of the TLS manager options, shared with PrepareReloadConfig so the + // open and reload paths cannot drift apart. + conf Config + addr string httpsEnabled bool @@ -71,28 +75,8 @@ type Service struct { unixSocketGroup int bindSocket string - // httpsCertificate is the initial TLS certificate to load if httpsEnabled is true. This should not be - // exposed to client code because a different certificate might be loaded on a config reload. - httpsCertificate string - - // httpsPrivateKey is the initial TLS private key to load if httpsEnabled is true. This should not be - // exposed to client code because a different private key might be loaded on a config reload. - httpsPrivateKey string - - // httpsInsecureCertificate is true if certificate file permissions should be ignored. - httpsInsecureCertificate bool - - // httpsClientAuthType defines the type of client authentication required (aka - // mTLS). A nil value leaves the base TLS config's ClientAuth in place. - httpsClientAuthType *tls.ClientAuthType - - // httpsClientCA configures the CA pool used to verify client certificates - // (aka mTLS). A nil value leaves the base TLS config's client pool in place. - httpsClientCA *tlsconfig.CAConfig - - limit int - tlsConfig *tls.Config - err chan error + limit int + err chan error closeFunc func() error @@ -127,39 +111,23 @@ type Service struct { func NewService(c Config, certMonitor *tlsconfig.TLSCertMonitor) *Service { handler := NewHandler(c) - // Convert the optional client-auth type to a *tls.ClientAuthType so a nil - // (unset) config leaves the base TLS config's ClientAuth in place. - var clientAuthType *tls.ClientAuthType - if c.HTTPSClientAuthType != nil { - auth := tls.ClientAuthType(*c.HTTPSClientAuthType) - clientAuthType = &auth - } - s := &Service{ - certMonitor: certMonitor, - addr: c.BindAddress, - httpsEnabled: c.HTTPSEnabled, - httpsCertificate: c.HTTPSCertificate, - httpsPrivateKey: c.HTTPSPrivateKey, - httpsInsecureCertificate: c.HTTPSInsecureCertificate, - httpsClientAuthType: clientAuthType, - httpsClientCA: c.HTTPSClientCA, - limit: c.MaxConnectionLimit, - tlsConfig: c.TLS, - err: make(chan error, 2), // There could be two serve calls that fail. - unixSocket: c.UnixSocketEnabled, - unixSocketPerm: uint32(c.UnixSocketPermissions), - bindSocket: c.BindSocket, - Handler: handler, + certMonitor: certMonitor, + conf: c, + addr: c.BindAddress, + httpsEnabled: c.HTTPSEnabled, + limit: c.MaxConnectionLimit, + err: make(chan error, 2), // There could be two serve calls that fail. + unixSocket: c.UnixSocketEnabled, + unixSocketPerm: uint32(c.UnixSocketPermissions), + bindSocket: c.BindSocket, + Handler: handler, httpServer: http.Server{ Handler: handler, }, Logger: zap.NewNop(), } s.closeFunc = sync.OnceValue(s.doClose) - if s.tlsConfig == nil { - s.tlsConfig = new(tls.Config) - } if c.UnixSocketGroup != nil { s.unixSocketGroup = int(*c.UnixSocketGroup) } @@ -183,13 +151,9 @@ func (s *Service) Open() error { // Open listener. tm, err := tlsconfig.NewServerTLSConfigManager( s.certMonitor, - tlsconfig.WithUseTLS(s.httpsEnabled), - tlsconfig.WithBaseConfig(s.tlsConfig), - tlsconfig.WithServerCertificate(s.httpsCertificate, s.httpsPrivateKey), - tlsconfig.WithIgnoreFilePermissions(s.httpsInsecureCertificate), - tlsconfig.WithClientAuthPtr(s.httpsClientAuthType), - tlsconfig.WithClientCA(s.httpsClientCA), - tlsconfig.WithLogger(s.Logger)) + append( + s.conf.TLSManagerOpts(), + tlsconfig.WithLogger(s.Logger))...) if err != nil { return fmt.Errorf("httpd: error creating TLS manager: %w", err) } @@ -338,12 +302,14 @@ func (s *Service) PrepareReloadConfig(c Config) (func() error, error) { return nil, errors.New("httpd: no TLS manager available") } - // Make sure the specified certificate will load correctly and return an apply function. - if apply, err := s.tlsManager.PrepareReconfigure( - tlsconfig.WithServerCertificate(c.HTTPSCertificate, c.HTTPSPrivateKey)); err == nil { + // Reconfigure from the whole of c, not just the certificate: the + // listener resolves its configuration per connection, so every TLS + // setting reloads. https-enabled is rejected above, so the useTLS in + // these options always matches what the listener was bound with. + if apply, err := s.tlsManager.PrepareReconfigure(c.TLSManagerOpts()...); err == nil { return apply, nil } else { - return nil, fmt.Errorf("httpd: error loading certificate at (%q, %q): %w", c.HTTPSCertificate, c.HTTPSPrivateKey, err) + return nil, fmt.Errorf("httpd: error reloading TLS configuration (certificate %q, key %q): %w", c.HTTPSCertificate, c.HTTPSPrivateKey, err) } } diff --git a/services/httpd/service_test.go b/services/httpd/service_test.go index 4eaf509d325..d3dac77aa91 100644 --- a/services/httpd/service_test.go +++ b/services/httpd/service_test.go @@ -187,7 +187,8 @@ func TestService_VerifyReloadedConfig(t *testing.T) { HTTPSPrivateKey: "/nonexistent/path/key.pem", } applyFunc, err := s.PrepareReloadConfig(newConfig) - require.ErrorContains(t, err, "error loading certificate") + require.ErrorContains(t, err, "error reloading TLS configuration") + require.ErrorContains(t, err, "/nonexistent/path/cert.pem", "the error should name the path that failed") require.Nil(t, applyFunc) }) diff --git a/services/httpd/tlsreload_test.go b/services/httpd/tlsreload_test.go new file mode 100644 index 00000000000..cc8bdfc4a6c --- /dev/null +++ b/services/httpd/tlsreload_test.go @@ -0,0 +1,247 @@ +package httpd_test + +// Tests that a config reload applies every TLS setting, not just the +// certificate. Open and PrepareReloadConfig both build the manager from +// Config.TLSManagerOpts, and the listener resolves its configuration per +// connection, so a reloaded setting takes effect on the next connection without +// the listener being rebound. + +import ( + "crypto/tls" + "crypto/x509" + "os" + "testing" + + th "github.com/influxdata/influxdb/pkg/testing/helper" + "github.com/influxdata/influxdb/pkg/testing/selfsigned" + "github.com/influxdata/influxdb/pkg/tlsconfig" + "github.com/influxdata/influxdb/services/httpd" + "github.com/influxdata/influxdb/toml" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +// openHTTPS starts an httpd service with TLS on an ephemeral port. +func openHTTPS(t *testing.T, c httpd.Config) *httpd.Service { + t.Helper() + + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + c.BindAddress = "127.0.0.1:0" + c.HTTPSEnabled = true + + s := httpd.NewService(c, certMonitor) + s.WithLogger(zap.NewNop()) + require.NoError(t, s.Open()) + t.Cleanup(th.CheckedClose(t, s)) + return s +} + +// clientCertPool builds a pool trusting ss's CA. +func clientCertPool(t *testing.T, ss *selfsigned.Cert) *x509.CertPool { + t.Helper() + pool := x509.NewCertPool() + pem, err := os.ReadFile(ss.CACertPath) + require.NoError(t, err) + require.True(t, pool.AppendCertsFromPEM(pem)) + return pool +} + +// handshake dials the service and reports the server's view of the handshake. +// The client's own error is not enough: under TLS 1.3 a client can finish its +// handshake before the server's rejection arrives, so a read is needed to +// surface it. +func handshake(t *testing.T, s *httpd.Service, clientCfg *tls.Config) error { + t.Helper() + + clientCfg.InsecureSkipVerify = true + conn, err := tls.Dial("tcp", s.Addr().String(), clientCfg) + if err != nil { + return err + } + defer conn.Close() + + if err := conn.Handshake(); err != nil { + return err + } + // Force the server's alert, if any, to surface. + if _, err := conn.Write([]byte("GET /ping HTTP/1.0\r\n\r\n")); err != nil { + return err + } + buf := make([]byte, 1) + if _, err := conn.Read(buf); err != nil { + return err + } + return nil +} + +func TestService_ReloadTLSConfig_ClientAuth(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("c", "Client CA")) + + // Start with client certificates not required. + s := openHTTPS(t, httpd.Config{ + HTTPSCertificate: serverSS.CertPath, + HTTPSPrivateKey: serverSS.KeyPath, + }) + + require.NoError(t, handshake(t, s, &tls.Config{}), + "a client without a certificate should be accepted before the reload") + + // Reload requiring and verifying client certificates. + required := toml.TlsClientAuthType(tls.RequireAndVerifyClientCert) + newConfig := httpd.Config{ + BindAddress: "127.0.0.1:0", + HTTPSEnabled: true, + HTTPSCertificate: serverSS.CertPath, + HTTPSPrivateKey: serverSS.KeyPath, + HTTPSClientAuthType: &required, + HTTPSClientCA: &tlsconfig.CAConfig{Paths: []string{clientSS.CACertPath}}, + } + + apply, err := s.PrepareReloadConfig(newConfig) + require.NoError(t, err) + require.NotNil(t, apply) + + // Preparing only validates. + require.NoError(t, handshake(t, s, &tls.Config{}), "PrepareReloadConfig must not change the listener") + + require.NoError(t, apply()) + + // The listener was never rebound, but the new client auth policy applies. + require.Error(t, handshake(t, s, &tls.Config{}), + "a client without a certificate should be rejected after the reload") + + clientCert, err := tls.LoadX509KeyPair(clientSS.CertPath, clientSS.KeyPath) + require.NoError(t, err) + require.NoError(t, handshake(t, s, &tls.Config{Certificates: []tls.Certificate{clientCert}}), + "a client with a certificate signed by the reloaded CA should be accepted") +} + +func TestService_ReloadTLSConfig_ClientCA(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + caA := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("a", "Client CA A")) + caB := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("b", "Client CA B")) + + required := toml.TlsClientAuthType(tls.RequireAndVerifyClientCert) + base := httpd.Config{ + BindAddress: "127.0.0.1:0", + HTTPSEnabled: true, + HTTPSCertificate: serverSS.CertPath, + HTTPSPrivateKey: serverSS.KeyPath, + HTTPSClientAuthType: &required, + HTTPSClientCA: &tlsconfig.CAConfig{Paths: []string{caA.CACertPath}}, + } + s := openHTTPS(t, base) + + certA, err := tls.LoadX509KeyPair(caA.CertPath, caA.KeyPath) + require.NoError(t, err) + certB, err := tls.LoadX509KeyPair(caB.CertPath, caB.KeyPath) + require.NoError(t, err) + + require.NoError(t, handshake(t, s, &tls.Config{Certificates: []tls.Certificate{certA}})) + require.Error(t, handshake(t, s, &tls.Config{Certificates: []tls.Certificate{certB}}), + "CA B is not trusted before the reload") + + // Swap the trusted client CA from A to B. + newConfig := base + newConfig.HTTPSClientCA = &tlsconfig.CAConfig{Paths: []string{caB.CACertPath}} + + apply, err := s.PrepareReloadConfig(newConfig) + require.NoError(t, err) + require.NoError(t, apply()) + + require.NoError(t, handshake(t, s, &tls.Config{Certificates: []tls.Certificate{certB}}), + "CA B should be trusted after the reload") + require.Error(t, handshake(t, s, &tls.Config{Certificates: []tls.Certificate{certA}}), + "CA A should no longer be trusted after the reload") +} + +func TestService_ReloadTLSConfig_MinVersion(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + + s := openHTTPS(t, httpd.Config{ + HTTPSCertificate: serverSS.CertPath, + HTTPSPrivateKey: serverSS.KeyPath, + }) + + require.NoError(t, handshake(t, s, &tls.Config{MaxVersion: tls.VersionTLS12}), + "TLS 1.2 should be accepted before the reload") + + // Raise the minimum version through the base TLS config. + newConfig := httpd.Config{ + BindAddress: "127.0.0.1:0", + HTTPSEnabled: true, + HTTPSCertificate: serverSS.CertPath, + HTTPSPrivateKey: serverSS.KeyPath, + TLS: &tls.Config{MinVersion: tls.VersionTLS13}, + } + + apply, err := s.PrepareReloadConfig(newConfig) + require.NoError(t, err) + require.NoError(t, apply()) + + require.Error(t, handshake(t, s, &tls.Config{MaxVersion: tls.VersionTLS12}), + "TLS 1.2 should be rejected after raising min-version") + require.NoError(t, handshake(t, s, &tls.Config{MinVersion: tls.VersionTLS13}), + "TLS 1.3 should still be accepted") +} + +func TestService_ReloadTLSConfig_FailureKeepsPreviousConfig(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("c", "Client CA")) + + s := openHTTPS(t, httpd.Config{ + HTTPSCertificate: serverSS.CertPath, + HTTPSPrivateKey: serverSS.KeyPath, + }) + + // A client CA that cannot be read must not disturb the running listener. + newConfig := httpd.Config{ + BindAddress: "127.0.0.1:0", + HTTPSEnabled: true, + HTTPSCertificate: serverSS.CertPath, + HTTPSPrivateKey: serverSS.KeyPath, + HTTPSClientCA: &tlsconfig.CAConfig{Paths: []string{"/nonexistent/ca.pem"}}, + } + + apply, err := s.PrepareReloadConfig(newConfig) + require.Error(t, err) + require.Nil(t, apply) + + require.NoError(t, handshake(t, s, &tls.Config{}), + "a failed reload must leave the listener serving the previous configuration") + + // Sanity check that the pool the test relies on is otherwise usable. + require.NotNil(t, clientCertPool(t, clientSS)) +} + +// TestService_TLSUsage covers the service naming itself to the certificate +// monitor. The monitor groups its warnings by usage, so a service that does not +// name itself is reported as an anonymous ".server". +func TestService_TLSUsage(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + + core, logs := observer.New(zapcore.InfoLevel) + certMonitor := tlsconfig.NewTLSCertMonitor(tlsconfig.WithMonitorLogger(zap.New(core))) + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + s := httpd.NewService(httpd.Config{ + BindAddress: "127.0.0.1:0", + HTTPSEnabled: true, + HTTPSCertificate: serverSS.CertPath, + HTTPSPrivateKey: serverSS.KeyPath, + }, certMonitor) + s.WithLogger(zap.NewNop()) + require.NoError(t, s.Open()) + t.Cleanup(th.CheckedClose(t, s)) + + entries := logs.FilterMessage("Registered certificate loader").TakeAll() + require.Len(t, entries, 1) + require.Equal(t, "httpd.server", entries[0].ContextMap()["usage"]) +} diff --git a/services/opentsdb/service.go b/services/opentsdb/service.go index 1ee9aa46621..9de75e5027e 100644 --- a/services/opentsdb/service.go +++ b/services/opentsdb/service.go @@ -155,9 +155,12 @@ func (s *Service) Open() error { s.wg.Add(1) go func() { defer s.wg.Done(); s.processBatches(s.batcher) }() - // Open listener. + // Open listener. The usage carries the bind address because a server can run + // several OpenTSDB services at once, and the certificate monitor groups its + // warnings by usage. cm, err := tlsconfig.NewServerTLSConfigManager( s.certMonitor, + tlsconfig.WithUsage(fmt.Sprintf("opentsdb(%s)", s.BindAddress)), tlsconfig.WithUseTLS(s.tls), tlsconfig.WithBaseConfig(s.tlsConfig), tlsconfig.WithServerCertificate(s.cert, s.privateKey), diff --git a/services/opentsdb/service_test.go b/services/opentsdb/service_test.go index daac98f6cee..dc7fc2b5077 100644 --- a/services/opentsdb/service_test.go +++ b/services/opentsdb/service_test.go @@ -25,6 +25,9 @@ import ( "github.com/influxdata/influxdb/toml" "github.com/influxdata/influxdb/tsdb" "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" ) func Test_Service_OpenClose(t *testing.T) { @@ -268,7 +271,9 @@ type TestService struct { func NewTestService(t *testing.T, database string, bind string) *TestService { certMonitor := tlsconfig.NewTLSCertMonitor() require.NoError(t, certMonitor.Open()) - defer th.CheckedClose(t, certMonitor)() + // The monitor has to outlive this helper, so close it when the test ends + // rather than when the helper returns. + t.Cleanup(th.CheckedClose(t, certMonitor)) s, err := NewService(Config{ BindAddress: bind, @@ -337,7 +342,7 @@ func TestService_ClientCertAuth(t *testing.T) { certMonitor := tlsconfig.NewTLSCertMonitor() require.NoError(t, certMonitor.Open()) - defer th.CheckedClose(t, certMonitor) + defer th.CheckedClose(t, certMonitor)() authType := toml.TlsClientAuthType(tls.RequireAndVerifyClientCert) s, err := NewService(Config{ @@ -401,3 +406,38 @@ func TestService_ClientCertAuth(t *testing.T) { require.Error(t, err, "server should reject a client with an untrusted certificate") }) } + +// TestService_TLSUsage covers the service naming itself to the certificate +// monitor. A server can run several OpenTSDB services at once, so the usage +// carries the bind address; otherwise the monitor, which groups its warnings by +// usage, could not tell two of them apart. +func TestService_TLSUsage(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + + core, logs := observer.New(zapcore.InfoLevel) + certMonitor := tlsconfig.NewTLSCertMonitor(tlsconfig.WithMonitorLogger(zap.New(core))) + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + const bind = "127.0.0.1:0" + s, err := NewService(Config{ + BindAddress: bind, + Database: "db0", + ConsistencyLevel: "one", + TLSEnabled: true, + Certificate: serverSS.CertPath, + PrivateKey: serverSS.KeyPath, + TLS: new(tls.Config), + }, certMonitor) + require.NoError(t, err) + + s.MetaClient = &internal.MetaClientMock{ + CreateDatabaseFn: func(string) (*meta.DatabaseInfo, error) { return nil, nil }, + } + require.NoError(t, s.Open()) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + + entries := logs.FilterMessage("Registered certificate loader").TakeAll() + require.Len(t, entries, 1) + require.Equal(t, "opentsdb("+bind+").server", entries[0].ContextMap()["usage"]) +} diff --git a/services/subscriber/config.go b/services/subscriber/config.go index 53660e50219..18a08c1da92 100644 --- a/services/subscriber/config.go +++ b/services/subscriber/config.go @@ -129,6 +129,7 @@ func (c Config) effectiveRootCA() *tlsconfig.CAConfig { // TLSManagerOpts returns the list of TLS manager options specified by c. func (c Config) TLSManagerOpts() []tlsconfig.TLSConfigManagerOpt { return []tlsconfig.TLSConfigManagerOpt{ + tlsconfig.WithUsage("subscriber"), tlsconfig.WithUseTLS(true), tlsconfig.WithBaseConfig(c.TLS), tlsconfig.WithAllowInsecure(c.InsecureSkipVerify), diff --git a/services/subscriber/tlsreload_internal_test.go b/services/subscriber/tlsreload_internal_test.go index 954846075f9..3d47cc79ae0 100644 --- a/services/subscriber/tlsreload_internal_test.go +++ b/services/subscriber/tlsreload_internal_test.go @@ -23,6 +23,9 @@ import ( "github.com/influxdata/influxdb/pkg/tlsconfig" "github.com/influxdata/influxdb/toml" "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" ) // certSerial returns the serial of ss's leaf certificate. @@ -405,3 +408,28 @@ func TestService_ReloadTLSConfig_DisabledService(t *testing.T) { require.NotNil(t, apply) require.NoError(t, apply()) } + +// TestService_TLSUsage covers the service naming itself to the certificate +// monitor, which groups its warnings by usage. +func TestService_TLSUsage(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + core, logs := observer.New(zapcore.InfoLevel) + certMonitor := tlsconfig.NewTLSCertMonitor(tlsconfig.WithMonitorLogger(zap.New(core))) + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + conf := NewConfig() + conf.Certificate = ss.CertPath + conf.PrivateKey = ss.KeyPath + + s := NewService(conf, certMonitor) + s.MetaClient = stubMetaClient{} + require.NoError(t, s.Open()) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + + entries := logs.FilterMessage("Registered certificate loader").TakeAll() + require.Len(t, entries, 1) + require.Equal(t, "subscriber.client", entries[0].ContextMap()["usage"], + "the subscriber holds a client certificate, not a server one") +} From 6b1012a704b438e3409e0a0a08b71f8a852b29c3 Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 13:13:45 -0500 Subject: [PATCH 07/18] feat: add server certificate sanity checking. - Server certificates must now have the server use EKU set. - Added options to turn the sanity check errors into warnings. --- etc/config.sample.toml | 15 ++ pkg/testing/selfsigned/selfsigned.go | 26 +++- pkg/tlsconfig/certconfig.go | 76 ++++++++-- pkg/tlsconfig/certconfig_test.go | 214 +++++++++++++++++++++++++++ pkg/tlsconfig/configmanager.go | 16 ++ pkg/tlsconfig/loadcert.go | 27 ++++ pkg/tlsconfig/loadcert_test.go | 76 ++++++++++ services/httpd/config.go | 80 +++++----- services/httpd/tlsreload_test.go | 39 +++++ services/opentsdb/config.go | 35 +++-- services/opentsdb/service.go | 51 ++++--- services/opentsdb/service_test.go | 47 ++++++ services/subscriber/config.go | 7 + 13 files changed, 618 insertions(+), 91 deletions(-) diff --git a/etc/config.sample.toml b/etc/config.sample.toml index c284a144c61..e55bc48fdd8 100644 --- a/etc/config.sample.toml +++ b/etc/config.sample.toml @@ -389,6 +389,11 @@ # Allows insecure file permissions for https-certificate and https-private-key when enabled. # https-insecure-certificate = false + # Loads https-certificate even when it fails the checks for whether a server can use it, + # such as the certificate not permitting server authentication. The failed checks are + # logged instead. A missing or unparseable certificate is still an error. + # https-ignore-sanity-checks = false + # The type of client certificate authentication (mutual TLS) to require. One of # NoClientCert, RequestClientCert, RequireAnyClientCert, VerifyClientCertIfGiven, or # RequireAndVerifyClientCert. Unset leaves client authentication disabled (NoClientCert). @@ -492,6 +497,11 @@ # Allows insecure file permissions on certificate and private-key when enabled. # insecure-certificate = false + # Loads certificate even when it fails the checks for whether a server can use it, such as + # the certificate not permitting server authentication. The failed checks are logged + # instead. A missing or unparseable certificate is still an error. + # ignore-sanity-checks = false + # The number of writer goroutines processing the write channel. # write-concurrency = 40 @@ -612,6 +622,11 @@ # Allows insecure file permissions on certificate and private-key when enabled. # insecure-certificate = false + # Loads certificate even when it fails the checks for whether a server can use it, such as + # the certificate not permitting server authentication. The failed checks are logged + # instead. A missing or unparseable certificate is still an error. + # ignore-sanity-checks = false + # The type of client certificate authentication (mutual TLS) to require. One of # NoClientCert, RequestClientCert, RequireAnyClientCert, VerifyClientCertIfGiven, or # RequireAndVerifyClientCert. Unset leaves client authentication disabled (NoClientCert). diff --git a/pkg/testing/selfsigned/selfsigned.go b/pkg/testing/selfsigned/selfsigned.go index 6f1517aadd7..58729cd3274 100644 --- a/pkg/testing/selfsigned/selfsigned.go +++ b/pkg/testing/selfsigned/selfsigned.go @@ -48,6 +48,14 @@ type CertOptions struct { // CACommonName sets the CA certificate's Subject.CommonName field CACommonName string + + // ExtKeyUsage replaces the leaf certificate's extended key usages. It is + // only consulted when ExtKeyUsageSet is true, so that an empty value can + // omit the extension entirely rather than mean "unset". + ExtKeyUsage []x509.ExtKeyUsage + + // ExtKeyUsageSet indicates ExtKeyUsage was configured with WithExtKeyUsage. + ExtKeyUsageSet bool } type CertOpt func(*CertOptions) @@ -88,6 +96,16 @@ func WithCombinedFile() CertOpt { } } +// WithExtKeyUsage replaces the leaf certificate's extended key usages, which +// default to server and client authentication. Passing no usages omits the +// extension entirely, leaving the certificate unrestricted. +func WithExtKeyUsage(eku ...x509.ExtKeyUsage) CertOpt { + return func(o *CertOptions) { + o.ExtKeyUsage = eku + o.ExtKeyUsageSet = true + } +} + func WithCASubject(organization, commonName string) CertOpt { return func(o *CertOptions) { o.CAOrganization = organization @@ -172,6 +190,12 @@ func NewSelfSignedCert(t *testing.T, opts ...CertOpt) *Cert { serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) require.NoError(t, err) + + extKeyUsage := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth} + if options.ExtKeyUsageSet { + extKeyUsage = options.ExtKeyUsage + } + // Basically the same as the CA template, but its own serial, and with ip addresses and dns names. template := x509.Certificate{ SerialNumber: serial, @@ -179,7 +203,7 @@ func NewSelfSignedCert(t *testing.T, opts ...CertOpt) *Cert { NotAfter: options.NotAfter, KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + ExtKeyUsage: extKeyUsage, BasicConstraintsValid: true, diff --git a/pkg/tlsconfig/certconfig.go b/pkg/tlsconfig/certconfig.go index 4d1ca5dc7a6..00ce9de86b8 100644 --- a/pkg/tlsconfig/certconfig.go +++ b/pkg/tlsconfig/certconfig.go @@ -30,6 +30,7 @@ var ( ErrCertificateNil = errors.New("TLS certificate is nil") ErrCertificateEmpty = errors.New("TLS certificate is empty") ErrCertificateInvalid = errors.New("TLS certificate is invalid") + ErrCertificateNotServerAuth = errors.New("TLS certificate does not permit server authentication") ErrCertificateRequestInfoNil = errors.New("CertificateRequestInfo is nil") ErrLoadedCertificateInvalid = errors.New("LoadedCertificate is invalid") ErrNoCertificateMonitor = errors.New("no certificate monitor") @@ -81,16 +82,18 @@ type tlsCertLoaderConfig struct { // usage is the descriptive usage of the cert loader. usage string - // ignoreFilePermissions is true if file permission checks should be bypassed. - // It is during a reconfiguration if ignoreFilePermissionsSet is false. + // ignoreFilePermissions is true if file permission checks should be bypassed + // when loading certificates. ignoreFilePermissions bool - // logger is the logger to use for logging. It is ignored during reconfiguration - // if loggerSet is false. - logger *zap.Logger + // ignoreSanityChecks is true if failed certificate sanity checks should be + // logged and overlooked instead of failing the load. + ignoreSanityChecks bool - // loggerSet is only true if WithCertLoaderLogger was used. - loggerSet bool + // logger is the logger to use for logging. It is only applied when the cert + // loader is constructed: a reconfiguration through PrepareLoad keeps the + // logger the loader already has. + logger *zap.Logger } // TLSCertLoaderOpt is a function to configure a TLSCertLoader. @@ -105,11 +108,11 @@ func WithCertLoaderCertificate(certPath string, keyPath string) TLSCertLoaderOpt } } -// WithCertLoaderLogger assigns a logger for to use. +// WithCertLoaderLogger assigns a logger to use. It only takes effect when given +// to NewTLSCertLoader; a loader keeps its original logger through a PrepareLoad. func WithCertLoaderLogger(logger *zap.Logger) TLSCertLoaderOpt { return func(c *tlsCertLoaderConfig) { c.logger = logger - c.loggerSet = true } } @@ -127,6 +130,17 @@ func WithCertLoaderIgnoreFilePermissions(ignore bool) TLSCertLoaderOpt { } } +// WithCertLoaderIgnoreSanityChecks logs failed certificate sanity checks and +// loads the certificate anyway, instead of failing the load. It is an escape +// hatch for a certificate this package judges unusable but a deployment relies +// on; it does not relax the checks that a certificate be present and parseable, +// which are faults rather than judgments. +func WithCertLoaderIgnoreSanityChecks(ignore bool) TLSCertLoaderOpt { + return func(c *tlsCertLoaderConfig) { + c.ignoreSanityChecks = ignore + } +} + // NewTLSCertLoader creates a TLSCertLoader loaded with the certificate found in certPath and keyPath. // Only trusted input (standard configuration files) should be used for certPath and keyPath. // If the certificate can not be loaded, an error is returned. On success, a monitor is setup to @@ -338,10 +352,46 @@ func (cl *TLSCertLoader) PrepareLoad(opts ...TLSCertLoaderOpt) (func() error, er return nil, err } - if loadedCert.IsEmpty() && cl.role.IsServerRole() { - err := fmt.Errorf("%s: can not use an empty certificate for a server: %w", cl.usage, ErrCertificateEmpty) - logLoadError(err) - return nil, err + // Sanity check the certificate against how a server will use it. A + // certificate a server can not present, or that every client will refuse, is + // caught here rather than at the first handshake, where it would surface + // per-connection as an error naming neither the certificate nor the reason. + if cl.role.IsServerRole() { + // These two are faults rather than judgments, so they fail the load even + // when sanity checks are ignored: a server using TLS always needs a + // certificate, and a leaf that will not parse is a real problem. + if loadedCert.IsEmpty() { + err := fmt.Errorf("%s: can not use an empty certificate for a server: %w", cl.usage, ErrCertificateEmpty) + logLoadError(err) + return nil, err + } + + leaf, err := loadedCert.GetLeaf() + if err != nil { + err = fmt.Errorf("%s: %w", cl.usage, err) + logLoadError(err) + return nil, err + } + + // Everything below is a sanity check: the certificate is real, but this + // package judges a server unable to use it. Collect them all so one load + // reports every problem rather than only the first. + var sanityErrs []error + + if !leaf.SupportsServerAuth() { + sanityErrs = append(sanityErrs, fmt.Errorf("%s: can not use a certificate that does not permit server authentication for a server: %w", + cl.usage, ErrCertificateNotServerAuth)) + } + + if len(sanityErrs) > 0 { + sanityErr := errors.Join(sanityErrs...) + if !c.ignoreSanityChecks { + logLoadError(sanityErr) + return nil, sanityErr + } + log.Warn("TLS certificate failed sanity checks, loading it anyway because sanity checks are being ignored", + zap.Error(sanityErr)) + } } loadedCert.WithLogContext(log).Info("Successfully loaded TLS certificate") diff --git a/pkg/tlsconfig/certconfig_test.go b/pkg/tlsconfig/certconfig_test.go index 709ab890c03..dc0a40bc1ec 100644 --- a/pkg/tlsconfig/certconfig_test.go +++ b/pkg/tlsconfig/certconfig_test.go @@ -885,3 +885,217 @@ func TestRole_Predicates(t *testing.T) { }) } } + +func TestTLSCertLoader_ServerCertificateMustSupportServerAuth(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + // A certificate issued only for client authentication. Every compliant TLS + // client refuses it from a server, so a server must not load it. + clientOnlySS := selfsigned.NewSelfSignedCert(t, + selfsigned.WithExtKeyUsage(x509.ExtKeyUsageClientAuth)) + serverSS := selfsigned.NewSelfSignedCert(t) + + t.Run("server role rejects a client-only certificate", func(t *testing.T) { + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderUsage("httpd.server"), + WithCertLoaderCertificate(clientOnlySS.CertPath, clientOnlySS.KeyPath)) + require.ErrorIs(t, err, ErrCertificateNotServerAuth) + require.ErrorContains(t, err, "httpd.server: ", "the error should name the usage that failed") + require.Nil(t, cl) + }) + + t.Run("client role accepts a client-only certificate", func(t *testing.T) { + // The check is specific to how a server uses the certificate; a client + // presenting it is exactly what it was issued for. + cl, err := NewTLSCertLoader( + ClientOnlyRole, + monitor, + WithCertLoaderCertificate(clientOnlySS.CertPath, clientOnlySS.KeyPath)) + require.NoError(t, err) + require.NotNil(t, cl) + defer th.CheckedClose(t, cl)() + }) + + t.Run("a certificate with no extended key usage is unrestricted", func(t *testing.T) { + unrestrictedSS := selfsigned.NewSelfSignedCert(t, selfsigned.WithExtKeyUsage()) + + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(unrestrictedSS.CertPath, unrestrictedSS.KeyPath)) + require.NoError(t, err) + require.NotNil(t, cl) + defer th.CheckedClose(t, cl)() + }) + + t.Run("reload to a client-only certificate is refused", func(t *testing.T) { + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(serverSS.CertPath, serverSS.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, cl)() + + apply, err := cl.PrepareLoad(WithCertLoaderCertificate(clientOnlySS.CertPath, clientOnlySS.KeyPath)) + require.ErrorIs(t, err, ErrCertificateNotServerAuth) + require.Nil(t, apply) + + // The working certificate stays in place. + certPath, _ := cl.Paths() + require.Equal(t, serverSS.CertPath, certPath) + }) +} + +func TestTLSConfigManager_ServerCertificateMustSupportServerAuth(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + clientOnlySS := selfsigned.NewSelfSignedCert(t, + selfsigned.WithExtKeyUsage(x509.ExtKeyUsageClientAuth)) + + t.Run("server manager refuses it", func(t *testing.T) { + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(clientOnlySS.CertPath, clientOnlySS.KeyPath)) + require.ErrorIs(t, err, ErrCertificateNotServerAuth) + require.Nil(t, manager) + }) + + t.Run("client manager accepts it as a client certificate", func(t *testing.T) { + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithClientCertificate(clientOnlySS.CertPath, clientOnlySS.KeyPath)) + require.NoError(t, err) + require.NotNil(t, manager) + defer th.CheckedClose(t, manager)() + }) +} + +func TestTLSCertLoader_IgnoreSanityChecks(t *testing.T) { + // A certificate issued only for client authentication: real and parseable, + // but a server should not be presenting it. + clientOnlySS := selfsigned.NewSelfSignedCert(t, + selfsigned.WithExtKeyUsage(x509.ExtKeyUsageClientAuth)) + + t.Run("loads the certificate and warns", func(t *testing.T) { + core, logs := observer.New(zapcore.InfoLevel) + + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderUsage("httpd.server"), + WithCertLoaderLogger(zap.New(core)), + WithCertLoaderIgnoreSanityChecks(true), + WithCertLoaderCertificate(clientOnlySS.CertPath, clientOnlySS.KeyPath)) + require.NoError(t, err, "an ignored sanity check must not fail the load") + require.NotNil(t, cl) + defer th.CheckedClose(t, cl)() + + // The certificate is genuinely in use, not merely accepted. + certPath, _ := cl.Paths() + require.Equal(t, clientOnlySS.CertPath, certPath) + require.NotNil(t, cl.Certificate()) + + warnings := logs.FilterMessage("TLS certificate failed sanity checks, loading it anyway because sanity checks are being ignored").TakeAll() + require.Len(t, warnings, 1, "ignoring a sanity check must still say so") + require.Equal(t, zap.WarnLevel, warnings[0].Level) + require.Contains(t, warnings[0].ContextMap()["error"], ErrCertificateNotServerAuth.Error(), + "the warning should say which check failed") + }) + + t.Run("still fails when not ignored", func(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderIgnoreSanityChecks(false), + WithCertLoaderCertificate(clientOnlySS.CertPath, clientOnlySS.KeyPath)) + require.ErrorIs(t, err, ErrCertificateNotServerAuth) + require.Nil(t, cl) + }) + + t.Run("an empty certificate still fails", func(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + ss := selfsigned.NewSelfSignedCert(t) + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderIgnoreSanityChecks(true), + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, cl)() + + // A server always needs a certificate: that is a fault, not a sanity + // check, so ignoring sanity checks does not excuse it. + apply, err := cl.PrepareLoad(WithCertLoaderCertificate("", "")) + require.ErrorIs(t, err, ErrCertificateEmpty) + require.Nil(t, apply) + }) + + t.Run("reload can ignore sanity checks", func(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + ss := selfsigned.NewSelfSignedCert(t) + cl, err := NewTLSCertLoader( + ServerOnlyRole, + monitor, + WithCertLoaderCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, cl)() + + // The option carries through a reload like the rest of the config. + apply, err := cl.PrepareLoad( + WithCertLoaderIgnoreSanityChecks(true), + WithCertLoaderCertificate(clientOnlySS.CertPath, clientOnlySS.KeyPath)) + require.NoError(t, err) + require.NoError(t, apply()) + + certPath, _ := cl.Paths() + require.Equal(t, clientOnlySS.CertPath, certPath) + }) +} + +func TestTLSConfigManager_WithIgnoreSanityChecks(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + clientOnlySS := selfsigned.NewSelfSignedCert(t, + selfsigned.WithExtKeyUsage(x509.ExtKeyUsageClientAuth)) + + t.Run("passes through to the cert loader", func(t *testing.T) { + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithIgnoreSanityChecks(true), + WithServerCertificate(clientOnlySS.CertPath, clientOnlySS.KeyPath)) + require.NoError(t, err, "the manager option must reach the cert loader") + require.NotNil(t, manager) + defer th.CheckedClose(t, manager)() + + certPath, _ := manager.serverCertLoader.Paths() + require.Equal(t, clientOnlySS.CertPath, certPath) + require.NotNil(t, manager.TLSConfig().GetCertificate) + }) + + t.Run("defaults to enforcing", func(t *testing.T) { + manager, err := NewServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(clientOnlySS.CertPath, clientOnlySS.KeyPath)) + require.ErrorIs(t, err, ErrCertificateNotServerAuth) + require.Nil(t, manager) + }) +} diff --git a/pkg/tlsconfig/configmanager.go b/pkg/tlsconfig/configmanager.go index 384414a6696..beb0dd735ca 100644 --- a/pkg/tlsconfig/configmanager.go +++ b/pkg/tlsconfig/configmanager.go @@ -273,6 +273,10 @@ type tlsConfigManagerConfig struct { // ignoreFilePermissions indicates if cert loaders should ignore file permissions. ignoreFilePermissions bool + + // ignoreSanityChecks indicates if cert loaders should log and overlook failed + // certificate sanity checks instead of failing the load. + ignoreSanityChecks bool } // commonCertLoaderOpts returns a common list of options for the TLSCertLoader. @@ -280,6 +284,7 @@ func (c *tlsConfigManagerConfig) commonCertLoaderOpts() []TLSCertLoaderOpt { return []TLSCertLoaderOpt{ WithCertLoaderLogger(c.logger), WithCertLoaderIgnoreFilePermissions(c.ignoreFilePermissions), + WithCertLoaderIgnoreSanityChecks(c.ignoreSanityChecks), } } @@ -408,6 +413,17 @@ func WithIgnoreFilePermissions(ignore bool) TLSConfigManagerOpt { } } +// WithIgnoreSanityChecks logs failed certificate sanity checks and loads the +// certificate anyway, instead of failing. It is an escape hatch for a +// certificate this package judges unusable but a deployment relies on. It does +// not relax the requirement that a server have a present, parseable +// certificate, which is a fault rather than a judgment. +func WithIgnoreSanityChecks(ignore bool) TLSConfigManagerOpt { + return func(cl *tlsConfigManagerConfig) { + cl.ignoreSanityChecks = ignore + } +} + // WithUsage sets the config manager descriptive usage. func WithUsage(usage string) TLSConfigManagerOpt { return func(c *tlsConfigManagerConfig) { diff --git a/pkg/tlsconfig/loadcert.go b/pkg/tlsconfig/loadcert.go index 41b4dc1ea58..5a71ff71d84 100644 --- a/pkg/tlsconfig/loadcert.go +++ b/pkg/tlsconfig/loadcert.go @@ -58,6 +58,33 @@ func (xc *X509Certificate) ExpiresSoon(expirationAdvanced time.Duration) (bool, return false, 0 } +// SupportsServerAuth reports whether xc may be presented by a TLS server. +// +// A certificate with no extended key usage extension is unrestricted and may be +// used for any purpose. Once the extension is present it is exhaustive, so it +// must name server authentication (or anyExtendedKeyUsage) for a peer to accept +// the certificate. Go's verifier rejects the rest with "x509: certificate +// specifies an incompatible key usage". +func (xc *X509Certificate) SupportsServerAuth() bool { + if xc == nil || xc.Certificate == nil { + return false + } + + // Both lists must be empty to conclude the extension is absent: an + // extension naming only unrecognized OIDs still restricts the certificate, + // and leaves ExtKeyUsage empty. + if len(xc.ExtKeyUsage) == 0 && len(xc.UnknownExtKeyUsage) == 0 { + return true + } + + for _, eku := range xc.ExtKeyUsage { + if eku == x509.ExtKeyUsageServerAuth || eku == x509.ExtKeyUsageAny { + return true + } + } + return false +} + // logIssues logs issues with xc. Issues include: // - expired certificate // - certificates that are about to expire diff --git a/pkg/tlsconfig/loadcert_test.go b/pkg/tlsconfig/loadcert_test.go index 5a9cdecfa38..817118d98d0 100644 --- a/pkg/tlsconfig/loadcert_test.go +++ b/pkg/tlsconfig/loadcert_test.go @@ -1,6 +1,8 @@ package tlsconfig import ( + "crypto/x509" + "encoding/asn1" "os" "path" "testing" @@ -299,3 +301,77 @@ func TestLoadCertificate_MalformedFiles(t *testing.T) { require.False(t, lc.IsValid()) }) } + +func TestX509Certificate_SupportsServerAuth(t *testing.T) { + tests := []struct { + name string + eku []x509.ExtKeyUsage + unknown []asn1.ObjectIdentifier + supported bool + }{ + { + name: "no extension is unrestricted", + supported: true, + }, + { + name: "server auth", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + supported: true, + }, + { + name: "server and client auth", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + supported: true, + }, + { + name: "any extended key usage", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + supported: true, + }, + { + name: "server auth among others", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageCodeSigning, x509.ExtKeyUsageServerAuth}, + supported: true, + }, + { + name: "client auth only", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + supported: false, + }, + { + name: "unrelated usage only", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageEmailProtection}, + supported: false, + }, + { + // The extension is present but names only OIDs x509 does not + // recognize, which still restricts the certificate. ExtKeyUsage + // being empty is therefore not enough to call it unrestricted. + name: "only unrecognized usages", + unknown: []asn1.ObjectIdentifier{{1, 3, 6, 1, 4, 1, 99999, 1}}, + supported: false, + }, + { + name: "server auth alongside an unrecognized usage", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + unknown: []asn1.ObjectIdentifier{{1, 3, 6, 1, 4, 1, 99999, 1}}, + supported: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + xc := &X509Certificate{Certificate: &x509.Certificate{ + ExtKeyUsage: tt.eku, + UnknownExtKeyUsage: tt.unknown, + }} + require.Equal(t, tt.supported, xc.SupportsServerAuth()) + }) + } + + t.Run("nil certificate", func(t *testing.T) { + var xc *X509Certificate + require.False(t, xc.SupportsServerAuth()) + require.False(t, (&X509Certificate{}).SupportsServerAuth()) + }) +} diff --git a/services/httpd/config.go b/services/httpd/config.go index 2feca99a4a4..61f5868626f 100644 --- a/services/httpd/config.go +++ b/services/httpd/config.go @@ -33,43 +33,46 @@ const ( // Config represents a configuration for a HTTP service. type Config struct { - Enabled bool `toml:"enabled"` - BindAddress string `toml:"bind-address"` - AuthEnabled bool `toml:"auth-enabled"` - LogEnabled bool `toml:"log-enabled"` - SuppressWriteLog bool `toml:"suppress-write-log"` - WriteTracing bool `toml:"write-tracing"` - FluxEnabled bool `toml:"flux-enabled"` - FluxLogEnabled bool `toml:"flux-log-enabled"` - FluxTesting bool `toml:"flux-testing"` - PprofEnabled bool `toml:"pprof-enabled"` - PprofAuthEnabled bool `toml:"pprof-auth-enabled"` - DebugPprofEnabled bool `toml:"debug-pprof-enabled"` - PingAuthEnabled bool `toml:"ping-auth-enabled"` - PromReadAuthEnabled bool `toml:"prom-read-auth-enabled"` - HTTPHeaders map[string]string `toml:"headers"` - HTTPSEnabled bool `toml:"https-enabled"` - HTTPSCertificate string `toml:"https-certificate"` - HTTPSPrivateKey string `toml:"https-private-key"` - HTTPSInsecureCertificate bool `toml:"https-insecure-certificate"` - HTTPSClientAuthType *toml.TlsClientAuthType `toml:"https-client-auth-type"` - HTTPSClientCA *tlsconfig.CAConfig `toml:"https-client-ca"` - MaxRowLimit int `toml:"max-row-limit"` - MaxConnectionLimit int `toml:"max-connection-limit"` - SharedSecret string `toml:"shared-secret"` - Realm string `toml:"realm"` - UnixSocketEnabled bool `toml:"unix-socket-enabled"` - UnixSocketGroup *toml.Group `toml:"unix-socket-group"` - UnixSocketPermissions toml.FileMode `toml:"unix-socket-permissions"` - BindSocket string `toml:"bind-socket"` - MaxBodySize toml.SSize `toml:"max-body-size"` - AccessLogPath string `toml:"access-log-path"` - AccessLogStatusFilters []StatusFilter `toml:"access-log-status-filters"` - MaxConcurrentWriteLimit int `toml:"max-concurrent-write-limit"` - MaxEnqueuedWriteLimit int `toml:"max-enqueued-write-limit"` - EnqueuedWriteTimeout time.Duration `toml:"enqueued-write-timeout"` - UserQueryBytesEnabled bool `toml:"user-query-bytes-enabled"` - TLS *tls.Config `toml:"-"` + Enabled bool `toml:"enabled"` + BindAddress string `toml:"bind-address"` + AuthEnabled bool `toml:"auth-enabled"` + LogEnabled bool `toml:"log-enabled"` + SuppressWriteLog bool `toml:"suppress-write-log"` + WriteTracing bool `toml:"write-tracing"` + FluxEnabled bool `toml:"flux-enabled"` + FluxLogEnabled bool `toml:"flux-log-enabled"` + FluxTesting bool `toml:"flux-testing"` + PprofEnabled bool `toml:"pprof-enabled"` + PprofAuthEnabled bool `toml:"pprof-auth-enabled"` + DebugPprofEnabled bool `toml:"debug-pprof-enabled"` + PingAuthEnabled bool `toml:"ping-auth-enabled"` + PromReadAuthEnabled bool `toml:"prom-read-auth-enabled"` + HTTPHeaders map[string]string `toml:"headers"` + HTTPSEnabled bool `toml:"https-enabled"` + HTTPSCertificate string `toml:"https-certificate"` + HTTPSPrivateKey string `toml:"https-private-key"` + HTTPSInsecureCertificate bool `toml:"https-insecure-certificate"` + // HTTPSIgnoreSanityChecks loads the certificate even when it fails the + // checks that decide whether a server can use it at all. + HTTPSIgnoreSanityChecks bool `toml:"https-ignore-sanity-checks"` + HTTPSClientAuthType *toml.TlsClientAuthType `toml:"https-client-auth-type"` + HTTPSClientCA *tlsconfig.CAConfig `toml:"https-client-ca"` + MaxRowLimit int `toml:"max-row-limit"` + MaxConnectionLimit int `toml:"max-connection-limit"` + SharedSecret string `toml:"shared-secret"` + Realm string `toml:"realm"` + UnixSocketEnabled bool `toml:"unix-socket-enabled"` + UnixSocketGroup *toml.Group `toml:"unix-socket-group"` + UnixSocketPermissions toml.FileMode `toml:"unix-socket-permissions"` + BindSocket string `toml:"bind-socket"` + MaxBodySize toml.SSize `toml:"max-body-size"` + AccessLogPath string `toml:"access-log-path"` + AccessLogStatusFilters []StatusFilter `toml:"access-log-status-filters"` + MaxConcurrentWriteLimit int `toml:"max-concurrent-write-limit"` + MaxEnqueuedWriteLimit int `toml:"max-enqueued-write-limit"` + EnqueuedWriteTimeout time.Duration `toml:"enqueued-write-timeout"` + UserQueryBytesEnabled bool `toml:"user-query-bytes-enabled"` + TLS *tls.Config `toml:"-"` } // NewConfig returns a new Config with default settings. @@ -89,6 +92,7 @@ func NewConfig() Config { HTTPSEnabled: false, HTTPSCertificate: "/etc/ssl/influxdb.pem", HTTPSInsecureCertificate: false, + HTTPSIgnoreSanityChecks: false, // A nil HTTPSClientAuthType or HTTPSClientCA means "not configured" and // leaves the base TLS config's value in place. When set, tlsconfig // resolves and validates them (see tlsconfig.CAConfig and WithClientAuthPtr). @@ -118,6 +122,7 @@ func (c Config) Diagnostics() (*diagnostics.Diagnostics, error) { "bind-address": c.BindAddress, "https-enabled": c.HTTPSEnabled, "https-insecure-certificate": c.HTTPSInsecureCertificate, + "https-ignore-sanity-checks": c.HTTPSIgnoreSanityChecks, "max-row-limit": c.MaxRowLimit, "max-connection-limit": c.MaxConnectionLimit, "access-log-path": c.AccessLogPath, @@ -236,6 +241,7 @@ func (c Config) TLSManagerOpts() []tlsconfig.TLSConfigManagerOpt { tlsconfig.WithBaseConfig(c.TLS), tlsconfig.WithServerCertificate(c.HTTPSCertificate, c.HTTPSPrivateKey), tlsconfig.WithIgnoreFilePermissions(c.HTTPSInsecureCertificate), + tlsconfig.WithIgnoreSanityChecks(c.HTTPSIgnoreSanityChecks), tlsconfig.WithClientAuthPtr(clientAuthType), tlsconfig.WithClientCA(c.HTTPSClientCA), } diff --git a/services/httpd/tlsreload_test.go b/services/httpd/tlsreload_test.go index cc8bdfc4a6c..a27671d7022 100644 --- a/services/httpd/tlsreload_test.go +++ b/services/httpd/tlsreload_test.go @@ -245,3 +245,42 @@ func TestService_TLSUsage(t *testing.T) { require.Len(t, entries, 1) require.Equal(t, "httpd.server", entries[0].ContextMap()["usage"]) } + +// TestConfig_IgnoreSanityChecks covers the config option reaching the +// TLS manager. A certificate issued only for client authentication fails the +// server sanity checks, so it loads only when the option is set. +func TestConfig_IgnoreSanityChecks(t *testing.T) { + clientOnlySS := selfsigned.NewSelfSignedCert(t, + selfsigned.WithExtKeyUsage(x509.ExtKeyUsageClientAuth)) + + newService := func(t *testing.T, ignore bool) error { + t.Helper() + + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + s := httpd.NewService(httpd.Config{ + BindAddress: "127.0.0.1:0", + HTTPSEnabled: true, + HTTPSCertificate: clientOnlySS.CertPath, + HTTPSPrivateKey: clientOnlySS.KeyPath, + HTTPSIgnoreSanityChecks: ignore, + }, certMonitor) + s.WithLogger(zap.NewNop()) + + err := s.Open() + if err == nil { + t.Cleanup(th.CheckedClose(t, s)) + } + return err + } + + t.Run("enforced by default", func(t *testing.T) { + require.ErrorContains(t, newService(t, false), tlsconfig.ErrCertificateNotServerAuth.Error()) + }) + + t.Run("ignored when configured", func(t *testing.T) { + require.NoError(t, newService(t, true)) + }) +} diff --git a/services/opentsdb/config.go b/services/opentsdb/config.go index 644bc3ea937..76a3d56666a 100644 --- a/services/opentsdb/config.go +++ b/services/opentsdb/config.go @@ -38,22 +38,25 @@ const ( // Config represents the configuration of the OpenTSDB service. type Config struct { - Enabled bool `toml:"enabled"` - BindAddress string `toml:"bind-address"` - Database string `toml:"database"` - RetentionPolicy string `toml:"retention-policy"` - ConsistencyLevel string `toml:"consistency-level"` - TLSEnabled bool `toml:"tls-enabled"` - Certificate string `toml:"certificate"` - PrivateKey string `toml:"private-key"` - InsecureCertificate bool `toml:"insecure-certificate"` - ClientAuthType *toml.TlsClientAuthType `toml:"client-auth-type"` - ClientCA *tlsconfig.CAConfig `toml:"client-ca"` - BatchSize int `toml:"batch-size"` - BatchPending int `toml:"batch-pending"` - BatchTimeout toml.Duration `toml:"batch-timeout"` - LogPointErrors bool `toml:"log-point-errors"` - TLS *tls.Config `toml:"-"` + Enabled bool `toml:"enabled"` + BindAddress string `toml:"bind-address"` + Database string `toml:"database"` + RetentionPolicy string `toml:"retention-policy"` + ConsistencyLevel string `toml:"consistency-level"` + TLSEnabled bool `toml:"tls-enabled"` + Certificate string `toml:"certificate"` + PrivateKey string `toml:"private-key"` + InsecureCertificate bool `toml:"insecure-certificate"` + // IgnoreSanityChecks loads the certificate even when it fails the + // checks that decide whether a server can use it at all. + IgnoreSanityChecks bool `toml:"ignore-sanity-checks"` + ClientAuthType *toml.TlsClientAuthType `toml:"client-auth-type"` + ClientCA *tlsconfig.CAConfig `toml:"client-ca"` + BatchSize int `toml:"batch-size"` + BatchPending int `toml:"batch-pending"` + BatchTimeout toml.Duration `toml:"batch-timeout"` + LogPointErrors bool `toml:"log-point-errors"` + TLS *tls.Config `toml:"-"` } // Compile-time check that *Config implements toml.Defaulter so it can be used diff --git a/services/opentsdb/service.go b/services/opentsdb/service.go index 9de75e5027e..79ae25ab464 100644 --- a/services/opentsdb/service.go +++ b/services/opentsdb/service.go @@ -54,12 +54,13 @@ type Service struct { certMonitor *tlsconfig.TLSCertMonitor - tls bool - tlsManager *tlsconfig.TLSConfigManager - tlsConfig *tls.Config - cert string - privateKey string - insecureCert bool + tls bool + tlsManager *tlsconfig.TLSConfigManager + tlsConfig *tls.Config + cert string + privateKey string + insecureCert bool + ignoreSanityChecks bool // clientAuthType controls TLS client-certificate authentication (mTLS). A // nil value leaves the base TLS config's ClientAuth in place. @@ -110,24 +111,25 @@ func NewService(c Config, certMonitor *tlsconfig.TLSCertMonitor) (*Service, erro } s := &Service{ - certMonitor: certMonitor, - tls: d.TLSEnabled, - tlsConfig: d.TLS, - cert: d.Certificate, - privateKey: d.PrivateKey, - insecureCert: d.InsecureCertificate, - clientAuthType: clientAuthType, - clientCA: d.ClientCA, - BindAddress: d.BindAddress, - Database: d.Database, - RetentionPolicy: d.RetentionPolicy, - batchSize: d.BatchSize, - batchPending: d.BatchPending, - batchTimeout: time.Duration(d.BatchTimeout), - Logger: zap.NewNop(), - LogPointErrors: d.LogPointErrors, - stats: &Statistics{}, - defaultTags: models.StatisticTags{"bind": d.BindAddress}, + certMonitor: certMonitor, + tls: d.TLSEnabled, + tlsConfig: d.TLS, + cert: d.Certificate, + privateKey: d.PrivateKey, + insecureCert: d.InsecureCertificate, + ignoreSanityChecks: d.IgnoreSanityChecks, + clientAuthType: clientAuthType, + clientCA: d.ClientCA, + BindAddress: d.BindAddress, + Database: d.Database, + RetentionPolicy: d.RetentionPolicy, + batchSize: d.BatchSize, + batchPending: d.BatchPending, + batchTimeout: time.Duration(d.BatchTimeout), + Logger: zap.NewNop(), + LogPointErrors: d.LogPointErrors, + stats: &Statistics{}, + defaultTags: models.StatisticTags{"bind": d.BindAddress}, } if s.tlsConfig == nil { s.tlsConfig = new(tls.Config) @@ -165,6 +167,7 @@ func (s *Service) Open() error { tlsconfig.WithBaseConfig(s.tlsConfig), tlsconfig.WithServerCertificate(s.cert, s.privateKey), tlsconfig.WithIgnoreFilePermissions(s.insecureCert), + tlsconfig.WithIgnoreSanityChecks(s.ignoreSanityChecks), tlsconfig.WithClientAuthPtr(s.clientAuthType), tlsconfig.WithClientCA(s.clientCA), tlsconfig.WithLogger(s.Logger)) diff --git a/services/opentsdb/service_test.go b/services/opentsdb/service_test.go index dc7fc2b5077..8ad8347dc9a 100644 --- a/services/opentsdb/service_test.go +++ b/services/opentsdb/service_test.go @@ -2,6 +2,7 @@ package opentsdb import ( "crypto/tls" + "crypto/x509" "errors" "fmt" "net" @@ -441,3 +442,49 @@ func TestService_TLSUsage(t *testing.T) { require.Len(t, entries, 1) require.Equal(t, "opentsdb("+bind+").server", entries[0].ContextMap()["usage"]) } + +// TestConfig_IgnoreSanityChecks covers the config option reaching the +// TLS manager. A certificate issued only for client authentication fails the +// server sanity checks, so it opens only when the option is set. +func TestConfig_IgnoreSanityChecks(t *testing.T) { + clientOnlySS := selfsigned.NewSelfSignedCert(t, + selfsigned.WithExtKeyUsage(x509.ExtKeyUsageClientAuth)) + + openService := func(t *testing.T, ignore bool) error { + t.Helper() + + certMonitor := tlsconfig.NewTLSCertMonitor() + require.NoError(t, certMonitor.Open()) + t.Cleanup(th.CheckedClose(t, certMonitor)) + + s, err := NewService(Config{ + BindAddress: "127.0.0.1:0", + Database: "db0", + ConsistencyLevel: "one", + TLSEnabled: true, + Certificate: clientOnlySS.CertPath, + PrivateKey: clientOnlySS.KeyPath, + IgnoreSanityChecks: ignore, + TLS: new(tls.Config), + }, certMonitor) + require.NoError(t, err) + + s.MetaClient = &internal.MetaClientMock{ + CreateDatabaseFn: func(string) (*meta.DatabaseInfo, error) { return nil, nil }, + } + + openErr := s.Open() + if openErr == nil { + t.Cleanup(func() { require.NoError(t, s.Close()) }) + } + return openErr + } + + t.Run("enforced by default", func(t *testing.T) { + require.ErrorContains(t, openService(t, false), tlsconfig.ErrCertificateNotServerAuth.Error()) + }) + + t.Run("ignored when configured", func(t *testing.T) { + require.NoError(t, openService(t, true)) + }) +} diff --git a/services/subscriber/config.go b/services/subscriber/config.go index 18a08c1da92..da024e3f4bf 100644 --- a/services/subscriber/config.go +++ b/services/subscriber/config.go @@ -54,6 +54,12 @@ type Config struct { // should be ignored when it is loaded. InsecureCertificate bool `toml:"insecure-certificate"` + // IgnoreSanityChecks loads the certificate even when it fails the + // checks that decide whether it can be used at all. The checks currently + // only cover server certificates, so this has no effect on the subscriber's + // client certificate today. + IgnoreSanityChecks bool `toml:"ignore-sanity-checks"` + // The number of writer goroutines processing the write channel. WriteConcurrency int `toml:"write-concurrency"` @@ -136,6 +142,7 @@ func (c Config) TLSManagerOpts() []tlsconfig.TLSConfigManagerOpt { tlsconfig.WithClientCertificate(c.Certificate, c.PrivateKey), tlsconfig.WithRootCA(c.effectiveRootCA()), tlsconfig.WithIgnoreFilePermissions(c.InsecureCertificate), + tlsconfig.WithIgnoreSanityChecks(c.IgnoreSanityChecks), } } From f77fdde83b7d160c728c1ca50b232222cb07b399 Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 13:56:30 -0500 Subject: [PATCH 08/18] feat: add client certificate sanity checks - Add checks for client certificate client usage EKU. This check is more subtle than the server certificate server usage EKU check in order to avoiding currently working configurations. --- pkg/tlsconfig/configmanager.go | 181 +++++++++++++++++++++++++- pkg/tlsconfig/configmanager_test.go | 191 ++++++++++++++++++++++++++++ pkg/tlsconfig/loadcert.go | 25 +++- pkg/tlsconfig/loadcert_test.go | 55 ++++++++ 4 files changed, 440 insertions(+), 12 deletions(-) diff --git a/pkg/tlsconfig/configmanager.go b/pkg/tlsconfig/configmanager.go index beb0dd735ca..35628c9fb66 100644 --- a/pkg/tlsconfig/configmanager.go +++ b/pkg/tlsconfig/configmanager.go @@ -46,6 +46,10 @@ var ( // that has no TLS configuration to serve it. ErrNoTLSConfig = errors.New("no TLS configuration available") + // ErrCertificateNotClientAuth indicates a certificate a client would present does + // not permit client authentication. + ErrCertificateNotClientAuth = errors.New("TLS certificate does not permit client authentication") + // ErrNotSupportedServer indicates that an operation is not supported by a server role // config manager. ErrNotSupportedServer = errors.New("operation not supported by server role TLS manager") @@ -117,6 +121,11 @@ type TLSConfigManager struct { // usage is the descriptive usage for this config manager. usage string + // logger is the logger used for logging status. It is always usable and + // already carries the usage, so callers can log through it directly. It can + // only be set at construction time using WithLogger. + logger *zap.Logger + // serverCertLoader is the cert loader for server certificates. It is only // created if role is a server role. It is only modified on instantiation and // does not require a mutex because it has its own internal locking. @@ -298,13 +307,37 @@ func (c *tlsConfigManagerConfig) serverCertLoaderOpts() []TLSCertLoaderOpt { // clientCertLoaderOpts returns the options needed for the client TLSCertLoader. // If the client certificate is not set, then the server certificate will // be used as a fallback. -func (c *tlsConfigManagerConfig) clientCertLoaderOpts() []TLSCertLoaderOpt { - certPath := c.clientCertPath - keyPath := c.clientKeyPath - if certPath == "" && keyPath == "" { - certPath = c.serverCertPath - keyPath = c.serverKeyPath +// clientCertificatePaths resolves the certificate the client will present. When +// no client certificate is configured the server pair is used instead, which +// usingFallback reports. Setting either client path suppresses the fallback. +func (c *tlsConfigManagerConfig) clientCertificatePaths() (certPath, keyPath string, usingFallback bool) { + if c.clientCertPath == "" && c.clientKeyPath == "" { + return c.serverCertPath, c.serverKeyPath, true + } + return c.clientCertPath, c.clientKeyPath, false +} + +// verifiesClientCertificates reports whether the effective client +// authentication makes a server verify the certificates clients present, which +// is what makes the client extended key usage matter. Below +// tls.VerifyClientCertIfGiven the certificate is never verified, so its usages +// are never examined, even when one is demanded. +// +// The effective value is resolved the same way prepareConfigure applies it: the +// base config's setting, overridden by WithClientAuth when it was given. +func (c *tlsConfigManagerConfig) verifiesClientCertificates() bool { + clientAuth := tls.NoClientCert + if c.baseConfig != nil { + clientAuth = c.baseConfig.ClientAuth + } + if c.clientAuth != nil { + clientAuth = *c.clientAuth } + return clientAuth >= tls.VerifyClientCertIfGiven +} + +func (c *tlsConfigManagerConfig) clientCertLoaderOpts() []TLSCertLoaderOpt { + certPath, keyPath, _ := c.clientCertificatePaths() return append(c.commonCertLoaderOpts(), WithCertLoaderCertificate(certPath, keyPath), ) @@ -517,6 +550,10 @@ func newTLSConfigManager(opts ...TLSConfigManagerOpt) (*TLSConfigManager, error) clientCertLoader: clientCertLoader, } + // Set the logger even when none was configured, so that everything below has + // a usable one. + cm.setLogger(c.logger) + if apply, err := cm.prepareConfigure(&c); err != nil { return nil, fmt.Errorf("error configuring new TLSConfigManager: %w", err) } else if err := apply(); err != nil { @@ -526,6 +563,131 @@ func newTLSConfigManager(opts ...TLSConfigManagerOpt) (*TLSConfigManager, error) return cm, nil } +// setLogger sets the current logger and adds context to it. +func (cm *TLSConfigManager) setLogger(logger *zap.Logger) { + cm.logger = logger + if cm.logger == nil { + cm.logger = zap.NewNop() + } + + // Add usage to logger. + cm.logger = cm.logger.With(zap.String(logUsageContext, cm.usage)) +} + +// checkClientCertificate sanity checks the certificate the client will present +// for client authentication, returning an error if the load should fail. +// +// The checks live here rather than in the cert loader because they need what +// only the manager knows: whether the certificate is the client's own or +// borrowed from the server, and how client certificates are authenticated. +// +// How firmly a failure is treated depends on what the certificate is and how it +// will be used: +// +// - A certificate configured with WithClientCertificate exists to be presented +// by the client, so failing a check is an error. +// - A client that has fallen back to the server's certificate is a different +// matter. Server certificates that do not name client authentication are +// common, and such a deployment works today wherever no peer verifies the +// certificates clients present. Failing it there would break working +// configurations, so it is only ever a warning. +// - Once client certificates are verified, a borrowed certificate is going to +// be rejected, so failing a check is an error again. +// +// The client authentication setting describes how this manager's server treats +// incoming clients, and is read here as a statement about the deployment: a +// cluster that verifies the clients it accepts is taken to be one whose peers +// will verify this client in turn. +// +// An error is downgraded to a warning when sanity checks are being ignored. +func (cm *TLSConfigManager) checkClientCertificate(c *tlsConfigManagerConfig) error { + certPath, keyPath, usingFallback := c.clientCertificatePaths() + + // Both of these only mean something for a manager that is also a server: a + // client-only manager has no server whose certificate it could have + // borrowed, and no client authentication setting of its own to read the + // deployment from. For it, a client certificate is simply a client + // certificate. + peersVerify := cm.role.IsServerRole() && c.verifiesClientCertificates() + borrowedFromServer := cm.role.IsServerRole() && usingFallback + + var ( + // sanityErrs are failures that matter however the certificate is used. + sanityErrs []error + + // verifiedOnlyErrs are failures that only matter once a peer verifies + // the certificates clients present. They are kept apart from sanityErrs + // so that excusing a borrowed certificate excuses only the checks that + // depend on that verification, and not every other check with them. + verifiedOnlyErrs []error + ) + + if certPath == "" && keyPath == "" { + // Presenting no client certificate is a valid configuration, until a + // peer demands one and checks it. + if !peersVerify { + return nil + } + sanityErrs = append(sanityErrs, fmt.Errorf( + "%s: no client certificate is configured, but client certificates are verified: %w", + cm.usage, ErrCertificateEmpty)) + } else { + // The cert loader has already prepared this same certificate, so a read + // failure here is a race with a changing file rather than something for + // this check to report. + loadedCert, err := LoadCertificate(certPath, keyPath, + WithLoadCertificateIgnoreFilePermissions(c.ignoreFilePermissions)) + if err != nil { + return nil + } + + // A leaf that will not parse is a fault rather than a judgment, so it + // fails the load even when sanity checks are ignored. + leaf, err := loadedCert.GetLeaf() + if err != nil { + return fmt.Errorf("%s: %w", cm.usage, err) + } + + // A peer only examines the extended key usage when it verifies the + // certificate, so this failure is moot until one does. + if !leaf.SupportsClientAuth() { + verifiedOnlyErrs = append(verifiedOnlyErrs, fmt.Errorf( + "%s: certificate presented for client authentication does not permit it: %w", + cm.usage, ErrCertificateNotClientAuth)) + } + } + + log := cm.logger.With( + zap.String(logCertContext, certPath), + zap.String(logKeyContext, keyPath)) + + // A borrowed server certificate that no peer will verify keeps working, and + // failing it would break configurations that predate these checks. Only the + // checks that hinge on that verification are excused. + if borrowedFromServer && !peersVerify { + if len(verifiedOnlyErrs) > 0 { + log.Warn("Server TLS certificate is also being used as the client certificate and failed client sanity checks; "+ + "this is only a problem against a peer that verifies client certificates", + zap.Error(errors.Join(verifiedOnlyErrs...))) + } + } else { + sanityErrs = append(sanityErrs, verifiedOnlyErrs...) + } + + if len(sanityErrs) == 0 { + return nil + } + sanityErr := errors.Join(sanityErrs...) + + if c.ignoreSanityChecks { + log.Warn("TLS certificate failed sanity checks, loading it anyway because sanity checks are being ignored", + zap.Error(sanityErr)) + return nil + } + + return sanityErr +} + // prepareConfigure changes the configuration of cm based on func (cm *TLSConfigManager) prepareConfigure(c *tlsConfigManagerConfig) (func() error, error) { // Create and setup base tls.Config @@ -614,6 +776,13 @@ func (cm *TLSConfigManager) prepareConfigure(c *tlsConfigManagerConfig) (func() } else { applyClientCert = apply } + + // The client sanity check lives here rather than in the cert loader + // because it needs what only the manager knows: whether the certificate + // is the client's own or the server's, and how clients are authenticated. + if err := cm.checkClientCertificate(c); err != nil { + return nil, err + } } return func() error { diff --git a/pkg/tlsconfig/configmanager_test.go b/pkg/tlsconfig/configmanager_test.go index 9696466196a..f3520eb756a 100644 --- a/pkg/tlsconfig/configmanager_test.go +++ b/pkg/tlsconfig/configmanager_test.go @@ -2437,3 +2437,194 @@ func TestTLSConfigManager_ListenSessionResumption(t *testing.T) { require.Contains(t, resumed, true, "per-connection configs must not break session resumption") } + +// serverOnlyCert is a certificate that permits server authentication but not +// client authentication, so a client presenting it fails the client checks. +func serverOnlyCert(t *testing.T) *selfsigned.Cert { + t.Helper() + return selfsigned.NewSelfSignedCert(t, selfsigned.WithExtKeyUsage(x509.ExtKeyUsageServerAuth)) +} + +func TestTLSConfigManager_ClientCertificateSanityChecks(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + t.Run("client-only rejects a certificate without client auth", func(t *testing.T) { + ss := serverOnlyCert(t) + + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithClientCertificate(ss.CertPath, ss.KeyPath)) + require.ErrorIs(t, err, ErrCertificateNotClientAuth) + require.Nil(t, manager) + }) + + t.Run("client-only with no certificate is fine", func(t *testing.T) { + // The client simply presents nothing. A client-only manager has no + // client auth setting of its own to contradict that. + manager, err := NewClientTLSConfigManager(monitor, WithUseTLS(true)) + require.NoError(t, err) + require.NotNil(t, manager) + defer th.CheckedClose(t, manager)() + }) + + t.Run("client-only accepts a proper client certificate", func(t *testing.T) { + ss := selfsigned.NewSelfSignedCert(t) + + manager, err := NewClientTLSConfigManager( + monitor, + WithUseTLS(true), + WithClientCertificate(ss.CertPath, ss.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + }) + + t.Run("client/server rejects an explicit certificate without client auth", func(t *testing.T) { + serverSS := selfsigned.NewSelfSignedCert(t) + clientSS := serverOnlyCert(t) + + // The certificate was configured to be the client's, so it is held to + // the client checks regardless of client authentication. + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCertificate(clientSS.CertPath, clientSS.KeyPath)) + require.ErrorIs(t, err, ErrCertificateNotClientAuth) + require.Nil(t, manager) + }) + + t.Run("client/server warns when a borrowed certificate is never verified", func(t *testing.T) { + serverSS := serverOnlyCert(t) + + core, logs := observer.New(zapcore.InfoLevel) + + // This is the shape of every deployment that predates these checks: a + // server certificate reused by the client, with no client auth. It must + // keep working even though sanity checks are enforced. + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithLogger(zap.New(core)), + WithIgnoreSanityChecks(false), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath)) + require.NoError(t, err, "a borrowed certificate no peer verifies must not fail the load") + require.NotNil(t, manager) + defer th.CheckedClose(t, manager)() + + warnings := logs.FilterMessage("Server TLS certificate is also being used as the client certificate and failed client sanity checks; " + + "this is only a problem against a peer that verifies client certificates").TakeAll() + require.Len(t, warnings, 1, "the load should still say what is wrong") + require.Equal(t, zap.WarnLevel, warnings[0].Level) + require.Contains(t, warnings[0].ContextMap()["error"], ErrCertificateNotClientAuth.Error()) + }) + + t.Run("client/server rejects a borrowed certificate once clients are verified", func(t *testing.T) { + serverSS := serverOnlyCert(t) + clientCASS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("c", "Client CA")) + + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCA(&CAConfig{Paths: []string{clientCASS.CACertPath}}), + WithClientAuth(tls.RequireAndVerifyClientCert)) + require.ErrorIs(t, err, ErrCertificateNotClientAuth) + require.Nil(t, manager) + }) + + t.Run("verification is what matters, not demanding a certificate", func(t *testing.T) { + serverSS := serverOnlyCert(t) + + // RequireAnyClientCert demands a certificate but never verifies it, so + // the extended key usage is never examined and the load succeeds. + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientAuth(tls.RequireAnyClientCert)) + require.NoError(t, err, "RequireAnyClientCert does not verify, so the usage is moot") + defer th.CheckedClose(t, manager)() + + // VerifyClientCertIfGiven does not demand one but does verify it. + manager, err = NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientAuth(tls.VerifyClientCertIfGiven)) + require.ErrorIs(t, err, ErrCertificateNotClientAuth, + "VerifyClientCertIfGiven verifies, so the usage matters") + require.Nil(t, manager) + }) + + t.Run("ignoring sanity checks downgrades the error", func(t *testing.T) { + serverSS := serverOnlyCert(t) + clientCASS := selfsigned.NewSelfSignedCert(t, selfsigned.WithCASubject("c", "Client CA")) + + core, logs := observer.New(zapcore.InfoLevel) + + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithLogger(zap.New(core)), + WithIgnoreSanityChecks(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), + WithClientCA(&CAConfig{Paths: []string{clientCASS.CACertPath}}), + WithClientAuth(tls.RequireAndVerifyClientCert)) + require.NoError(t, err) + require.NotNil(t, manager) + defer th.CheckedClose(t, manager)() + + require.Len(t, logs.FilterMessage("TLS certificate failed sanity checks, loading it anyway because sanity checks are being ignored").TakeAll(), 1) + }) +} + +// TestTLSConfigManager_LoggerCarriesUsage covers the manager's logger being +// usable and already carrying the usage, so that code logging through it does +// not have to add the context itself. +func TestTLSConfigManager_LoggerCarriesUsage(t *testing.T) { + monitor := newTestCertMonitor(t) + defer th.CheckedClose(t, monitor)() + + t.Run("usage is added once at construction", func(t *testing.T) { + serverSS := serverOnlyCert(t) + + core, logs := observer.New(zapcore.InfoLevel) + + // A borrowed server certificate without client auth warns, which is a + // convenient way to see what the manager's logger carries. + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithUsage("meta"), + WithLogger(zap.New(core)), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath)) + require.NoError(t, err) + defer th.CheckedClose(t, manager)() + + entries := logs.FilterMessageSnippet("also being used as the client certificate").TakeAll() + require.Len(t, entries, 1) + + ctx := entries[0].ContextMap() + require.Equal(t, "meta", ctx["usage"], "the manager's logger should carry its usage") + require.Equal(t, serverSS.CertPath, ctx["cert"]) + require.Equal(t, serverSS.KeyPath, ctx["key"]) + }) + + t.Run("no logger configured is still usable", func(t *testing.T) { + serverSS := serverOnlyCert(t) + + // Without WithLogger the manager must still log somewhere rather than + // panic on a nil logger. + require.NotPanics(t, func() { + manager, err := NewClientServerTLSConfigManager( + monitor, + WithUseTLS(true), + WithServerCertificate(serverSS.CertPath, serverSS.KeyPath)) + require.NoError(t, err) + require.NotNil(t, manager.logger) + require.NoError(t, manager.Close()) + }) + }) +} diff --git a/pkg/tlsconfig/loadcert.go b/pkg/tlsconfig/loadcert.go index 5a71ff71d84..7fe72ffb93b 100644 --- a/pkg/tlsconfig/loadcert.go +++ b/pkg/tlsconfig/loadcert.go @@ -58,14 +58,14 @@ func (xc *X509Certificate) ExpiresSoon(expirationAdvanced time.Duration) (bool, return false, 0 } -// SupportsServerAuth reports whether xc may be presented by a TLS server. +// supportsExtKeyUsage reports whether xc permits want. // // A certificate with no extended key usage extension is unrestricted and may be // used for any purpose. Once the extension is present it is exhaustive, so it -// must name server authentication (or anyExtendedKeyUsage) for a peer to accept -// the certificate. Go's verifier rejects the rest with "x509: certificate -// specifies an incompatible key usage". -func (xc *X509Certificate) SupportsServerAuth() bool { +// must name want (or anyExtendedKeyUsage) for a peer to accept the certificate. +// Go's verifier rejects the rest with "x509: certificate specifies an +// incompatible key usage". +func (xc *X509Certificate) supportsExtKeyUsage(want x509.ExtKeyUsage) bool { if xc == nil || xc.Certificate == nil { return false } @@ -78,13 +78,26 @@ func (xc *X509Certificate) SupportsServerAuth() bool { } for _, eku := range xc.ExtKeyUsage { - if eku == x509.ExtKeyUsageServerAuth || eku == x509.ExtKeyUsageAny { + if eku == want || eku == x509.ExtKeyUsageAny { return true } } return false } +// SupportsServerAuth reports whether xc may be presented by a TLS server. +func (xc *X509Certificate) SupportsServerAuth() bool { + return xc.supportsExtKeyUsage(x509.ExtKeyUsageServerAuth) +} + +// SupportsClientAuth reports whether xc may be presented by a TLS client for +// client authentication. It only matters against a peer that verifies the +// certificates clients present; below tls.VerifyClientCertIfGiven the usages +// are never examined. +func (xc *X509Certificate) SupportsClientAuth() bool { + return xc.supportsExtKeyUsage(x509.ExtKeyUsageClientAuth) +} + // logIssues logs issues with xc. Issues include: // - expired certificate // - certificates that are about to expire diff --git a/pkg/tlsconfig/loadcert_test.go b/pkg/tlsconfig/loadcert_test.go index 817118d98d0..33360756b7c 100644 --- a/pkg/tlsconfig/loadcert_test.go +++ b/pkg/tlsconfig/loadcert_test.go @@ -302,6 +302,61 @@ func TestLoadCertificate_MalformedFiles(t *testing.T) { }) } +func TestX509Certificate_SupportsClientAuth(t *testing.T) { + tests := []struct { + name string + eku []x509.ExtKeyUsage + unknown []asn1.ObjectIdentifier + supported bool + }{ + { + name: "no extension is unrestricted", + supported: true, + }, + { + name: "client auth", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + supported: true, + }, + { + name: "server and client auth", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + supported: true, + }, + { + name: "any extended key usage", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + supported: true, + }, + { + name: "server auth only", + eku: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + supported: false, + }, + { + name: "only unrecognized usages", + unknown: []asn1.ObjectIdentifier{{1, 3, 6, 1, 4, 1, 99999, 1}}, + supported: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + xc := &X509Certificate{Certificate: &x509.Certificate{ + ExtKeyUsage: tt.eku, + UnknownExtKeyUsage: tt.unknown, + }} + require.Equal(t, tt.supported, xc.SupportsClientAuth()) + }) + } + + t.Run("nil certificate", func(t *testing.T) { + var xc *X509Certificate + require.False(t, xc.SupportsClientAuth()) + require.False(t, (&X509Certificate{}).SupportsClientAuth()) + }) +} + func TestX509Certificate_SupportsServerAuth(t *testing.T) { tests := []struct { name string From a88759ad3c3a7572444484fd19793ec6c38325fe Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 16:07:56 -0500 Subject: [PATCH 09/18] fix: disabled TLSConfigManager can dial and listen Fixed NewDisabledTLSConfigManager so that the returned TLSConfigManager can both dial and listen plaintext. --- pkg/tlsconfig/configmanager.go | 15 ++++- pkg/tlsconfig/configmanager_test.go | 89 ++++++++++++++++++++++++++--- 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/pkg/tlsconfig/configmanager.go b/pkg/tlsconfig/configmanager.go index 35628c9fb66..7132adc6e4a 100644 --- a/pkg/tlsconfig/configmanager.go +++ b/pkg/tlsconfig/configmanager.go @@ -853,7 +853,20 @@ func NewClientServerTLSConfigManager(monitor *TLSCertMonitor, opts ...TLSConfigM // config manager can not be reconfigured to enable TLS later. It is primarily useful for tests // that do not require TLS. func NewDisabledTLSConfigManager() *TLSConfigManager { - return &TLSConfigManager{disabled: true} + cm := &TLSConfigManager{ + disabled: true, + + // role is set to ServerAndClientRole because a disabled TLSConfigManager should + // still be able to do plaintext dials and listens. + role: ServerAndClientRole, + } + + // A disabled manager has nothing to say today, but every manager is built + // with a usable logger so that anything logging through one does not have to + // ask whether it exists. + cm.setLogger(nil) + + return cm } // TLSConfig returns a tls.Config for use with dial and listen functions. When TLS is disabled the return is nil. diff --git a/pkg/tlsconfig/configmanager_test.go b/pkg/tlsconfig/configmanager_test.go index f3520eb756a..0ad1f37a745 100644 --- a/pkg/tlsconfig/configmanager_test.go +++ b/pkg/tlsconfig/configmanager_test.go @@ -588,7 +588,7 @@ func TestTLSConfigManager_Listen(t *testing.T) { require.NoError(t, manager.Close()) }() - _, err = manager.Listen("tcp", "invalid:address:format") + _, err := manager.Listen("tcp", "invalid:address:format") require.ErrorContains(t, err, "address invalid:address:format: too many colons in address") }) } @@ -714,15 +714,88 @@ func TestTLSConfigManager_Dial(t *testing.T) { } func TestNewDisabledTLSConfigManager(t *testing.T) { - monitor := newTestCertMonitor(t) - defer th.CheckedClose(t, monitor)() - + // A disabled manager takes no certificate monitor and owns nothing that has + // to be closed, which is the whole point of it for tests that do not want + // TLS. disabled := NewDisabledTLSConfigManager() - defer th.CheckedClose(t, disabled)() - require.NotNil(t, disabled) - require.False(t, disabled.UseTLS()) - require.Nil(t, disabled.TLSConfig()) + + t.Run("TLS is off", func(t *testing.T) { + require.False(t, disabled.UseTLS()) + require.Nil(t, disabled.TLSConfig()) + }) + + t.Run("has a usable logger like any other manager", func(t *testing.T) { + // Nothing logs through a disabled manager today, so this guards the + // invariant rather than any current behavior: the next thing that does + // must not have to nil check first. + require.NotNil(t, disabled.logger) + require.NotPanics(t, func() { disabled.logger.Info("disabled manager logger is usable") }) + }) + + t.Run("listens in plaintext", func(t *testing.T) { + listener, err := disabled.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + // Plain TCP rather than a TLS listener wrapping it. + require.IsType(t, &net.TCPListener{}, listener) + }) + + t.Run("serves a plaintext connection end to end", func(t *testing.T) { + listener, err := disabled.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + testData := []byte("hello") + serverDone := make(chan error, 1) + go simpleEchoServer(serverDone, listener, len(testData)) + + // The same disabled manager both listens and dials, so it has to hold + // both roles. + conn, err := disabled.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer th.CheckedClose(t, conn)() + require.IsType(t, &net.TCPConn{}, conn) + + _, err = conn.Write(testData) + require.NoError(t, err) + + buf := make([]byte, len(testData)) + _, err = conn.Read(buf) + require.NoError(t, err) + require.Equal(t, testData, buf) + require.NoError(t, <-serverDone) + }) + + t.Run("DialContext dials in plaintext", func(t *testing.T) { + listener, err := disabled.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + conn, err := disabled.DialContext(context.Background(), "tcp", listener.Addr().String()) + require.NoError(t, err) + require.IsType(t, &net.TCPConn{}, conn) + require.NoError(t, conn.Close()) + }) + + t.Run("DialWithDialer dials in plaintext", func(t *testing.T) { + listener, err := disabled.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer th.CheckedClose(t, listener)() + + conn, err := disabled.DialWithDialer(&net.Dialer{}, "tcp", listener.Addr().String()) + require.NoError(t, err) + require.IsType(t, &net.TCPConn{}, conn) + require.NoError(t, conn.Close()) + }) + + t.Run("Close is a harmless no-op", func(t *testing.T) { + // There are no cert loaders to close, but Close must stay safe to call + // so a disabled manager can stand in for a real one. + require.NoError(t, disabled.Close()) + require.NoError(t, disabled.Close()) + }) } func TestNewClientTLSConfigManager(t *testing.T) { From ebbd11a83f0b07e0cee0ef22e5cf1df0f1d1ff70 Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 16:23:34 -0500 Subject: [PATCH 10/18] chore: fix issues with comments --- cmd/influxd/run/server.go | 3 +-- pkg/tlsconfig/configmanager.go | 12 ++++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/cmd/influxd/run/server.go b/cmd/influxd/run/server.go index c779fce55fa..6debf0121e6 100644 --- a/cmd/influxd/run/server.go +++ b/cmd/influxd/run/server.go @@ -630,8 +630,7 @@ func (s *Server) ApplyReloadedConfig(config *Config, log *zap.Logger) error { } } - // Reload the subscriber's client TLS certificate at its currently - // configured path so rotated certificates are picked up. + // Reload the subscriber's TLS configuration, including certificates, CAs, and other TLS settings. if s.Subscriber != nil { if af, err := s.Subscriber.PrepareReloadTLSCertificates(config.Subscriber); err == nil { applyFuncs = append(applyFuncs, af) diff --git a/pkg/tlsconfig/configmanager.go b/pkg/tlsconfig/configmanager.go index 7132adc6e4a..acc285e04be 100644 --- a/pkg/tlsconfig/configmanager.go +++ b/pkg/tlsconfig/configmanager.go @@ -27,7 +27,7 @@ const ( var ( // ErrConfigureDisabledManager is returned when an attempt is made to reconfigure - // a disabled config managaer. + // a disabled config manager. ErrConfigureDisabledManager = errors.New("cannot configure disabled TLS manager") // ErrClientListen is returned when attempt is made to have a client role manager @@ -814,16 +814,12 @@ func (cm *TLSConfigManager) prepareConfigure(c *tlsConfigManagerConfig) (func() // use. // // Previously, many options were required parameters to this function. They are still -// required, but given using With*() parameters. This has the advantaged of allowing +// required, but given using With*() parameters. This has the advantage of allowing // a single function to convert a TOML configuration to a slice of With* parameters for -// both construction and re-configuration. It does make missing an option a run-time +// both construction and reconfiguration. It does make missing an option a run-time // error instead of a compile-time error. // -// The following are require options. It is an error if they are missing. -// - WithUseTLS -// - WithBaseConfig -// -// If WithUseTLS(true) is given, then WithCertificate must also be given. +// If WithUseTLS(true) is given, then WithServerCertificate must also be given. // // All options given as direct positional parameters are required and can not be changed // after construction. From 8c059304dcc30c4af94913b95d396d0f8db42c61 Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 16:29:07 -0500 Subject: [PATCH 11/18] chore: fix compilation issue, slight test improvement --- pkg/tlsconfig/configmanager_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/tlsconfig/configmanager_test.go b/pkg/tlsconfig/configmanager_test.go index 0ad1f37a745..34ba9c19cde 100644 --- a/pkg/tlsconfig/configmanager_test.go +++ b/pkg/tlsconfig/configmanager_test.go @@ -588,8 +588,9 @@ func TestTLSConfigManager_Listen(t *testing.T) { require.NoError(t, manager.Close()) }() - _, err := manager.Listen("tcp", "invalid:address:format") + conn, err := manager.Listen("tcp", "invalid:address:format") require.ErrorContains(t, err, "address invalid:address:format: too many colons in address") + require.Nil(t, conn) }) } @@ -2022,8 +2023,9 @@ func TestTLSConfigManager_WithUsage(t *testing.T) { defer th.CheckedClose(t, manager)() // A server manager cannot dial; the message should say which manager. - _, err = manager.Dial("tcp", "127.0.0.1:0") + conn, err := manager.Dial("tcp", "127.0.0.1:0") require.ErrorContains(t, err, "httpd: ") + require.Nil(t, conn) }) t.Run("unset usage still names the loaders", func(t *testing.T) { From 1784768dbf1ea398431279433a28d75e65375fb6 Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 16:29:38 -0500 Subject: [PATCH 12/18] fix: mutex protected fields accessed without lock - Fix mutex protected fields access without lock. - Add comments explaining why trigger channel sends occur outside of lock. --- pkg/tlsconfig/certmonitor.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/tlsconfig/certmonitor.go b/pkg/tlsconfig/certmonitor.go index cac60243d05..b63b04c5b30 100644 --- a/pkg/tlsconfig/certmonitor.go +++ b/pkg/tlsconfig/certmonitor.go @@ -248,10 +248,12 @@ func (m *TLSCertMonitor) checkInterval() time.Duration { func (m *TLSCertMonitor) SetExpirationAdvanced(expirationAdvanced time.Duration) { m.mu.Lock() + triggerDelay := m.triggerDelayInterval m.certExpirationAdvanced = expirationAdvanced m.mu.Unlock() - m.resetTrigger(m.triggerDelayInterval) + // Send on channel outside of lock to prevent deadlocks. + m.resetTrigger(triggerDelay) } func (m *TLSCertMonitor) expirationAdvanced() time.Duration { @@ -274,10 +276,12 @@ func (m *TLSCertMonitor) SetTriggerDelay(d time.Duration) { // have certificates have been reloaded. func (m *TLSCertMonitor) QueueWarnIssues(cl *TLSCertLoader) { m.mu.Lock() + triggerDelay := m.triggerDelayInterval m.queuedCertLoaders = append(m.queuedCertLoaders, cl) m.mu.Unlock() - m.resetTrigger(m.triggerDelayInterval) + // Send on channel outside of lock to prevent deadlocks. + m.resetTrigger(triggerDelay) } // takeQueuedCertLoaders returns and clears TLSCertLoader objects queued From 27a416c83af7fd2fa3a7f51a123b7d0da652f08d Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Wed, 15 Jul 2026 18:06:00 -0500 Subject: [PATCH 13/18] chore: rename `ignore-sanity-checks` to `ignore-cert-sanity-checks` Renamed config options `ignore-sanity-checks` for subscriber and opentsdb services to `ignore-cert-sanity-checks`. This makes it obvious the sanity checks being ignored are for certificates and not general sanity checks. This naming matches other projects. --- etc/config.sample.toml | 4 ++-- services/opentsdb/config.go | 18 +++++++++--------- services/opentsdb/service.go | 2 +- services/opentsdb/service_test.go | 20 ++++++++++---------- services/subscriber/config.go | 6 +++--- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/etc/config.sample.toml b/etc/config.sample.toml index e55bc48fdd8..9cf07228a17 100644 --- a/etc/config.sample.toml +++ b/etc/config.sample.toml @@ -500,7 +500,7 @@ # Loads certificate even when it fails the checks for whether a server can use it, such as # the certificate not permitting server authentication. The failed checks are logged # instead. A missing or unparseable certificate is still an error. - # ignore-sanity-checks = false + # ignore-cert-sanity-checks = false # The number of writer goroutines processing the write channel. # write-concurrency = 40 @@ -625,7 +625,7 @@ # Loads certificate even when it fails the checks for whether a server can use it, such as # the certificate not permitting server authentication. The failed checks are logged # instead. A missing or unparseable certificate is still an error. - # ignore-sanity-checks = false + # ignore-cert-sanity-checks = false # The type of client certificate authentication (mutual TLS) to require. One of # NoClientCert, RequestClientCert, RequireAnyClientCert, VerifyClientCertIfGiven, or diff --git a/services/opentsdb/config.go b/services/opentsdb/config.go index 76a3d56666a..1f0c8141aa0 100644 --- a/services/opentsdb/config.go +++ b/services/opentsdb/config.go @@ -47,16 +47,16 @@ type Config struct { Certificate string `toml:"certificate"` PrivateKey string `toml:"private-key"` InsecureCertificate bool `toml:"insecure-certificate"` - // IgnoreSanityChecks loads the certificate even when it fails the + // IgnoreCertSanityChecks loads the certificate even when it fails the // checks that decide whether a server can use it at all. - IgnoreSanityChecks bool `toml:"ignore-sanity-checks"` - ClientAuthType *toml.TlsClientAuthType `toml:"client-auth-type"` - ClientCA *tlsconfig.CAConfig `toml:"client-ca"` - BatchSize int `toml:"batch-size"` - BatchPending int `toml:"batch-pending"` - BatchTimeout toml.Duration `toml:"batch-timeout"` - LogPointErrors bool `toml:"log-point-errors"` - TLS *tls.Config `toml:"-"` + IgnoreCertSanityChecks bool `toml:"ignore-cert-sanity-checks"` + ClientAuthType *toml.TlsClientAuthType `toml:"client-auth-type"` + ClientCA *tlsconfig.CAConfig `toml:"client-ca"` + BatchSize int `toml:"batch-size"` + BatchPending int `toml:"batch-pending"` + BatchTimeout toml.Duration `toml:"batch-timeout"` + LogPointErrors bool `toml:"log-point-errors"` + TLS *tls.Config `toml:"-"` } // Compile-time check that *Config implements toml.Defaulter so it can be used diff --git a/services/opentsdb/service.go b/services/opentsdb/service.go index 79ae25ab464..062d3cce6ad 100644 --- a/services/opentsdb/service.go +++ b/services/opentsdb/service.go @@ -117,7 +117,7 @@ func NewService(c Config, certMonitor *tlsconfig.TLSCertMonitor) (*Service, erro cert: d.Certificate, privateKey: d.PrivateKey, insecureCert: d.InsecureCertificate, - ignoreSanityChecks: d.IgnoreSanityChecks, + ignoreSanityChecks: d.IgnoreCertSanityChecks, clientAuthType: clientAuthType, clientCA: d.ClientCA, BindAddress: d.BindAddress, diff --git a/services/opentsdb/service_test.go b/services/opentsdb/service_test.go index 8ad8347dc9a..2460e90565e 100644 --- a/services/opentsdb/service_test.go +++ b/services/opentsdb/service_test.go @@ -443,10 +443,10 @@ func TestService_TLSUsage(t *testing.T) { require.Equal(t, "opentsdb("+bind+").server", entries[0].ContextMap()["usage"]) } -// TestConfig_IgnoreSanityChecks covers the config option reaching the +// TestConfig_IgnoreCertSanityChecks covers the config option reaching the // TLS manager. A certificate issued only for client authentication fails the // server sanity checks, so it opens only when the option is set. -func TestConfig_IgnoreSanityChecks(t *testing.T) { +func TestConfig_IgnoreCertSanityChecks(t *testing.T) { clientOnlySS := selfsigned.NewSelfSignedCert(t, selfsigned.WithExtKeyUsage(x509.ExtKeyUsageClientAuth)) @@ -458,14 +458,14 @@ func TestConfig_IgnoreSanityChecks(t *testing.T) { t.Cleanup(th.CheckedClose(t, certMonitor)) s, err := NewService(Config{ - BindAddress: "127.0.0.1:0", - Database: "db0", - ConsistencyLevel: "one", - TLSEnabled: true, - Certificate: clientOnlySS.CertPath, - PrivateKey: clientOnlySS.KeyPath, - IgnoreSanityChecks: ignore, - TLS: new(tls.Config), + BindAddress: "127.0.0.1:0", + Database: "db0", + ConsistencyLevel: "one", + TLSEnabled: true, + Certificate: clientOnlySS.CertPath, + PrivateKey: clientOnlySS.KeyPath, + IgnoreCertSanityChecks: ignore, + TLS: new(tls.Config), }, certMonitor) require.NoError(t, err) diff --git a/services/subscriber/config.go b/services/subscriber/config.go index da024e3f4bf..708ed90ee3b 100644 --- a/services/subscriber/config.go +++ b/services/subscriber/config.go @@ -54,11 +54,11 @@ type Config struct { // should be ignored when it is loaded. InsecureCertificate bool `toml:"insecure-certificate"` - // IgnoreSanityChecks loads the certificate even when it fails the + // IgnoreCertSanityChecks loads the certificate even when it fails the // checks that decide whether it can be used at all. The checks currently // only cover server certificates, so this has no effect on the subscriber's // client certificate today. - IgnoreSanityChecks bool `toml:"ignore-sanity-checks"` + IgnoreCertSanityChecks bool `toml:"ignore-cert-sanity-checks"` // The number of writer goroutines processing the write channel. WriteConcurrency int `toml:"write-concurrency"` @@ -142,7 +142,7 @@ func (c Config) TLSManagerOpts() []tlsconfig.TLSConfigManagerOpt { tlsconfig.WithClientCertificate(c.Certificate, c.PrivateKey), tlsconfig.WithRootCA(c.effectiveRootCA()), tlsconfig.WithIgnoreFilePermissions(c.InsecureCertificate), - tlsconfig.WithIgnoreSanityChecks(c.IgnoreSanityChecks), + tlsconfig.WithIgnoreSanityChecks(c.IgnoreCertSanityChecks), } } From 9bec6555f20d0313299a7b888b03c73eb76e10be Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Thu, 16 Jul 2026 15:05:56 -0500 Subject: [PATCH 14/18] fix: logger safety in TLSCertMonitor --- pkg/tlsconfig/certmonitor.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/tlsconfig/certmonitor.go b/pkg/tlsconfig/certmonitor.go index b63b04c5b30..5f4e8f8703e 100644 --- a/pkg/tlsconfig/certmonitor.go +++ b/pkg/tlsconfig/certmonitor.go @@ -205,7 +205,11 @@ func (m *TLSCertMonitor) logger() *zap.Logger { func (m *TLSCertMonitor) SetLogger(log *zap.Logger) { m.mu.Lock() defer m.mu.Unlock() - m.log = log + if log != nil { + m.log = log + } else { + m.log = zap.NewNop() + } } // isClosed returns true if the monitor has been closed. @@ -427,7 +431,7 @@ func (m *TLSCertMonitor) checkCerts(filter []*TLSCertLoader) { // monitor periodically logs certificate errors with the currently registered // TLSCertLoader objects. func (m *TLSCertMonitor) monitorCerts(startWg *sync.WaitGroup, stopWg *sync.WaitGroup) { - m.log.Info("Starting TLS certificate monitor") + m.logger().Info("Starting TLS certificate monitor") ticker := time.NewTicker(m.checkInterval()) defer ticker.Stop() @@ -451,7 +455,7 @@ func (m *TLSCertMonitor) monitorCerts(startWg *sync.WaitGroup, stopWg *sync.Wait m.triggerDelay.Reset(d) case <-m.closeCh: - m.log.Info("Stopping TLS certificate monitor") + m.logger().Info("Stopping TLS certificate monitor") ticker.Stop() m.triggerDelay.Stop() if stopWg != nil { From 3de4c7f56a600236869b2e4cc2027293d0c9ad49 Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Thu, 16 Jul 2026 16:07:41 -0500 Subject: [PATCH 15/18] feat: add mTLS support to `influx` CLI command --- cmd/influx/cli/cli.go | 121 +++++++++++++++++++++++++------- cmd/influx/cli/cli_mtls_test.go | 112 +++++++++++++++++++++++++++++ cmd/influx/main.go | 15 ++++ 3 files changed, 224 insertions(+), 24 deletions(-) create mode 100644 cmd/influx/cli/cli_mtls_test.go diff --git a/cmd/influx/cli/cli.go b/cmd/influx/cli/cli.go index ebaa9d68b2b..2bb146d289f 100644 --- a/cmd/influx/cli/cli.go +++ b/cmd/influx/cli/cli.go @@ -4,6 +4,7 @@ package cli // import "github.com/influxdata/influxdb/cmd/influx/cli" import ( "bytes" "context" + "crypto/tls" "encoding/csv" "encoding/json" "errors" @@ -29,6 +30,7 @@ import ( fluxClient "github.com/influxdata/influxdb/flux/client" v8 "github.com/influxdata/influxdb/importer/v8" "github.com/influxdata/influxdb/models" + "github.com/influxdata/influxdb/pkg/tlsconfig" "github.com/influxdata/influxql" "github.com/peterh/liner" "golang.org/x/term" @@ -39,30 +41,39 @@ var ErrBlankCommand = errors.New("empty input") // CommandLine holds CLI configuration and state. type CommandLine struct { - Line *liner.State - URL url.URL - Host string - Port int - PathPrefix string - Database string - Type QueryLanguage - Ssl bool - RetentionPolicy string - ClientVersion string - ServerVersion string - Pretty bool // controls pretty print for json - Format string // controls the output format. Valid values are json, csv, or column - Execute string - ShowVersion bool - Import bool - Chunked bool - ChunkSize int - NodeID int - Quit chan struct{} - IgnoreSignals bool // Ignore signals normally caught by this process (used primarily for testing) - ForceTTY bool // Force the CLI to act as if it were connected to a TTY - osSignals chan os.Signal - historyFilePath string + Line *liner.State + URL url.URL + Host string + Port int + PathPrefix string + Database string + Type QueryLanguage + Ssl bool + ClientCert string // path to the client certificate presented for mutual TLS + ClientKey string // path to the client private key for mutual TLS + RootCA string // path to the CA bundle used to verify the server certificate + // InsecureCertificate ignores file permission checks when loading the client + // certificate and key. + InsecureCertificate bool + // IgnoreCertSanityChecks loads the client certificate even when it fails the + // sanity checks that decide whether it is usable for client authentication. + IgnoreCertSanityChecks bool + RetentionPolicy string + ClientVersion string + ServerVersion string + Pretty bool // controls pretty print for json + Format string // controls the output format. Valid values are json, csv, or column + Execute string + ShowVersion bool + Import bool + Chunked bool + ChunkSize int + NodeID int + Quit chan struct{} + IgnoreSignals bool // Ignore signals normally caught by this process (used primarily for testing) + ForceTTY bool // Force the CLI to act as if it were connected to a TTY + osSignals chan os.Signal + historyFilePath string Client *client.Client ClientConfig client.Config // Client config options. @@ -118,6 +129,32 @@ func (c *CommandLine) Run() error { c.ClientConfig.Password = os.Getenv("INFLUX_PASSWORD") } + // Read TLS settings from the environment when the corresponding flag was not + // given, mirroring the username/password precedence above. + if c.ClientCert == "" { + c.ClientCert = os.Getenv("INFLUX_CERT") + } + if c.ClientKey == "" { + c.ClientKey = os.Getenv("INFLUX_KEY") + } + if c.RootCA == "" { + c.RootCA = os.Getenv("INFLUX_ROOT_CA") + } + if !c.InsecureCertificate { + c.InsecureCertificate, _ = strconv.ParseBool(os.Getenv("INFLUX_INSECURE_CERTIFICATE")) + } + if !c.IgnoreCertSanityChecks { + c.IgnoreCertSanityChecks, _ = strconv.ParseBool(os.Getenv("INFLUX_IGNORE_CERT_SANITY_CHECKS")) + } + + // Build the mutual TLS config once, up front, so a bad certificate or key is + // reported before any connection is attempted. + tlsConfig, err := c.buildClientTLSConfig() + if err != nil { + return err + } + c.ClientConfig.TLS = tlsConfig + addr := fmt.Sprintf("%s:%d/%s", c.Host, c.Port, c.PathPrefix) url, err := client.ParseConnectionString(addr, c.Ssl) if err != nil { @@ -337,6 +374,42 @@ func (c *CommandLine) ParseCommand(cmd string) error { return ErrBlankCommand } +// buildClientTLSConfig builds the *tls.Config used for mutual TLS from the +// configured client certificate, private key, and root CA. It returns nil when +// none of them are set, leaving the client's default TLS behavior unchanged. +// +// InsecureSkipVerify is intentionally not set here; the client applies it from +// the -unsafeSsl flag, which the SSL-detection retry logic in Run toggles after +// this config has been built. +func (c *CommandLine) buildClientTLSConfig() (*tls.Config, error) { + if c.ClientCert == "" && c.ClientKey == "" && c.RootCA == "" { + return nil, nil + } + + var rootCA *tlsconfig.CAConfig + if c.RootCA != "" { + rootCA = &tlsconfig.CAConfig{Paths: []string{c.RootCA}} + } + + // The influx CLI is a one-shot client, so the certificate monitor is never + // started; it exists only because the config manager requires one to build + // its client certificate loader. + monitor := tlsconfig.NewTLSCertMonitor() + cm, err := tlsconfig.NewClientTLSConfigManager( + monitor, + tlsconfig.WithUsage("influx"), + tlsconfig.WithUseTLS(true), + tlsconfig.WithClientCertificate(c.ClientCert, c.ClientKey), + tlsconfig.WithRootCA(rootCA), + tlsconfig.WithIgnoreFilePermissions(c.InsecureCertificate), + tlsconfig.WithIgnoreSanityChecks(c.IgnoreCertSanityChecks), + ) + if err != nil { + return nil, err + } + return cm.TLSConfig(), nil +} + // Connect connects to a server. func (c *CommandLine) Connect(cmd string) error { // normalize cmd diff --git a/cmd/influx/cli/cli_mtls_test.go b/cmd/influx/cli/cli_mtls_test.go new file mode 100644 index 00000000000..21bd8c940b7 --- /dev/null +++ b/cmd/influx/cli/cli_mtls_test.go @@ -0,0 +1,112 @@ +package cli_test + +import ( + "crypto/tls" + "crypto/x509" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strconv" + "testing" + + "github.com/influxdata/influxdb/cmd/influx/cli" + "github.com/influxdata/influxdb/pkg/testing/selfsigned" + "github.com/stretchr/testify/require" +) + +// mtlsTestServer starts an HTTPS test server that requires and verifies a client +// certificate signed by cert's CA. It returns the host and port the CLI should +// dial. +func mtlsTestServer(t *testing.T, cert *selfsigned.Cert) (host string, port int) { + t.Helper() + + serverPair, err := tls.LoadX509KeyPair(cert.CertPath, cert.KeyPath) + require.NoError(t, err) + + caPEM, err := os.ReadFile(cert.CACertPath) + require.NoError(t, err) + pool := x509.NewCertPool() + require.True(t, pool.AppendCertsFromPEM(caPEM), "failed to append CA certificate") + + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Influxdb-Version", SERVER_VERSION) + })) + ts.TLS = &tls.Config{ + Certificates: []tls.Certificate{serverPair}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: pool, + } + ts.StartTLS() + t.Cleanup(ts.Close) + + u, err := url.Parse(ts.URL) + require.NoError(t, err) + h, p, err := net.SplitHostPort(u.Host) + require.NoError(t, err) + port, err = strconv.Atoi(p) + require.NoError(t, err) + return h, port +} + +// TestRunCLI_MutualTLS verifies the CLI presents its client certificate and +// verifies the server against the configured root CA. +func TestRunCLI_MutualTLS(t *testing.T) { + cert := selfsigned.NewSelfSignedCert(t) + host, port := mtlsTestServer(t, cert) + + c := cli.New(CLIENT_VERSION) + c.Host = host + c.Port = port + c.Ssl = true + c.ClientCert = cert.CertPath + c.ClientKey = cert.KeyPath + c.RootCA = cert.CACertPath + c.Format = "column" + c.ClientConfig.Precision = "ns" + c.Execute = "SHOW DATABASES" + c.IgnoreSignals = true + c.ForceTTY = true + + require.NoError(t, c.Run()) + require.Equal(t, SERVER_VERSION, c.ServerVersion) +} + +// TestRunCLI_MutualTLS_MissingClientCert verifies the connection is refused when +// the CLI trusts the server but presents no client certificate. +func TestRunCLI_MutualTLS_MissingClientCert(t *testing.T) { + cert := selfsigned.NewSelfSignedCert(t) + host, port := mtlsTestServer(t, cert) + + c := cli.New(CLIENT_VERSION) + c.Host = host + c.Port = port + c.Ssl = true + c.RootCA = cert.CACertPath // trust the server, but present no client cert + c.Execute = "SHOW DATABASES" + c.IgnoreSignals = true + c.ForceTTY = true + + require.Error(t, c.Run()) +} + +// TestRunCLI_MutualTLS_UntrustedServer verifies the server certificate is +// rejected when no root CA that signed it is configured. +func TestRunCLI_MutualTLS_UntrustedServer(t *testing.T) { + cert := selfsigned.NewSelfSignedCert(t) + host, port := mtlsTestServer(t, cert) + + c := cli.New(CLIENT_VERSION) + c.Host = host + c.Port = port + c.Ssl = true + c.ClientCert = cert.CertPath + c.ClientKey = cert.KeyPath + // No RootCA: the self-signed server is not in the system pool. + c.Execute = "SHOW DATABASES" + c.IgnoreSignals = true + c.ForceTTY = true + + require.Error(t, c.Run()) +} diff --git a/cmd/influx/main.go b/cmd/influx/main.go index 80ec786e597..8c96ddd626e 100644 --- a/cmd/influx/main.go +++ b/cmd/influx/main.go @@ -49,6 +49,11 @@ func main() { fs.Var(&c.Type, "type", "query language for executing commands or invoking the REPL: influxql, flux") fs.BoolVar(&c.Ssl, "ssl", false, "Use https for connecting to cluster.") fs.BoolVar(&c.ClientConfig.UnsafeSsl, "unsafeSsl", false, "Set this when connecting to the cluster using https and not use SSL verification.") + fs.StringVar(&c.ClientCert, "cert", "", "Path to the client certificate file (PEM) presented for mutual TLS. Also settable via INFLUX_CERT.") + fs.StringVar(&c.ClientKey, "key", "", "Path to the client private key file (PEM) for mutual TLS. Also settable via INFLUX_KEY.") + fs.StringVar(&c.RootCA, "root-ca", "", "Path to the CA bundle (PEM) used to verify the server certificate. Also settable via INFLUX_ROOT_CA.") + fs.BoolVar(&c.InsecureCertificate, "insecure-certificate", false, "Ignore file permission checks when loading the client certificate and key. Also settable via INFLUX_INSECURE_CERTIFICATE.") + fs.BoolVar(&c.IgnoreCertSanityChecks, "ignore-cert-sanity-checks", false, "Load the client certificate even if it fails client-authentication sanity checks. Also settable via INFLUX_IGNORE_CERT_SANITY_CHECKS.") fs.StringVar(&c.Format, "format", defaultFormat, "Format specifies the format of the server responses: json, csv, or column.") fs.StringVar(&c.ClientConfig.Precision, "precision", defaultPrecision, "Precision specifies the format of the timestamp: rfc3339,h,m,s,ms,u or ns.") fs.StringVar(&c.ClientConfig.WriteConsistency, "consistency", "all", "Set write consistency level: any, one, quorum, or all.") @@ -84,6 +89,16 @@ func main() { Use https for requests. -unsafeSsl Set this when connecting to the cluster using https and not use SSL verification. + -cert 'path' + Path to the client certificate file (PEM) presented for mutual TLS. + -key 'path' + Path to the client private key file (PEM) for mutual TLS. + -root-ca 'path' + Path to the CA bundle (PEM) used to verify the server certificate. + -insecure-certificate + Ignore file permission checks when loading the client certificate and key. + -ignore-cert-sanity-checks + Load the client certificate even if it fails client-authentication sanity checks. -execute 'command' Execute command and quit. -type 'influxql|flux' From 5ab7d5cd04eef81b9b45c010f9999fac381a1d96 Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Thu, 16 Jul 2026 16:12:08 -0500 Subject: [PATCH 16/18] chore: update man page for `influx` CLI command --- man/influx.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/man/influx.txt b/man/influx.txt index 66e556d736d..e753ae47461 100644 --- a/man/influx.txt +++ b/man/influx.txt @@ -48,6 +48,21 @@ OPTIONS -unsafeSsl:: Set this with '-ssl' to allow unsafe connections. +-cert :: + Path to the client certificate file (PEM) presented to the server for mutual TLS. If the file also contains the private key, '-key' may be omitted. May also be set with the 'INFLUX_CERT' environment variable. + +-key :: + Path to the client private key file (PEM) for mutual TLS, when the key is stored separately from the certificate given by '-cert'. May also be set with the 'INFLUX_KEY' environment variable. + +-root-ca :: + Path to the CA bundle file (PEM) used to verify the server certificate. May also be set with the 'INFLUX_ROOT_CA' environment variable. + +-insecure-certificate:: + Ignore file permission checks when loading the client certificate and key. May also be set with the 'INFLUX_INSECURE_CERTIFICATE' environment variable. + +-ignore-cert-sanity-checks:: + Load the client certificate even if it fails client-authentication sanity checks. May also be set with the 'INFLUX_IGNORE_CERT_SANITY_CHECKS' environment variable. + -execute :: Executes the command and exits. From 69c311a4b3c3ef5d0f7ccf53c1826cca416263b9 Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Thu, 16 Jul 2026 16:36:06 -0500 Subject: [PATCH 17/18] chore: test improvements and fix spelling mistakes - Improve error checking in tests. - Use th.CheckedClose in more places. - Fix some spelling mistakes. --- pkg/tlsconfig/certconfig.go | 8 ++++---- pkg/tlsconfig/certconfig_test.go | 18 ++++++------------ pkg/tlsconfig/clientcert_test.go | 4 ++-- pkg/tlsconfig/configmanager.go | 4 ++-- pkg/tlsconfig/configmanager_test.go | 10 ++++++---- services/httpd/service.go | 2 +- services/httpd/service_test.go | 4 ++-- 7 files changed, 23 insertions(+), 27 deletions(-) diff --git a/pkg/tlsconfig/certconfig.go b/pkg/tlsconfig/certconfig.go index 00ce9de86b8..fbadfc269c5 100644 --- a/pkg/tlsconfig/certconfig.go +++ b/pkg/tlsconfig/certconfig.go @@ -143,7 +143,7 @@ func WithCertLoaderIgnoreSanityChecks(ignore bool) TLSCertLoaderOpt { // NewTLSCertLoader creates a TLSCertLoader loaded with the certificate found in certPath and keyPath. // Only trusted input (standard configuration files) should be used for certPath and keyPath. -// If the certificate can not be loaded, an error is returned. On success, a monitor is setup to +// If the certificate cannot be loaded, an error is returned. On success, a monitor is setup to // periodically check the certificate for expiration. func NewTLSCertLoader(role Role, monitor *TLSCertMonitor, opts ...TLSCertLoaderOpt) (rCertLoader *TLSCertLoader, rErr error) { cl := &TLSCertLoader{ @@ -353,7 +353,7 @@ func (cl *TLSCertLoader) PrepareLoad(opts ...TLSCertLoaderOpt) (func() error, er } // Sanity check the certificate against how a server will use it. A - // certificate a server can not present, or that every client will refuse, is + // certificate a server cannot present, or that every client will refuse, is // caught here rather than at the first handshake, where it would surface // per-connection as an error naming neither the certificate nor the reason. if cl.role.IsServerRole() { @@ -361,7 +361,7 @@ func (cl *TLSCertLoader) PrepareLoad(opts ...TLSCertLoaderOpt) (func() error, er // when sanity checks are ignored: a server using TLS always needs a // certificate, and a leaf that will not parse is a real problem. if loadedCert.IsEmpty() { - err := fmt.Errorf("%s: can not use an empty certificate for a server: %w", cl.usage, ErrCertificateEmpty) + err := fmt.Errorf("%s: cannot use an empty certificate for a server: %w", cl.usage, ErrCertificateEmpty) logLoadError(err) return nil, err } @@ -379,7 +379,7 @@ func (cl *TLSCertLoader) PrepareLoad(opts ...TLSCertLoaderOpt) (func() error, er var sanityErrs []error if !leaf.SupportsServerAuth() { - sanityErrs = append(sanityErrs, fmt.Errorf("%s: can not use a certificate that does not permit server authentication for a server: %w", + sanityErrs = append(sanityErrs, fmt.Errorf("%s: cannot use a certificate that does not permit server authentication for a server: %w", cl.usage, ErrCertificateNotServerAuth)) } diff --git a/pkg/tlsconfig/certconfig_test.go b/pkg/tlsconfig/certconfig_test.go index dc0a40bc1ec..d0ec61ece89 100644 --- a/pkg/tlsconfig/certconfig_test.go +++ b/pkg/tlsconfig/certconfig_test.go @@ -154,9 +154,7 @@ func TestTLSCertLoader_GoodCertPersists(t *testing.T) { WithCertLoaderLogger(logger)) require.NoError(t, err) require.NotNil(t, cl) - defer func() { - require.NoError(t, cl.Close()) - }() + defer th.CheckedClose(t, cl)() var goodSerial big.Int { @@ -252,7 +250,7 @@ func TestTLSCertLoader_FileNotFound(t *testing.T) { ServerOnlyRole, monitor, WithCertLoaderCertificate("/nonexistent/path/to/cert.pem", ss.KeyPath)) - require.ErrorContains(t, err, "no such file or directory") + require.ErrorIs(t, err, os.ErrNotExist) require.Nil(t, cl) // Non-existent key file @@ -260,7 +258,7 @@ func TestTLSCertLoader_FileNotFound(t *testing.T) { ServerOnlyRole, monitor, WithCertLoaderCertificate(ss.CertPath, "/nonexistent/path/to/key.pem")) - require.ErrorContains(t, err, "no such file or directory") + require.ErrorIs(t, err, os.ErrNotExist) require.Nil(t, cl) // Both files non-existent @@ -268,7 +266,7 @@ func TestTLSCertLoader_FileNotFound(t *testing.T) { ServerOnlyRole, monitor, WithCertLoaderCertificate("/nonexistent/cert.pem", "/nonexistent/key.pem")) - require.ErrorContains(t, err, "no such file or directory") + require.ErrorIs(t, err, os.ErrNotExist) require.Nil(t, cl) } @@ -316,9 +314,7 @@ func TestTLSCertLoader_CombinedFile(t *testing.T) { WithCertLoaderCertificate(ss.CertPath, "")) require.NoError(t, err) require.NotNil(t, cl) - defer func() { - require.NoError(t, cl.Close()) - }() + defer th.CheckedClose(t, cl)() // Get certificate and verify it cert, err := cl.GetCertificate(nil) @@ -444,9 +440,7 @@ func TestTLSCertLoader_ExpiredCertificateLogging(t *testing.T) { WithCertLoaderLogger(logger)) require.NoError(t, err) require.NotNil(t, cl) - defer func() { - require.NoError(t, cl.Close()) - }() + defer th.CheckedClose(t, cl)() checkWarning := func(t *testing.T) { warning := logs.FilterMessage("Certificate is expired").TakeAll() diff --git a/pkg/tlsconfig/clientcert_test.go b/pkg/tlsconfig/clientcert_test.go index 252f9f36e26..62f07dc37e9 100644 --- a/pkg/tlsconfig/clientcert_test.go +++ b/pkg/tlsconfig/clientcert_test.go @@ -180,7 +180,7 @@ func TestTLSConfigManager_WithClientCertificate_Paths(t *testing.T) { WithServerCertificate(serverSS.CertPath, serverSS.KeyPath), WithClientCertificate(missing, missing)) require.ErrorContains(t, err, "error configuring client cert loader") - require.ErrorContains(t, err, "no such file or directory") + require.ErrorIs(t, err, os.ErrNotExist) require.Nil(t, manager) }) } @@ -379,7 +379,7 @@ func TestTLSConfigManager_ReconfigureClientCertificate(t *testing.T) { missing := path.Join(t.TempDir(), "absent.pem") apply, err := manager.PrepareReconfigure(WithClientCertificate(missing, missing)) - require.ErrorContains(t, err, "no such file or directory") + require.ErrorIs(t, err, os.ErrNotExist) require.Nil(t, apply) clientCert, _ := manager.clientCertLoader.Paths() diff --git a/pkg/tlsconfig/configmanager.go b/pkg/tlsconfig/configmanager.go index acc285e04be..bbb60124b5a 100644 --- a/pkg/tlsconfig/configmanager.go +++ b/pkg/tlsconfig/configmanager.go @@ -821,7 +821,7 @@ func (cm *TLSConfigManager) prepareConfigure(c *tlsConfigManagerConfig) (func() // // If WithUseTLS(true) is given, then WithServerCertificate must also be given. // -// All options given as direct positional parameters are required and can not be changed +// All options given as direct positional parameters are required and cannot be changed // after construction. // // The returned TLSConfigManager can be used for server operations (e.g. Listen), but not for client operations (e.g. Dial). @@ -846,7 +846,7 @@ func NewClientServerTLSConfigManager(monitor *TLSCertMonitor, opts ...TLSConfigM } // NewDisabledTLSConfigManager creates a TLSConfigManager that has TLS disabled. A disabled -// config manager can not be reconfigured to enable TLS later. It is primarily useful for tests +// config manager cannot be reconfigured to enable TLS later. It is primarily useful for tests // that do not require TLS. func NewDisabledTLSConfigManager() *TLSConfigManager { cm := &TLSConfigManager{ diff --git a/pkg/tlsconfig/configmanager_test.go b/pkg/tlsconfig/configmanager_test.go index 34ba9c19cde..6966c8db6c8 100644 --- a/pkg/tlsconfig/configmanager_test.go +++ b/pkg/tlsconfig/configmanager_test.go @@ -161,7 +161,8 @@ func TestTLSConfigManager_ConstructorError(t *testing.T) { WithUseTLS(true), WithBaseConfig(nil), WithServerCertificate("/nonexistent/cert.pem", "/nonexistent/key.pem")) - require.ErrorContains(t, err, "LoadCertificate: error opening \"/nonexistent/cert.pem\" for reading: open /nonexistent/cert.pem: no such file or directory") + require.ErrorIs(t, err, os.ErrNotExist) + require.ErrorContains(t, err, "LoadCertificate: error opening \"/nonexistent/cert.pem\" for reading:") require.Nil(t, manager) } @@ -473,7 +474,8 @@ func TestTLSConfigManager_PrepareCertificateLoad(t *testing.T) { // Prepare loading with nonexistent paths should fail callback, err := manager.PrepareReconfigure( WithServerCertificate("/nonexistent/cert.pem", "/nonexistent/key.pem")) - require.ErrorContains(t, err, "LoadCertificate: error opening \"/nonexistent/cert.pem\" for reading: open /nonexistent/cert.pem: no such file or directory") + require.ErrorIs(t, err, os.ErrNotExist) + require.ErrorContains(t, err, "LoadCertificate: error opening \"/nonexistent/cert.pem\" for reading:") require.Nil(t, callback) }) @@ -1101,7 +1103,7 @@ func TestTLSConfigManager_WithRootCAFiles(t *testing.T) { WithUseTLS(true), WithRootCA(&CAConfig{Paths: []string{"/nonexistent/ca.pem"}})) require.ErrorIs(t, err, os.ErrNotExist) - require.ErrorContains(t, err, "error creating root CA pool: error reading file \"/nonexistent/ca.pem\" for CA store: open /nonexistent/ca.pem: no such file or directory") + require.ErrorContains(t, err, "error creating root CA pool: error reading file \"/nonexistent/ca.pem\" for CA store:") require.Nil(t, manager) }) @@ -1366,7 +1368,7 @@ func TestTLSConfigManager_WithClientCAFiles(t *testing.T) { WithClientAuth(tls.RequireAndVerifyClientCert), WithClientCA(&CAConfig{Paths: []string{"/nonexistent/ca.pem"}})) require.ErrorIs(t, err, os.ErrNotExist) - require.ErrorContains(t, err, "error creating client CA pool: error reading file \"/nonexistent/ca.pem\" for CA store: open /nonexistent/ca.pem: no such file or directory") + require.ErrorContains(t, err, "error creating client CA pool: error reading file \"/nonexistent/ca.pem\" for CA store:") require.Nil(t, manager) }) diff --git a/services/httpd/service.go b/services/httpd/service.go index e7b395ff10c..d9f28bd2512 100644 --- a/services/httpd/service.go +++ b/services/httpd/service.go @@ -291,7 +291,7 @@ func (s *Service) PrepareReloadConfig(c Config) (func() error, error) { // Let the user know that changing the https-enabled setting doesn't work. if s.httpsEnabled != c.HTTPSEnabled { - return nil, fmt.Errorf("httpd: can not change https-enabled on a running server") + return nil, fmt.Errorf("httpd: cannot change https-enabled on a running server") } if s.httpsEnabled { diff --git a/services/httpd/service_test.go b/services/httpd/service_test.go index d3dac77aa91..ea75bc7ebe6 100644 --- a/services/httpd/service_test.go +++ b/services/httpd/service_test.go @@ -62,7 +62,7 @@ func TestService_VerifyReloadedConfig(t *testing.T) { HTTPSEnabled: false, } applyFunc, err := s.PrepareReloadConfig(newConfig) - require.ErrorContains(t, err, "can not change https-enabled on a running server") + require.ErrorContains(t, err, "cannot change https-enabled on a running server") require.Nil(t, applyFunc) }) @@ -84,7 +84,7 @@ func TestService_VerifyReloadedConfig(t *testing.T) { HTTPSPrivateKey: ss.KeyPath, } applyFunc, err := s.PrepareReloadConfig(newConfig) - require.ErrorContains(t, err, "can not change https-enabled on a running server") + require.ErrorContains(t, err, "cannot change https-enabled on a running server") require.Nil(t, applyFunc) }) From 630d2244c5c7b682117db558846bbb292cba2bab Mon Sep 17 00:00:00 2001 From: Geoffrey Wossum Date: Thu, 16 Jul 2026 17:15:05 -0500 Subject: [PATCH 18/18] chore: test improvements --- cmd/influx/cli/cli.go | 2 +- cmd/influx/cli/cli_mtls_test.go | 10 ++++++++-- pkg/tlsconfig/certconfig_test.go | 28 +++++++++++++++++++++++++++- services/httpd/tlsreload_test.go | 14 ++++++++++++-- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/cmd/influx/cli/cli.go b/cmd/influx/cli/cli.go index 2bb146d289f..a6650363771 100644 --- a/cmd/influx/cli/cli.go +++ b/cmd/influx/cli/cli.go @@ -183,7 +183,7 @@ func (c *CommandLine) Run() error { } c.ClientConfig.UnsafeSsl = false } - return fmt.Errorf("Failed to connect to %s: %s\n%s", c.Client.Addr(), err.Error(), msg) + return fmt.Errorf("Failed to connect to %s: %w\n%s", c.Client.Addr(), err, msg) } // Modify precision. diff --git a/cmd/influx/cli/cli_mtls_test.go b/cmd/influx/cli/cli_mtls_test.go index 21bd8c940b7..d58799637cb 100644 --- a/cmd/influx/cli/cli_mtls_test.go +++ b/cmd/influx/cli/cli_mtls_test.go @@ -88,7 +88,8 @@ func TestRunCLI_MutualTLS_MissingClientCert(t *testing.T) { c.IgnoreSignals = true c.ForceTTY = true - require.Error(t, c.Run()) + // The server aborts the handshake because no client certificate was sent. + require.ErrorContains(t, c.Run(), "certificate required") } // TestRunCLI_MutualTLS_UntrustedServer verifies the server certificate is @@ -108,5 +109,10 @@ func TestRunCLI_MutualTLS_UntrustedServer(t *testing.T) { c.IgnoreSignals = true c.ForceTTY = true - require.Error(t, c.Run()) + // The server certificate cannot be verified against any trusted CA, which + // surfaces as a TLS certificate-verification error in the wrapped chain. + err := c.Run() + var certErr *tls.CertificateVerificationError + require.ErrorAs(t, err, &certErr) + require.ErrorContains(t, err, "certificate signed by unknown authority") } diff --git a/pkg/tlsconfig/certconfig_test.go b/pkg/tlsconfig/certconfig_test.go index d0ec61ece89..a4adea0fa50 100644 --- a/pkg/tlsconfig/certconfig_test.go +++ b/pkg/tlsconfig/certconfig_test.go @@ -202,7 +202,33 @@ func TestTLSCertLoader_GoodCertPersists(t *testing.T) { require.NotNil(t, x509Cert.SerialNumber) require.Equal(t, goodSerial, *x509Cert.SerialNumber) - // TODO: Check log lines + // The failed reload should log that it kept the previously loaded + // certificate, naming both the certificate that failed and the one still + // in use (with its serial), so operators can tell the reload was rejected + // rather than silently applied. + require.Equal(t, 3, logs.Len()) + logLines := logs.TakeAll() + require.Equal(t, "Loading TLS certificate (start)", logLines[0].Message) + require.Equal(t, "Error loading TLS certificate, continuing to use previously loaded certificate", logLines[1].Message) + require.Equal(t, "Loading TLS certificate (end)", logLines[2].Message) + + // The start and end lines name the certificate whose load was attempted. + for _, l := range []observer.LoggedEntry{logLines[0], logLines[2]} { + ctx := l.ContextMap() + require.Equal(t, emptyPath, ctx["cert"]) + require.Equal(t, emptyPath, ctx["key"]) + } + + // The error line is logged at error level and distinguishes the failed + // certificate from the active one that remains in use. + require.Equal(t, zapcore.ErrorLevel, logLines[1].Level) + cm := logLines[1].ContextMap() + require.Equal(t, emptyPath, cm["failedCert"]) + require.Equal(t, emptyPath, cm["failedKey"]) + require.Equal(t, ss.CertPath, cm["activeCert"]) + require.Equal(t, ss.KeyPath, cm["activeKey"]) + require.Equal(t, goodSerial.String(), cm["activeCertSerial"]) + require.Contains(t, cm["error"], "failed to find any PEM data") } } diff --git a/services/httpd/tlsreload_test.go b/services/httpd/tlsreload_test.go index a27671d7022..1c306fa73ab 100644 --- a/services/httpd/tlsreload_test.go +++ b/services/httpd/tlsreload_test.go @@ -55,7 +55,7 @@ func clientCertPool(t *testing.T, ss *selfsigned.Cert) *x509.CertPool { // The client's own error is not enough: under TLS 1.3 a client can finish its // handshake before the server's rejection arrives, so a read is needed to // surface it. -func handshake(t *testing.T, s *httpd.Service, clientCfg *tls.Config) error { +func handshake(t *testing.T, s *httpd.Service, clientCfg *tls.Config) (rErr error) { t.Helper() clientCfg.InsecureSkipVerify = true @@ -63,7 +63,17 @@ func handshake(t *testing.T, s *httpd.Service, clientCfg *tls.Config) error { if err != nil { return err } - defer conn.Close() + defer func() { + // If a handshake error occurs, conn is automatically closed and we might + // get an error close because the connection is automatically closed. + // The closeErr doesn't give any indication it was due to a TLS error + // and the error is not consistent. The best we can do is say there should + // not be an error if the overall function did not return an error. + closeErr := conn.Close() + if rErr == nil { + require.NoError(t, closeErr) + } + }() if err := conn.Handshake(); err != nil { return err