From 6fc701b15999d4453377071e20a34d28d86320ff Mon Sep 17 00:00:00 2001 From: TheophileDiot Date: Fri, 6 Mar 2026 15:16:29 +0100 Subject: [PATCH 01/59] feat: add context7 configuration file with URL and public key --- context7.json | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 context7.json diff --git a/context7.json b/context7.json new file mode 100644 index 0000000000..d99e7a2b06 --- /dev/null +++ b/context7.json @@ -0,0 +1,4 @@ +{ + "url": "https://context7.com/bunkerity/bunkerweb", + "public_key": "pk_LvmBTDFeSHkvz22IYNQlv" +} From ab24f60211e51c4c684b2ca97d935728e108dfb5 Mon Sep 17 00:00:00 2001 From: TheophileDiot Date: Fri, 6 Mar 2026 15:17:42 +0100 Subject: [PATCH 02/59] Revert "feat: add context7 configuration file with URL and public key" This reverts commit 6fc701b15999d4453377071e20a34d28d86320ff. --- context7.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 context7.json diff --git a/context7.json b/context7.json deleted file mode 100644 index d99e7a2b06..0000000000 --- a/context7.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "url": "https://context7.com/bunkerity/bunkerweb", - "public_key": "pk_LvmBTDFeSHkvz22IYNQlv" -} From 230f84083927d43cab8cdc05eee7bd43d9c1615d Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 12 Mar 2026 16:58:19 +0100 Subject: [PATCH 03/59] use OCSP files for stapling responses --- .../server-http/ssl-certificate-lua.conf | 161 +++++- src/common/core/ssl/jobs/ocsp-refresh.py | 465 ++++++++++++++++++ src/common/core/ssl/plugin.json | 19 +- 3 files changed, 619 insertions(+), 26 deletions(-) create mode 100644 src/common/core/ssl/jobs/ocsp-refresh.py diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 74e44d0a4c..7924618792 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -74,7 +74,8 @@ ssl_certificate_by_lua_block { local cdatastore = require "bunkerweb.datastore" local cclusterstore = require "bunkerweb.clusterstore" local cjson = require "cjson" - local ssl = require "ngx.ssl" + local ssl = require "ngx.ssl" + local ocsp = require "ngx.ocsp" local ngx = ngx local ngx_req = ngx.req @@ -101,37 +102,144 @@ ssl_certificate_by_lua_block { return end - local function get_phase_order(ord, phase, server_name) - if ord.per_site and server_name and ord.per_site[server_name] and ord.per_site[server_name][phase] then - return ord.per_site[server_name][phase] - elseif ord.global and ord.global[phase] then - return ord.global[phase] + -- Resolve per-site plugin order + local function get_phase_order(ord, phase, server_name) + if ord.per_site and server_name and ord.per_site[server_name] and ord.per_site[server_name][phase] then + return ord.per_site[server_name][phase] + elseif ord.global and ord.global[phase] then + return ord.global[phase] + end + return ord[phase] + end + + local server_name = ssl.server_name() + local phase_order = get_phase_order(order, "ssl_certificate", server_name) + + -- Helper: check if OCSP stapling is enabled for this site + local function is_ocsp_stapling_enabled() + -- Try to get SSL_USE_OCSP_STAPLING from variables, default to "yes" if not found + local use_ocsp, err = get_variable("SSL_USE_OCSP_STAPLING", server_name) + if not use_ocsp then + -- If we can't get the variable, log a warning but continue (graceful fallback) + logger:log(WARN, "OCSP can't get SSL_USE_OCSP_STAPLING variable: " .. (err or "unknown")) + return true -- Continue attempting OCSP stapling anyway + end + return use_ocsp == "yes" + end + + -- Helper: set OCSP stapling from cache (Redis + local shared dict + ocsp.der files) + local function set_ocsp_from_cache() + logger:log(INFO, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) + + if not is_ocsp_stapling_enabled() then + logger:log(INFO, "OCSP stapling disabled via SSL_USE_OCSP_STAPLING setting") + return + end + + if not server_name or server_name == "" then + logger:log(ERR, "OCSP no server_name available") + return + end + + local cache_key = "SSL:ocsp_status:" .. server_name + + local resp + + -- 1) Local shared dict lookup (per-worker cache) + if not resp then + local db_val, gerr = internalstore:get(cache_key, true) + if not db_val and gerr and gerr ~= "not found" then + logger:log(ERR, "OCSP error while getting response from internalstore: " .. gerr) + end + if db_val then + resp = db_val end - return ord[phase] end - local server_name = ssl.server_name() - local phase_order = get_phase_order(order, "ssl_certificate", server_name) + -- 2) Fallback to on-disk OCSP response (/var/cache/bunkerweb/ssl/{domain}/ocsp.der, scheduler-managed from database) + if not resp then + logger:log(INFO, "OCSP trying to load from disk files for server " .. (server_name or "nil")) + local base_path = "/var/cache/bunkerweb/ssl/" + local candidates = { + server_name, + server_name .. "-ecdsa", + server_name .. "-rsa", + } + + for _, name in ipairs(candidates) do + if name and name ~= "" then + local path = base_path .. name .. "/ocsp.der" + logger:log(INFO, "OCSP checking path: " .. path) + local f = io.open(path, "rb") + if f then + logger:log(INFO, "OCSP file found: " .. path) + local data = f:read("*a") + f:close() + if data and #data > 0 then + logger:log(INFO, "OCSP read " .. #data .. " bytes from " .. path) + resp = data + -- Cache in shared memory to avoid disk I/O on every handshake + local ok_set, serr = internalstore:set(cache_key, resp, 3600, true) + if not ok_set then + logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) + end + + -- Also backfill Redis if available so other workers benefit. + local cluster2 = cclusterstore:new() + local ok_connect2, rerr2 = cluster2:connect(true) + if not ok_connect2 then + logger:log(ERR, "OCSP error while connecting to Redis for file backfill: " .. (rerr2 or "unknown")) + else + local ok_setex2, set_err2 = cluster2:call("setex", cache_key, 3600, resp) + if set_err2 then + logger:log(ERR, "OCSP error while backfilling Redis from file: " .. set_err2) + elseif not ok_setex2 then + logger:log(ERR, "OCSP Redis SETEX from file returned falsy for " .. server_name) + else + logger:log(INFO, "OCSP backfilled Redis from file for " .. server_name) + end + cluster2:close() + end + logger:log(INFO, "OCSP loaded response from file " .. path .. " for " .. server_name) + break + end + end + end + end + end + + if not resp then + logger:log(INFO, "OCSP no response found from any cache/disk source for " .. server_name) + return + end + + local ok_set, oerr = ocsp.set_ocsp_status_resp(resp) + if not ok_set then + logger:log(ERR, "OCSP failed to set stapling: " .. oerr) + else + logger:log(INFO, "OCSP stapling set from cache for " .. server_name) + end + end -- Call ssl_certificate() methods logger:log(INFO, "calling ssl_certificate() methods of plugins ...") for i, plugin_id in ipairs(phase_order) do -- Require call - local plugin_lua, err = require_plugin(plugin_id) + local plugin_lua, perr = require_plugin(plugin_id) if plugin_lua == false then - logger:log(ERR, err) + logger:log(ERR, perr) elseif plugin_lua == nil then - logger:log(INFO, err) + logger:log(INFO, perr) else -- Check if plugin has ssl_certificate method if plugin_lua.ssl_certificate ~= nil then -- New call - local ok, plugin_obj = new_plugin(plugin_lua) - if not ok then + local ok_new, plugin_obj = new_plugin(plugin_lua) + if not ok_new then logger:log(ERR, plugin_obj) else - local ok, ret = call_plugin(plugin_obj, "ssl_certificate") - if not ok then + local ok_call, ret = call_plugin(plugin_obj, "ssl_certificate") + if not ok_call then logger:log(ERR, ret) elseif not ret.ret then logger:log(ERR, plugin_id .. ":ssl_certificate() call failed : " .. ret.msg) @@ -139,18 +247,21 @@ ssl_certificate_by_lua_block { logger:log(INFO, plugin_id .. ":ssl_certificate() call successful : " .. ret.msg) if ret.status then logger:log(INFO, plugin_id .. " is setting certificate/key : " .. ret.msg) - local ok, err = clear_certs() - if not ok then - logger:log(ERR, "error while clearing certificates : " .. err) + local ok_clear, cerr = clear_certs() + if not ok_clear then + logger:log(ERR, "error while clearing certificates : " .. cerr) end - ok, err = set_cert(ret.status[1]) - if not ok then - logger:log(ERR, "error while setting certificate : " .. err) + local ok_cert, serr = set_cert(ret.status[1]) + if not ok_cert then + logger:log(ERR, "error while setting certificate : " .. serr) else - local ok, err = set_priv_key(ret.status[2]) - if not ok then - logger:log(ERR, "error while setting private key : " .. err) + local ok_key, kerr = set_priv_key(ret.status[2]) + if not ok_key then + logger:log(ERR, "error while setting private key : " .. kerr) else + -- Try to set OCSP stapling from cache (if enabled) + logger:log(INFO, "DEBUG: About to call set_ocsp_from_cache() for " .. server_name) + set_ocsp_from_cache() logger:log(INFO, "certificate set by " .. plugin_id) return true end diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py new file mode 100644 index 0000000000..34c35b58c5 --- /dev/null +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 + +import hashlib +import logging +import os +import shlex +import subprocess +import sys as _sys +import tempfile +import time +from datetime import datetime, timezone +from pathlib import Path +from sys import exit as sys_exit, path as sys_path +from typing import Any, List, Optional, Tuple + +# Add BunkerWeb Python deps (Job, logger, Database) to path +for deps_path in [ + Path(os.sep, "usr", "share", "bunkerweb", *paths).as_posix() + for paths in (("deps", "python"), ("utils",), ("db",)) +]: + if deps_path not in sys_path: + sys_path.append(deps_path) + +# Gracefully handle import failures +try: + from jobs import Job # type: ignore +except ImportError as e: + print(f"FATAL: Could not import Job: {e}", file=_sys.stderr) + _sys.exit(1) + +try: + from logger import getLogger # type: ignore +except ImportError as e: + print(f"FATAL: Could not import logger: {e}", file=_sys.stderr) + _sys.exit(1) + +# Optional: Database support for OCSP storage +try: + from Database import Database # type: ignore +except ImportError as e: + print(f"WARNING: Database not available: {e}", file=_sys.stderr) + Database = None # type: ignore + +# Dedicated logger for OCSP refresh job. +# Log level is controlled by the standard BunkerWeb LOG_LEVEL / CUSTOM_LOG_LEVEL env vars. +try: + LOG = getLogger("SSL.OCSP-REFRESH") +except Exception as e: + print(f"FATAL: Could not initialize logger: {e}", file=_sys.stderr) + _sys.exit(1) + +status = 0 + +# Base paths +LIVE_BASE = Path(os.sep, "var", "cache", "bunkerweb", "letsencrypt", "etc", "live") +# Use scheduler-managed cache directory (automatically synced from database on restart) +CONFIGS_SSL_BASE = Path(os.sep, "var", "cache", "bunkerweb", "ssl") + + +def run_cmd(cmd: List[str]) -> Tuple[int, str, str]: + """ + Run a subprocess command and return (rc, stdout, stderr). + When LOG is in debug level, the command line is logged. + """ + LOG.debug("OCSP running command: %s", " ".join(shlex.quote(c) for c in cmd)) + proc = subprocess.run( + cmd, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + env={"PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", "")}, + ) + return proc.returncode, proc.stdout, proc.stderr + + +def cert_supports_ocsp(fullchain: Path) -> Optional[str]: + """ + First function: test if the certificate supports OCSP. + Returns OCSP responder URL if present, else None. + """ + LOG.debug("OCSP checking support for certificate %s", fullchain) + rc, out, err = run_cmd( + ["openssl", "x509", "-noout", "-ocsp_uri", "-in", fullchain.as_posix()] + ) + if rc != 0: + LOG.debug("OCSP openssl -ocsp_uri failed for %s: %s", fullchain, err.strip()) + return None + url = out.strip() + if not url: + LOG.debug("OCSP no responder URL advertised in %s", fullchain) + return None + LOG.debug("OCSP found responder URL for %s: %s", fullchain, url) + return url + + +def _normalize_dn_line(line: str, prefix: str) -> str: + """ + Normalize an OpenSSL subject/issuer line like 'issuer= CN = R3, O = Let's Encrypt, C = US' + so comparison is more robust to spacing. + """ + line = line.strip() + if line.lower().startswith(prefix.lower()): + line = line[len(prefix) :].strip() + # Remove repeated spaces around '=' and commas + parts = [part.strip() for part in line.replace(", ", ",").split(",")] + return ",".join(parts).lower() + + +def is_cert_issuer(leaf_path: Path, issuer_path: Path) -> bool: + """ + Verify that issuer_path is the issuer of leaf_path by comparing + leaf issuer DN with issuer subject DN using openssl x509. + """ + rc1, issuer_line, err1 = run_cmd( + ["openssl", "x509", "-noout", "-issuer", "-in", leaf_path.as_posix()] + ) + if rc1 != 0: + LOG.debug("OCSP openssl -issuer failed for %s: %s", leaf_path, err1.strip()) + return False + + rc2, subject_line, err2 = run_cmd( + ["openssl", "x509", "-noout", "-subject", "-in", issuer_path.as_posix()] + ) + if rc2 != 0: + LOG.debug("OCSP openssl -subject failed for %s: %s", issuer_path, err2.strip()) + return False + + issuer_dn = _normalize_dn_line(issuer_line, "issuer=") + subject_dn = _normalize_dn_line(subject_line, "subject=") + + if issuer_dn != subject_dn: + LOG.debug( + "OCSP issuer mismatch for leaf %s and candidate %s: issuer_dn=%r subject_dn=%r", + leaf_path, + issuer_path, + issuer_dn, + subject_dn, + ) + return False + + LOG.debug("OCSP verified issuer for leaf %s is %s", leaf_path, issuer_path) + return True + + +def split_chain(fullchain: Path) -> Tuple[Path, Path]: + """ + Split fullchain.pem into leaf.pem and issuer.pem temporary files. + Assumes first cert = leaf, second = issuer. + """ + LOG.debug("OCSP splitting fullchain %s into leaf and issuer candidates", fullchain) + text = fullchain.read_text(encoding="utf-8") + parts = text.strip().split("-----END CERTIFICATE-----") + cert_blobs = [p for p in parts if "BEGIN CERTIFICATE" in p] + if len(cert_blobs) < 2: + raise RuntimeError(f"fullchain {fullchain} does not contain at least two certs") + + # Write the leaf cert (first block) to a temp file + leaf_pem = cert_blobs[0] + "-----END CERTIFICATE-----\n" + leaf_fd, leaf_path_str = tempfile.mkstemp(suffix=".pem", prefix="ocsp-leaf-") + os.close(leaf_fd) + leaf_path = Path(leaf_path_str) + leaf_path.write_text(leaf_pem, encoding="utf-8") + + # By default, assume the second cert is the issuer + issuer_path: Optional[Path] = None + + # Try to find a blob whose subject matches the leaf issuer + for idx in range(1, len(cert_blobs)): + pem = cert_blobs[idx] + "-----END CERTIFICATE-----\n" + tmp_fd, tmp_path_str = tempfile.mkstemp(suffix=".pem", prefix="ocsp-issuer-") + os.close(tmp_fd) + tmp_path = Path(tmp_path_str) + tmp_path.write_text(pem, encoding="utf-8") + + try: + if is_cert_issuer(leaf_path, tmp_path): + issuer_path = tmp_path + LOG.debug( + "OCSP selected issuer certificate index %d from fullchain %s", + idx, + fullchain, + ) + break + except Exception as e: + LOG.debug("OCSP error while verifying issuer for %s: %s", fullchain, e) + + # Not the issuer: clean up this temp file + try: + tmp_path.unlink(missing_ok=True) + except Exception: + pass + + # Fallback: if we could not verify, still use the second cert as issuer + if issuer_path is None: + pem = cert_blobs[1] + "-----END CERTIFICATE-----\n" + tmp_fd, tmp_path_str = tempfile.mkstemp(suffix=".pem", prefix="ocsp-issuer-") + os.close(tmp_fd) + issuer_path = Path(tmp_path_str) + issuer_path.write_text(pem, encoding="utf-8") + LOG.warning( + "Could not verify issuer chain for %s, falling back to second certificate as issuer " + "(this usually means the chain is non-standard or OpenSSL output could not be parsed cleanly)", + fullchain, + ) + + return leaf_path, issuer_path + + +def parse_next_update_ttl(ocsp_text: str) -> int: + """ + Parse Next Update from openssl ocsp -text output and return TTL in seconds. + Falls back to 3600s on error. + """ + next_update_str: Optional[str] = None + for line in ocsp_text.splitlines(): + line = line.strip() + if "Next Update:" in line: + parts = line.split("Next Update:", 1) + if len(parts) == 2: + next_update_str = parts[1].strip() + break + + if not next_update_str: + LOG.debug("OCSP no Next Update field found in response, falling back to 3600s TTL") + return 3600 + + # Common OpenSSL OCSP time format: "Jan 1 00:00:00 2026 GMT" + try: + dt = datetime.strptime(next_update_str, "%b %d %H:%M:%S %Y %Z") + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + now = datetime.now(timezone.utc) + ttl = int((dt - now).total_seconds()) + if ttl <= 0: + LOG.debug("OCSP parsed Next Update is in the past, using 300s TTL instead") + return 300 + # Cap TTL to 7 days to avoid very long-lived cache entries + return min(ttl, 7 * 24 * 3600) + except Exception as e: + LOG.debug("OCSP failed to parse Next Update from response: %s", e) + return 3600 + + +def fetch_ocsp_response(fullchain: Path, ocsp_url: str, timeout: int = 10) -> Tuple[Optional[bytes], int]: + """ + Fetch OCSP response using openssl ocsp with a network timeout. + Returns (raw DER bytes or None, ttl_seconds). + """ + leaf, issuer = split_chain(fullchain) + ocsp_der_fd, ocsp_der_path = tempfile.mkstemp(suffix=".der", prefix="ocsp-resp-") + os.close(ocsp_der_fd) + ocsp_der = Path(ocsp_der_path) + + try: + cmd = [ + "openssl", + "ocsp", + "-issuer", + issuer.as_posix(), + "-cert", + leaf.as_posix(), + "-url", + ocsp_url, + "-no_nonce", + "-timeout", + str(timeout), + "-respout", + ocsp_der.as_posix(), + "-text", + ] + rc, out, err = run_cmd(cmd) + if rc != 0: + LOG.error("OCSP openssl ocsp failed for %s: rc=%s err=%s", fullchain, rc, err.strip()) + return None, 0 + + if not ocsp_der.is_file() or ocsp_der.stat().st_size == 0: + LOG.error("OCSP no response written for %s (empty respout file)", fullchain) + return None, 0 + + ttl = parse_next_update_ttl(out) + return ocsp_der.read_bytes(), ttl + finally: + try: + leaf.unlink(missing_ok=True) + issuer.unlink(missing_ok=True) + ocsp_der.unlink(missing_ok=True) + except Exception: + pass + + +def extract_san_dns(fullchain: Path) -> List[str]: + """ + Try to extract DNS names from certificate SAN to use as SNI keys. + This is best-effort; if it fails, we fall back to the cert directory basename. + """ + rc, out, err = run_cmd( + ["openssl", "x509", "-noout", "-text", "-in", fullchain.as_posix()] + ) + if rc != 0: + LOG.debug("OCSP openssl -text failed for %s: %s", fullchain, err.strip()) + return [] + + names: List[str] = [] + in_san = False + for line in out.splitlines(): + line = line.strip() + if "X509v3 Subject Alternative Name" in line: + in_san = True + continue + if in_san: + if not line.startswith("DNS:") and "DNS:" not in line: + break + parts = line.split(",") + for part in parts: + part = part.strip() + if part.startswith("DNS:"): + dns = part[4:].strip() + if dns: + names.append(dns) + + return sorted(set(names)) + + +def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> None: + fullchain = cert_dir / "fullchain.pem" + if not fullchain.is_file(): + return + + cert_name = cert_dir.name + LOG.info("OCSP processing certificate directory %s", cert_name) + + ocsp_url = cert_supports_ocsp(fullchain) + if not ocsp_url: + LOG.info("OCSP certificate %s has no responder, skipping fetch", cert_name) + return + + LOG.info("OCSP responder for certificate %s: %s", cert_name, ocsp_url) + + ocsp_der: Optional[bytes] = None + ttl: int = 0 + + # Use a timeout of 10 seconds per attempt, retry once after 10 seconds + for attempt in (1, 2): + ocsp_der, ttl = fetch_ocsp_response(fullchain, ocsp_url, timeout=10) + if ocsp_der: + LOG.debug("OCSP successfully fetched response for %s on attempt %d (TTL=%ds)", cert_name, attempt, ttl) + break + if attempt == 1: + LOG.warning("OCSP fetch failed for %s, retrying once after 10 seconds ...", cert_name) + time.sleep(10) + + if not ocsp_der: + LOG.error("OCSP failed to fetch response for %s after retries", cert_name) + return + + LOG.debug("OCSP final TTL for %s is %ds (from Next Update parsing)", cert_name, ttl) + + # Calculate checksum for integrity verification + checksum = hashlib.sha256(ocsp_der).hexdigest() + + # === Store OCSP response in database (new) === + if db: + cache_key = f"ocsp/{cert_name}" + try: + # Store in database + err = db.upsert_job_cache( + service_id=None, # Global cache entry + file_name=cache_key, + data=ocsp_der, + job_name="ocsp-refresh", + checksum=checksum, + ) + + if err: + LOG.error("OCSP error while storing response for %s in database: %s", cert_name, err) + else: + LOG.info("OCSP stored response for %s in database (TTL=%ds, checksum=%s)", + cert_name, ttl, checksum[:8]) + except Exception as e: + LOG.error("OCSP exception while storing response for %s in database: %s", cert_name, e) + + # === Datastore sync would happen via scheduler sync (not needed here) === + # The scheduler will sync OCSP responses from database to datastore on workers + # For now, disk storage is sufficient and always works + + # === Also store on disk as fallback (/etc/bunkerweb/configs/ssl/{cert-name}/ocsp.der) === + # Create the SSL configs directory for this certificate + ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + try: + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + except Exception as e: + LOG.error("OCSP error while creating directory %s: %s", ocsp_cert_dir, e) + return + + # Write OCSP response to configs/ssl/{cert-name}/ocsp.der + ocsp_path = ocsp_cert_dir / "ocsp.der" + try: + ocsp_path.write_bytes(ocsp_der) + ocsp_path.chmod(0o644) # Readable by nginx + LOG.info("OCSP saved response for %s to disk at %s", cert_name, ocsp_path) + except Exception as e: + LOG.error("OCSP error while writing response for %s to disk: %s", cert_name, e) + + +def main() -> int: + global status + db: Optional[Any] = None + try: + # CRITICAL: Log immediately to confirm script is running + try: + LOG.error("OCSP DEBUG: script started, initializing Job") + except Exception as e: + print(f"ERROR: Could not log initial message: {e}", file=_sys.stderr) + + try: + job = Job(LOG, __file__) + LOG.debug("OCSP Job initialized successfully") + except Exception as e: + LOG.error("OCSP could not initialize Job: %s", e) + return 2 + + # Initialize database connection (optional) + if Database is not None: + try: + db = Database(LOG) + LOG.debug("OCSP database connection established") + except Exception as e: + LOG.warning("OCSP could not establish database connection, will use disk-only storage: %s", e) + db = None + else: + LOG.debug("OCSP Database module not available, will use disk-only storage") + + if not LIVE_BASE.is_dir(): + LOG.info("OCSP live certificate directory %s does not exist, nothing to do", LIVE_BASE) + return 0 + + cert_dirs = [d for d in sorted(LIVE_BASE.iterdir()) if d.is_dir()] + LOG.info("OCSP found %d live certificate directories under %s", len(cert_dirs), LIVE_BASE) + + for cert_dir in cert_dirs: + fullchain = cert_dir / "fullchain.pem" + if not fullchain.is_file(): + LOG.debug("OCSP skipping %s because fullchain.pem is missing", cert_dir) + continue + process_one_cert_dir(job, cert_dir, db) + + return status + except BaseException as e: + LOG.exception("OCSP exception in ocsp-refresh.py") + LOG.error("OCSP exception while running ocsp-refresh.py: %s", e) + return 2 + finally: + # Ensure database connection is properly closed + if db: + try: + db.close() + except Exception: + pass + + +if __name__ == "__main__": + sys_exit(main()) + diff --git a/src/common/core/ssl/plugin.json b/src/common/core/ssl/plugin.json index cb38e7dd39..9891b21160 100644 --- a/src/common/core/ssl/plugin.json +++ b/src/common/core/ssl/plugin.json @@ -95,6 +95,23 @@ "label": "SSL Session Cache Size", "regex": "^(off|none|[1-9][0-9]*[kKmMgG]?)$", "type": "text" + }, + "SSL_USE_OCSP_STAPLING": { + "context": "multisite", + "default": "yes", + "help": "Enable OCSP stapling for this site when certificates advertise an OCSP responder. Applies to Let's Encrypt and custom/self-signed certificates that support OCSP.", + "id": "ssl-use-ocsp-stapling", + "label": "Use OCSP stapling", + "regex": "^(yes|no)$", + "type": "check" + } + }, + "jobs": [ + { + "name": "ocsp-refresh", + "file": "ocsp-refresh.py", + "every": "hour", + "reload": false } - } + ] } From 80236ed100aa4d0ebabd630142ae120edbe221bf Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 12 Mar 2026 17:25:29 +0100 Subject: [PATCH 04/59] Update ssl-certificate-lua.conf TODO: Check SSL_USE_OCSP_STAPLING variable when variable loading is available in ssl_certificate phase --- src/common/confs/server-http/ssl-certificate-lua.conf | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 7924618792..62705dd469 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -117,14 +117,9 @@ ssl_certificate_by_lua_block { -- Helper: check if OCSP stapling is enabled for this site local function is_ocsp_stapling_enabled() - -- Try to get SSL_USE_OCSP_STAPLING from variables, default to "yes" if not found - local use_ocsp, err = get_variable("SSL_USE_OCSP_STAPLING", server_name) - if not use_ocsp then - -- If we can't get the variable, log a warning but continue (graceful fallback) - logger:log(WARN, "OCSP can't get SSL_USE_OCSP_STAPLING variable: " .. (err or "unknown")) - return true -- Continue attempting OCSP stapling anyway - end - return use_ocsp == "yes" + -- TODO: Check SSL_USE_OCSP_STAPLING variable when variable loading is available in ssl_certificate phase + -- For now, always attempt OCSP stapling if files exist (graceful fallback if not) + return true end -- Helper: set OCSP stapling from cache (Redis + local shared dict + ocsp.der files) From 6d75d8ad69d08d217e3f83be467530c06505b2a9 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 12 Mar 2026 17:35:53 +0100 Subject: [PATCH 05/59] remove redis integration --- .../server-http/ssl-certificate-lua.conf | 42 +++++-------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 62705dd469..8013cf4818 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -72,7 +72,6 @@ ssl_certificate_by_lua_block { local helpers = require "bunkerweb.helpers" local utils = require "bunkerweb.utils" local cdatastore = require "bunkerweb.datastore" - local cclusterstore = require "bunkerweb.clusterstore" local cjson = require "cjson" local ssl = require "ngx.ssl" local ocsp = require "ngx.ocsp" @@ -122,7 +121,7 @@ ssl_certificate_by_lua_block { return true end - -- Helper: set OCSP stapling from cache (Redis + local shared dict + ocsp.der files) + -- Helper: set OCSP stapling from cache (internalstore + ocsp.der disk files) local function set_ocsp_from_cache() logger:log(INFO, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) @@ -179,22 +178,8 @@ ssl_certificate_by_lua_block { logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) end - -- Also backfill Redis if available so other workers benefit. - local cluster2 = cclusterstore:new() - local ok_connect2, rerr2 = cluster2:connect(true) - if not ok_connect2 then - logger:log(ERR, "OCSP error while connecting to Redis for file backfill: " .. (rerr2 or "unknown")) - else - local ok_setex2, set_err2 = cluster2:call("setex", cache_key, 3600, resp) - if set_err2 then - logger:log(ERR, "OCSP error while backfilling Redis from file: " .. set_err2) - elseif not ok_setex2 then - logger:log(ERR, "OCSP Redis SETEX from file returned falsy for " .. server_name) - else - logger:log(INFO, "OCSP backfilled Redis from file for " .. server_name) - end - cluster2:close() - end + end + end logger:log(INFO, "OCSP loaded response from file " .. path .. " for " .. server_name) break end @@ -211,8 +196,7 @@ ssl_certificate_by_lua_block { local ok_set, oerr = ocsp.set_ocsp_status_resp(resp) if not ok_set then logger:log(ERR, "OCSP failed to set stapling: " .. oerr) - else - logger:log(INFO, "OCSP stapling set from cache for " .. server_name) + logger:log(INFO, "OCSP stapling set from cache for " .. server_name) end end @@ -225,21 +209,18 @@ ssl_certificate_by_lua_block { logger:log(ERR, perr) elseif plugin_lua == nil then logger:log(INFO, perr) - else - -- Check if plugin has ssl_certificate method + -- Check if plugin has ssl_certificate method if plugin_lua.ssl_certificate ~= nil then -- New call local ok_new, plugin_obj = new_plugin(plugin_lua) if not ok_new then logger:log(ERR, plugin_obj) - else - local ok_call, ret = call_plugin(plugin_obj, "ssl_certificate") + local ok_call, ret = call_plugin(plugin_obj, "ssl_certificate") if not ok_call then logger:log(ERR, ret) elseif not ret.ret then logger:log(ERR, plugin_id .. ":ssl_certificate() call failed : " .. ret.msg) - else - logger:log(INFO, plugin_id .. ":ssl_certificate() call successful : " .. ret.msg) + logger:log(INFO, plugin_id .. ":ssl_certificate() call successful : " .. ret.msg) if ret.status then logger:log(INFO, plugin_id .. " is setting certificate/key : " .. ret.msg) local ok_clear, cerr = clear_certs() @@ -249,12 +230,10 @@ ssl_certificate_by_lua_block { local ok_cert, serr = set_cert(ret.status[1]) if not ok_cert then logger:log(ERR, "error while setting certificate : " .. serr) - else - local ok_key, kerr = set_priv_key(ret.status[2]) + local ok_key, kerr = set_priv_key(ret.status[2]) if not ok_key then logger:log(ERR, "error while setting private key : " .. kerr) - else - -- Try to set OCSP stapling from cache (if enabled) + -- Try to set OCSP stapling from cache (if enabled) logger:log(INFO, "DEBUG: About to call set_ocsp_from_cache() for " .. server_name) set_ocsp_from_cache() logger:log(INFO, "certificate set by " .. plugin_id) @@ -264,8 +243,7 @@ ssl_certificate_by_lua_block { end end end - else - logger:log(INFO, "skipped execution of " .. plugin_id .. " because method ssl_certificate() is not defined") + logger:log(INFO, "skipped execution of " .. plugin_id .. " because method ssl_certificate() is not defined") end end end From d444061439de6dd2ccc81ff7184edc5b8b8df5df Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 12 Mar 2026 17:59:58 +0100 Subject: [PATCH 06/59] fix "end" entry --- src/common/confs/server-http/ssl-certificate-lua.conf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 8013cf4818..d3f3cacce8 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -178,9 +178,7 @@ ssl_certificate_by_lua_block { logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) end - end - end - logger:log(INFO, "OCSP loaded response from file " .. path .. " for " .. server_name) + logger:log(INFO, "OCSP loaded response from file " .. path .. " for " .. server_name) break end end From 18b58deda022e3f60bd5282cd88ec80282eda384 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 12 Mar 2026 18:03:03 +0100 Subject: [PATCH 07/59] fix tab errors --- .../server-http/ssl-certificate-lua.conf | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index d3f3cacce8..c5981136ae 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -136,7 +136,6 @@ ssl_certificate_by_lua_block { end local cache_key = "SSL:ocsp_status:" .. server_name - local resp -- 1) Local shared dict lookup (per-worker cache) @@ -150,7 +149,7 @@ ssl_certificate_by_lua_block { end end - -- 2) Fallback to on-disk OCSP response (/var/cache/bunkerweb/ssl/{domain}/ocsp.der, scheduler-managed from database) + -- 2) Fallback to on-disk OCSP response (/var/cache/bunkerweb/ssl/{domain}/ocsp.der) if not resp then logger:log(INFO, "OCSP trying to load from disk files for server " .. (server_name or "nil")) local base_path = "/var/cache/bunkerweb/ssl/" @@ -177,8 +176,7 @@ ssl_certificate_by_lua_block { if not ok_set then logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) end - - logger:log(INFO, "OCSP loaded response from file " .. path .. " for " .. server_name) + logger:log(INFO, "OCSP loaded response from file " .. path .. " for " .. server_name) break end end @@ -194,7 +192,8 @@ ssl_certificate_by_lua_block { local ok_set, oerr = ocsp.set_ocsp_status_resp(resp) if not ok_set then logger:log(ERR, "OCSP failed to set stapling: " .. oerr) - logger:log(INFO, "OCSP stapling set from cache for " .. server_name) + else + logger:log(INFO, "OCSP stapling set from cache for " .. server_name) end end @@ -202,36 +201,41 @@ ssl_certificate_by_lua_block { logger:log(INFO, "calling ssl_certificate() methods of plugins ...") for i, plugin_id in ipairs(phase_order) do -- Require call - local plugin_lua, perr = require_plugin(plugin_id) + local plugin_lua, err = require_plugin(plugin_id) if plugin_lua == false then - logger:log(ERR, perr) + logger:log(ERR, err) elseif plugin_lua == nil then - logger:log(INFO, perr) - -- Check if plugin has ssl_certificate method + logger:log(INFO, err) + else + -- Check if plugin has ssl_certificate method if plugin_lua.ssl_certificate ~= nil then -- New call - local ok_new, plugin_obj = new_plugin(plugin_lua) - if not ok_new then + local ok, plugin_obj = new_plugin(plugin_lua) + if not ok then logger:log(ERR, plugin_obj) - local ok_call, ret = call_plugin(plugin_obj, "ssl_certificate") - if not ok_call then + else + local ok, ret = call_plugin(plugin_obj, "ssl_certificate") + if not ok then logger:log(ERR, ret) elseif not ret.ret then logger:log(ERR, plugin_id .. ":ssl_certificate() call failed : " .. ret.msg) - logger:log(INFO, plugin_id .. ":ssl_certificate() call successful : " .. ret.msg) + else + logger:log(INFO, plugin_id .. ":ssl_certificate() call successful : " .. ret.msg) if ret.status then logger:log(INFO, plugin_id .. " is setting certificate/key : " .. ret.msg) - local ok_clear, cerr = clear_certs() - if not ok_clear then - logger:log(ERR, "error while clearing certificates : " .. cerr) + local ok, err = clear_certs() + if not ok then + logger:log(ERR, "error while clearing certificates : " .. err) end - local ok_cert, serr = set_cert(ret.status[1]) - if not ok_cert then - logger:log(ERR, "error while setting certificate : " .. serr) - local ok_key, kerr = set_priv_key(ret.status[2]) - if not ok_key then - logger:log(ERR, "error while setting private key : " .. kerr) - -- Try to set OCSP stapling from cache (if enabled) + ok, err = set_cert(ret.status[1]) + if not ok then + logger:log(ERR, "error while setting certificate : " .. err) + else + local ok, err = set_priv_key(ret.status[2]) + if not ok then + logger:log(ERR, "error while setting private key : " .. err) + else + -- Try to set OCSP stapling from cache (if enabled) logger:log(INFO, "DEBUG: About to call set_ocsp_from_cache() for " .. server_name) set_ocsp_from_cache() logger:log(INFO, "certificate set by " .. plugin_id) @@ -241,7 +245,8 @@ ssl_certificate_by_lua_block { end end end - logger:log(INFO, "skipped execution of " .. plugin_id .. " because method ssl_certificate() is not defined") + else + logger:log(INFO, "skipped execution of " .. plugin_id .. " because method ssl_certificate() is not defined") end end end From a731fe8390ab3fda832756494457a6118d2c3d5b Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 08:39:14 +0100 Subject: [PATCH 08/59] Add a 3 days TTL cache for OCSP responses Do not fetch OCSP response when TTL of current is more than 3 days --- src/common/core/ssl/jobs/ocsp-refresh.py | 137 +++++++++++++++++++++-- 1 file changed, 130 insertions(+), 7 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 34c35b58c5..685c8ab864 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -323,6 +323,108 @@ def extract_san_dns(fullchain: Path) -> List[str]: return sorted(set(names)) +def get_cached_ocsp_ttl(cert_name: str) -> Optional[int]: + """ + Check if cached OCSP file exists and return its remaining TTL in seconds. + Returns None if file doesn't exist or TTL cannot be determined. + + This allows skipping fetch if the cached response is still valid for >3 days. + """ + ocsp_path = CONFIGS_SSL_BASE / cert_name / "ocsp.der" + LOG.info("OCSP TTL check: checking cache for %s at %s", cert_name, ocsp_path) + + if not ocsp_path.is_file(): + LOG.info("OCSP TTL check: cache miss - file not found for %s", cert_name) + return None + + LOG.debug("OCSP cached file found for %s, reading TTL...", cert_name) + try: + # Use openssl ocsp -respin to read cached DER and extract Next Update + # Use -noverify to skip signature verification (we only care about the TTL) + rc, out, err = run_cmd( + ["openssl", "ocsp", "-respin", ocsp_path.as_posix(), "-text", "-noverify"] + ) + if rc != 0: + LOG.warning("OCSP failed to read cached DER for %s (rc=%d): %s", cert_name, rc, err.strip()) + return None + + ttl = parse_next_update_ttl(out) + three_days_sec = 259200 + if ttl > three_days_sec: + LOG.info( + "OCSP cached response for %s is fresh: TTL=%ds (%.1f days) > 3 days threshold", + cert_name, + ttl, + ttl / 86400.0, + ) + else: + LOG.info( + "OCSP cached response for %s needs refresh: TTL=%ds (%.1f days) <= 3 days threshold", + cert_name, + ttl, + ttl / 86400.0, + ) + return ttl + except Exception as e: + LOG.warning("OCSP exception while reading cached TTL for %s: %s", cert_name, e) + return None + + +def restore_ocsp_from_database(db: Optional[Any] = None) -> None: + """ + Restore cached OCSP responses from database to disk. + Called at startup to ensure disk cache is populated from database. + This handles ephemeral storage (tmpfs, etc.) by restoring files on each run. + """ + if not db: + LOG.debug("OCSP database not available, skipping cache restoration") + return + + LOG.info("OCSP restoring cached responses from database to disk...") + try: + restored_count = 0 + + # Get all possible cert names from live directory and restore from database + if LIVE_BASE.is_dir(): + for cert_dir in sorted(LIVE_BASE.iterdir()): + if not cert_dir.is_dir(): + continue + cert_name = cert_dir.name + cache_key = f"ocsp/{cert_name}" + + try: + # Use the Database.get_job_cache_file() method to retrieve cached OCSP response + # Returns bytes directly if found, None if not found + data = db.get_job_cache_file( + job_name="ocsp-refresh", + file_name=cache_key, + with_data=True, + with_info=False, + ) + + if not data: + continue + + # Restore to disk + ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + ocsp_path = ocsp_cert_dir / "ocsp.der" + ocsp_path.write_bytes(data) + ocsp_path.chmod(0o644) + restored_count += 1 + LOG.debug("OCSP restored cached response for %s from database", cert_name) + except Exception as e: + LOG.debug("OCSP could not restore cache for %s: %s", cert_name, e) + continue + + if restored_count > 0: + LOG.info("OCSP restored %d cached responses from database to disk", restored_count) + else: + LOG.debug("OCSP no cached responses restored from database (cache may be cold)") + except Exception as e: + LOG.debug("OCSP exception while attempting cache restoration: %s", e) + + def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> None: fullchain = cert_dir / "fullchain.pem" if not fullchain.is_file(): @@ -338,6 +440,18 @@ def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> LOG.info("OCSP responder for certificate %s: %s", cert_name, ocsp_url) + # === Check if cached OCSP response is still fresh (>3 days TTL) === + # TTL-based optimization: skip fetch if cached response valid for >3 days (259200 seconds) + cached_ttl = get_cached_ocsp_ttl(cert_name) + if cached_ttl is not None and cached_ttl > 259200: + LOG.info( + "OCSP cached response for %s still valid for %ds (%.1f days), skipping fetch", + cert_name, + cached_ttl, + cached_ttl / 86400.0, + ) + return + ocsp_der: Optional[bytes] = None ttl: int = 0 @@ -360,7 +474,8 @@ def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> # Calculate checksum for integrity verification checksum = hashlib.sha256(ocsp_der).hexdigest() - # === Store OCSP response in database (new) === + # === Store OCSP response in database === + db_stored = False if db: cache_key = f"ocsp/{cert_name}" try: @@ -378,14 +493,19 @@ def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> else: LOG.info("OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, checksum[:8]) + db_stored = True except Exception as e: LOG.error("OCSP exception while storing response for %s in database: %s", cert_name, e) + else: + # If no database available, still proceed with disk storage + db_stored = True + + # === Only write to disk AFTER successful database storage === + # This prevents overwriting existing OCSP files if the fetch failed or had issues + if not db_stored: + LOG.warning("OCSP skipping disk storage for %s due to database storage failure", cert_name) + return - # === Datastore sync would happen via scheduler sync (not needed here) === - # The scheduler will sync OCSP responses from database to datastore on workers - # For now, disk storage is sufficient and always works - - # === Also store on disk as fallback (/etc/bunkerweb/configs/ssl/{cert-name}/ocsp.der) === # Create the SSL configs directory for this certificate ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name try: @@ -394,7 +514,7 @@ def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> LOG.error("OCSP error while creating directory %s: %s", ocsp_cert_dir, e) return - # Write OCSP response to configs/ssl/{cert-name}/ocsp.der + # Write OCSP response to cache/ssl/{cert-name}/ocsp.der ocsp_path = ocsp_cert_dir / "ocsp.der" try: ocsp_path.write_bytes(ocsp_der) @@ -436,6 +556,9 @@ def main() -> int: LOG.info("OCSP live certificate directory %s does not exist, nothing to do", LIVE_BASE) return 0 + # Restore cached OCSP responses from database (handles ephemeral storage) + restore_ocsp_from_database(db) + cert_dirs = [d for d in sorted(LIVE_BASE.iterdir()) if d.is_dir()] LOG.info("OCSP found %d live certificate directories under %s", len(cert_dirs), LIVE_BASE) From 05c6926598702c6a4b97836a1097f3c2f52611ef Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 10:27:25 +0100 Subject: [PATCH 09/59] add customcert processing + refresh TTL - add customcert processing - use refresh TTL at 50% of lifetime --- .../server-http/ssl-certificate-lua.conf | 17 +- src/common/core/ssl/jobs/ocsp-refresh.py | 423 +++++++++++++++--- 2 files changed, 366 insertions(+), 74 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index c5981136ae..ae9872ca72 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -151,41 +151,46 @@ ssl_certificate_by_lua_block { -- 2) Fallback to on-disk OCSP response (/var/cache/bunkerweb/ssl/{domain}/ocsp.der) if not resp then - logger:log(INFO, "OCSP trying to load from disk files for server " .. (server_name or "nil")) local base_path = "/var/cache/bunkerweb/ssl/" local candidates = { server_name, server_name .. "-ecdsa", server_name .. "-rsa", + "customcert-" .. server_name, + "customcert-" .. server_name .. "-ecdsa", + "customcert-" .. server_name .. "-rsa", } for _, name in ipairs(candidates) do if name and name ~= "" then local path = base_path .. name .. "/ocsp.der" - logger:log(INFO, "OCSP checking path: " .. path) local f = io.open(path, "rb") if f then - logger:log(INFO, "OCSP file found: " .. path) local data = f:read("*a") f:close() if data and #data > 0 then - logger:log(INFO, "OCSP read " .. #data .. " bytes from " .. path) + logger:log(INFO, "OCSP loaded response (" .. #data .. " bytes) from " .. path) resp = data -- Cache in shared memory to avoid disk I/O on every handshake local ok_set, serr = internalstore:set(cache_key, resp, 3600, true) if not ok_set then logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) end - logger:log(INFO, "OCSP loaded response from file " .. path .. " for " .. server_name) break end end + else + -- Check if file exists but couldn't be read (permission denied) + local file_exists = os.execute("test -f " .. path .. " 2>/dev/null") + if file_exists == 0 then + logger:log(ERR, "OCSP permission denied reading " .. path .. " (check file permissions)") + end end end end if not resp then - logger:log(INFO, "OCSP no response found from any cache/disk source for " .. server_name) + logger:log(ngx.DEBUG, "OCSP not found for " .. server_name) return end diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 685c8ab864..5dc8273330 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -62,7 +62,7 @@ def run_cmd(cmd: List[str]) -> Tuple[int, str, str]: Run a subprocess command and return (rc, stdout, stderr). When LOG is in debug level, the command line is logged. """ - LOG.debug("OCSP running command: %s", " ".join(shlex.quote(c) for c in cmd)) + LOG.debug("๐Ÿ”„ OCSP running command: %s", " ".join(shlex.quote(c) for c in cmd)) proc = subprocess.run( cmd, stdin=subprocess.DEVNULL, @@ -80,18 +80,18 @@ def cert_supports_ocsp(fullchain: Path) -> Optional[str]: First function: test if the certificate supports OCSP. Returns OCSP responder URL if present, else None. """ - LOG.debug("OCSP checking support for certificate %s", fullchain) + LOG.debug("๐Ÿ”’ OCSP checking support for certificate %s", fullchain) rc, out, err = run_cmd( ["openssl", "x509", "-noout", "-ocsp_uri", "-in", fullchain.as_posix()] ) if rc != 0: - LOG.debug("OCSP openssl -ocsp_uri failed for %s: %s", fullchain, err.strip()) + LOG.debug("๐Ÿ”’ OCSP openssl -ocsp_uri failed for %s: %s", fullchain, err.strip()) return None url = out.strip() if not url: - LOG.debug("OCSP no responder URL advertised in %s", fullchain) + LOG.debug("๐Ÿ”’ OCSP no responder URL advertised in %s", fullchain) return None - LOG.debug("OCSP found responder URL for %s: %s", fullchain, url) + LOG.debug("๐ŸŒ OCSP found responder URL for %s: %s", fullchain, url) return url @@ -117,14 +117,14 @@ def is_cert_issuer(leaf_path: Path, issuer_path: Path) -> bool: ["openssl", "x509", "-noout", "-issuer", "-in", leaf_path.as_posix()] ) if rc1 != 0: - LOG.debug("OCSP openssl -issuer failed for %s: %s", leaf_path, err1.strip()) + LOG.debug("๐Ÿ”’ OCSP openssl -issuer failed for %s: %s", leaf_path, err1.strip()) return False rc2, subject_line, err2 = run_cmd( ["openssl", "x509", "-noout", "-subject", "-in", issuer_path.as_posix()] ) if rc2 != 0: - LOG.debug("OCSP openssl -subject failed for %s: %s", issuer_path, err2.strip()) + LOG.debug("๐Ÿ”’ OCSP openssl -subject failed for %s: %s", issuer_path, err2.strip()) return False issuer_dn = _normalize_dn_line(issuer_line, "issuer=") @@ -132,7 +132,7 @@ def is_cert_issuer(leaf_path: Path, issuer_path: Path) -> bool: if issuer_dn != subject_dn: LOG.debug( - "OCSP issuer mismatch for leaf %s and candidate %s: issuer_dn=%r subject_dn=%r", + "๐Ÿ”’ OCSP issuer mismatch for leaf %s and candidate %s: issuer_dn=%r subject_dn=%r", leaf_path, issuer_path, issuer_dn, @@ -140,7 +140,7 @@ def is_cert_issuer(leaf_path: Path, issuer_path: Path) -> bool: ) return False - LOG.debug("OCSP verified issuer for leaf %s is %s", leaf_path, issuer_path) + LOG.debug("โœ“ OCSP verified issuer for leaf %s is %s", leaf_path, issuer_path) return True @@ -149,7 +149,7 @@ def split_chain(fullchain: Path) -> Tuple[Path, Path]: Split fullchain.pem into leaf.pem and issuer.pem temporary files. Assumes first cert = leaf, second = issuer. """ - LOG.debug("OCSP splitting fullchain %s into leaf and issuer candidates", fullchain) + LOG.debug("๐Ÿ“œ OCSP splitting fullchain %s into leaf and issuer candidates", fullchain) text = fullchain.read_text(encoding="utf-8") parts = text.strip().split("-----END CERTIFICATE-----") cert_blobs = [p for p in parts if "BEGIN CERTIFICATE" in p] @@ -178,13 +178,13 @@ def split_chain(fullchain: Path) -> Tuple[Path, Path]: if is_cert_issuer(leaf_path, tmp_path): issuer_path = tmp_path LOG.debug( - "OCSP selected issuer certificate index %d from fullchain %s", + "โœ“ OCSP selected issuer certificate index %d from fullchain %s", idx, fullchain, ) break except Exception as e: - LOG.debug("OCSP error while verifying issuer for %s: %s", fullchain, e) + LOG.debug("๐Ÿ”’ OCSP error while verifying issuer for %s: %s", fullchain, e) # Not the issuer: clean up this temp file try: @@ -211,7 +211,7 @@ def split_chain(fullchain: Path) -> Tuple[Path, Path]: def parse_next_update_ttl(ocsp_text: str) -> int: """ Parse Next Update from openssl ocsp -text output and return TTL in seconds. - Falls back to 3600s on error. + Falls back to 86400s (24 hours, RFC standard) on error. """ next_update_str: Optional[str] = None for line in ocsp_text.splitlines(): @@ -223,8 +223,8 @@ def parse_next_update_ttl(ocsp_text: str) -> int: break if not next_update_str: - LOG.debug("OCSP no Next Update field found in response, falling back to 3600s TTL") - return 3600 + LOG.debug("โšก OCSP no Next Update field found in response, falling back to RFC standard 86400s (24 hours) TTL") + return 86400 # Common OpenSSL OCSP time format: "Jan 1 00:00:00 2026 GMT" try: @@ -234,13 +234,13 @@ def parse_next_update_ttl(ocsp_text: str) -> int: now = datetime.now(timezone.utc) ttl = int((dt - now).total_seconds()) if ttl <= 0: - LOG.debug("OCSP parsed Next Update is in the past, using 300s TTL instead") + LOG.debug("โšก OCSP parsed Next Update is in the past, using 300s TTL instead") return 300 # Cap TTL to 7 days to avoid very long-lived cache entries return min(ttl, 7 * 24 * 3600) except Exception as e: - LOG.debug("OCSP failed to parse Next Update from response: %s", e) - return 3600 + LOG.debug("โšก OCSP failed to parse Next Update from response: %s, falling back to RFC standard 86400s (24 hours)", e) + return 86400 def fetch_ocsp_response(fullchain: Path, ocsp_url: str, timeout: int = 10) -> Tuple[Optional[bytes], int]: @@ -272,11 +272,11 @@ def fetch_ocsp_response(fullchain: Path, ocsp_url: str, timeout: int = 10) -> Tu ] rc, out, err = run_cmd(cmd) if rc != 0: - LOG.error("OCSP openssl ocsp failed for %s: rc=%s err=%s", fullchain, rc, err.strip()) + LOG.error("โŒ OCSP openssl ocsp failed for %s: rc=%s err=%s", fullchain, rc, err.strip()) return None, 0 if not ocsp_der.is_file() or ocsp_der.stat().st_size == 0: - LOG.error("OCSP no response written for %s (empty respout file)", fullchain) + LOG.error("โŒ OCSP no response written for %s (empty respout file)", fullchain) return None, 0 ttl = parse_next_update_ttl(out) @@ -331,13 +331,13 @@ def get_cached_ocsp_ttl(cert_name: str) -> Optional[int]: This allows skipping fetch if the cached response is still valid for >3 days. """ ocsp_path = CONFIGS_SSL_BASE / cert_name / "ocsp.der" - LOG.info("OCSP TTL check: checking cache for %s at %s", cert_name, ocsp_path) + LOG.info("โšก OCSP TTL check: checking cache for %s at %s", cert_name, ocsp_path) if not ocsp_path.is_file(): - LOG.info("OCSP TTL check: cache miss - file not found for %s", cert_name) + LOG.info("โšก OCSP TTL check: cache miss - file not found for %s", cert_name) return None - LOG.debug("OCSP cached file found for %s, reading TTL...", cert_name) + LOG.debug("โšก OCSP cached file found for %s, reading TTL...", cert_name) try: # Use openssl ocsp -respin to read cached DER and extract Next Update # Use -noverify to skip signature verification (we only care about the TTL) @@ -352,24 +352,236 @@ def get_cached_ocsp_ttl(cert_name: str) -> Optional[int]: three_days_sec = 259200 if ttl > three_days_sec: LOG.info( - "OCSP cached response for %s is fresh: TTL=%ds (%.1f days) > 3 days threshold", + "โœ“ OCSP cached response for %s is fresh: TTL=%ds (%.1f days) > 3 days threshold", cert_name, ttl, ttl / 86400.0, ) else: LOG.info( - "OCSP cached response for %s needs refresh: TTL=%ds (%.1f days) <= 3 days threshold", + "๐Ÿ”„ OCSP cached response for %s needs refresh: TTL=%ds (%.1f days) <= 3 days threshold", cert_name, ttl, ttl / 86400.0, ) return ttl except Exception as e: - LOG.warning("OCSP exception while reading cached TTL for %s: %s", cert_name, e) + LOG.warning("โš ๏ธ OCSP exception while reading cached TTL for %s: %s", cert_name, e) return None +def validate_cert_file(cert_file: Path, key_file: Path) -> Tuple[Optional[Path], Optional[Path]]: + """ + Validate cert and key file order. Tries both orderings if the first one fails. + Accepts certificate chains (fullchain.pem with multiple certs) - OpenSSL reads the first one. + Returns (valid_cert_path, valid_key_path) or (None, None) if both fail. + """ + for attempt in (1, 2): + test_cert = cert_file if attempt == 1 else key_file + test_key = key_file if attempt == 1 else cert_file + + # Try to read the file as a certificate (openssl x509 reads first cert from chain) + rc, _, _ = run_cmd( + ["openssl", "x509", "-noout", "-in", test_cert.as_posix()] + ) + + if rc == 0: + # File is a valid certificate or certificate chain + LOG.debug("โœ“ OCSP validated cert/key ordering: cert=%s, key=%s", test_cert, test_key) + return test_cert, test_key + + # Both orderings failed + LOG.warning("โŒ OCSP could not find valid certificate in either file ordering") + return None, None + + +def extract_ocsp_url_from_cert(cert_file: Path) -> Optional[str]: + """ + Extract OCSP responder URL from a certificate file. + Returns the URL if present, None otherwise. + """ + rc, out, err = run_cmd( + ["openssl", "x509", "-noout", "-ocsp_uri", "-in", cert_file.as_posix()] + ) + if rc != 0: + LOG.debug("๐Ÿ”’ OCSP could not extract OCSP URL from %s: %s", cert_file, err.strip()) + return None + url = out.strip() + if not url: + LOG.debug("๐Ÿ”’ OCSP no responder URL found in %s", cert_file) + return None + return url + + +def process_custom_certs(job: Job, db: Optional[Any] = None, stats: Optional[dict] = None) -> None: + """ + Process OCSP for custom certificates by auto-discovering from filesystem. + Scans /var/cache/bunkerweb/customcert/ for existing service directories. + """ + if stats is None: + stats = {} + + try: + customcert_base = Path(os.sep, "var", "cache", "bunkerweb", "customcert") + + # Auto-discover services from filesystem instead of environment variables + if not customcert_base.is_dir(): + LOG.info("โ„น๏ธ OCSP custom cert directory %s does not exist", customcert_base) + return + + service_dirs = sorted([d for d in customcert_base.iterdir() if d.is_dir()]) + if not service_dirs: + LOG.info("โ„น๏ธ OCSP no custom certificate directories found under %s", customcert_base) + return + + all_domains = [d.name for d in service_dirs] + LOG.info("๐Ÿ”„ OCSP auto-discovered %d custom certificate service(s) from filesystem", len(all_domains)) + stats["custom_certs_processed"] = len(all_domains) + + for service_name in all_domains: + # Service directory exists, so custom certs are configured for this service + # Try to find custom certificate files (main, -ecdsa, or -rsa variants) + base_dir = Path(os.sep, "var", "cache", "bunkerweb", "customcert", service_name) + cert_variants = ["cert.pem", "cert-ecdsa.pem", "cert-rsa.pem"] + key_variants = ["key.pem", "key-ecdsa.pem", "key-rsa.pem"] + + valid_cert = None + + # Try to find valid cert/key pair among all variants + for cert_variant in cert_variants: + cert_path = base_dir / cert_variant + if not cert_path.is_file(): + continue + + for key_variant in key_variants: + key_path = base_dir / key_variant + if not key_path.is_file(): + continue + + LOG.debug("๐Ÿ”„ OCSP trying custom cert variant: %s / %s", cert_variant, key_variant) + valid_cert, _ = validate_cert_file(cert_path, key_path) + if valid_cert: + LOG.info("โœ“ OCSP found valid custom certificate for service %s: %s", service_name, valid_cert) + break + + if valid_cert: + break + + if not valid_cert: + LOG.debug("โ„น๏ธ OCSP custom cert cache not found for %s in %s", service_name, base_dir) + stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + 1 + continue + + LOG.info("๐Ÿ”„ OCSP processing custom certificate for service %s", service_name) + + # Extract OCSP URL from certificate + ocsp_url = extract_ocsp_url_from_cert(valid_cert) + if not ocsp_url: + LOG.info("โ„น๏ธ OCSP custom cert %s has no responder, skipping", service_name) + stats["custom_certs_no_ocsp"] = stats.get("custom_certs_no_ocsp", 0) + 1 + continue + + LOG.info("๐ŸŒ OCSP responder for custom cert %s: %s", service_name, ocsp_url) + + # Use customcert- prefix for OCSP cache key + cert_name = f"customcert-{service_name}" + + # Check if cached OCSP response is still fresh + # Skip fetch if cached response still valid for > 50% of its lifetime + cached_ttl = get_cached_ocsp_ttl(cert_name) + if cached_ttl is not None: + refresh_threshold = int(cached_ttl * 0.5) + if cached_ttl > refresh_threshold: + LOG.info( + "โœ“ OCSP cached response for custom cert %s still valid for %ds (%.1f days), refresh at 50%% threshold=%ds, skipping fetch", + service_name, + cached_ttl, + cached_ttl / 86400.0, + refresh_threshold, + ) + stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 + continue + + ocsp_der: Optional[bytes] = None + ttl: int = 0 + + # Use a timeout of 10 seconds per attempt, retry once after 10 seconds + for attempt in (1, 2): + ocsp_der, ttl = fetch_ocsp_response(valid_cert, ocsp_url, timeout=10) + if ocsp_der: + LOG.debug("โœ“ OCSP successfully fetched response for custom cert %s on attempt %d (TTL=%ds)", + service_name, attempt, ttl) + break + if attempt == 1: + LOG.warning("โš ๏ธ OCSP fetch failed for custom cert %s, retrying once after 10 seconds ...", service_name) + time.sleep(10) + + if not ocsp_der: + LOG.error("โŒ OCSP failed to fetch response for custom cert %s after retries", service_name) + stats["errors"] = stats.get("errors", 0) + 1 + continue + + LOG.debug("โšก OCSP final TTL for custom cert %s is %ds", service_name, ttl) + + # Calculate checksum for integrity verification + checksum = hashlib.sha256(ocsp_der).hexdigest() + + # Store OCSP response in database + db_stored = False + if db: + cache_key = f"ocsp/{cert_name}" + try: + err = db.upsert_job_cache( + service_id=None, # Global cache entry + file_name=cache_key, + data=ocsp_der, + job_name="ocsp-refresh", + checksum=checksum, + ) + + if err: + LOG.error("โŒ OCSP error storing custom cert %s response in database: %s", service_name, err) + stats["errors"] = stats.get("errors", 0) + 1 + else: + LOG.info("โœ“ OCSP stored custom cert %s response in database (TTL=%ds, checksum=%s)", + service_name, ttl, checksum[:8]) + db_stored = True + except Exception as e: + LOG.error("โŒ OCSP exception storing custom cert %s in database: %s", service_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + else: + db_stored = True + + # Write to disk only after successful database storage + if not db_stored: + LOG.warning("โš ๏ธ OCSP skipping disk storage for custom cert %s due to database storage failure", service_name) + continue + + # Create SSL configs directory for this certificate + ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + try: + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + except Exception as e: + LOG.error("โŒ OCSP error creating directory %s: %s", ocsp_cert_dir, e) + stats["errors"] = stats.get("errors", 0) + 1 + continue + + # Write OCSP response to cache + ocsp_path = ocsp_cert_dir / "ocsp.der" + try: + ocsp_path.write_bytes(ocsp_der) + ocsp_path.chmod(0o644) # Readable by nginx + LOG.info("โœ“ OCSP saved custom cert %s response to disk at %s", service_name, ocsp_path) + stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 + except Exception as e: + LOG.error("โŒ OCSP error writing custom cert %s response to disk: %s", service_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + + except Exception as e: + LOG.error("OCSP exception while processing custom certificates: %s", e) + stats["errors"] = stats.get("errors", 0) + 1 + + def restore_ocsp_from_database(db: Optional[Any] = None) -> None: """ Restore cached OCSP responses from database to disk. @@ -377,10 +589,10 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: This handles ephemeral storage (tmpfs, etc.) by restoring files on each run. """ if not db: - LOG.debug("OCSP database not available, skipping cache restoration") + LOG.debug("โ„น๏ธ OCSP database not available, skipping cache restoration") return - LOG.info("OCSP restoring cached responses from database to disk...") + LOG.info("๐Ÿ”„ OCSP restoring cached responses from database to disk...") try: restored_count = 0 @@ -412,45 +624,82 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: ocsp_path.write_bytes(data) ocsp_path.chmod(0o644) restored_count += 1 - LOG.debug("OCSP restored cached response for %s from database", cert_name) + LOG.debug("โœ“ OCSP restored cached response for %s from database", cert_name) except Exception as e: - LOG.debug("OCSP could not restore cache for %s: %s", cert_name, e) + LOG.debug("โš ๏ธ OCSP could not restore cache for %s: %s", cert_name, e) continue + # Also restore custom certificate OCSP responses (using filesystem auto-discovery) + try: + customcert_base = Path(os.sep, "var", "cache", "bunkerweb", "customcert") + if customcert_base.is_dir(): + service_dirs = sorted([d for d in customcert_base.iterdir() if d.is_dir()]) + for service_dir in service_dirs: + service_name = service_dir.name + cache_key = f"ocsp/customcert-{service_name}" + try: + data = db.get_job_cache_file( + job_name="ocsp-refresh", + file_name=cache_key, + with_data=True, + with_info=False, + ) + if not data: + continue + ocsp_cert_dir = CONFIGS_SSL_BASE / f"customcert-{service_name}" + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + ocsp_path = ocsp_cert_dir / "ocsp.der" + ocsp_path.write_bytes(data) + ocsp_path.chmod(0o644) + restored_count += 1 + LOG.debug("โœ“ OCSP restored cached custom cert %s from database", service_name) + except Exception as e: + LOG.debug("โš ๏ธ OCSP could not restore custom cert cache for %s: %s", service_name, e) + continue + except Exception as e: + LOG.debug("โš ๏ธ OCSP exception while restoring custom cert cache: %s", e) + if restored_count > 0: - LOG.info("OCSP restored %d cached responses from database to disk", restored_count) + LOG.info("โœ“ OCSP restored %d cached responses from database to disk", restored_count) else: - LOG.debug("OCSP no cached responses restored from database (cache may be cold)") + LOG.debug("โ„น๏ธ OCSP no cached responses restored from database (cache may be cold)") except Exception as e: LOG.debug("OCSP exception while attempting cache restoration: %s", e) -def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> None: +def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None, stats: Optional[dict] = None) -> None: + if stats is None: + stats = {} fullchain = cert_dir / "fullchain.pem" if not fullchain.is_file(): return cert_name = cert_dir.name - LOG.info("OCSP processing certificate directory %s", cert_name) + LOG.info("๐Ÿ”„ OCSP processing certificate directory %s", cert_name) ocsp_url = cert_supports_ocsp(fullchain) if not ocsp_url: - LOG.info("OCSP certificate %s has no responder, skipping fetch", cert_name) + LOG.info("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) + stats["le_certs_no_ocsp"] = stats.get("le_certs_no_ocsp", 0) + 1 return - LOG.info("OCSP responder for certificate %s: %s", cert_name, ocsp_url) + LOG.info("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) - # === Check if cached OCSP response is still fresh (>3 days TTL) === - # TTL-based optimization: skip fetch if cached response valid for >3 days (259200 seconds) + # === Check if cached OCSP response is still fresh === + # TTL-based optimization: skip fetch if cached response still valid for > 50% of its lifetime cached_ttl = get_cached_ocsp_ttl(cert_name) - if cached_ttl is not None and cached_ttl > 259200: - LOG.info( - "OCSP cached response for %s still valid for %ds (%.1f days), skipping fetch", - cert_name, - cached_ttl, - cached_ttl / 86400.0, - ) - return + if cached_ttl is not None: + refresh_threshold = int(cached_ttl * 0.5) + if cached_ttl > refresh_threshold: + LOG.info( + "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), refresh at 50%% threshold=%ds, skipping fetch", + cert_name, + cached_ttl, + cached_ttl / 86400.0, + refresh_threshold, + ) + stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 + return ocsp_der: Optional[bytes] = None ttl: int = 0 @@ -459,17 +708,18 @@ def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> for attempt in (1, 2): ocsp_der, ttl = fetch_ocsp_response(fullchain, ocsp_url, timeout=10) if ocsp_der: - LOG.debug("OCSP successfully fetched response for %s on attempt %d (TTL=%ds)", cert_name, attempt, ttl) + LOG.debug("โœ“ OCSP successfully fetched response for %s on attempt %d (TTL=%ds)", cert_name, attempt, ttl) break if attempt == 1: - LOG.warning("OCSP fetch failed for %s, retrying once after 10 seconds ...", cert_name) + LOG.warning("โš ๏ธ OCSP fetch failed for %s, retrying once after 10 seconds ...", cert_name) time.sleep(10) if not ocsp_der: - LOG.error("OCSP failed to fetch response for %s after retries", cert_name) + LOG.error("โŒ OCSP failed to fetch response for %s after retries", cert_name) + stats["errors"] = stats.get("errors", 0) + 1 return - LOG.debug("OCSP final TTL for %s is %ds (from Next Update parsing)", cert_name, ttl) + LOG.debug("โšก OCSP final TTL for %s is %ds (from Next Update parsing)", cert_name, ttl) # Calculate checksum for integrity verification checksum = hashlib.sha256(ocsp_der).hexdigest() @@ -489,13 +739,15 @@ def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> ) if err: - LOG.error("OCSP error while storing response for %s in database: %s", cert_name, err) + LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) + stats["errors"] = stats.get("errors", 0) + 1 else: - LOG.info("OCSP stored response for %s in database (TTL=%ds, checksum=%s)", + LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, checksum[:8]) db_stored = True except Exception as e: - LOG.error("OCSP exception while storing response for %s in database: %s", cert_name, e) + LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 else: # If no database available, still proceed with disk storage db_stored = True @@ -503,7 +755,7 @@ def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> # === Only write to disk AFTER successful database storage === # This prevents overwriting existing OCSP files if the fetch failed or had issues if not db_stored: - LOG.warning("OCSP skipping disk storage for %s due to database storage failure", cert_name) + LOG.warning("โš ๏ธ OCSP skipping disk storage for %s due to database storage failure", cert_name) return # Create the SSL configs directory for this certificate @@ -511,7 +763,8 @@ def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> try: ocsp_cert_dir.mkdir(parents=True, exist_ok=True) except Exception as e: - LOG.error("OCSP error while creating directory %s: %s", ocsp_cert_dir, e) + LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) + stats["errors"] = stats.get("errors", 0) + 1 return # Write OCSP response to cache/ssl/{cert-name}/ocsp.der @@ -519,60 +772,94 @@ def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None) -> try: ocsp_path.write_bytes(ocsp_der) ocsp_path.chmod(0o644) # Readable by nginx - LOG.info("OCSP saved response for %s to disk at %s", cert_name, ocsp_path) + LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) + stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 except Exception as e: - LOG.error("OCSP error while writing response for %s to disk: %s", cert_name, e) + LOG.error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 def main() -> int: global status db: Optional[Any] = None + + # Statistics tracking + stats = { + "le_certs_processed": 0, + "le_certs_skipped": 0, + "le_certs_no_ocsp": 0, + "custom_certs_processed": 0, + "custom_certs_skipped": 0, + "custom_certs_no_ocsp": 0, + "ocsp_cached_responses": 0, + "ocsp_fetched_responses": 0, + "errors": 0, + } + try: # CRITICAL: Log immediately to confirm script is running try: - LOG.error("OCSP DEBUG: script started, initializing Job") + LOG.error("๐Ÿ”„ OCSP DEBUG: script started, initializing Job") except Exception as e: print(f"ERROR: Could not log initial message: {e}", file=_sys.stderr) try: job = Job(LOG, __file__) - LOG.debug("OCSP Job initialized successfully") + LOG.debug("โœ“ OCSP Job initialized successfully") except Exception as e: - LOG.error("OCSP could not initialize Job: %s", e) + LOG.error("โŒ OCSP could not initialize Job: %s", e) return 2 # Initialize database connection (optional) if Database is not None: try: db = Database(LOG) - LOG.debug("OCSP database connection established") + LOG.debug("โœ“ OCSP database connection established") except Exception as e: - LOG.warning("OCSP could not establish database connection, will use disk-only storage: %s", e) + LOG.warning("โš ๏ธ OCSP could not establish database connection, will use disk-only storage: %s", e) db = None else: - LOG.debug("OCSP Database module not available, will use disk-only storage") + LOG.debug("โ„น๏ธ OCSP Database module not available, will use disk-only storage") if not LIVE_BASE.is_dir(): - LOG.info("OCSP live certificate directory %s does not exist, nothing to do", LIVE_BASE) + LOG.info("โ„น๏ธ OCSP live certificate directory %s does not exist, nothing to do", LIVE_BASE) return 0 # Restore cached OCSP responses from database (handles ephemeral storage) restore_ocsp_from_database(db) cert_dirs = [d for d in sorted(LIVE_BASE.iterdir()) if d.is_dir()] - LOG.info("OCSP found %d live certificate directories under %s", len(cert_dirs), LIVE_BASE) + LOG.info("โ„น๏ธ OCSP found %d live certificate directories under %s", len(cert_dirs), LIVE_BASE) + stats["le_certs_processed"] = len(cert_dirs) for cert_dir in cert_dirs: fullchain = cert_dir / "fullchain.pem" if not fullchain.is_file(): LOG.debug("OCSP skipping %s because fullchain.pem is missing", cert_dir) + stats["le_certs_skipped"] += 1 continue - process_one_cert_dir(job, cert_dir, db) + process_one_cert_dir(job, cert_dir, db, stats) + + # Process custom certificates + process_custom_certs(job, db, stats) + # Summary message with statistics + LOG.info("โœ“ OCSP refresh job completed successfully") + LOG.info( + "๐Ÿ“Š Statistics: ๐Ÿ” LE certs=%d (skipped=%d, no OCSP=%d) | ๐Ÿ” Custom certs=%d (no OCSP=%d) | ๐Ÿ”„ Fetched=%d | โœ“ Cached=%d | โŒ Errors=%d", + stats["le_certs_processed"], + stats["le_certs_skipped"], + stats["le_certs_no_ocsp"], + stats["custom_certs_processed"], + stats["custom_certs_no_ocsp"], + stats["ocsp_fetched_responses"], + stats["ocsp_cached_responses"], + stats["errors"], + ) return status except BaseException as e: - LOG.exception("OCSP exception in ocsp-refresh.py") - LOG.error("OCSP exception while running ocsp-refresh.py: %s", e) + LOG.exception("โŒ OCSP exception in ocsp-refresh.py") + LOG.error("โŒ OCSP exception while running ocsp-refresh.py: %s", e) return 2 finally: # Ensure database connection is properly closed From f824b41ef9a3f1fabc04dd60c7b6f81857d50bc3 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 11:25:10 +0100 Subject: [PATCH 10/59] fix cleanup run, fix job run - job cleaned up existing directories - run as main() --- src/common/core/ssl/jobs/ocsp-refresh.py | 29 +++++++++++++++--------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 5dc8273330..6d581e8c75 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -799,18 +799,12 @@ def main() -> int: try: # CRITICAL: Log immediately to confirm script is running try: - LOG.error("๐Ÿ”„ OCSP DEBUG: script started, initializing Job") + LOG.error("๐Ÿ”„ OCSP DEBUG: script started") except Exception as e: print(f"ERROR: Could not log initial message: {e}", file=_sys.stderr) - try: - job = Job(LOG, __file__) - LOG.debug("โœ“ OCSP Job initialized successfully") - except Exception as e: - LOG.error("โŒ OCSP could not initialize Job: %s", e) - return 2 - - # Initialize database connection (optional) + # Initialize database connection FIRST to ensure we can restore cached responses + db = None if Database is not None: try: db = Database(LOG) @@ -821,11 +815,24 @@ def main() -> int: else: LOG.debug("โ„น๏ธ OCSP Database module not available, will use disk-only storage") + # IMPORTANT: Skip Job initialization (and automatic cleanup) entirely for safety + # Keeping cached OCSP files ensures we have data available even if errors occur + # The database restoration ensures consistency without needing to delete files + # Create a minimal Job-like placeholder object (job parameter is passed to functions but not used) + class MinimalJob: + def __init__(self, log): + self.log = log + + job = MinimalJob(LOG) + LOG.debug("โœ“ OCSP skipping Job initialization (cleanup disabled for data safety)") + if not LIVE_BASE.is_dir(): LOG.info("โ„น๏ธ OCSP live certificate directory %s does not exist, nothing to do", LIVE_BASE) return 0 # Restore cached OCSP responses from database (handles ephemeral storage) + # This ensures consistency by restoring from persistent database storage + # Particularly important for ephemeral storage (tmpfs) or container restarts restore_ocsp_from_database(db) cert_dirs = [d for d in sorted(LIVE_BASE.iterdir()) if d.is_dir()] @@ -870,6 +877,6 @@ def main() -> int: pass -if __name__ == "__main__": - sys_exit(main()) +# run it +sys_exit(main()) From 3762f36dc485c9ec6a95d8d430a53e0e847ec465 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 13:32:08 +0100 Subject: [PATCH 11/59] use python cryptography + add cleanup + SAN fix + fix race condition - migrate to native python cryptography module so we don't have openssl dependency - add cleanup function when SSL_USE_OCSP_STAPLING=no - add symlinks in case of multiple SAN entries - fix race condition during service reload: use database for certs extraction --- src/common/core/ssl/jobs/ocsp-refresh.py | 1353 ++++++++++++---------- 1 file changed, 712 insertions(+), 641 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 6d581e8c75..742a3da781 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -1,17 +1,26 @@ #!/usr/bin/env python3 import hashlib -import logging +import io import os -import shlex -import subprocess +import re +import shutil import sys as _sys -import tempfile +import tarfile import time -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from pathlib import Path from sys import exit as sys_exit, path as sys_path -from typing import Any, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple +from urllib.error import URLError +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.serialization import Encoding +from cryptography.x509 import ocsp as x509_ocsp +from cryptography.x509.oid import ExtensionOID, AuthorityInformationAccessOID # Add BunkerWeb Python deps (Job, logger, Database) to path for deps_path in [ @@ -51,535 +60,384 @@ status = 0 -# Base paths -LIVE_BASE = Path(os.sep, "var", "cache", "bunkerweb", "letsencrypt", "etc", "live") # Use scheduler-managed cache directory (automatically synced from database on restart) CONFIGS_SSL_BASE = Path(os.sep, "var", "cache", "bunkerweb", "ssl") -def run_cmd(cmd: List[str]) -> Tuple[int, str, str]: - """ - Run a subprocess command and return (rc, stdout, stderr). - When LOG is in debug level, the command line is logged. - """ - LOG.debug("๐Ÿ”„ OCSP running command: %s", " ".join(shlex.quote(c) for c in cmd)) - proc = subprocess.run( - cmd, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=False, - env={"PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", "")}, - ) - return proc.returncode, proc.stdout, proc.stderr - - -def cert_supports_ocsp(fullchain: Path) -> Optional[str]: +def extract_ocsp_url(pem_data: bytes, cert_name: str = "") -> Optional[str]: """ - First function: test if the certificate supports OCSP. - Returns OCSP responder URL if present, else None. + Extract OCSP responder URL from PEM certificate data using the cryptography library. + Validates the URL scheme is http:// or https://. + Returns OCSP responder URL if present and valid, else None. """ - LOG.debug("๐Ÿ”’ OCSP checking support for certificate %s", fullchain) - rc, out, err = run_cmd( - ["openssl", "x509", "-noout", "-ocsp_uri", "-in", fullchain.as_posix()] - ) - if rc != 0: - LOG.debug("๐Ÿ”’ OCSP openssl -ocsp_uri failed for %s: %s", fullchain, err.strip()) + LOG.debug("๐Ÿ”’ OCSP checking support for certificate %s", cert_name) + try: + cert = x509.load_pem_x509_certificate(pem_data) + + aia = cert.extensions.get_extension_for_oid(ExtensionOID.AUTHORITY_INFORMATION_ACCESS) + for access_description in aia.value: + if access_description.access_method == AuthorityInformationAccessOID.OCSP: + url = access_description.access_location.value + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + LOG.warning("โš ๏ธ OCSP URL has invalid scheme for %s: %s", cert_name, url) + return None + LOG.debug("๐ŸŒ OCSP found responder URL for %s: %s", cert_name, url) + return url + + LOG.debug("๐Ÿ”’ OCSP no responder URL advertised in %s", cert_name) return None - url = out.strip() - if not url: - LOG.debug("๐Ÿ”’ OCSP no responder URL advertised in %s", fullchain) + except x509.ExtensionNotFound: + LOG.debug("๐Ÿ”’ OCSP no AIA extension found in %s", cert_name) + return None + except Exception as e: + LOG.debug("๐Ÿ”’ OCSP failed to extract OCSP URL from %s: %s", cert_name, e) return None - LOG.debug("๐ŸŒ OCSP found responder URL for %s: %s", fullchain, url) - return url - - -def _normalize_dn_line(line: str, prefix: str) -> str: - """ - Normalize an OpenSSL subject/issuer line like 'issuer= CN = R3, O = Let's Encrypt, C = US' - so comparison is more robust to spacing. - """ - line = line.strip() - if line.lower().startswith(prefix.lower()): - line = line[len(prefix) :].strip() - # Remove repeated spaces around '=' and commas - parts = [part.strip() for part in line.replace(", ", ",").split(",")] - return ",".join(parts).lower() -def is_cert_issuer(leaf_path: Path, issuer_path: Path) -> bool: +def _fetch_issuer_from_aia(leaf: x509.Certificate, cert_name: str = "") -> Optional[x509.Certificate]: """ - Verify that issuer_path is the issuer of leaf_path by comparing - leaf issuer DN with issuer subject DN using openssl x509. + Fetch the issuer certificate from the AIA caIssuers URL in the leaf certificate. + Returns the issuer certificate or None if unavailable. """ - rc1, issuer_line, err1 = run_cmd( - ["openssl", "x509", "-noout", "-issuer", "-in", leaf_path.as_posix()] - ) - if rc1 != 0: - LOG.debug("๐Ÿ”’ OCSP openssl -issuer failed for %s: %s", leaf_path, err1.strip()) - return False - - rc2, subject_line, err2 = run_cmd( - ["openssl", "x509", "-noout", "-subject", "-in", issuer_path.as_posix()] - ) - if rc2 != 0: - LOG.debug("๐Ÿ”’ OCSP openssl -subject failed for %s: %s", issuer_path, err2.strip()) - return False - - issuer_dn = _normalize_dn_line(issuer_line, "issuer=") - subject_dn = _normalize_dn_line(subject_line, "subject=") - - if issuer_dn != subject_dn: - LOG.debug( - "๐Ÿ”’ OCSP issuer mismatch for leaf %s and candidate %s: issuer_dn=%r subject_dn=%r", - leaf_path, - issuer_path, - issuer_dn, - subject_dn, - ) - return False - - LOG.debug("โœ“ OCSP verified issuer for leaf %s is %s", leaf_path, issuer_path) - return True + try: + aia = leaf.extensions.get_extension_for_oid(ExtensionOID.AUTHORITY_INFORMATION_ACCESS) + for access_description in aia.value: + if access_description.access_method == AuthorityInformationAccessOID.CA_ISSUERS: + url = access_description.access_location.value + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + continue + LOG.debug("๐Ÿ”„ OCSP fetching issuer for %s from %s", cert_name, url) + req = Request(url, headers={"Accept": "application/pkix-cert, application/x-x509-ca-cert"}) + response = urlopen(req, timeout=10) + issuer_data = response.read() + # Try DER first (most common for caIssuers), then PEM + try: + return x509.load_der_x509_certificate(issuer_data) + except Exception: + return x509.load_pem_x509_certificate(issuer_data) + except x509.ExtensionNotFound: + LOG.debug("๐Ÿ”’ OCSP no AIA extension in leaf cert for %s", cert_name) + except Exception as e: + LOG.warning("โš ๏ธ OCSP failed to fetch issuer from AIA for %s: %s", cert_name, e) + return None -def split_chain(fullchain: Path) -> Tuple[Path, Path]: +def _parse_chain(pem_data: bytes, cert_name: str = "") -> Tuple[x509.Certificate, x509.Certificate]: """ - Split fullchain.pem into leaf.pem and issuer.pem temporary files. - Assumes first cert = leaf, second = issuer. + Parse fullchain PEM data and return (leaf_cert, issuer_cert). + The leaf is the first certificate; the issuer is identified by matching + the leaf's issuer DN against the subject DNs of the remaining certificates. + Falls back to the second certificate if no DN match is found. """ - LOG.debug("๐Ÿ“œ OCSP splitting fullchain %s into leaf and issuer candidates", fullchain) - text = fullchain.read_text(encoding="utf-8") - parts = text.strip().split("-----END CERTIFICATE-----") - cert_blobs = [p for p in parts if "BEGIN CERTIFICATE" in p] - if len(cert_blobs) < 2: - raise RuntimeError(f"fullchain {fullchain} does not contain at least two certs") - - # Write the leaf cert (first block) to a temp file - leaf_pem = cert_blobs[0] + "-----END CERTIFICATE-----\n" - leaf_fd, leaf_path_str = tempfile.mkstemp(suffix=".pem", prefix="ocsp-leaf-") - os.close(leaf_fd) - leaf_path = Path(leaf_path_str) - leaf_path.write_text(leaf_pem, encoding="utf-8") - - # By default, assume the second cert is the issuer - issuer_path: Optional[Path] = None - - # Try to find a blob whose subject matches the leaf issuer - for idx in range(1, len(cert_blobs)): - pem = cert_blobs[idx] + "-----END CERTIFICATE-----\n" - tmp_fd, tmp_path_str = tempfile.mkstemp(suffix=".pem", prefix="ocsp-issuer-") - os.close(tmp_fd) - tmp_path = Path(tmp_path_str) - tmp_path.write_text(pem, encoding="utf-8") + certs = x509.load_pem_x509_certificates(pem_data) + leaf = certs[0] - try: - if is_cert_issuer(leaf_path, tmp_path): - issuer_path = tmp_path - LOG.debug( - "โœ“ OCSP selected issuer certificate index %d from fullchain %s", - idx, - fullchain, - ) - break - except Exception as e: - LOG.debug("๐Ÿ”’ OCSP error while verifying issuer for %s: %s", fullchain, e) + if len(certs) >= 2: + # Find the issuer by matching leaf.issuer to candidate.subject + for idx, candidate in enumerate(certs[1:], start=1): + if leaf.issuer == candidate.subject: + LOG.debug("โœ“ OCSP selected issuer certificate index %d for %s", idx, cert_name) + return leaf, candidate - # Not the issuer: clean up this temp file - try: - tmp_path.unlink(missing_ok=True) - except Exception: - pass - - # Fallback: if we could not verify, still use the second cert as issuer - if issuer_path is None: - pem = cert_blobs[1] + "-----END CERTIFICATE-----\n" - tmp_fd, tmp_path_str = tempfile.mkstemp(suffix=".pem", prefix="ocsp-issuer-") - os.close(tmp_fd) - issuer_path = Path(tmp_path_str) - issuer_path.write_text(pem, encoding="utf-8") + # Fallback: use the second certificate LOG.warning( - "Could not verify issuer chain for %s, falling back to second certificate as issuer " - "(this usually means the chain is non-standard or OpenSSL output could not be parsed cleanly)", - fullchain, + "โš ๏ธ OCSP could not verify issuer chain for %s by DN match, falling back to second certificate", + cert_name, ) + return leaf, certs[1] - return leaf_path, issuer_path + # Single cert (no chain): try to fetch issuer from AIA caIssuers URL + LOG.debug("๐Ÿ”„ OCSP single cert for %s, attempting to fetch issuer from AIA caIssuers", cert_name) + issuer = _fetch_issuer_from_aia(leaf, cert_name) + if issuer: + return leaf, issuer + raise RuntimeError(f"fullchain for {cert_name} does not contain issuer and AIA fetch failed") -def parse_next_update_ttl(ocsp_text: str) -> int: + +def _ocsp_response_lifetimes(ocsp_response: x509_ocsp.OCSPResponse) -> Tuple[Optional[int], Optional[int]]: """ - Parse Next Update from openssl ocsp -text output and return TTL in seconds. - Falls back to 86400s (24 hours, RFC standard) on error. + Extract (remaining_ttl_seconds, total_lifetime_seconds) from a parsed OCSP response. + Returns (None, None) if the timing fields are unavailable or invalid. """ - next_update_str: Optional[str] = None - for line in ocsp_text.splitlines(): - line = line.strip() - if "Next Update:" in line: - parts = line.split("Next Update:", 1) - if len(parts) == 2: - next_update_str = parts[1].strip() - break + # Prefer *_utc properties (cryptography 42.0+) to avoid deprecation warnings + this_update = getattr(ocsp_response, "this_update_utc", None) or ocsp_response.this_update + if this_update is None: + return None, None - if not next_update_str: - LOG.debug("โšก OCSP no Next Update field found in response, falling back to RFC standard 86400s (24 hours) TTL") - return 86400 + if this_update.tzinfo is None: + this_update = this_update.replace(tzinfo=timezone.utc) - # Common OpenSSL OCSP time format: "Jan 1 00:00:00 2026 GMT" - try: - dt = datetime.strptime(next_update_str, "%b %d %H:%M:%S %Y %Z") - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - now = datetime.now(timezone.utc) - ttl = int((dt - now).total_seconds()) - if ttl <= 0: - LOG.debug("โšก OCSP parsed Next Update is in the past, using 300s TTL instead") - return 300 - # Cap TTL to 7 days to avoid very long-lived cache entries - return min(ttl, 7 * 24 * 3600) - except Exception as e: - LOG.debug("โšก OCSP failed to parse Next Update from response: %s, falling back to RFC standard 86400s (24 hours)", e) - return 86400 + next_update = getattr(ocsp_response, "next_update_utc", None) or ocsp_response.next_update + if next_update is None: + # No Next Update: treat lifetime as 24 hours from This Update (RFC standard fallback) + LOG.debug("โšก OCSP Next Update missing, using default lifetime of 24h from This Update") + next_update = this_update + timedelta(hours=24) + elif next_update.tzinfo is None: + next_update = next_update.replace(tzinfo=timezone.utc) + + total_lifetime = int((next_update - this_update).total_seconds()) + if total_lifetime <= 0: + LOG.debug("โšก OCSP invalid lifetime: Next Update is not after This Update") + return None, None + + now = datetime.now(timezone.utc) + remaining = max(0, int((next_update - now).total_seconds())) + return remaining, total_lifetime -def fetch_ocsp_response(fullchain: Path, ocsp_url: str, timeout: int = 10) -> Tuple[Optional[bytes], int]: +def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", timeout: int = 10) -> Tuple[Optional[bytes], int]: """ - Fetch OCSP response using openssl ocsp with a network timeout. + Fetch OCSP response using cryptography + urllib. Returns (raw DER bytes or None, ttl_seconds). """ - leaf, issuer = split_chain(fullchain) - ocsp_der_fd, ocsp_der_path = tempfile.mkstemp(suffix=".der", prefix="ocsp-resp-") - os.close(ocsp_der_fd) - ocsp_der = Path(ocsp_der_path) + try: + leaf, issuer = _parse_chain(pem_data, cert_name) + except Exception as e: + LOG.error("โŒ OCSP failed to parse chain for %s: %s", cert_name, e) + return None, 0 try: - cmd = [ - "openssl", - "ocsp", - "-issuer", - issuer.as_posix(), - "-cert", - leaf.as_posix(), - "-url", + # Build OCSP request + builder = x509_ocsp.OCSPRequestBuilder() + builder = builder.add_certificate(leaf, issuer, hashes.SHA256()) + ocsp_request = builder.build() + ocsp_request_data = ocsp_request.public_bytes(Encoding.DER) + + # Send OCSP request via HTTP POST + req = Request( ocsp_url, - "-no_nonce", - "-timeout", - str(timeout), - "-respout", - ocsp_der.as_posix(), - "-text", - ] - rc, out, err = run_cmd(cmd) - if rc != 0: - LOG.error("โŒ OCSP openssl ocsp failed for %s: rc=%s err=%s", fullchain, rc, err.strip()) - return None, 0 + data=ocsp_request_data, + headers={"Content-Type": "application/ocsp-request"}, + method="POST", + ) + response = urlopen(req, timeout=timeout) + ocsp_response_data = response.read() - if not ocsp_der.is_file() or ocsp_der.stat().st_size == 0: - LOG.error("โŒ OCSP no response written for %s (empty respout file)", fullchain) + # Parse response and validate status + ocsp_response = x509_ocsp.load_der_ocsp_response(ocsp_response_data) + if ocsp_response.response_status != x509_ocsp.OCSPResponseStatus.SUCCESSFUL: + LOG.error("โŒ OCSP response status is not successful for %s: %s", cert_name, ocsp_response.response_status) return None, 0 - ttl = parse_next_update_ttl(out) - return ocsp_der.read_bytes(), ttl - finally: - try: - leaf.unlink(missing_ok=True) - issuer.unlink(missing_ok=True) - ocsp_der.unlink(missing_ok=True) - except Exception: - pass + # Extract TTL from response timing fields + remaining, total_lifetime = _ocsp_response_lifetimes(ocsp_response) + if remaining is not None: + ttl = min(remaining, 7 * 24 * 3600) # Cap to 7 days + else: + ttl = 86400 # RFC standard fallback + + return ocsp_response_data, ttl + except URLError as e: + LOG.error("โŒ OCSP network error fetching response for %s from %s: %s", cert_name, ocsp_url, e) + return None, 0 + except Exception as e: + LOG.error("โŒ OCSP failed to fetch response for %s: %s", cert_name, e) + return None, 0 -def extract_san_dns(fullchain: Path) -> List[str]: +def extract_san_dns(pem_data: bytes, cert_name: str = "") -> List[str]: """ - Try to extract DNS names from certificate SAN to use as SNI keys. - This is best-effort; if it fails, we fall back to the cert directory basename. + Extract DNS names from certificate SAN extension. + Best-effort; returns empty list on failure. """ - rc, out, err = run_cmd( - ["openssl", "x509", "-noout", "-text", "-in", fullchain.as_posix()] - ) - if rc != 0: - LOG.debug("OCSP openssl -text failed for %s: %s", fullchain, err.strip()) + try: + cert = x509.load_pem_x509_certificate(pem_data) + san = cert.extensions.get_extension_for_oid(ExtensionOID.SUBJECT_ALTERNATIVE_NAME) + names = san.value.get_values_for_type(x509.DNSName) + return sorted(set(names)) + except x509.ExtensionNotFound: + return [] + except Exception as e: + LOG.debug("OCSP failed to extract SAN from %s: %s", cert_name, e) return [] - - names: List[str] = [] - in_san = False - for line in out.splitlines(): - line = line.strip() - if "X509v3 Subject Alternative Name" in line: - in_san = True - continue - if in_san: - if not line.startswith("DNS:") and "DNS:" not in line: - break - parts = line.split(",") - for part in parts: - part = part.strip() - if part.startswith("DNS:"): - dns = part[4:].strip() - if dns: - names.append(dns) - - return sorted(set(names)) -def get_cached_ocsp_ttl(cert_name: str) -> Optional[int]: +def get_cached_ocsp_ttl(cert_name: str) -> Tuple[Optional[int], Optional[int]]: """ - Check if cached OCSP file exists and return its remaining TTL in seconds. - Returns None if file doesn't exist or TTL cannot be determined. - - This allows skipping fetch if the cached response is still valid for >3 days. + Check if cached OCSP DER file exists and return: + - remaining TTL in seconds (until Next Update) + - total lifetime in seconds (Next Update - This Update) + Both values are None if they cannot be determined. """ ocsp_path = CONFIGS_SSL_BASE / cert_name / "ocsp.der" LOG.info("โšก OCSP TTL check: checking cache for %s at %s", cert_name, ocsp_path) if not ocsp_path.is_file(): LOG.info("โšก OCSP TTL check: cache miss - file not found for %s", cert_name) - return None + return None, None - LOG.debug("โšก OCSP cached file found for %s, reading TTL...", cert_name) + LOG.debug("โšก OCSP cached file found for %s, reading This/Next Update...", cert_name) try: - # Use openssl ocsp -respin to read cached DER and extract Next Update - # Use -noverify to skip signature verification (we only care about the TTL) - rc, out, err = run_cmd( - ["openssl", "ocsp", "-respin", ocsp_path.as_posix(), "-text", "-noverify"] + ocsp_data = ocsp_path.read_bytes() + ocsp_response = x509_ocsp.load_der_ocsp_response(ocsp_data) + + remaining, total_lifetime = _ocsp_response_lifetimes(ocsp_response) + if remaining is None or total_lifetime is None: + LOG.info("๐Ÿ”„ OCSP could not determine precise lifetime for %s from cached response", cert_name) + return None, None + + LOG.info( + "โšก OCSP cached response for %s: remaining=%ds (%.1f days), total_lifetime=%ds (%.1f days)", + cert_name, + remaining, + remaining / 86400.0, + total_lifetime, + total_lifetime / 86400.0, ) - if rc != 0: - LOG.warning("OCSP failed to read cached DER for %s (rc=%d): %s", cert_name, rc, err.strip()) - return None - - ttl = parse_next_update_ttl(out) - three_days_sec = 259200 - if ttl > three_days_sec: - LOG.info( - "โœ“ OCSP cached response for %s is fresh: TTL=%ds (%.1f days) > 3 days threshold", - cert_name, - ttl, - ttl / 86400.0, - ) - else: - LOG.info( - "๐Ÿ”„ OCSP cached response for %s needs refresh: TTL=%ds (%.1f days) <= 3 days threshold", - cert_name, - ttl, - ttl / 86400.0, - ) - return ttl + return remaining, total_lifetime except Exception as e: LOG.warning("โš ๏ธ OCSP exception while reading cached TTL for %s: %s", cert_name, e) - return None + return None, None -def validate_cert_file(cert_file: Path, key_file: Path) -> Tuple[Optional[Path], Optional[Path]]: +def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> None: """ - Validate cert and key file order. Tries both orderings if the first one fails. - Accepts certificate chains (fullchain.pem with multiple certs) - OpenSSL reads the first one. - Returns (valid_cert_path, valid_key_path) or (None, None) if both fail. + Create symlinks for each SAN in the certificate so that OCSP responses + can be found by any SNI name, not just the cert directory name. + E.g., if cert is for example.com + www.example.com and stored under example.com/ocsp.der, + creates www.example.com/ocsp.der -> example.com/ocsp.der """ - for attempt in (1, 2): - test_cert = cert_file if attempt == 1 else key_file - test_key = key_file if attempt == 1 else cert_file - - # Try to read the file as a certificate (openssl x509 reads first cert from chain) - rc, _, _ = run_cmd( - ["openssl", "x509", "-noout", "-in", test_cert.as_posix()] - ) + sans = extract_san_dns(pem_data, cert_name) + # Strip -rsa/-ecdsa suffix from cert_name for comparison + base_name = _service_name_from_dir(cert_name) - if rc == 0: - # File is a valid certificate or certificate chain - LOG.debug("โœ“ OCSP validated cert/key ordering: cert=%s, key=%s", test_cert, test_key) - return test_cert, test_key + for san in sans: + if san == base_name or san == cert_name: + continue - # Both orderings failed - LOG.warning("โŒ OCSP could not find valid certificate in either file ordering") - return None, None + san_dir = CONFIGS_SSL_BASE / san + san_ocsp = san_dir / "ocsp.der" + # Skip if target already exists (real file or valid symlink) + if san_ocsp.exists(): + continue -def extract_ocsp_url_from_cert(cert_file: Path) -> Optional[str]: - """ - Extract OCSP responder URL from a certificate file. - Returns the URL if present, None otherwise. - """ - rc, out, err = run_cmd( - ["openssl", "x509", "-noout", "-ocsp_uri", "-in", cert_file.as_posix()] - ) - if rc != 0: - LOG.debug("๐Ÿ”’ OCSP could not extract OCSP URL from %s: %s", cert_file, err.strip()) - return None - url = out.strip() - if not url: - LOG.debug("๐Ÿ”’ OCSP no responder URL found in %s", cert_file) - return None - return url + try: + san_dir.mkdir(parents=True, exist_ok=True) + # Use relative symlink: ../cert_name/ocsp.der + rel_target = Path("..") / cert_name / "ocsp.der" + san_ocsp.symlink_to(rel_target) + LOG.debug("๐Ÿ”— OCSP created SAN symlink %s -> %s", san_ocsp, rel_target) + except Exception as e: + LOG.debug("โš ๏ธ OCSP could not create SAN symlink for %s: %s", san, e) -def process_custom_certs(job: Job, db: Optional[Any] = None, stats: Optional[dict] = None) -> None: - """ - Process OCSP for custom certificates by auto-discovering from filesystem. - Scans /var/cache/bunkerweb/customcert/ for existing service directories. +def _load_le_certs_from_db(db: Any) -> Dict[str, bytes]: """ - if stats is None: - stats = {} - - try: - customcert_base = Path(os.sep, "var", "cache", "bunkerweb", "customcert") + Load Let's Encrypt fullchain PEM data from the database tarball. + The certbot-new job stores the entire /var/cache/bunkerweb/letsencrypt/etc directory + as a tarball via cache_dir(). We extract fullchain.pem for each cert directory. - # Auto-discover services from filesystem instead of environment variables - if not customcert_base.is_dir(): - LOG.info("โ„น๏ธ OCSP custom cert directory %s does not exist", customcert_base) - return - - service_dirs = sorted([d for d in customcert_base.iterdir() if d.is_dir()]) - if not service_dirs: - LOG.info("โ„น๏ธ OCSP no custom certificate directories found under %s", customcert_base) - return + Returns dict: {cert_name: fullchain_pem_bytes} + """ + result: Dict[str, bytes] = {} - all_domains = [d.name for d in service_dirs] - LOG.info("๐Ÿ”„ OCSP auto-discovered %d custom certificate service(s) from filesystem", len(all_domains)) - stats["custom_certs_processed"] = len(all_domains) + # The tarball is stored with file_name = "folder:/var/cache/bunkerweb/letsencrypt/etc.tgz" + # It may be stored by either certbot-new or certbot-renew depending on which ran last + tgz_file_name = "folder:/var/cache/bunkerweb/letsencrypt/etc.tgz" - for service_name in all_domains: - # Service directory exists, so custom certs are configured for this service - # Try to find custom certificate files (main, -ecdsa, or -rsa variants) - base_dir = Path(os.sep, "var", "cache", "bunkerweb", "customcert", service_name) - cert_variants = ["cert.pem", "cert-ecdsa.pem", "cert-rsa.pem"] - key_variants = ["key.pem", "key-ecdsa.pem", "key-rsa.pem"] + tgz_data = None + for job_name in ("certbot-renew", "certbot-new"): + try: + tgz_data = db.get_job_cache_file( + job_name=job_name, + file_name=tgz_file_name, + with_data=True, + with_info=False, + ) + except Exception as e: + LOG.warning("โš ๏ธ OCSP failed to query database for LE tarball (job=%s): %s", job_name, e) + continue + if tgz_data: + LOG.debug("โœ“ OCSP found LE tarball from %s job", job_name) + break - valid_cert = None + if not tgz_data: + LOG.debug("โ„น๏ธ OCSP no LE tarball found in database (certbot-new cache)") + return result - # Try to find valid cert/key pair among all variants - for cert_variant in cert_variants: - cert_path = base_dir / cert_variant - if not cert_path.is_file(): + try: + with tarfile.open(fileobj=io.BytesIO(tgz_data), mode="r:gz") as tar: + for member in tar.getmembers(): + # Match live//fullchain.pem symlinks โ€” tarfile.extractfile() + # follows symlinks within the archive to read the actual archive/ data + if not member.issym(): continue + parts = Path(member.name).parts + # Match pattern: ./live//fullchain.pem + if len(parts) < 3: + continue + idx = 1 if parts[0] == "." else 0 + if len(parts) == idx + 3 and parts[idx] == "live" and parts[idx + 2] == "fullchain.pem": + cert_name = parts[idx + 1] + try: + f = tar.extractfile(member) + if f: + pem_data = f.read() + if not pem_data or b"-----BEGIN" not in pem_data: + LOG.warning("โš ๏ธ OCSP LE fullchain for %s is empty or not valid PEM, skipping", cert_name) + continue + result[cert_name] = pem_data + LOG.debug("โœ“ OCSP extracted LE fullchain for %s from database tarball", cert_name) + else: + LOG.warning("โš ๏ธ OCSP could not read LE fullchain for %s from tarball (extractfile returned None)", cert_name) + except KeyError: + LOG.warning("โš ๏ธ OCSP symlink target missing in tarball for %s", cert_name) + except Exception as e: + LOG.warning("โš ๏ธ OCSP failed to extract LE fullchain for %s: %s", cert_name, e) + except tarfile.TarError as e: + LOG.error("โŒ OCSP LE tarball is corrupted or invalid: %s", e) + except Exception as e: + LOG.error("โŒ OCSP failed to extract LE certificates from database tarball: %s", e) - for key_variant in key_variants: - key_path = base_dir / key_variant - if not key_path.is_file(): - continue - - LOG.debug("๐Ÿ”„ OCSP trying custom cert variant: %s / %s", cert_variant, key_variant) - valid_cert, _ = validate_cert_file(cert_path, key_path) - if valid_cert: - LOG.info("โœ“ OCSP found valid custom certificate for service %s: %s", service_name, valid_cert) - break + return result - if valid_cert: - break - if not valid_cert: - LOG.debug("โ„น๏ธ OCSP custom cert cache not found for %s in %s", service_name, base_dir) - stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + 1 - continue +def _load_custom_certs_from_db(db: Any) -> Dict[str, bytes]: + """ + Load custom certificate PEM data from the database. + The custom-cert job stores cert.pem per service_id via cache_file(). - LOG.info("๐Ÿ”„ OCSP processing custom certificate for service %s", service_name) + Returns dict: {service_name: cert_pem_bytes} + """ + result: Dict[str, bytes] = {} - # Extract OCSP URL from certificate - ocsp_url = extract_ocsp_url_from_cert(valid_cert) - if not ocsp_url: - LOG.info("โ„น๏ธ OCSP custom cert %s has no responder, skipping", service_name) - stats["custom_certs_no_ocsp"] = stats.get("custom_certs_no_ocsp", 0) + 1 - continue + try: + cache_files = db.get_jobs_cache_files(job_name="custom-cert", with_data=True) + except Exception as e: + LOG.error("โŒ OCSP failed to query database for custom certificates: %s", e) + return result - LOG.info("๐ŸŒ OCSP responder for custom cert %s: %s", service_name, ocsp_url) - - # Use customcert- prefix for OCSP cache key - cert_name = f"customcert-{service_name}" - - # Check if cached OCSP response is still fresh - # Skip fetch if cached response still valid for > 50% of its lifetime - cached_ttl = get_cached_ocsp_ttl(cert_name) - if cached_ttl is not None: - refresh_threshold = int(cached_ttl * 0.5) - if cached_ttl > refresh_threshold: - LOG.info( - "โœ“ OCSP cached response for custom cert %s still valid for %ds (%.1f days), refresh at 50%% threshold=%ds, skipping fetch", - service_name, - cached_ttl, - cached_ttl / 86400.0, - refresh_threshold, - ) - stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 - continue + if not cache_files: + LOG.debug("โ„น๏ธ OCSP no custom-cert cache entries found in database") + return result - ocsp_der: Optional[bytes] = None - ttl: int = 0 - - # Use a timeout of 10 seconds per attempt, retry once after 10 seconds - for attempt in (1, 2): - ocsp_der, ttl = fetch_ocsp_response(valid_cert, ocsp_url, timeout=10) - if ocsp_der: - LOG.debug("โœ“ OCSP successfully fetched response for custom cert %s on attempt %d (TTL=%ds)", - service_name, attempt, ttl) - break - if attempt == 1: - LOG.warning("โš ๏ธ OCSP fetch failed for custom cert %s, retrying once after 10 seconds ...", service_name) - time.sleep(10) - - if not ocsp_der: - LOG.error("โŒ OCSP failed to fetch response for custom cert %s after retries", service_name) - stats["errors"] = stats.get("errors", 0) + 1 + for entry in cache_files: + try: + file_name = entry.get("file_name", "") + service_id = entry.get("service_id", "") + # Match cert.pem, cert-ecdsa.pem, cert-rsa.pem + if not (file_name.startswith("cert") and file_name.endswith(".pem")): continue - - LOG.debug("โšก OCSP final TTL for custom cert %s is %ds", service_name, ttl) - - # Calculate checksum for integrity verification - checksum = hashlib.sha256(ocsp_der).hexdigest() - - # Store OCSP response in database - db_stored = False - if db: - cache_key = f"ocsp/{cert_name}" - try: - err = db.upsert_job_cache( - service_id=None, # Global cache entry - file_name=cache_key, - data=ocsp_der, - job_name="ocsp-refresh", - checksum=checksum, - ) - - if err: - LOG.error("โŒ OCSP error storing custom cert %s response in database: %s", service_name, err) - stats["errors"] = stats.get("errors", 0) + 1 - else: - LOG.info("โœ“ OCSP stored custom cert %s response in database (TTL=%ds, checksum=%s)", - service_name, ttl, checksum[:8]) - db_stored = True - except Exception as e: - LOG.error("โŒ OCSP exception storing custom cert %s in database: %s", service_name, e) - stats["errors"] = stats.get("errors", 0) + 1 - else: - db_stored = True - - # Write to disk only after successful database storage - if not db_stored: - LOG.warning("โš ๏ธ OCSP skipping disk storage for custom cert %s due to database storage failure", service_name) + if not service_id: + LOG.debug("โš ๏ธ OCSP custom cert entry %s has no service_id, skipping", file_name) continue - - # Create SSL configs directory for this certificate - ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name - try: - ocsp_cert_dir.mkdir(parents=True, exist_ok=True) - except Exception as e: - LOG.error("โŒ OCSP error creating directory %s: %s", ocsp_cert_dir, e) - stats["errors"] = stats.get("errors", 0) + 1 + data = entry.get("data") + if not data: + LOG.warning("โš ๏ธ OCSP custom cert %s for service %s has no data, skipping", file_name, service_id) continue + # Derive suffix from filename: cert-ecdsa.pem -> -ecdsa, cert.pem -> "" + suffix = file_name.replace("cert", "").replace(".pem", "") # e.g. "-ecdsa", "-rsa", "" + key = f"{service_id}{suffix}" + result[key] = data + LOG.debug("โœ“ OCSP loaded custom cert for %s from database (file=%s, size=%d)", key, file_name, len(data)) + except Exception as e: + LOG.warning("โš ๏ธ OCSP failed to process custom cert entry %s: %s", entry.get("file_name", "?"), e) - # Write OCSP response to cache - ocsp_path = ocsp_cert_dir / "ocsp.der" - try: - ocsp_path.write_bytes(ocsp_der) - ocsp_path.chmod(0o644) # Readable by nginx - LOG.info("โœ“ OCSP saved custom cert %s response to disk at %s", service_name, ocsp_path) - stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 - except Exception as e: - LOG.error("โŒ OCSP error writing custom cert %s response to disk: %s", service_name, e) - stats["errors"] = stats.get("errors", 0) + 1 - - except Exception as e: - LOG.error("OCSP exception while processing custom certificates: %s", e) - stats["errors"] = stats.get("errors", 0) + 1 + return result def restore_ocsp_from_database(db: Optional[Any] = None) -> None: @@ -596,68 +454,24 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: try: restored_count = 0 - # Get all possible cert names from live directory and restore from database - if LIVE_BASE.is_dir(): - for cert_dir in sorted(LIVE_BASE.iterdir()): - if not cert_dir.is_dir(): - continue - cert_name = cert_dir.name - cache_key = f"ocsp/{cert_name}" - - try: - # Use the Database.get_job_cache_file() method to retrieve cached OCSP response - # Returns bytes directly if found, None if not found - data = db.get_job_cache_file( - job_name="ocsp-refresh", - file_name=cache_key, - with_data=True, - with_info=False, - ) - - if not data: - continue - - # Restore to disk - ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name - ocsp_cert_dir.mkdir(parents=True, exist_ok=True) - ocsp_path = ocsp_cert_dir / "ocsp.der" - ocsp_path.write_bytes(data) - ocsp_path.chmod(0o644) - restored_count += 1 - LOG.debug("โœ“ OCSP restored cached response for %s from database", cert_name) - except Exception as e: - LOG.debug("โš ๏ธ OCSP could not restore cache for %s: %s", cert_name, e) - continue + # Get all OCSP cache entries from database for this job + cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=True) + for entry in cache_files: + file_name = entry.get("file_name", "") + if not file_name.startswith("ocsp/") or not entry.get("data"): + continue - # Also restore custom certificate OCSP responses (using filesystem auto-discovery) - try: - customcert_base = Path(os.sep, "var", "cache", "bunkerweb", "customcert") - if customcert_base.is_dir(): - service_dirs = sorted([d for d in customcert_base.iterdir() if d.is_dir()]) - for service_dir in service_dirs: - service_name = service_dir.name - cache_key = f"ocsp/customcert-{service_name}" - try: - data = db.get_job_cache_file( - job_name="ocsp-refresh", - file_name=cache_key, - with_data=True, - with_info=False, - ) - if not data: - continue - ocsp_cert_dir = CONFIGS_SSL_BASE / f"customcert-{service_name}" - ocsp_cert_dir.mkdir(parents=True, exist_ok=True) - ocsp_path = ocsp_cert_dir / "ocsp.der" - ocsp_path.write_bytes(data) - ocsp_path.chmod(0o644) - restored_count += 1 - LOG.debug("โœ“ OCSP restored cached custom cert %s from database", service_name) - except Exception as e: - LOG.debug("โš ๏ธ OCSP could not restore custom cert cache for %s: %s", service_name, e) - continue - except Exception as e: - LOG.debug("โš ๏ธ OCSP exception while restoring custom cert cache: %s", e) + cert_name = file_name[len("ocsp/"):] + try: + ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + ocsp_path = ocsp_cert_dir / "ocsp.der" + ocsp_path.write_bytes(entry["data"]) + ocsp_path.chmod(0o644) + restored_count += 1 + LOG.debug("โœ“ OCSP restored cached response for %s from database", cert_name) + except Exception as e: + LOG.debug("โš ๏ธ OCSP could not restore cache for %s: %s", cert_name, e) if restored_count > 0: LOG.info("โœ“ OCSP restored %d cached responses from database to disk", restored_count) @@ -667,118 +481,388 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: LOG.debug("OCSP exception while attempting cache restoration: %s", e) -def process_one_cert_dir(job: Job, cert_dir: Path, db: Optional[Any] = None, stats: Optional[dict] = None) -> None: +def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None) -> None: + """Process a single certificate for OCSP stapling. Works with in-memory PEM data.""" if stats is None: stats = {} - fullchain = cert_dir / "fullchain.pem" - if not fullchain.is_file(): - return - cert_name = cert_dir.name - LOG.info("๐Ÿ”„ OCSP processing certificate directory %s", cert_name) + service_name = _service_name_from_dir(cert_name) - ocsp_url = cert_supports_ocsp(fullchain) - if not ocsp_url: - LOG.info("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) - stats["le_certs_no_ocsp"] = stats.get("le_certs_no_ocsp", 0) + 1 + # Check per-service OCSP stapling setting + if not _is_ocsp_enabled_for_service(service_name): + LOG.info("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) + cleanup_ocsp_cache(db, cert_name) + stats["le_certs_skipped"] = stats.get("le_certs_skipped", 0) + 1 return - LOG.info("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) - - # === Check if cached OCSP response is still fresh === - # TTL-based optimization: skip fetch if cached response still valid for > 50% of its lifetime - cached_ttl = get_cached_ocsp_ttl(cert_name) - if cached_ttl is not None: - refresh_threshold = int(cached_ttl * 0.5) - if cached_ttl > refresh_threshold: - LOG.info( - "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), refresh at 50%% threshold=%ds, skipping fetch", - cert_name, - cached_ttl, - cached_ttl / 86400.0, - refresh_threshold, - ) - stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 + LOG.info("๐Ÿ”„ OCSP processing certificate %s", cert_name) + + try: + ocsp_url = extract_ocsp_url(pem_data, cert_name) + if not ocsp_url: + LOG.info("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) + stats["le_certs_no_ocsp"] = stats.get("le_certs_no_ocsp", 0) + 1 return - ocsp_der: Optional[bytes] = None - ttl: int = 0 + LOG.info("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) + + # === Check if cached OCSP response is still fresh === + # Refresh when remaining TTL is at or below 50% of the original lifetime. + cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name) + if cached_ttl is not None and total_lifetime is not None and total_lifetime > 0: + half_lifetime = total_lifetime // 2 + if cached_ttl > half_lifetime: + LOG.info( + "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), above 50%% lifetime threshold=%ds (%.1f days), skipping fetch", + cert_name, + cached_ttl, + cached_ttl / 86400.0, + half_lifetime, + half_lifetime / 86400.0, + ) + stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 + return - # Use a timeout of 10 seconds per attempt, retry once after 10 seconds - for attempt in (1, 2): - ocsp_der, ttl = fetch_ocsp_response(fullchain, ocsp_url, timeout=10) - if ocsp_der: - LOG.debug("โœ“ OCSP successfully fetched response for %s on attempt %d (TTL=%ds)", cert_name, attempt, ttl) - break - if attempt == 1: - LOG.warning("โš ๏ธ OCSP fetch failed for %s, retrying once after 10 seconds ...", cert_name) - time.sleep(10) + ocsp_der: Optional[bytes] = None + ttl: int = 0 - if not ocsp_der: - LOG.error("โŒ OCSP failed to fetch response for %s after retries", cert_name) - stats["errors"] = stats.get("errors", 0) + 1 - return + # Use a timeout of 10 seconds per attempt, retry once after 10 seconds + for attempt in (1, 2): + ocsp_der, ttl = fetch_ocsp_response(pem_data, ocsp_url, cert_name=cert_name, timeout=10) + if ocsp_der: + LOG.debug("โœ“ OCSP successfully fetched response for %s on attempt %d (TTL=%ds)", cert_name, attempt, ttl) + break + if attempt == 1: + LOG.warning("โš ๏ธ OCSP fetch failed for %s, retrying once after 10 seconds ...", cert_name) + time.sleep(10) + + if not ocsp_der: + LOG.error("โŒ OCSP failed to fetch response for %s after retries", cert_name) + stats["errors"] = stats.get("errors", 0) + 1 + return - LOG.debug("โšก OCSP final TTL for %s is %ds (from Next Update parsing)", cert_name, ttl) + LOG.debug("โšก OCSP final TTL for %s is %ds (from Next Update parsing)", cert_name, ttl) - # Calculate checksum for integrity verification - checksum = hashlib.sha256(ocsp_der).hexdigest() + # Calculate checksum for integrity verification + checksum = hashlib.sha256(ocsp_der).hexdigest() - # === Store OCSP response in database === - db_stored = False - if db: - cache_key = f"ocsp/{cert_name}" - try: - # Store in database - err = db.upsert_job_cache( - service_id=None, # Global cache entry - file_name=cache_key, - data=ocsp_der, - job_name="ocsp-refresh", - checksum=checksum, - ) + # === Store OCSP response in database === + db_stored = False + if db: + cache_key = f"ocsp/{cert_name}" + try: + err = db.upsert_job_cache( + service_id=None, # Global cache entry + file_name=cache_key, + data=ocsp_der, + job_name="ocsp-refresh", + checksum=checksum, + ) - if err: - LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) + if err: + LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) + stats["errors"] = stats.get("errors", 0) + 1 + else: + LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, checksum[:8]) + db_stored = True + except Exception as e: + LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 - else: - LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", - cert_name, ttl, checksum[:8]) - db_stored = True + else: + # If no database available, still proceed with disk storage + db_stored = True + + # === Only write to disk AFTER successful database storage === + if not db_stored: + LOG.warning("โš ๏ธ OCSP skipping disk storage for %s due to database storage failure", cert_name) + return + + # Create the SSL configs directory for this certificate + ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + try: + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) except Exception as e: - LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) + LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) stats["errors"] = stats.get("errors", 0) + 1 - else: - # If no database available, still proceed with disk storage - db_stored = True + return - # === Only write to disk AFTER successful database storage === - # This prevents overwriting existing OCSP files if the fetch failed or had issues - if not db_stored: - LOG.warning("โš ๏ธ OCSP skipping disk storage for %s due to database storage failure", cert_name) - return + # Write OCSP response to cache/ssl/{cert-name}/ocsp.der + ocsp_path = ocsp_cert_dir / "ocsp.der" + try: + ocsp_path.write_bytes(ocsp_der) + ocsp_path.chmod(0o644) # Readable by nginx + LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) + stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 - # Create the SSL configs directory for this certificate - ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name - try: - ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + # Create symlinks for each SAN so OCSP is found by any SNI name + _create_san_symlinks(pem_data, ocsp_path, cert_name) + except Exception as e: + LOG.error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 except Exception as e: - LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) + LOG.error("โŒ OCSP exception while processing certificate %s: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 + + +def process_custom_certs(db: Optional[Any] = None, stats: Optional[dict] = None) -> None: + """ + Process OCSP for custom certificates using certificate data from the database. + """ + if stats is None: + stats = {} + + if not db: + LOG.info("โ„น๏ธ OCSP database not available, cannot process custom certificates") return - # Write OCSP response to cache/ssl/{cert-name}/ocsp.der - ocsp_path = ocsp_cert_dir / "ocsp.der" try: - ocsp_path.write_bytes(ocsp_der) - ocsp_path.chmod(0o644) # Readable by nginx - LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) - stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 + custom_certs = _load_custom_certs_from_db(db) + if not custom_certs: + LOG.info("โ„น๏ธ OCSP no custom certificates found in database") + return + + LOG.info("๐Ÿ”„ OCSP loaded %d custom certificate(s) from database", len(custom_certs)) + stats["custom_certs_processed"] = len(custom_certs) + + for service_name, cert_pem in sorted(custom_certs.items()): + # service_name is e.g. "opdash.net-ecdsa" โ€” strip suffix to get actual service + svc = _service_name_from_dir(service_name) + if not _is_ocsp_enabled_for_service(svc): + LOG.info("๐Ÿงน OCSP stapling disabled for service %s, cleaning up custom cert cache for %s", svc, service_name) + cleanup_ocsp_cache(db, f"customcert-{service_name}") + stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + 1 + continue + + try: + # Warn and strip embedded private keys (should never be in cert file) + if b"-----BEGIN" in cert_pem and b"PRIVATE KEY" in cert_pem: + LOG.warning("โš ๏ธ OCSP custom cert for %s contains an embedded private key! This is a security risk โ€” private keys should not be stored in cert files.", service_name) + # Remove all private key PEM blocks to avoid logging secrets + cert_pem = re.sub(rb"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----\s*", b"", cert_pem) + + # Strip comment lines (lines starting with #) and content before first -----BEGIN + cert_pem = b"\n".join(line for line in cert_pem.split(b"\n") if not line.startswith(b"#")) + pem_start = cert_pem.find(b"-----BEGIN") + if pem_start > 0: + cert_pem = cert_pem[pem_start:] + elif pem_start < 0: + LOG.warning("โš ๏ธ OCSP custom cert for %s has no PEM BEGIN marker, skipping", service_name) + stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + 1 + continue + + # Validate it's a real certificate + try: + x509.load_pem_x509_certificate(cert_pem) + except Exception as e: + LOG.warning("โš ๏ธ OCSP custom cert for %s is not a valid PEM certificate (len=%d): %s", service_name, len(cert_pem), e) + LOG.debug("๐Ÿ” OCSP custom cert %s raw data (first 10000 bytes): %s", service_name, cert_pem[:10000]) + stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + 1 + continue + + LOG.info("๐Ÿ”„ OCSP processing custom certificate for service %s", service_name) + + # Use customcert- prefix for OCSP cache key + cert_name = f"customcert-{service_name}" + + # Extract OCSP URL from certificate + ocsp_url = extract_ocsp_url(cert_pem, cert_name) + if not ocsp_url: + LOG.info("โ„น๏ธ OCSP custom cert %s has no responder, skipping", cert_name) + stats["custom_certs_no_ocsp"] = stats.get("custom_certs_no_ocsp", 0) + 1 + continue + + LOG.info("๐ŸŒ OCSP responder for custom cert %s: %s", cert_name, ocsp_url) + + # Check if cached OCSP response is still fresh + cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name) + if cached_ttl is not None and total_lifetime is not None and total_lifetime > 0: + half_lifetime = total_lifetime // 2 + if cached_ttl > half_lifetime: + LOG.info( + "โœ“ OCSP cached response for custom cert %s still valid for %ds (%.1f days), above 50%% lifetime threshold=%ds (%.1f days), skipping fetch", + cert_name, + cached_ttl, + cached_ttl / 86400.0, + half_lifetime, + half_lifetime / 86400.0, + ) + stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 + continue + + ocsp_der: Optional[bytes] = None + ttl: int = 0 + + # Use a timeout of 10 seconds per attempt, retry once after 10 seconds + for attempt in (1, 2): + ocsp_der, ttl = fetch_ocsp_response(cert_pem, ocsp_url, cert_name=cert_name, timeout=10) + if ocsp_der: + LOG.debug("โœ“ OCSP successfully fetched response for custom cert %s on attempt %d (TTL=%ds)", cert_name, attempt, ttl) + break + if attempt == 1: + LOG.warning("โš ๏ธ OCSP fetch failed for custom cert %s, retrying once after 10 seconds ...", cert_name) + time.sleep(10) + + if not ocsp_der: + LOG.error("โŒ OCSP failed to fetch response for custom cert %s after retries", cert_name) + stats["errors"] = stats.get("errors", 0) + 1 + continue + + LOG.debug("โšก OCSP final TTL for custom cert %s is %ds", cert_name, ttl) + + # Calculate checksum for integrity verification + checksum = hashlib.sha256(ocsp_der).hexdigest() + + # Store OCSP response in database + db_stored = False + cache_key = f"ocsp/{cert_name}" + try: + err = db.upsert_job_cache( + service_id=None, + file_name=cache_key, + data=ocsp_der, + job_name="ocsp-refresh", + checksum=checksum, + ) + + if err: + LOG.error("โŒ OCSP error storing custom cert %s response in database: %s", cert_name, err) + stats["errors"] = stats.get("errors", 0) + 1 + else: + LOG.info("โœ“ OCSP stored custom cert %s response in database (TTL=%ds, checksum=%s)", cert_name, ttl, checksum[:8]) + db_stored = True + except Exception as e: + LOG.error("โŒ OCSP exception storing custom cert %s in database: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + + # Write to disk only after successful database storage + if not db_stored: + LOG.warning("โš ๏ธ OCSP skipping disk storage for custom cert %s due to database storage failure", cert_name) + continue + + # Create SSL configs directory for this certificate + ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + try: + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + except Exception as e: + LOG.error("โŒ OCSP error creating directory %s: %s", ocsp_cert_dir, e) + stats["errors"] = stats.get("errors", 0) + 1 + continue + + # Write OCSP response to cache + ocsp_path = ocsp_cert_dir / "ocsp.der" + try: + ocsp_path.write_bytes(ocsp_der) + ocsp_path.chmod(0o644) # Readable by nginx + LOG.info("โœ“ OCSP saved custom cert %s response to disk at %s", cert_name, ocsp_path) + stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 + + # Create symlinks for each SAN so OCSP is found by any SNI name + _create_san_symlinks(cert_pem, ocsp_path, cert_name) + except Exception as e: + LOG.error("โŒ OCSP error writing custom cert %s response to disk: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + + except Exception as e: + LOG.error("OCSP exception while processing custom certificate for %s: %s", service_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + except Exception as e: - LOG.error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) + LOG.error("OCSP exception while processing custom certificates: %s", e) stats["errors"] = stats.get("errors", 0) + 1 +def _service_name_from_dir(dir_name: str) -> str: + """Strip -rsa or -ecdsa suffix from a directory name to get the service name.""" + for suffix in ("-rsa", "-ecdsa"): + if dir_name.endswith(suffix): + return dir_name[: -len(suffix)] + return dir_name + + +def _is_ocsp_enabled_for_service(service_name: str) -> bool: + """Check if OCSP stapling is enabled for a specific service (multisite setting).""" + # Check service-specific setting first, fall back to global + value = os.getenv(f"{service_name}_SSL_USE_OCSP_STAPLING", os.getenv("SSL_USE_OCSP_STAPLING", "yes")) + return value.lower() == "yes" + + +def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None, purge_db: bool = True) -> None: + """ + Remove OCSP stapling leftovers from disk and optionally database. + If cert_name is provided, only clean up that specific certificate. + If cert_name is None, clean up ALL OCSP caches. + If purge_db is False, only disk files are removed โ€” database entries are preserved + so cached responses can be quickly restored when OCSP is re-enabled. + """ + if cert_name: + # Clean up a single certificate's OCSP cache + ocsp_dir = CONFIGS_SSL_BASE / cert_name + if ocsp_dir.is_dir(): + shutil.rmtree(ocsp_dir, ignore_errors=True) + LOG.info("๐Ÿงน OCSP removed cache directory for %s", cert_name) + + # Also remove any SAN symlinks that point to this cert's OCSP cache + if CONFIGS_SSL_BASE.is_dir(): + target_rel = Path("..") / cert_name / "ocsp.der" + for entry in CONFIGS_SSL_BASE.iterdir(): + if not entry.is_dir(): + continue + san_ocsp = entry / "ocsp.der" + if san_ocsp.is_symlink(): + try: + if san_ocsp.readlink() == target_rel: + san_ocsp.unlink() + LOG.debug("๐Ÿงน OCSP removed SAN symlink %s", san_ocsp) + # Remove empty directory + try: + entry.rmdir() + except OSError: + pass + except Exception: + pass + + if db and purge_db: + try: + cache_key = f"ocsp/{cert_name}" + db.delete_job_cache(file_name=cache_key, job_name="ocsp-refresh") + LOG.debug("๐Ÿงน OCSP removed database entry for %s", cert_name) + except Exception as e: + LOG.debug("๐Ÿงน OCSP could not remove database entry for %s: %s", cert_name, e) + else: + # Clean up ALL OCSP caches (including symlinks) + if CONFIGS_SSL_BASE.is_dir(): + for entry in sorted(CONFIGS_SSL_BASE.iterdir()): + if not entry.is_dir(): + continue + ocsp_file = entry / "ocsp.der" + if ocsp_file.is_symlink() or ocsp_file.is_file(): + ocsp_file.unlink(missing_ok=True) + LOG.info("๐Ÿงน OCSP removed cached response %s", ocsp_file) + # Remove the directory if it's now empty + try: + entry.rmdir() + except OSError: + pass # Directory not empty, leave it + + if db and purge_db: + try: + # Remove all OCSP cache entries from database + job_cache_files = db.get_jobs_cache_files(plugin_id="ssl") + for cache_file in job_cache_files: + if cache_file.get("file_name", "").startswith("ocsp/"): + try: + db.delete_job_cache(file_name=cache_file["file_name"], job_name="ocsp-refresh") + LOG.debug("๐Ÿงน OCSP removed database entry %s", cache_file["file_name"]) + except Exception as e: + LOG.debug("๐Ÿงน OCSP could not remove database entry %s: %s", cache_file["file_name"], e) + except Exception as e: + LOG.debug("๐Ÿงน OCSP could not clean database entries: %s", e) + elif not purge_db: + LOG.info("๐Ÿงน OCSP disk caches cleaned up (database entries preserved for quick restoration)") + + LOG.info("๐Ÿงน OCSP all stapling caches cleaned up") + + def main() -> int: global status db: Optional[Any] = None @@ -797,61 +881,54 @@ def main() -> int: } try: - # CRITICAL: Log immediately to confirm script is running - try: - LOG.error("๐Ÿ”„ OCSP DEBUG: script started") - except Exception as e: - print(f"ERROR: Could not log initial message: {e}", file=_sys.stderr) + LOG.info("๐Ÿ”„ OCSP refresh job started") - # Initialize database connection FIRST to ensure we can restore cached responses + # Initialize database connection โ€” this is our primary data source db = None if Database is not None: try: db = Database(LOG) LOG.debug("โœ“ OCSP database connection established") except Exception as e: - LOG.warning("โš ๏ธ OCSP could not establish database connection, will use disk-only storage: %s", e) - db = None + LOG.error("โŒ OCSP could not establish database connection: %s", e) + return 2 else: - LOG.debug("โ„น๏ธ OCSP Database module not available, will use disk-only storage") - - # IMPORTANT: Skip Job initialization (and automatic cleanup) entirely for safety - # Keeping cached OCSP files ensures we have data available even if errors occur - # The database restoration ensures consistency without needing to delete files - # Create a minimal Job-like placeholder object (job parameter is passed to functions but not used) - class MinimalJob: - def __init__(self, log): - self.log = log - - job = MinimalJob(LOG) - LOG.debug("โœ“ OCSP skipping Job initialization (cleanup disabled for data safety)") - - if not LIVE_BASE.is_dir(): - LOG.info("โ„น๏ธ OCSP live certificate directory %s does not exist, nothing to do", LIVE_BASE) + LOG.error("โŒ OCSP Database module not available, cannot proceed") + return 2 + + # Check if OCSP stapling is globally disabled + ocsp_enabled = os.getenv("SSL_USE_OCSP_STAPLING", "yes").lower() + if ocsp_enabled != "yes": + LOG.info("๐Ÿงน OCSP stapling is globally disabled (SSL_USE_OCSP_STAPLING=%s), cleaning up disk caches (preserving database for quick re-enable)", ocsp_enabled) + cleanup_ocsp_cache(db, purge_db=False) return 0 # Restore cached OCSP responses from database (handles ephemeral storage) - # This ensures consistency by restoring from persistent database storage - # Particularly important for ephemeral storage (tmpfs) or container restarts restore_ocsp_from_database(db) - cert_dirs = [d for d in sorted(LIVE_BASE.iterdir()) if d.is_dir()] - LOG.info("โ„น๏ธ OCSP found %d live certificate directories under %s", len(cert_dirs), LIVE_BASE) - stats["le_certs_processed"] = len(cert_dirs) + # Process Let's Encrypt certificates from database tarball + le_certs = _load_le_certs_from_db(db) + if le_certs: + LOG.info("โ„น๏ธ OCSP loaded %d LE certificate(s) from database", len(le_certs)) + stats["le_certs_processed"] = len(le_certs) - for cert_dir in cert_dirs: - fullchain = cert_dir / "fullchain.pem" - if not fullchain.is_file(): - LOG.debug("OCSP skipping %s because fullchain.pem is missing", cert_dir) - stats["le_certs_skipped"] += 1 - continue - process_one_cert_dir(job, cert_dir, db, stats) + for cert_name, pem_data in sorted(le_certs.items()): + _process_cert(cert_name, pem_data, db, stats) + else: + LOG.info("โ„น๏ธ OCSP no LE certificates found in database") + + # Process custom certificates from database + process_custom_certs(db, stats) + + # Decide exit status based on results + if stats["errors"] > 0: + status = 2 - # Process custom certificates - process_custom_certs(job, db, stats) + if stats["errors"] == 0: + LOG.info("โœ“ OCSP refresh job completed successfully") + else: + LOG.warning("โš ๏ธ OCSP refresh job completed with %d error(s)", stats["errors"]) - # Summary message with statistics - LOG.info("โœ“ OCSP refresh job completed successfully") LOG.info( "๐Ÿ“Š Statistics: ๐Ÿ” LE certs=%d (skipped=%d, no OCSP=%d) | ๐Ÿ” Custom certs=%d (no OCSP=%d) | ๐Ÿ”„ Fetched=%d | โœ“ Cached=%d | โŒ Errors=%d", stats["le_certs_processed"], @@ -869,14 +946,8 @@ def __init__(self, log): LOG.error("โŒ OCSP exception while running ocsp-refresh.py: %s", e) return 2 finally: - # Ensure database connection is properly closed - if db: - try: - db.close() - except Exception: - pass + pass # run it sys_exit(main()) - From a4b10afd089f4f8e5a57e6cc288a9c8feadac7f9 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 14:11:43 +0100 Subject: [PATCH 12/59] add checksum checking, cleanup function --- src/common/core/ssl/jobs/ocsp-refresh.py | 121 ++++++++++++++++++++--- 1 file changed, 108 insertions(+), 13 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 742a3da781..dce095eedf 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -450,9 +450,11 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: LOG.debug("โ„น๏ธ OCSP database not available, skipping cache restoration") return - LOG.info("๐Ÿ”„ OCSP restoring cached responses from database to disk...") + LOG.info("๐Ÿ”„ OCSP syncing cached responses from database to disk...") try: restored_count = 0 + replaced_count = 0 + ok_count = 0 # Get all OCSP cache entries from database for this job cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=True) @@ -462,23 +464,41 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: continue cert_name = file_name[len("ocsp/"):] + db_data = entry["data"] + db_checksum = entry.get("checksum") or hashlib.sha256(db_data).hexdigest() + try: ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name - ocsp_cert_dir.mkdir(parents=True, exist_ok=True) ocsp_path = ocsp_cert_dir / "ocsp.der" - ocsp_path.write_bytes(entry["data"]) - ocsp_path.chmod(0o644) - restored_count += 1 - LOG.debug("โœ“ OCSP restored cached response for %s from database", cert_name) + + if ocsp_path.is_file(): + # File exists โ€” compare checksum + disk_checksum = hashlib.sha256(ocsp_path.read_bytes()).hexdigest() + if disk_checksum == db_checksum: + ok_count += 1 + LOG.debug("โœ“ OCSP disk file for %s matches database (checksum=%s)", cert_name, db_checksum[:8]) + continue + # Checksum mismatch โ€” replace with database version + LOG.info("๐Ÿ”„ OCSP disk file for %s has wrong checksum (disk=%s, db=%s), replacing", cert_name, disk_checksum[:8], db_checksum[:8]) + ocsp_path.write_bytes(db_data) + ocsp_path.chmod(0o644) + replaced_count += 1 + else: + # File missing โ€” restore from database + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + ocsp_path.write_bytes(db_data) + ocsp_path.chmod(0o644) + restored_count += 1 + LOG.debug("โœ“ OCSP restored cached response for %s from database", cert_name) except Exception as e: - LOG.debug("โš ๏ธ OCSP could not restore cache for %s: %s", cert_name, e) + LOG.debug("โš ๏ธ OCSP could not sync cache for %s: %s", cert_name, e) - if restored_count > 0: - LOG.info("โœ“ OCSP restored %d cached responses from database to disk", restored_count) + if restored_count > 0 or replaced_count > 0: + LOG.info("โœ“ OCSP sync complete: restored=%d, replaced=%d, unchanged=%d", restored_count, replaced_count, ok_count) else: - LOG.debug("โ„น๏ธ OCSP no cached responses restored from database (cache may be cold)") + LOG.debug("โ„น๏ธ OCSP sync complete: all %d disk files match database (restored=0, replaced=0)", ok_count) except Exception as e: - LOG.debug("OCSP exception while attempting cache restoration: %s", e) + LOG.debug("OCSP exception while attempting cache sync: %s", e) def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None) -> None: @@ -863,6 +883,71 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None LOG.info("๐Ÿงน OCSP all stapling caches cleaned up") +def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: Optional[dict] = None) -> None: + """ + Remove OCSP cache entries (disk + database) for services that no longer have + certificates in the database. This handles deleted services. + """ + if stats is None: + stats = {} + + # Build set of valid cert names from LE certs + valid_cert_names: set = set(le_certs.keys()) + + # Add custom cert names (with customcert- prefix) + if db: + try: + custom_certs = _load_custom_certs_from_db(db) + for key in custom_certs: + valid_cert_names.add(f"customcert-{key}") + except Exception as e: + LOG.warning("โš ๏ธ OCSP could not load custom certs for orphan check: %s", e) + return # Don't clean up if we can't verify what's valid + + if not valid_cert_names: + LOG.debug("โ„น๏ธ OCSP no valid certs found, skipping orphan cleanup to avoid accidental deletion") + return + + orphaned_count = 0 + + # Check database OCSP entries + if db: + try: + cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=False) + for entry in cache_files: + file_name = entry.get("file_name", "") + if not file_name.startswith("ocsp/"): + continue + cert_name = file_name[len("ocsp/"):] + if cert_name not in valid_cert_names: + LOG.info("๐Ÿงน OCSP removing orphaned entry for deleted service: %s", cert_name) + cleanup_ocsp_cache(db, cert_name) + orphaned_count += 1 + except Exception as e: + LOG.warning("โš ๏ธ OCSP could not check database for orphaned entries: %s", e) + + # Check disk directories for orphaned OCSP files + if CONFIGS_SSL_BASE.is_dir(): + for entry in sorted(CONFIGS_SSL_BASE.iterdir()): + if not entry.is_dir(): + continue + ocsp_file = entry / "ocsp.der" + if not ocsp_file.is_file() and not ocsp_file.is_symlink(): + continue + cert_name = entry.name + if cert_name not in valid_cert_names: + # Check if it's a SAN symlink (those are managed by _create_san_symlinks) + if ocsp_file.is_symlink(): + continue + LOG.info("๐Ÿงน OCSP removing orphaned disk cache for deleted service: %s", cert_name) + cleanup_ocsp_cache(db, cert_name) + orphaned_count += 1 + + if orphaned_count > 0: + LOG.info("๐Ÿงน OCSP cleaned up %d orphaned cache entries for deleted services", orphaned_count) + stats["orphaned_cleaned"] = orphaned_count + + def main() -> int: global status db: Optional[Any] = None @@ -903,7 +988,13 @@ def main() -> int: cleanup_ocsp_cache(db, purge_db=False) return 0 - # Restore cached OCSP responses from database (handles ephemeral storage) + # Wait for scheduler's directory purge to finish after service restart, + # then restore cached OCSP responses from database to disk. + # This handles ephemeral storage and post-restart cache directory cleanup. + ocsp_files_exist = any((CONFIGS_SSL_BASE / d / "ocsp.der").is_file() for d in CONFIGS_SSL_BASE.iterdir()) if CONFIGS_SSL_BASE.is_dir() else False + if not ocsp_files_exist: + LOG.info("๐Ÿ”„ OCSP no cached files on disk, waiting 10s for scheduler purge to finish before restoring from database...") + time.sleep(10) restore_ocsp_from_database(db) # Process Let's Encrypt certificates from database tarball @@ -920,6 +1011,9 @@ def main() -> int: # Process custom certificates from database process_custom_certs(db, stats) + # Clean up orphaned OCSP entries for deleted services + _cleanup_orphaned_ocsp(db, le_certs or {}, stats) + # Decide exit status based on results if stats["errors"] > 0: status = 2 @@ -930,7 +1024,7 @@ def main() -> int: LOG.warning("โš ๏ธ OCSP refresh job completed with %d error(s)", stats["errors"]) LOG.info( - "๐Ÿ“Š Statistics: ๐Ÿ” LE certs=%d (skipped=%d, no OCSP=%d) | ๐Ÿ” Custom certs=%d (no OCSP=%d) | ๐Ÿ”„ Fetched=%d | โœ“ Cached=%d | โŒ Errors=%d", + "๐Ÿ“Š Statistics: ๐Ÿ” LE certs=%d (skipped=%d, no OCSP=%d) | ๐Ÿ” Custom certs=%d (no OCSP=%d) | ๐Ÿ”„ Fetched=%d | โœ“ Cached=%d | ๐Ÿงน Orphaned=%d | โŒ Errors=%d", stats["le_certs_processed"], stats["le_certs_skipped"], stats["le_certs_no_ocsp"], @@ -938,6 +1032,7 @@ def main() -> int: stats["custom_certs_no_ocsp"], stats["ocsp_fetched_responses"], stats["ocsp_cached_responses"], + stats.get("orphaned_cleaned", 0), stats["errors"], ) return status From 55df73d8be1590587bf99a2582fdb25ed916773e Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 14:14:03 +0100 Subject: [PATCH 13/59] Trigger OCSP stapling refresh after successful order of a LE cert or a custom cert upload trigger an OCSP stapling refresh so we can use OCSP stapling instantly --- .../core/customcert/jobs/custom-cert.py | 18 +++++++++++++++++ .../core/letsencrypt/jobs/certbot-new.py | 19 ++++++++++++++++++ .../core/letsencrypt/jobs/certbot-renew.py | 20 ++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/common/core/customcert/jobs/custom-cert.py b/src/common/core/customcert/jobs/custom-cert.py index e0f2e56863..e62e2c258c 100644 --- a/src/common/core/customcert/jobs/custom-cert.py +++ b/src/common/core/customcert/jobs/custom-cert.py @@ -218,6 +218,24 @@ def check_cert(cert_file: Union[Path, bytes], key_file: Union[Path, bytes], firs for first_server in skipped_servers: JOB.del_cache("cert.pem", service_id=first_server) JOB.del_cache("key.pem", service_id=first_server) + + # Trigger OCSP stapling refresh when certificates changed + if status == 1 and getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": + LOGGER.info("๐Ÿ”„ OCSP triggering refresh after custom certificate change") + try: + import sys + + ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") + result = run([sys.executable, ocsp_script], stdin=DEVNULL, capture_output=True, text=True, timeout=300) + if result.returncode == 0: + LOGGER.info("โœ“ OCSP refresh completed successfully after custom cert change") + else: + LOGGER.warning(f"โš ๏ธ OCSP refresh returned exit code {result.returncode}") + if result.stderr: + for line in result.stderr.strip().splitlines(): + LOGGER.debug(f"OCSP: {line}") + except Exception as e: + LOGGER.warning(f"โš ๏ธ OCSP post-upload refresh failed (non-fatal): {e}") except SystemExit as e: status = e.code except BaseException as e: diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index 006ad1c3aa..25390e62ab 100644 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -1107,6 +1107,25 @@ def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, D LOGGER.error(f"Error while saving data to db cache : {err}") else: LOGGER.info("Successfully saved data to db cache") + + # * Trigger OCSP stapling refresh for newly issued certificates + newly_issued = [svc for svc, cfg in services.items() if cfg.get("exists")] + if newly_issued and getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": + LOGGER.info(f"๐Ÿ”„ OCSP triggering refresh for {len(newly_issued)} newly issued cert(s): {', '.join(newly_issued)}") + try: + import sys + + ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") + result = run([sys.executable, ocsp_script], stdin=DEVNULL, capture_output=True, text=True, timeout=300) + if result.returncode == 0: + LOGGER.info("โœ“ OCSP refresh completed successfully after cert issuance") + else: + LOGGER.warning(f"โš ๏ธ OCSP refresh returned exit code {result.returncode}") + if result.stderr: + for line in result.stderr.strip().splitlines(): + LOGGER.debug(f"OCSP: {line}") + except Exception as e: + LOGGER.warning(f"โš ๏ธ OCSP post-issuance refresh failed (non-fatal): {e}") except SystemExit as e: status = e.code except BaseException as e: diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index e4430953fc..a7a274e6c9 100644 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -2,7 +2,7 @@ from os import getenv, sep from os.path import join -from subprocess import DEVNULL, PIPE, Popen +from subprocess import DEVNULL, PIPE, Popen, run from sys import exit as sys_exit, path as sys_path from traceback import format_exc @@ -99,6 +99,24 @@ LOGGER.error(f"Error while saving Let's Encrypt data to db cache : {err}") else: LOGGER.info("Successfully saved Let's Encrypt data to db cache") + + # Refresh OCSP stapling after successful renewal + if process.returncode == 0 and getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": + LOGGER.info("๐Ÿ”„ OCSP triggering refresh after certificate renewal") + try: + import sys + + ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") + result = run([sys.executable, ocsp_script], stdin=DEVNULL, capture_output=True, text=True, timeout=300) + if result.returncode == 0: + LOGGER.info("โœ“ OCSP refresh completed successfully after renewal") + else: + LOGGER.warning(f"โš ๏ธ OCSP refresh returned exit code {result.returncode}") + if result.stderr: + for line in result.stderr.strip().splitlines(): + LOGGER.debug(f"OCSP: {line}") + except Exception as e: + LOGGER.warning(f"โš ๏ธ OCSP post-renewal refresh failed (non-fatal): {e}") except SystemExit as e: status = e.code except BaseException as e: From d6ee17348d21f254d5a9f74811d926082cfd518c Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 14:35:00 +0100 Subject: [PATCH 14/59] OCSP Response Signature is NOT Cryptographically Verified Addressing the high-severity security issue where OCSP response signatures were not being cryptographically verified. --- src/common/core/ssl/jobs/ocsp-refresh.py | 42 ++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index dce095eedf..bd1dd230c2 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -23,6 +23,8 @@ from cryptography.x509.oid import ExtensionOID, AuthorityInformationAccessOID # Add BunkerWeb Python deps (Job, logger, Database) to path +# Add current script's parent directory to path for local imports +sys_path.append(str(Path(__file__).resolve().parent.parent.parent)) for deps_path in [ Path(os.sep, "usr", "share", "bunkerweb", *paths).as_posix() for paths in (("deps", "python"), ("utils",), ("db",)) @@ -222,8 +224,44 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim LOG.error("โŒ OCSP response status is not successful for %s: %s", cert_name, ocsp_response.response_status) return None, 0 - # Extract TTL from response timing fields - remaining, total_lifetime = _ocsp_response_lifetimes(ocsp_response) + # === SECURE OCSP RESPONSE VERIFICATION === + # Use OpenSSL to cryptographically verify the OCSP response signature. + # This prevents an attacker from MITM-ing the HTTP request and feeding a fake "SUCCESSFUL" payload. + import tempfile + import subprocess + from cryptography.hazmat.primitives.serialization import Encoding + + with tempfile.NamedTemporaryFile(suffix=".der", delete=True) as f_der, \ + tempfile.NamedTemporaryFile(suffix=".pem", mode="w", delete=True) as f_issuer, \ + tempfile.NamedTemporaryFile(suffix=".pem", mode="w", delete=True) as f_leaf: + + f_der.write(ocsp_response_data) + f_der.flush() + + f_issuer.write(issuer.public_bytes(Encoding.PEM).decode("utf-8")) + f_issuer.flush() + + f_leaf.write(leaf.public_bytes(Encoding.PEM).decode("utf-8")) + f_leaf.flush() + + # -respin checks the response + # -partial_chain allows the issuer to act as a trust anchor even if it's an intermediate + cmd = [ + "openssl", "ocsp", + "-respin", f_der.name, + "-issuer", f_issuer.name, + "-cert", f_leaf.name, + "-CAfile", f_issuer.name, + "-partial_chain" + ] + + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if p.returncode != 0: + LOG.error("โŒ OCSP response cryptographic signature verification failed for %s. Discarding forged/invalid response. OpenSSL Error: %s", cert_name, p.stderr.strip() or p.stdout.strip()) + return None, 0 + + # Extract TTL + remaining, _ = _ocsp_response_lifetimes(ocsp_response) if remaining is not None: ttl = min(remaining, 7 * 24 * 3600) # Cap to 7 days else: From 6c60a9af0b329c8eca7c82d678f583184112eeca Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 14:41:04 +0100 Subject: [PATCH 15/59] fix multiple vulnerability issues - SSRF checks when verifying the OCSP Issuer - protections against Path Traversal via untrusted certificate Subject Alternative Names (SANs). --- src/common/core/ssl/jobs/ocsp-refresh.py | 53 ++++++++++++++++++------ 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index bd1dd230c2..1124ac8c96 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -106,19 +106,30 @@ def _fetch_issuer_from_aia(leaf: x509.Certificate, cert_name: str = "") -> Optio aia = leaf.extensions.get_extension_for_oid(ExtensionOID.AUTHORITY_INFORMATION_ACCESS) for access_description in aia.value: if access_description.access_method == AuthorityInformationAccessOID.CA_ISSUERS: - url = access_description.access_location.value - parsed = urlparse(url) + issuer_url = access_description.access_location.value + parsed = urlparse(issuer_url) if parsed.scheme not in ("http", "https"): continue - LOG.debug("๐Ÿ”„ OCSP fetching issuer for %s from %s", cert_name, url) - req = Request(url, headers={"Accept": "application/pkix-cert, application/x-x509-ca-cert"}) - response = urlopen(req, timeout=10) - issuer_data = response.read() - # Try DER first (most common for caIssuers), then PEM + LOG.info("๐ŸŒ OCSP fetching issuer certificate from AIA: %s", issuer_url) + + if not is_safe_url(issuer_url): + LOG.error("โŒ OCSP unsafe AIA issuer URL detected: %s", issuer_url) + return None + + req = Request(issuer_url, headers={"User-Agent": "BunkerWeb OCSP Fetcher (SSRF-Protected)"}) try: - return x509.load_der_x509_certificate(issuer_data) - except Exception: - return x509.load_pem_x509_certificate(issuer_data) + # Use a short timeout so we don't hang + with urlopen(req, timeout=10) as response: + issuer_der = response.read() + # Try DER first (most common for caIssuers), then PEM + try: + return x509.load_der_x509_certificate(issuer_der) + except Exception: + return x509.load_pem_x509_certificate(issuer_der) + except URLError as e: + LOG.warning("โš ๏ธ OCSP network error fetching issuer from AIA for %s from %s: %s", cert_name, issuer_url, e) + except Exception as e: + LOG.warning("โš ๏ธ OCSP failed to fetch issuer from AIA for %s from %s: %s", cert_name, issuer_url, e) except x509.ExtensionNotFound: LOG.debug("๐Ÿ”’ OCSP no AIA extension in leaf cert for %s", cert_name) except Exception as e: @@ -208,11 +219,18 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim ocsp_request = builder.build() ocsp_request_data = ocsp_request.public_bytes(Encoding.DER) - # Send OCSP request via HTTP POST + # --- Build HTTP request --- + if not is_safe_url(ocsp_url): + LOG.error("โŒ OCSP unsafe responder URL detected: %s", ocsp_url) + return None, 0 + req = Request( ocsp_url, data=ocsp_request_data, - headers={"Content-Type": "application/ocsp-request"}, + headers={ + "Content-Type": "application/ocsp-request", + "User-Agent": "BunkerWeb OCSP Fetcher (SSRF-Protected)", + }, method="POST", ) response = urlopen(req, timeout=timeout) @@ -343,6 +361,12 @@ def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> No base_name = _service_name_from_dir(cert_name) for san in sans: + # Prevent Path Traversal by validating SAN format + # Allow alphanumeric, hyphens, dots, and wildcards (e.g., *.example.com) + if not re.match(r"^[A-Za-z0-9_.*-]+$", san): + LOG.warning("โš ๏ธ OCSP sanitization: skipping invalid/unsafe SAN %s", san) + continue + if san == base_name or san == cert_name: continue @@ -853,6 +877,11 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None so cached responses can be quickly restored when OCSP is re-enabled. """ if cert_name: + # Prevent Path Traversal during cleanup + if not re.match(r"^[A-Za-z0-9_.*-]+$", cert_name): + LOG.error("โŒ OCSP sanitization: refusing to clean up invalid/unsafe cert_name %s", cert_name) + return + # Clean up a single certificate's OCSP cache ocsp_dir = CONFIGS_SSL_BASE / cert_name if ocsp_dir.is_dir(): From 5fba3f45350926ddca854ea1d0287a0edec5b553 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 14:45:04 +0100 Subject: [PATCH 16/59] def is_safe_url --- src/common/core/ssl/jobs/ocsp-refresh.py | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 1124ac8c96..a8d523bd8f 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -66,6 +66,45 @@ CONFIGS_SSL_BASE = Path(os.sep, "var", "cache", "bunkerweb", "ssl") +def is_safe_url(url: str) -> bool: + """ + Validate that a URL is safe to fetch (HTTP/HTTPS only, no internal/private IPs). + Prevents Server-Side Request Forgery (SSRF) when fetching OCSP responses or issuer certs + from URLs embedded in untrusted certificates. + """ + import socket + import ipaddress + + try: + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + LOG.warning("โš ๏ธ SSRF protection: blocked URL with disallowed scheme: %s", url) + return False + + hostname = parsed.hostname + if not hostname: + LOG.warning("โš ๏ธ SSRF protection: blocked URL with no hostname: %s", url) + return False + + try: + addr_info = socket.getaddrinfo(hostname, None) + except socket.gaierror: + LOG.warning("โš ๏ธ SSRF protection: DNS resolution failed for %s", hostname) + return False + + for info in addr_info: + ip_str = info[4][0] + ip_obj = ipaddress.ip_address(ip_str) + if ip_obj.is_private or ip_obj.is_loopback or ip_obj.is_link_local or ip_obj.is_multicast or ip_obj.is_reserved: + LOG.warning("โš ๏ธ SSRF protection: blocked request to %s (resolves to internal IP %s)", hostname, ip_str) + return False + + return True + except Exception as e: + LOG.debug("SSRF protection: error validating URL %s: %s", url, e) + return False + + def extract_ocsp_url(pem_data: bytes, cert_name: str = "") -> Optional[str]: """ Extract OCSP responder URL from PEM certificate data using the cryptography library. From 6b5bef6209be7cfbda9332761830ea0abeba77df Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 14:51:41 +0100 Subject: [PATCH 17/59] Update ocsp-refresh.py - Path traversal via tarball cert_name and DB restore cert - Redundant import - Debug log data leak, unbounded HTTP response reads --- src/common/core/ssl/jobs/ocsp-refresh.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index a8d523bd8f..26fc427dd7 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -5,8 +5,10 @@ import os import re import shutil +import subprocess import sys as _sys import tarfile +import tempfile import time from datetime import datetime, timezone, timedelta from pathlib import Path @@ -159,7 +161,7 @@ def _fetch_issuer_from_aia(leaf: x509.Certificate, cert_name: str = "") -> Optio try: # Use a short timeout so we don't hang with urlopen(req, timeout=10) as response: - issuer_der = response.read() + issuer_der = response.read(1_048_576) # Cap at 1MB to prevent memory exhaustion # Try DER first (most common for caIssuers), then PEM try: return x509.load_der_x509_certificate(issuer_der) @@ -273,7 +275,7 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim method="POST", ) response = urlopen(req, timeout=timeout) - ocsp_response_data = response.read() + ocsp_response_data = response.read(102_400) # Cap at 100KB โ€” typical OCSP response is 1-4KB # Parse response and validate status ocsp_response = x509_ocsp.load_der_ocsp_response(ocsp_response_data) @@ -284,9 +286,8 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim # === SECURE OCSP RESPONSE VERIFICATION === # Use OpenSSL to cryptographically verify the OCSP response signature. # This prevents an attacker from MITM-ing the HTTP request and feeding a fake "SUCCESSFUL" payload. - import tempfile - import subprocess - from cryptography.hazmat.primitives.serialization import Encoding + # Verify the OCSP response signature using OpenSSL CLI. + # tempfile and subprocess are imported at the top of the file. with tempfile.NamedTemporaryFile(suffix=".der", delete=True) as f_der, \ tempfile.NamedTemporaryFile(suffix=".pem", mode="w", delete=True) as f_issuer, \ @@ -474,6 +475,10 @@ def _load_le_certs_from_db(db: Any) -> Dict[str, bytes]: idx = 1 if parts[0] == "." else 0 if len(parts) == idx + 3 and parts[idx] == "live" and parts[idx + 2] == "fullchain.pem": cert_name = parts[idx + 1] + # Prevent path traversal from crafted tarball entries + if not re.match(r"^[A-Za-z0-9_.*-]+$", cert_name): + LOG.warning("โš ๏ธ OCSP sanitization: skipping unsafe cert_name from tarball: %s", cert_name) + continue try: f = tar.extractfile(member) if f: @@ -565,6 +570,10 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: continue cert_name = file_name[len("ocsp/"):] + # Prevent path traversal from crafted database entries + if not re.match(r"^[A-Za-z0-9_.*-]+$", cert_name): + LOG.warning("โš ๏ธ OCSP sanitization: skipping unsafe cert_name from database: %s", cert_name) + continue db_data = entry["data"] db_checksum = entry.get("checksum") or hashlib.sha256(db_data).hexdigest() From 981178e2a10d245ab65161e2d4c447df18b7807e Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 15:01:00 +0100 Subject: [PATCH 18/59] Update ocsp-refresh.py - Path Traversal Prevention - Temporary File Permission Hardening - Debug Logging Reduction --- src/common/core/ssl/jobs/ocsp-refresh.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 26fc427dd7..ddac61ac91 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -292,13 +292,18 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim with tempfile.NamedTemporaryFile(suffix=".der", delete=True) as f_der, \ tempfile.NamedTemporaryFile(suffix=".pem", mode="w", delete=True) as f_issuer, \ tempfile.NamedTemporaryFile(suffix=".pem", mode="w", delete=True) as f_leaf: - + + # Secure temporary files with 0600 permissions (read/write for owner only) + os.chmod(f_der.name, 0o600) + os.chmod(f_issuer.name, 0o600) + os.chmod(f_leaf.name, 0o600) + f_der.write(ocsp_response_data) f_der.flush() - + f_issuer.write(issuer.public_bytes(Encoding.PEM).decode("utf-8")) f_issuer.flush() - + f_leaf.write(leaf.public_bytes(Encoding.PEM).decode("utf-8")) f_leaf.flush() @@ -531,6 +536,10 @@ def _load_custom_certs_from_db(db: Any) -> Dict[str, bytes]: if not service_id: LOG.debug("โš ๏ธ OCSP custom cert entry %s has no service_id, skipping", file_name) continue + # Prevent path traversal: validate service_id format + if not re.match(r"^[A-Za-z0-9_.*-]+$", service_id): + LOG.warning("โš ๏ธ OCSP sanitization: skipping custom cert with invalid service_id: %s", service_id) + continue data = entry.get("data") if not data: LOG.warning("โš ๏ธ OCSP custom cert %s for service %s has no data, skipping", file_name, service_id) @@ -538,6 +547,10 @@ def _load_custom_certs_from_db(db: Any) -> Dict[str, bytes]: # Derive suffix from filename: cert-ecdsa.pem -> -ecdsa, cert.pem -> "" suffix = file_name.replace("cert", "").replace(".pem", "") # e.g. "-ecdsa", "-rsa", "" key = f"{service_id}{suffix}" + # Double-check derived key is safe (defensive validation) + if not re.match(r"^[A-Za-z0-9_.*-]+$", key): + LOG.warning("โš ๏ธ OCSP sanitization: skipping custom cert with unsafe derived key: %s", key) + continue result[key] = data LOG.debug("โœ“ OCSP loaded custom cert for %s from database (file=%s, size=%d)", key, file_name, len(data)) except Exception as e: @@ -785,7 +798,7 @@ def process_custom_certs(db: Optional[Any] = None, stats: Optional[dict] = None) x509.load_pem_x509_certificate(cert_pem) except Exception as e: LOG.warning("โš ๏ธ OCSP custom cert for %s is not a valid PEM certificate (len=%d): %s", service_name, len(cert_pem), e) - LOG.debug("๐Ÿ” OCSP custom cert %s raw data (first 10000 bytes): %s", service_name, cert_pem[:10000]) + LOG.debug("๐Ÿ” OCSP custom cert %s raw data (first 200 bytes): %s", service_name, cert_pem[:200]) stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + 1 continue From ffa38d75319ee6a92fd356e0b1a986417c5a6ade Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 15:42:08 +0100 Subject: [PATCH 19/59] add locking, error handling, verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Acquire a file-based lock for a specific certificate to prevent race conditions when multiple scheduler instances fetch OCSP responses concurrently. - HTTP error code โ€” OCSP responder returned an error - End-of-job verification: ensure all OCSP files on disk match database checksums. Restores missing files from database to prevent OCSP stapling failures. This handles cases where OCSP cache directories are externally deleted or corrupted. --- src/common/core/ssl/jobs/ocsp-refresh.py | 378 +++++++++++++++++++++-- 1 file changed, 347 insertions(+), 31 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index ddac61ac91..002e386d34 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 +import fcntl import hashlib import io import os import re import shutil +import ssl import subprocess import sys as _sys import tarfile @@ -14,7 +16,7 @@ from pathlib import Path from sys import exit as sys_exit, path as sys_path from typing import Any, Dict, List, Optional, Tuple -from urllib.error import URLError +from urllib.error import HTTPError, URLError from urllib.parse import urlparse from urllib.request import Request, urlopen @@ -68,11 +70,125 @@ CONFIGS_SSL_BASE = Path(os.sep, "var", "cache", "bunkerweb", "ssl") +def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int = 600) -> Optional[int]: + """ + Acquire a file-based lock for a specific certificate to prevent race conditions + when multiple scheduler instances fetch OCSP responses concurrently. + + Lock files are stored in /tmp/bunkerweb/ and use timestamp-based staleness detection. + If a lock file hasn't been updated in stale_threshold seconds, it's considered stale + (indicating the previous process crashed or is hung). + + Args: + cert_name: Certificate identifier + timeout: Maximum seconds to wait for lock acquisition (default 300 = 5 minutes) + stale_threshold: Lock file age (seconds) to consider it stale (default 600 = 10 minutes) + + Returns: + File descriptor on success, None if lock unavailable or timed out + """ + lock_dir = Path(os.sep, "tmp", "bunkerweb") + try: + lock_dir.mkdir(parents=True, exist_ok=True, mode=0o755) + except Exception: + return None + + lock_file = lock_dir / f"ocsp-{cert_name}.lock" + start_time = time.time() + + while time.time() - start_time < timeout: + # Check if existing lock file is stale (crashed process) + try: + if lock_file.exists(): + mtime = lock_file.stat().st_mtime + age = time.time() - mtime + if age > stale_threshold: + LOG.warning( + "โš ๏ธ OCSP detected stale lock for %s (age %.0fs > threshold %ds). " + "Previous process may have crashed. Removing stale lock.", + cert_name, age, stale_threshold + ) + try: + lock_file.unlink() + except Exception: + pass # If we can't delete, we'll wait more + except Exception: + pass + + try: + fd = os.open(str(lock_file), os.O_CREAT | os.O_WRONLY, 0o600) + # Try non-blocking lock acquisition (LOCK_NB) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + # Lock acquired successfullyโ€”write timestamp to detect stale locks + try: + os.write(fd, str(int(time.time())).encode()) + except Exception: + pass # Non-critical timestamp write + return fd + except BlockingIOError: + # Lock held by another process, close fd and retry after brief delay + os.close(fd) + time.sleep(0.5) + continue + except Exception as e: + LOG.debug("โš ๏ธ OCSP lock acquisition attempt failed for %s: %s", cert_name, e) + time.sleep(0.5) + + # Timeout reached: log warning and proceed without lock + LOG.warning( + "โš ๏ธ OCSP could not acquire lock for %s after %ds. " + "If concurrent OCSP fetches happen, cryptographic verification protects data integrity. " + "Consider increasing scheduler interval or timeout if jobs consistently exceed %ds.", + cert_name, timeout, timeout + ) + return None + + +def _release_cert_lock(fd: Optional[int]) -> None: + """Release a file-based lock.""" + if fd is not None: + try: + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + except Exception: + pass + + +def _refresh_cert_lock(fd: Optional[int], cert_name: str) -> None: + """ + Refresh the lock file timestamp to prove the process is still alive. + Call this periodically while holding a lock during long-running operations. + Prevents stale lock detection from triggering on jobs that legitimately take >30 seconds. + + Args: + fd: File descriptor from _acquire_cert_lock + cert_name: Certificate identifier (for logging only) + """ + if fd is None: + return + + try: + # Seek to start and truncate to update modification time and content + os.lseek(fd, 0, os.SEEK_SET) + os.ftruncate(fd, 0) + timestamp = str(int(time.time())).encode() + os.write(fd, timestamp) + os.fsync(fd) # Ensure write is persisted + LOG.debug("๐Ÿ”’ OCSP refreshed lock timestamp for %s", cert_name) + except Exception as e: + LOG.debug("โš ๏ธ OCSP could not refresh lock timestamp for %s: %s", cert_name, e) + + def is_safe_url(url: str) -> bool: """ Validate that a URL is safe to fetch (HTTP/HTTPS only, no internal/private IPs). Prevents Server-Side Request Forgery (SSRF) when fetching OCSP responses or issuer certs from URLs embedded in untrusted certificates. + + Security note: Even if DNS rebinding occurs, the OCSP response is cryptographically + verified against the issuer's public key (see fetch_ocsp_response). An attacker cannot + forge a valid OCSP response without the issuer's private key. """ import socket import ipaddress @@ -167,6 +283,8 @@ def _fetch_issuer_from_aia(leaf: x509.Certificate, cert_name: str = "") -> Optio return x509.load_der_x509_certificate(issuer_der) except Exception: return x509.load_pem_x509_certificate(issuer_der) + except HTTPError as e: + LOG.warning("โš ๏ธ OCSP HTTP %d error fetching issuer from AIA for %s from %s: %s", e.code, cert_name, issuer_url, e.reason) except URLError as e: LOG.warning("โš ๏ธ OCSP network error fetching issuer from AIA for %s from %s: %s", cert_name, issuer_url, e) except Exception as e: @@ -274,7 +392,9 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim }, method="POST", ) - response = urlopen(req, timeout=timeout) + # Explicitly use default SSL context with certificate verification enabled + ssl_context = ssl.create_default_context() + response = urlopen(req, timeout=timeout, context=ssl_context) ocsp_response_data = response.read(102_400) # Cap at 100KB โ€” typical OCSP response is 1-4KB # Parse response and validate status @@ -331,7 +451,23 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim ttl = 86400 # RFC standard fallback return ocsp_response_data, ttl + except HTTPError as e: + # HTTP error code โ€” OCSP responder returned an error + error_desc = f"HTTP {e.code}" + if e.code == 404: + error_desc += " (Not Found - OCSP responder URL not available)" + elif e.code == 503: + error_desc += " (Service Unavailable - responder temporarily down)" + elif e.code == 500: + error_desc += " (Internal Server Error - responder misconfigured)" + elif 400 <= e.code < 500: + error_desc += f" (Client Error - {e.reason})" + elif e.code >= 500: + error_desc += f" (Server Error - {e.reason})" + LOG.error("โŒ OCSP %s from responder for %s at %s", error_desc, cert_name, ocsp_url) + return None, 0 except URLError as e: + # Network error โ€” DNS, connection refused, timeout, SSL error, etc. LOG.error("โŒ OCSP network error fetching response for %s from %s: %s", cert_name, ocsp_url, e) return None, 0 except Exception as e: @@ -720,36 +856,54 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta LOG.warning("โš ๏ธ OCSP skipping disk storage for %s due to database storage failure", cert_name) return - # Create the SSL configs directory for this certificate - ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + # Acquire lock to prevent race conditions with concurrent OCSP fetches + lock_fd = _acquire_cert_lock(cert_name) try: - ocsp_cert_dir.mkdir(parents=True, exist_ok=True) - except Exception as e: - LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) - stats["errors"] = stats.get("errors", 0) + 1 - return + # Create the SSL configs directory for this certificate + ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + try: + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + except Exception as e: + LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) + stats["errors"] = stats.get("errors", 0) + 1 + return - # Write OCSP response to cache/ssl/{cert-name}/ocsp.der - ocsp_path = ocsp_cert_dir / "ocsp.der" - try: - ocsp_path.write_bytes(ocsp_der) - ocsp_path.chmod(0o644) # Readable by nginx - LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) - stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 + # Write OCSP response to cache/ssl/{cert-name}/ocsp.der + ocsp_path = ocsp_cert_dir / "ocsp.der" + try: + ocsp_path.write_bytes(ocsp_der) + ocsp_path.chmod(0o644) # Readable by nginx + LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) + stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 - # Create symlinks for each SAN so OCSP is found by any SNI name - _create_san_symlinks(pem_data, ocsp_path, cert_name) - except Exception as e: - LOG.error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) - stats["errors"] = stats.get("errors", 0) + 1 + # Create symlinks for each SAN so OCSP is found by any SNI name + _create_san_symlinks(pem_data, ocsp_path, cert_name) + except Exception as e: + LOG.error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + finally: + _release_cert_lock(lock_fd) except Exception as e: LOG.error("โŒ OCSP exception while processing certificate %s: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 -def process_custom_certs(db: Optional[Any] = None, stats: Optional[dict] = None) -> None: +def process_custom_certs( + db: Optional[Any] = None, + stats: Optional[dict] = None, + lock_fd: Optional[int] = None, + refresh_fn: Optional[callable] = None, + timeout_fn: Optional[callable] = None, +) -> None: """ Process OCSP for custom certificates using certificate data from the database. + + Args: + db: Database connection + stats: Statistics dictionary + lock_fd: Optional file descriptor for lock refresh (for long operations) + refresh_fn: Optional callable to refresh lock (prevents stale detection) + timeout_fn: Optional callable to check job timeout """ if stats is None: stats = {} @@ -768,6 +922,13 @@ def process_custom_certs(db: Optional[Any] = None, stats: Optional[dict] = None) stats["custom_certs_processed"] = len(custom_certs) for service_name, cert_pem in sorted(custom_certs.items()): + # Check timeout before processing each certificate + if timeout_fn and timeout_fn(f"processing custom cert {service_name}"): + break + + # Refresh lock to prove we're still alive + if refresh_fn: + refresh_fn(service_name) # service_name is e.g. "opdash.net-ecdsa" โ€” strip suffix to get actual service svc = _service_name_from_dir(service_name) if not _is_ocsp_enabled_for_service(svc): @@ -798,7 +959,8 @@ def process_custom_certs(db: Optional[Any] = None, stats: Optional[dict] = None) x509.load_pem_x509_certificate(cert_pem) except Exception as e: LOG.warning("โš ๏ธ OCSP custom cert for %s is not a valid PEM certificate (len=%d): %s", service_name, len(cert_pem), e) - LOG.debug("๐Ÿ” OCSP custom cert %s raw data (first 200 bytes): %s", service_name, cert_pem[:200]) + cert_checksum = hashlib.sha256(cert_pem).hexdigest() + LOG.debug("๐Ÿ” OCSP custom cert %s checksum: %s", service_name, cert_checksum) stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + 1 continue @@ -995,7 +1157,7 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None if db and purge_db: try: # Remove all OCSP cache entries from database - job_cache_files = db.get_jobs_cache_files(plugin_id="ssl") + job_cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh") for cache_file in job_cache_files: if cache_file.get("file_name", "").startswith("ocsp/"): try: @@ -1076,10 +1238,138 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: stats["orphaned_cleaned"] = orphaned_count +def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dict] = None) -> None: + """ + End-of-job verification: ensure all OCSP files on disk match database checksums. + Restores missing files from database to prevent OCSP stapling failures. + + This handles cases where OCSP cache directories are externally deleted or corrupted. + Includes sleep before verification to allow concurrent writers to finish. + """ + if stats is None: + stats = {} + + if not db: + LOG.warning("โš ๏ธ OCSP cannot verify files without database connection") + return + + # Sleep briefly to avoid race conditions with concurrent writes to cache directories + LOG.debug("๐Ÿ’ค OCSP waiting 5s before verification (to allow concurrent writers to finish)...") + time.sleep(5) + + verify_count = 0 + restored_count = 0 + mismatch_count = 0 + + try: + # Get all OCSP entries from database + cache_entries = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=True) + if not cache_entries: + LOG.info("โ„น๏ธ OCSP no cache entries in database to verify") + return + + LOG.info("๐Ÿ” OCSP verifying %d cache entry(ies) from database", len(cache_entries)) + + for entry in cache_entries: + file_name = entry.get("file_name", "") + if not file_name.startswith("ocsp/"): + continue + + cert_name = file_name[len("ocsp/"):] + data = entry.get("data") + db_checksum = entry.get("checksum", "") + + if not data or not db_checksum: + LOG.warning("โš ๏ธ OCSP database entry %s has no data or checksum, skipping verification", cert_name) + continue + + verify_count += 1 + + # Check if file exists on disk + ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + ocsp_path = ocsp_cert_dir / "ocsp.der" + + try: + if not ocsp_path.is_file(): + # File missing: restore from database + LOG.warning("โš ๏ธ OCSP file missing for %s, restoring from database", cert_name) + try: + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + ocsp_path.write_bytes(data) + ocsp_path.chmod(0o644) + LOG.info("โœ“ OCSP restored %s from database", cert_name) + restored_count += 1 + except Exception as e: + LOG.error("โŒ OCSP could not restore %s from database: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + continue + + else: + # File exists: verify checksum + file_data = ocsp_path.read_bytes() + file_checksum = hashlib.sha256(file_data).hexdigest() + + if file_checksum != db_checksum: + LOG.warning( + "โš ๏ธ OCSP checksum mismatch for %s (file=%s, db=%s). " + "Restoring from database.", + cert_name, file_checksum[:8], db_checksum[:8] + ) + try: + ocsp_path.write_bytes(data) + ocsp_path.chmod(0o644) + LOG.info("โœ“ OCSP restored correct version of %s", cert_name) + mismatch_count += 1 + except Exception as e: + LOG.error("โŒ OCSP could not restore %s: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + else: + LOG.debug("โœ“ OCSP %s checksum verified (matches database)", cert_name) + + except Exception as e: + LOG.warning("โš ๏ธ OCSP error verifying %s: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + + if verify_count > 0: + LOG.info( + "๐Ÿ” OCSP verification complete: %d checked | โœ“ %d restored (missing) | ๐Ÿ”„ %d corrected (mismatch)", + verify_count, restored_count, mismatch_count + ) + stats["ocsp_verified"] = verify_count + stats["ocsp_restored"] = restored_count + stats["ocsp_corrected"] = mismatch_count + + except Exception as e: + LOG.warning("โš ๏ธ OCSP verification failed: %s", e) + stats["errors"] = stats.get("errors", 0) + 1 + + def main() -> int: global status db: Optional[Any] = None + # Job-level timeout: exit gracefully if exceeded (prevents deadlock on slow systems) + # Responses fetched so far are saved; next run continues where left off + JOB_TIMEOUT = 1500 # 25 minutes in seconds + job_start_time = time.time() + lock_fd_main = None + + def check_job_timeout(phase: str = "") -> bool: + """Check if job has exceeded timeout. Returns True if timeout exceeded.""" + elapsed = time.time() - job_start_time + if elapsed > JOB_TIMEOUT: + LOG.warning( + "โฑ๏ธ OCSP job timeout after %.0fs (%.1f minutes) %s. " + "Saved responses fetched so far; next run will continue processing remaining certificates.", + elapsed, elapsed / 60.0, phase + ) + return True + return False + + def refresh_job_lock(cert_name: str = "") -> None: + """Refresh the lock to prevent stale detection during long jobs.""" + _refresh_cert_lock(lock_fd_main, cert_name or "main") + # Statistics tracking stats = { "le_certs_processed": 0, @@ -1094,7 +1384,10 @@ def main() -> int: } try: - LOG.info("๐Ÿ”„ OCSP refresh job started") + LOG.info("๐Ÿ”„ OCSP refresh job started (timeout in %d minutes)", JOB_TIMEOUT // 60) + + # Acquire main lock for the entire OCSP refresh operation + lock_fd_main = _acquire_cert_lock("main", timeout=300, stale_threshold=1800) # Initialize database connection โ€” this is our primary data source db = None @@ -1132,15 +1425,32 @@ def main() -> int: stats["le_certs_processed"] = len(le_certs) for cert_name, pem_data in sorted(le_certs.items()): + # Check timeout before processing each certificate + if check_job_timeout(f"processing LE cert {cert_name}"): + break + + # Refresh lock to prove we're still alive + refresh_job_lock(cert_name) + _process_cert(cert_name, pem_data, db, stats) else: LOG.info("โ„น๏ธ OCSP no LE certificates found in database") - # Process custom certificates from database - process_custom_certs(db, stats) + # Check timeout before processing custom certs + if not check_job_timeout("before custom cert processing"): + # Process custom certificates from database + process_custom_certs(db, stats, lock_fd=lock_fd_main, refresh_fn=refresh_job_lock, timeout_fn=check_job_timeout) + + # Check timeout before cleanup + if not check_job_timeout("before orphaned cleanup"): + # Clean up orphaned OCSP entries for deleted services + _cleanup_orphaned_ocsp(db, le_certs or {}, stats) - # Clean up orphaned OCSP entries for deleted services - _cleanup_orphaned_ocsp(db, le_certs or {}, stats) + # End-of-job verification: ensure OCSP files match database checksums + # Restores missing files from database + if not check_job_timeout("before final verification"): + LOG.info("๐Ÿ” OCSP running end-of-job verification and restoration...") + _verify_and_restore_ocsp_files(db, stats) # Decide exit status based on results if stats["errors"] > 0: @@ -1151,8 +1461,10 @@ def main() -> int: else: LOG.warning("โš ๏ธ OCSP refresh job completed with %d error(s)", stats["errors"]) + elapsed = time.time() - job_start_time LOG.info( - "๐Ÿ“Š Statistics: ๐Ÿ” LE certs=%d (skipped=%d, no OCSP=%d) | ๐Ÿ” Custom certs=%d (no OCSP=%d) | ๐Ÿ”„ Fetched=%d | โœ“ Cached=%d | ๐Ÿงน Orphaned=%d | โŒ Errors=%d", + "๐Ÿ“Š Statistics (completed in %.0fs): ๐Ÿ” LE certs=%d (skipped=%d, no OCSP=%d) | ๐Ÿ” Custom certs=%d (no OCSP=%d) | ๐Ÿ”„ Fetched=%d | โœ“ Cached=%d | ๐Ÿงน Orphaned=%d | ๐Ÿ” Verified=%d (restored=%d, corrected=%d) | โŒ Errors=%d", + elapsed, stats["le_certs_processed"], stats["le_certs_skipped"], stats["le_certs_no_ocsp"], @@ -1161,6 +1473,9 @@ def main() -> int: stats["ocsp_fetched_responses"], stats["ocsp_cached_responses"], stats.get("orphaned_cleaned", 0), + stats.get("ocsp_verified", 0), + stats.get("ocsp_restored", 0), + stats.get("ocsp_corrected", 0), stats["errors"], ) return status @@ -1169,7 +1484,8 @@ def main() -> int: LOG.error("โŒ OCSP exception while running ocsp-refresh.py: %s", e) return 2 finally: - pass + # Always release the main lock + _release_cert_lock(lock_fd_main) # run it From 1321d90d670383c1419808e1a869371aed4a394c Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 15:57:30 +0100 Subject: [PATCH 20/59] fix tlinter errors --- src/common/core/ssl/jobs/ocsp-refresh.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 002e386d34..08c9243c00 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -15,7 +15,7 @@ from datetime import datetime, timezone, timedelta from pathlib import Path from sys import exit as sys_exit, path as sys_path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple, cast from urllib.error import HTTPError, URLError from urllib.parse import urlparse from urllib.request import Request, urlopen @@ -24,6 +24,7 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.serialization import Encoding from cryptography.x509 import ocsp as x509_ocsp +from cryptography.x509 import AuthorityInformationAccess, SubjectAlternativeName from cryptography.x509.oid import ExtensionOID, AuthorityInformationAccessOID # Add BunkerWeb Python deps (Job, logger, Database) to path @@ -234,7 +235,8 @@ def extract_ocsp_url(pem_data: bytes, cert_name: str = "") -> Optional[str]: cert = x509.load_pem_x509_certificate(pem_data) aia = cert.extensions.get_extension_for_oid(ExtensionOID.AUTHORITY_INFORMATION_ACCESS) - for access_description in aia.value: + aia_value = cast(AuthorityInformationAccess, aia.value) + for access_description in aia_value: if access_description.access_method == AuthorityInformationAccessOID.OCSP: url = access_description.access_location.value parsed = urlparse(url) @@ -261,7 +263,8 @@ def _fetch_issuer_from_aia(leaf: x509.Certificate, cert_name: str = "") -> Optio """ try: aia = leaf.extensions.get_extension_for_oid(ExtensionOID.AUTHORITY_INFORMATION_ACCESS) - for access_description in aia.value: + aia_value = cast(AuthorityInformationAccess, aia.value) + for access_description in aia_value: if access_description.access_method == AuthorityInformationAccessOID.CA_ISSUERS: issuer_url = access_description.access_location.value parsed = urlparse(issuer_url) @@ -483,7 +486,8 @@ def extract_san_dns(pem_data: bytes, cert_name: str = "") -> List[str]: try: cert = x509.load_pem_x509_certificate(pem_data) san = cert.extensions.get_extension_for_oid(ExtensionOID.SUBJECT_ALTERNATIVE_NAME) - names = san.value.get_values_for_type(x509.DNSName) + san_value = cast(SubjectAlternativeName, san.value) + names = san_value.get_values_for_type(x509.DNSName) return sorted(set(names)) except x509.ExtensionNotFound: return [] @@ -516,6 +520,10 @@ def get_cached_ocsp_ttl(cert_name: str) -> Tuple[Optional[int], Optional[int]]: LOG.info("๐Ÿ”„ OCSP could not determine precise lifetime for %s from cached response", cert_name) return None, None + # Type assertion: after None checks above, these are guaranteed to be int + assert remaining is not None + assert total_lifetime is not None + LOG.info( "โšก OCSP cached response for %s: remaining=%ds (%.1f days), total_lifetime=%ds (%.1f days)", cert_name, @@ -892,8 +900,8 @@ def process_custom_certs( db: Optional[Any] = None, stats: Optional[dict] = None, lock_fd: Optional[int] = None, - refresh_fn: Optional[callable] = None, - timeout_fn: Optional[callable] = None, + refresh_fn: Optional[Callable[[str], None]] = None, + timeout_fn: Optional[Callable[[str], bool]] = None, ) -> None: """ Process OCSP for custom certificates using certificate data from the database. @@ -1371,7 +1379,7 @@ def refresh_job_lock(cert_name: str = "") -> None: _refresh_cert_lock(lock_fd_main, cert_name or "main") # Statistics tracking - stats = { + stats: Dict[str, int] = { "le_certs_processed": 0, "le_certs_skipped": 0, "le_certs_no_ocsp": 0, From 426cc52dd6eb7dcc94e7a4ccc5e6dee1f6f52963 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 16:13:35 +0100 Subject: [PATCH 21/59] add rate limiting --- src/common/core/ssl/jobs/ocsp-refresh.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 08c9243c00..987bd551ad 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -453,6 +453,10 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim else: ttl = 86400 # RFC standard fallback + # Delay after successful fetch to prevent rate limiting + LOG.debug("โธ๏ธ OCSP delaying 5 seconds after fetch for %s to prevent rate limiting", cert_name) + time.sleep(5) + return ocsp_response_data, ttl except HTTPError as e: # HTTP error code โ€” OCSP responder returned an error From 3d259d45eddeeec91ec952a6c9172cb828362886 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 16:32:04 +0100 Subject: [PATCH 22/59] database: batch inserts - before: each OCSP request blocked the database - after: use a single insert into the database --- src/common/core/ssl/jobs/ocsp-refresh.py | 243 ++++++++++++----------- 1 file changed, 125 insertions(+), 118 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 987bd551ad..6ef9870f00 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -567,7 +567,20 @@ def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> No san_ocsp = san_dir / "ocsp.der" # Skip if target already exists (real file or valid symlink) - if san_ocsp.exists(): + if san_ocsp.exists() or san_ocsp.is_symlink(): + # Log details about what's already there for troubleshooting + if san_ocsp.is_symlink(): + try: + target = san_ocsp.readlink() + expected_target = Path("..") / cert_name / "ocsp.der" + if target == expected_target: + LOG.debug("๐Ÿ”— OCSP SAN symlink for %s already correct (%s -> %s)", san, san_ocsp, target) + else: + LOG.warning("โš ๏ธ OCSP SAN symlink for %s points to unexpected target (got %s, expected %s)", san, target, expected_target) + except Exception: + LOG.warning("โš ๏ธ OCSP SAN symlink for %s is broken or unreadable", san) + elif san_ocsp.is_file(): + LOG.warning("โš ๏ธ OCSP SAN path %s is a regular file (not a symlink); OCSP lookup for %s may fail if not configured correctly", san_ocsp, san) continue try: @@ -772,8 +785,13 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: LOG.debug("OCSP exception while attempting cache sync: %s", e) -def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None) -> None: - """Process a single certificate for OCSP stapling. Works with in-memory PEM data.""" +def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None) -> Optional[Tuple[str, bytes, int, str, bytes]]: + """ + Process a single certificate for OCSP stapling. Works with in-memory PEM data. + + Returns a tuple of (cert_name, ocsp_der, ttl, checksum, pem_data) for batched database writes, + or None if the cert was skipped/failed/cached. + """ if stats is None: stats = {} @@ -784,7 +802,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta LOG.info("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) cleanup_ocsp_cache(db, cert_name) stats["le_certs_skipped"] = stats.get("le_certs_skipped", 0) + 1 - return + return None LOG.info("๐Ÿ”„ OCSP processing certificate %s", cert_name) @@ -793,7 +811,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta if not ocsp_url: LOG.info("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) stats["le_certs_no_ocsp"] = stats.get("le_certs_no_ocsp", 0) + 1 - return + return None LOG.info("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) @@ -812,7 +830,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta half_lifetime / 86400.0, ) stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 - return + return None ocsp_der: Optional[bytes] = None ttl: int = 0 @@ -830,74 +848,21 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta if not ocsp_der: LOG.error("โŒ OCSP failed to fetch response for %s after retries", cert_name) stats["errors"] = stats.get("errors", 0) + 1 - return + return None LOG.debug("โšก OCSP final TTL for %s is %ds (from Next Update parsing)", cert_name, ttl) # Calculate checksum for integrity verification checksum = hashlib.sha256(ocsp_der).hexdigest() - # === Store OCSP response in database === - db_stored = False - if db: - cache_key = f"ocsp/{cert_name}" - try: - err = db.upsert_job_cache( - service_id=None, # Global cache entry - file_name=cache_key, - data=ocsp_der, - job_name="ocsp-refresh", - checksum=checksum, - ) + # === Return result for batched database writes === + # Database write will be done later in main() in a batch with all other certs + return (cert_name, ocsp_der, ttl, checksum, pem_data) - if err: - LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) - stats["errors"] = stats.get("errors", 0) + 1 - else: - LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, checksum[:8]) - db_stored = True - except Exception as e: - LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) - stats["errors"] = stats.get("errors", 0) + 1 - else: - # If no database available, still proceed with disk storage - db_stored = True - - # === Only write to disk AFTER successful database storage === - if not db_stored: - LOG.warning("โš ๏ธ OCSP skipping disk storage for %s due to database storage failure", cert_name) - return - - # Acquire lock to prevent race conditions with concurrent OCSP fetches - lock_fd = _acquire_cert_lock(cert_name) - try: - # Create the SSL configs directory for this certificate - ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name - try: - ocsp_cert_dir.mkdir(parents=True, exist_ok=True) - except Exception as e: - LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) - stats["errors"] = stats.get("errors", 0) + 1 - return - - # Write OCSP response to cache/ssl/{cert-name}/ocsp.der - ocsp_path = ocsp_cert_dir / "ocsp.der" - try: - ocsp_path.write_bytes(ocsp_der) - ocsp_path.chmod(0o644) # Readable by nginx - LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) - stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 - - # Create symlinks for each SAN so OCSP is found by any SNI name - _create_san_symlinks(pem_data, ocsp_path, cert_name) - except Exception as e: - LOG.error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) - stats["errors"] = stats.get("errors", 0) + 1 - finally: - _release_cert_lock(lock_fd) except Exception as e: LOG.error("โŒ OCSP exception while processing certificate %s: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 + return None def process_custom_certs( @@ -906,10 +871,12 @@ def process_custom_certs( lock_fd: Optional[int] = None, refresh_fn: Optional[Callable[[str], None]] = None, timeout_fn: Optional[Callable[[str], bool]] = None, -) -> None: +) -> List[Tuple[str, bytes, int, str, bytes]]: """ Process OCSP for custom certificates using certificate data from the database. + Returns a list of tuples (cert_name, ocsp_der, ttl, checksum, pem_data) for batched database writes. + Args: db: Database connection stats: Statistics dictionary @@ -920,15 +887,17 @@ def process_custom_certs( if stats is None: stats = {} + results: List[Tuple[str, bytes, int, str, bytes]] = [] + if not db: LOG.info("โ„น๏ธ OCSP database not available, cannot process custom certificates") - return + return results try: custom_certs = _load_custom_certs_from_db(db) if not custom_certs: LOG.info("โ„น๏ธ OCSP no custom certificates found in database") - return + return results LOG.info("๐Ÿ”„ OCSP loaded %d custom certificate(s) from database", len(custom_certs)) stats["custom_certs_processed"] = len(custom_certs) @@ -1029,55 +998,9 @@ def process_custom_certs( # Calculate checksum for integrity verification checksum = hashlib.sha256(ocsp_der).hexdigest() - # Store OCSP response in database - db_stored = False - cache_key = f"ocsp/{cert_name}" - try: - err = db.upsert_job_cache( - service_id=None, - file_name=cache_key, - data=ocsp_der, - job_name="ocsp-refresh", - checksum=checksum, - ) - - if err: - LOG.error("โŒ OCSP error storing custom cert %s response in database: %s", cert_name, err) - stats["errors"] = stats.get("errors", 0) + 1 - else: - LOG.info("โœ“ OCSP stored custom cert %s response in database (TTL=%ds, checksum=%s)", cert_name, ttl, checksum[:8]) - db_stored = True - except Exception as e: - LOG.error("โŒ OCSP exception storing custom cert %s in database: %s", cert_name, e) - stats["errors"] = stats.get("errors", 0) + 1 - - # Write to disk only after successful database storage - if not db_stored: - LOG.warning("โš ๏ธ OCSP skipping disk storage for custom cert %s due to database storage failure", cert_name) - continue - - # Create SSL configs directory for this certificate - ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name - try: - ocsp_cert_dir.mkdir(parents=True, exist_ok=True) - except Exception as e: - LOG.error("โŒ OCSP error creating directory %s: %s", ocsp_cert_dir, e) - stats["errors"] = stats.get("errors", 0) + 1 - continue - - # Write OCSP response to cache - ocsp_path = ocsp_cert_dir / "ocsp.der" - try: - ocsp_path.write_bytes(ocsp_der) - ocsp_path.chmod(0o644) # Readable by nginx - LOG.info("โœ“ OCSP saved custom cert %s response to disk at %s", cert_name, ocsp_path) - stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 - - # Create symlinks for each SAN so OCSP is found by any SNI name - _create_san_symlinks(cert_pem, ocsp_path, cert_name) - except Exception as e: - LOG.error("โŒ OCSP error writing custom cert %s response to disk: %s", cert_name, e) - stats["errors"] = stats.get("errors", 0) + 1 + # === Add to results for batched database writes === + # Database write will be done later in main() in a batch with all other certs + results.append((cert_name, ocsp_der, ttl, checksum, cert_pem)) except Exception as e: LOG.error("OCSP exception while processing custom certificate for %s: %s", service_name, e) @@ -1087,6 +1010,8 @@ def process_custom_certs( LOG.error("OCSP exception while processing custom certificates: %s", e) stats["errors"] = stats.get("errors", 0) + 1 + return results + def _service_name_from_dir(dir_name: str) -> str: """Strip -rsa or -ecdsa suffix from a directory name to get the service name.""" @@ -1414,6 +1339,25 @@ def refresh_job_lock(cert_name: str = "") -> None: LOG.error("โŒ OCSP Database module not available, cannot proceed") return 2 + # === Early validation: check cache directory permissions === + try: + CONFIGS_SSL_BASE.mkdir(parents=True, exist_ok=True) + # Verify we can write to the cache directory + test_file = CONFIGS_SSL_BASE / ".ocsp_write_test" + try: + test_file.touch() + test_file.unlink() + LOG.debug("โœ“ OCSP cache directory %s is readable and writable", CONFIGS_SSL_BASE) + except PermissionError: + LOG.error("โŒ OCSP cache directory %s is not writable (permission denied). Check directory ownership and permissions.", CONFIGS_SSL_BASE) + return 2 + except Exception as e: + LOG.error("โŒ OCSP could not verify write access to cache directory %s: %s", CONFIGS_SSL_BASE, e) + return 2 + except Exception as e: + LOG.error("โŒ OCSP could not create cache directory %s: %s", CONFIGS_SSL_BASE, e) + return 2 + # Check if OCSP stapling is globally disabled ocsp_enabled = os.getenv("SSL_USE_OCSP_STAPLING", "yes").lower() if ocsp_enabled != "yes": @@ -1430,6 +1374,9 @@ def refresh_job_lock(cert_name: str = "") -> None: time.sleep(10) restore_ocsp_from_database(db) + # === Collect all OCSP data for batched database writes === + all_ocsp_results: List[Tuple[str, bytes, int, str, bytes]] = [] + # Process Let's Encrypt certificates from database tarball le_certs = _load_le_certs_from_db(db) if le_certs: @@ -1444,14 +1391,74 @@ def refresh_job_lock(cert_name: str = "") -> None: # Refresh lock to prove we're still alive refresh_job_lock(cert_name) - _process_cert(cert_name, pem_data, db, stats) + result = _process_cert(cert_name, pem_data, db, stats) + if result: + all_ocsp_results.append(result) else: LOG.info("โ„น๏ธ OCSP no LE certificates found in database") # Check timeout before processing custom certs if not check_job_timeout("before custom cert processing"): # Process custom certificates from database - process_custom_certs(db, stats, lock_fd=lock_fd_main, refresh_fn=refresh_job_lock, timeout_fn=check_job_timeout) + custom_results = process_custom_certs(db, stats, lock_fd=lock_fd_main, refresh_fn=refresh_job_lock, timeout_fn=check_job_timeout) + all_ocsp_results.extend(custom_results) + + # === Batch write all OCSP responses to database === + if all_ocsp_results: + LOG.info("๐Ÿ”„ OCSP batching %d OCSP response(s) to database", len(all_ocsp_results)) + for cert_name, ocsp_der, ttl, checksum, pem_data in all_ocsp_results: + cache_key = f"ocsp/{cert_name}" + try: + err = db.upsert_job_cache( + service_id=None, # Global cache entry + file_name=cache_key, + data=ocsp_der, + job_name="ocsp-refresh", + checksum=checksum, + ) + + if err: + LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) + stats["errors"] = stats.get("errors", 0) + 1 + else: + LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, checksum[:8]) + except Exception as e: + LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + + # === Batch write all OCSP responses to disk === + for cert_name, ocsp_der, ttl, checksum, pem_data in all_ocsp_results: + try: + # Acquire lock to prevent race conditions with concurrent OCSP fetches + lock_fd = _acquire_cert_lock(cert_name) + try: + # Create the SSL configs directory for this certificate + ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + try: + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + except Exception as e: + LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) + stats["errors"] = stats.get("errors", 0) + 1 + continue + + # Write OCSP response to cache/ssl/{cert-name}/ocsp.der + ocsp_path = ocsp_cert_dir / "ocsp.der" + try: + ocsp_path.write_bytes(ocsp_der) + ocsp_path.chmod(0o644) # Readable by nginx + LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) + stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 + + # Create symlinks for each SAN so OCSP is found by any SNI name + _create_san_symlinks(pem_data, ocsp_path, cert_name) + except Exception as e: + LOG.error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + finally: + _release_cert_lock(lock_fd) + except Exception as e: + LOG.error("โŒ OCSP exception while writing response for %s to disk: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 # Check timeout before cleanup if not check_job_timeout("before orphaned cleanup"): From bfc6869c5408f366335e199b76b5ef7512034a3e Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 17:27:04 +0100 Subject: [PATCH 23/59] trigger OCSP update --- src/common/core/customcert/jobs/custom-cert.py | 15 +++++++++------ src/common/core/letsencrypt/jobs/certbot-new.py | 12 ++++++------ src/common/core/letsencrypt/jobs/certbot-renew.py | 9 ++++++--- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/common/core/customcert/jobs/custom-cert.py b/src/common/core/customcert/jobs/custom-cert.py index e62e2c258c..69d592b50f 100644 --- a/src/common/core/customcert/jobs/custom-cert.py +++ b/src/common/core/customcert/jobs/custom-cert.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from os import getenv, sep +from os import environ, getenv, sep from os.path import join from pathlib import Path from subprocess import DEVNULL, run @@ -159,6 +159,7 @@ def check_cert(cert_file: Union[Path, bytes], key_file: Union[Path, bytes], firs sys_exit(0) skipped_servers = [] + changed_domains = [] # Track which domains had certificate changes if not multisite: all_domains = [all_domains[0]] if getenv("USE_CUSTOM_SSL", "no") == "no": @@ -210,6 +211,7 @@ def check_cert(cert_file: Union[Path, bytes], key_file: Union[Path, bytes], firs continue elif need_reload: LOGGER.info(f"Detected change in {first_server}'s certificate") + changed_domains.append(first_server) # Track this domain as changed status = 1 continue @@ -219,23 +221,24 @@ def check_cert(cert_file: Union[Path, bytes], key_file: Union[Path, bytes], firs JOB.del_cache("cert.pem", service_id=first_server) JOB.del_cache("key.pem", service_id=first_server) - # Trigger OCSP stapling refresh when certificates changed - if status == 1 and getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": - LOGGER.info("๐Ÿ”„ OCSP triggering refresh after custom certificate change") + # Trigger OCSP stapling refresh when certificates changed (AFTER caching) + # OCSP job will compare new certs with cached ones and process differential updates + if changed_domains and getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": + LOGGER.info(f"๐Ÿ”„ OCSP triggering refresh for {len(changed_domains)} changed custom cert(s): {', '.join(changed_domains)}") try: import sys ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") result = run([sys.executable, ocsp_script], stdin=DEVNULL, capture_output=True, text=True, timeout=300) if result.returncode == 0: - LOGGER.info("โœ“ OCSP refresh completed successfully after custom cert change") + LOGGER.info("โœ“ OCSP refresh completed successfully after cert change") else: LOGGER.warning(f"โš ๏ธ OCSP refresh returned exit code {result.returncode}") if result.stderr: for line in result.stderr.strip().splitlines(): LOGGER.debug(f"OCSP: {line}") except Exception as e: - LOGGER.warning(f"โš ๏ธ OCSP post-upload refresh failed (non-fatal): {e}") + LOGGER.warning(f"โš ๏ธ OCSP post-change refresh failed (non-fatal): {e}") except SystemExit as e: status = e.code except BaseException as e: diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index 25390e62ab..e8092e785f 100644 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -1100,7 +1100,7 @@ def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, D save_zerossl_api_key_hashes(updated_zerossl_api_key_hashes) - # * Save data to db cache + # * Save data to db cache (full LE directory) if DATA_PATH.is_dir() and list(DATA_PATH.iterdir()): cached, err = JOB.cache_dir(DATA_PATH) if not cached: @@ -1108,17 +1108,17 @@ def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, D else: LOGGER.info("Successfully saved data to db cache") - # * Trigger OCSP stapling refresh for newly issued certificates - newly_issued = [svc for svc, cfg in services.items() if cfg.get("exists")] - if newly_issued and getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": - LOGGER.info(f"๐Ÿ”„ OCSP triggering refresh for {len(newly_issued)} newly issued cert(s): {', '.join(newly_issued)}") + # * Trigger OCSP stapling refresh for newly issued certificates (AFTER database save) + # OCSP job will compare new certs with cached ones and process differential updates + if getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": + LOGGER.info("๐Ÿ”„ OCSP triggering refresh for newly issued certificates") try: import sys ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") result = run([sys.executable, ocsp_script], stdin=DEVNULL, capture_output=True, text=True, timeout=300) if result.returncode == 0: - LOGGER.info("โœ“ OCSP refresh completed successfully after cert issuance") + LOGGER.info("โœ“ OCSP refresh completed successfully after issuance") else: LOGGER.warning(f"โš ๏ธ OCSP refresh returned exit code {result.returncode}") if result.stderr: diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index a7a274e6c9..314c1a999b 100644 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -30,6 +30,7 @@ LOGGER_CERTBOT = getLogger("LETS-ENCRYPT.RENEW.CERTBOT") status = 0 + try: # Check if we're using let's encrypt use_letsencrypt = False @@ -92,7 +93,7 @@ status = 2 LOGGER.error("Certificates renewal failed") - # Save Let's Encrypt data to db cache + # Save Let's Encrypt data to db cache (full directory) if DATA_PATH.is_dir() and list(DATA_PATH.iterdir()): cached, err = JOB.cache_dir(DATA_PATH) if not cached: @@ -100,9 +101,11 @@ else: LOGGER.info("Successfully saved Let's Encrypt data to db cache") - # Refresh OCSP stapling after successful renewal + # Trigger OCSP refresh after successful renewal (AFTER database save) + # OCSP job will compare new certs with cached ones and process differential updates if process.returncode == 0 and getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": - LOGGER.info("๐Ÿ”„ OCSP triggering refresh after certificate renewal") + LOGGER.info("๐Ÿ”„ OCSP triggering refresh for renewed certificates") + try: import sys From bc2da8769e75ba814f741f4117a9cb85ef40d32e Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 17:30:04 +0100 Subject: [PATCH 24/59] improve caching logic - add certificate checksum check - issue OCSP to new certs first, then changed certs, calculate TTL for the rest and run OCSP refresh if needed --- src/common/core/ssl/jobs/ocsp-refresh.py | 192 ++++++++++++++++++++++- 1 file changed, 187 insertions(+), 5 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 6ef9870f00..871410d6bc 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -593,6 +593,88 @@ def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> No LOG.debug("โš ๏ธ OCSP could not create SAN symlink for %s: %s", san, e) +def _get_cached_ocsp_certs(db: Any) -> set: + """ + Get the set of certificate names that have cached OCSP responses in the database. + Used to identify new vs. existing certificates for differential refresh strategy. + + Returns set of cert_name strings + """ + cached_certs = set() + + if db is None: + return cached_certs + + try: + cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=False) + for entry in cache_files: + file_name = entry.get("file_name", "") + if file_name.startswith("ocsp/"): + cert_name = file_name[len("ocsp/"):] + # Prevent path traversal from crafted database entries + if re.match(r"^[A-Za-z0-9_.*-]+$", cert_name): + cached_certs.add(cert_name) + except Exception as e: + LOG.warning("โš ๏ธ OCSP could not retrieve cached certificate list from database: %s", e) + + return cached_certs + + +def _get_cert_checksums(db: Any, cert_names: set) -> Dict[str, str]: + """ + Get previously stored checksums for certificates from the database. + Checksums are stored in cache entries like 'cert_checksum/cert_name'. + + Args: + db: Database connection + cert_names: Set of certificate names to retrieve checksums for + + Returns: + Dict of {cert_name: checksum_hex} + """ + checksums = {} + + if db is None or not cert_names: + return checksums + + try: + cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=False) + for entry in cache_files: + file_name = entry.get("file_name", "") + if file_name.startswith("cert_checksum/"): + cert_name = file_name[len("cert_checksum/"):] + if cert_name in cert_names: + # Retrieve the actual checksum data + try: + checksum_data = db.get_job_cache_file( + job_name="ocsp-refresh", + file_name=file_name, + with_data=True, + with_info=False, + ) + if checksum_data: + checksums[cert_name] = checksum_data.decode("utf-8").strip() + except Exception: + pass # Checksum entry doesn't exist or couldn't be retrieved + except Exception as e: + LOG.debug("โš ๏ธ OCSP could not retrieve certificate checksums from database: %s", e) + + return checksums + + +def _calculate_cert_checksum(pem_data: bytes) -> str: + """ + Calculate SHA256 checksum of certificate PEM data. + + Args: + pem_data: Certificate PEM content + + Returns: + Hex string of SHA256 hash + """ + return hashlib.sha256(pem_data).hexdigest() + + def _load_le_certs_from_db(db: Any) -> Dict[str, bytes]: """ Load Let's Encrypt fullchain PEM data from the database tarball. @@ -900,9 +982,35 @@ def process_custom_certs( return results LOG.info("๐Ÿ”„ OCSP loaded %d custom certificate(s) from database", len(custom_certs)) - stats["custom_certs_processed"] = len(custom_certs) - for service_name, cert_pem in sorted(custom_certs.items()): + # Check which custom certs have changed since last OCSP refresh + previous_custom_checksums = _get_cert_checksums(db, set(custom_certs.keys())) + changed_custom_certs = {} + unchanged_custom_certs = {} + + for cert_name, pem_data in custom_certs.items(): + current_checksum = _calculate_cert_checksum(pem_data) + previous_checksum = previous_custom_checksums.get(cert_name) + + if previous_checksum is None: + # No previous checksum, treat as changed + changed_custom_certs[cert_name] = pem_data + LOG.debug("โš ๏ธ OCSP no previous checksum found for custom cert %s, will refresh", cert_name) + elif current_checksum != previous_checksum: + # Certificate content has changed + changed_custom_certs[cert_name] = pem_data + LOG.debug("๐Ÿ”„ OCSP custom certificate content changed for %s", cert_name) + else: + # Certificate content unchanged + unchanged_custom_certs[cert_name] = pem_data + + if unchanged_custom_certs: + LOG.info("โœ“ OCSP found %d unchanged custom certificate(s) (skipping): %s", len(unchanged_custom_certs), ", ".join(sorted(unchanged_custom_certs.keys()))) + stats["custom_certs_unchanged"] = len(unchanged_custom_certs) + + stats["custom_certs_processed"] = len(changed_custom_certs) + + for service_name, cert_pem in sorted(changed_custom_certs.items()): # Check timeout before processing each certificate if timeout_fn and timeout_fn(f"processing custom cert {service_name}"): break @@ -1321,7 +1429,7 @@ def refresh_job_lock(cert_name: str = "") -> None: } try: - LOG.info("๐Ÿ”„ OCSP refresh job started (timeout in %d minutes)", JOB_TIMEOUT // 60) + LOG.info("๐Ÿ”„ OCSP refresh job started with differential update strategy (timeout in %d minutes)", JOB_TIMEOUT // 60) # Acquire main lock for the entire OCSP refresh operation lock_fd_main = _acquire_cert_lock("main", timeout=300, stale_threshold=1800) @@ -1339,6 +1447,10 @@ def refresh_job_lock(cert_name: str = "") -> None: LOG.error("โŒ OCSP Database module not available, cannot proceed") return 2 + # === Check for previously cached OCSP responses === + previous_ocsp_certs = _get_cached_ocsp_certs(db) + LOG.info("โ„น๏ธ OCSP found %d previously cached certificate(s)", len(previous_ocsp_certs)) + # === Early validation: check cache directory permissions === try: CONFIGS_SSL_BASE.mkdir(parents=True, exist_ok=True) @@ -1381,16 +1493,70 @@ def refresh_job_lock(cert_name: str = "") -> None: le_certs = _load_le_certs_from_db(db) if le_certs: LOG.info("โ„น๏ธ OCSP loaded %d LE certificate(s) from database", len(le_certs)) + + # Separate new certs from existing ones + new_le_certs = {k: v for k, v in le_certs.items() if k not in previous_ocsp_certs} + existing_le_certs = {k: v for k, v in le_certs.items() if k in previous_ocsp_certs} + + # For existing certs, check if certificate content has changed + previous_le_checksums = _get_cert_checksums(db, set(existing_le_certs.keys())) + changed_le_certs = {} + unchanged_le_certs = {} + + for cert_name, pem_data in existing_le_certs.items(): + current_checksum = _calculate_cert_checksum(pem_data) + previous_checksum = previous_le_checksums.get(cert_name) + + if previous_checksum is None: + # No previous checksum found, treat as changed (data loss or first refresh after upgrade) + changed_le_certs[cert_name] = pem_data + LOG.debug("โš ๏ธ OCSP no previous checksum found for %s, will refresh", cert_name) + elif current_checksum != previous_checksum: + # Certificate content has changed + changed_le_certs[cert_name] = pem_data + LOG.debug("๐Ÿ”„ OCSP certificate content changed for %s", cert_name) + else: + # Certificate content unchanged + unchanged_le_certs[cert_name] = pem_data + + if new_le_certs: + LOG.info("๐Ÿ†• OCSP found %d newly issued LE certificate(s): %s", len(new_le_certs), ", ".join(sorted(new_le_certs.keys()))) + stats["le_certs_new"] = len(new_le_certs) + + if changed_le_certs: + LOG.info("๐Ÿ”„ OCSP found %d LE certificate(s) with changed content: %s", len(changed_le_certs), ", ".join(sorted(changed_le_certs.keys()))) + stats["le_certs_changed"] = len(changed_le_certs) + + if unchanged_le_certs: + LOG.info("โœ“ OCSP found %d LE certificate(s) unchanged (skipping): %s", len(unchanged_le_certs), ", ".join(sorted(unchanged_le_certs.keys()))) + stats["le_certs_skipped"] = len(unchanged_le_certs) + stats["le_certs_processed"] = len(le_certs) - for cert_name, pem_data in sorted(le_certs.items()): + # Process new certs first with full OCSP refresh + for cert_name, pem_data in sorted(new_le_certs.items()): + # Check timeout before processing each certificate + if check_job_timeout(f"processing new LE cert {cert_name}"): + break + + # Refresh lock to prove we're still alive + refresh_job_lock(cert_name) + + LOG.debug("๐Ÿ†• OCSP refreshing new LE certificate: %s", cert_name) + result = _process_cert(cert_name, pem_data, db, stats) + if result: + all_ocsp_results.append(result) + + # Process changed certs with OCSP refresh + for cert_name, pem_data in sorted(changed_le_certs.items()): # Check timeout before processing each certificate - if check_job_timeout(f"processing LE cert {cert_name}"): + if check_job_timeout(f"processing changed LE cert {cert_name}"): break # Refresh lock to prove we're still alive refresh_job_lock(cert_name) + LOG.debug("๐Ÿ”„ OCSP refreshing changed LE certificate: %s", cert_name) result = _process_cert(cert_name, pem_data, db, stats) if result: all_ocsp_results.append(result) @@ -1426,6 +1592,22 @@ def refresh_job_lock(cert_name: str = "") -> None: LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 + # Store certificate content checksum for future differential checks + cert_checksum_key = f"cert_checksum/{cert_name}" + cert_checksum = _calculate_cert_checksum(pem_data) + try: + err = db.upsert_job_cache( + service_id=None, + file_name=cert_checksum_key, + data=cert_checksum.encode("utf-8"), + job_name="ocsp-refresh", + checksum=hashlib.sha256(cert_checksum.encode("utf-8")).hexdigest(), + ) + if err: + LOG.debug("โš ๏ธ OCSP could not store cert checksum for %s: %s", cert_name, err) + except Exception as e: + LOG.debug("โš ๏ธ OCSP exception while storing cert checksum for %s: %s", cert_name, e) + # === Batch write all OCSP responses to disk === for cert_name, ocsp_der, ttl, checksum, pem_data in all_ocsp_results: try: From a860bf554c8dd146dc36be20d7e3cdd8e92340c9 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 17:42:03 +0100 Subject: [PATCH 25/59] shorter TTL for caching --- src/common/confs/server-http/ssl-certificate-lua.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index ae9872ca72..02c55bb4b3 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -172,7 +172,7 @@ ssl_certificate_by_lua_block { logger:log(INFO, "OCSP loaded response (" .. #data .. " bytes) from " .. path) resp = data -- Cache in shared memory to avoid disk I/O on every handshake - local ok_set, serr = internalstore:set(cache_key, resp, 3600, true) + local ok_set, serr = internalstore:set(cache_key, resp, 300, true) if not ok_set then logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) end From 7f92fea84b8f9ec16618efb41884d4b3ed850f99 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 17:52:20 +0100 Subject: [PATCH 26/59] clean up expired OCSP entries --- src/common/core/ssl/jobs/ocsp-refresh.py | 69 ++++++++++++++++++++---- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 871410d6bc..c561c5d98f 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -69,6 +69,7 @@ # Use scheduler-managed cache directory (automatically synced from database on restart) CONFIGS_SSL_BASE = Path(os.sep, "var", "cache", "bunkerweb", "ssl") +MIN_TTL = 4500 # 75 minutes: aggressivly refresh/cleanup if TTL is below this def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int = 600) -> Optional[int]: @@ -898,21 +899,27 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta LOG.info("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) # === Check if cached OCSP response is still fresh === - # Refresh when remaining TTL is at or below 50% of the original lifetime. + # Refresh when remaining TTL is at or below 50% of the original lifetime, + # OR if it's below the 75-minute threshold (MIN_TTL). cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name) - if cached_ttl is not None and total_lifetime is not None and total_lifetime > 0: - half_lifetime = total_lifetime // 2 - if cached_ttl > half_lifetime: + if cached_ttl is not None: + half_lifetime = (total_lifetime // 2) if (total_lifetime and total_lifetime > 0) else 0 + + # Skip fetch ONLY if TTL is above MIN_TTL AND above 50% lifetime + if cached_ttl > MIN_TTL and cached_ttl > half_lifetime: LOG.info( - "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), above 50%% lifetime threshold=%ds (%.1f days), skipping fetch", + "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), above thresholds (MIN_TTL=%ds, 50%% threshold=%ds), skipping fetch", cert_name, cached_ttl, cached_ttl / 86400.0, + MIN_TTL, half_lifetime, - half_lifetime / 86400.0, ) stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 return None + + if cached_ttl <= MIN_TTL: + LOG.info("๐Ÿ”„ OCSP response for %s is near expiration (TTL=%ds < %ds), attempting aggressive refresh", cert_name, cached_ttl, MIN_TTL) ocsp_der: Optional[bytes] = None ttl: int = 0 @@ -929,6 +936,22 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta if not ocsp_der: LOG.error("โŒ OCSP failed to fetch response for %s after retries", cert_name) + + # AGGRESSIVE CLEANUP: If we had a cached response that is now below 75 mins (MIN_TTL), + # and the refresh failed, we MUST remove it now to prevent stapling expired data + # soon (since the next job runs in 60 mins). + if cached_ttl is not None and cached_ttl <= MIN_TTL: + LOG.warning( + "๐Ÿšจ๐Ÿšจ๐Ÿšจ OCSP CRITICAL WARNING: Cached response for %s is near expiration (TTL=%ds) " + "and refresh failed. Removing response from cache to prevent stapling expired data. " + "Security status: Stapling will be DISABLED for this certificate until refresh succeeds.", + cert_name, cached_ttl + ) + cleanup_ocsp_cache(db, cert_name) + elif cached_ttl is not None and cached_ttl <= 0: + LOG.warning("๐Ÿงน OCSP cached response for %s is already expired and refresh failed. Cleaning up invalid cache.", cert_name) + cleanup_ocsp_cache(db, cert_name) + stats["errors"] = stats.get("errors", 0) + 1 return None @@ -1012,11 +1035,11 @@ def process_custom_certs( for service_name, cert_pem in sorted(changed_custom_certs.items()): # Check timeout before processing each certificate - if timeout_fn and timeout_fn(f"processing custom cert {service_name}"): + if callable(timeout_fn) and timeout_fn(f"processing custom cert {service_name}"): break # Refresh lock to prove we're still alive - if refresh_fn: + if callable(refresh_fn): refresh_fn(service_name) # service_name is e.g. "opdash.net-ecdsa" โ€” strip suffix to get actual service svc = _service_name_from_dir(service_name) @@ -1334,6 +1357,23 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name ocsp_path = ocsp_cert_dir / "ocsp.der" + # Check if database response is already expired + try: + ocsp_response = x509_ocsp.load_der_ocsp_response(data) + remaining, _ = _ocsp_response_lifetimes(ocsp_response) + if remaining is not None and remaining <= 0: + LOG.warning("๐Ÿงน OCSP response in database for %s is expired (remaining=%ds). Skipping restoration to disk.", cert_name, remaining) + # Optionally remove from database to prevent future attempts + if db: + try: + db.delete_job_cache(file_name=file_name, job_name="ocsp-refresh") + LOG.debug("๐Ÿงน OCSP removed expired database entry %s", file_name) + except Exception: + pass + continue + except Exception as e: + LOG.warning("โš ๏ธ OCSP could not parse response from database for %s during verification: %s", cert_name, e) + try: if not ocsp_path.is_file(): # File missing: restore from database @@ -1380,9 +1420,9 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic "๐Ÿ” OCSP verification complete: %d checked | โœ“ %d restored (missing) | ๐Ÿ”„ %d corrected (mismatch)", verify_count, restored_count, mismatch_count ) - stats["ocsp_verified"] = verify_count - stats["ocsp_restored"] = restored_count - stats["ocsp_corrected"] = mismatch_count + stats["ocsp_verified"] = stats.get("ocsp_verified", 0) + verify_count + stats["ocsp_restored"] = stats.get("ocsp_restored", 0) + restored_count + stats["ocsp_corrected"] = stats.get("ocsp_corrected", 0) + mismatch_count except Exception as e: LOG.warning("โš ๏ธ OCSP verification failed: %s", e) @@ -1426,6 +1466,13 @@ def refresh_job_lock(cert_name: str = "") -> None: "ocsp_cached_responses": 0, "ocsp_fetched_responses": 0, "errors": 0, + "le_certs_new": 0, + "le_certs_changed": 0, + "custom_certs_unchanged": 0, + "ocsp_verified": 0, + "ocsp_restored": 0, + "ocsp_corrected": 0, + "orphaned_cleaned": 0, } try: From 3857c04101f7204b8b00fbfabb0a6bbc4fc5d82b Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 17:55:20 +0100 Subject: [PATCH 27/59] fix linter hints --- src/common/core/ssl/jobs/ocsp-refresh.py | 35 +++++++++++++----------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index c561c5d98f..65f20be74a 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -706,7 +706,7 @@ def _load_le_certs_from_db(db: Any) -> Dict[str, bytes]: LOG.debug("โœ“ OCSP found LE tarball from %s job", job_name) break - if not tgz_data: + if tgz_data is None: LOG.debug("โ„น๏ธ OCSP no LE tarball found in database (certbot-new cache)") return result @@ -940,17 +940,19 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta # AGGRESSIVE CLEANUP: If we had a cached response that is now below 75 mins (MIN_TTL), # and the refresh failed, we MUST remove it now to prevent stapling expired data # soon (since the next job runs in 60 mins). - if cached_ttl is not None and cached_ttl <= MIN_TTL: - LOG.warning( - "๐Ÿšจ๐Ÿšจ๐Ÿšจ OCSP CRITICAL WARNING: Cached response for %s is near expiration (TTL=%ds) " - "and refresh failed. Removing response from cache to prevent stapling expired data. " - "Security status: Stapling will be DISABLED for this certificate until refresh succeeds.", - cert_name, cached_ttl - ) - cleanup_ocsp_cache(db, cert_name) - elif cached_ttl is not None and cached_ttl <= 0: - LOG.warning("๐Ÿงน OCSP cached response for %s is already expired and refresh failed. Cleaning up invalid cache.", cert_name) - cleanup_ocsp_cache(db, cert_name) + if cached_ttl is not None: + current_ttl = cast(int, cached_ttl) + if current_ttl <= MIN_TTL: + LOG.warning( + "๐Ÿšจ๐Ÿšจ๐Ÿšจ OCSP CRITICAL WARNING: Cached response for %s is near expiration (TTL=%ds) " + "and refresh failed. Removing response from cache to prevent stapling expired data. " + "Security status: Stapling will be DISABLED for this certificate until refresh succeeds.", + cert_name, current_ttl + ) + cleanup_ocsp_cache(db, cert_name) + elif current_ttl <= 0: + LOG.warning("๐Ÿงน OCSP cached response for %s is already expired and refresh failed. Cleaning up invalid cache.", cert_name) + cleanup_ocsp_cache(db, cert_name) stats["errors"] = stats.get("errors", 0) + 1 return None @@ -1420,9 +1422,10 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic "๐Ÿ” OCSP verification complete: %d checked | โœ“ %d restored (missing) | ๐Ÿ”„ %d corrected (mismatch)", verify_count, restored_count, mismatch_count ) - stats["ocsp_verified"] = stats.get("ocsp_verified", 0) + verify_count - stats["ocsp_restored"] = stats.get("ocsp_restored", 0) + restored_count - stats["ocsp_corrected"] = stats.get("ocsp_corrected", 0) + mismatch_count + if stats is not None: + stats["ocsp_verified"] = stats.get("ocsp_verified", 0) + verify_count + stats["ocsp_restored"] = stats.get("ocsp_restored", 0) + restored_count + stats["ocsp_corrected"] = stats.get("ocsp_corrected", 0) + mismatch_count except Exception as e: LOG.warning("โš ๏ธ OCSP verification failed: %s", e) @@ -1617,7 +1620,7 @@ def refresh_job_lock(cert_name: str = "") -> None: all_ocsp_results.extend(custom_results) # === Batch write all OCSP responses to database === - if all_ocsp_results: + if db and all_ocsp_results: LOG.info("๐Ÿ”„ OCSP batching %d OCSP response(s) to database", len(all_ocsp_results)) for cert_name, ocsp_der, ttl, checksum, pem_data in all_ocsp_results: cache_key = f"ocsp/{cert_name}" From 1c57d48349f483b99586892ec427b6fd6340bdeb Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 18:11:44 +0100 Subject: [PATCH 28/59] fixing _get_cert_checksums errors --- src/common/core/ssl/jobs/ocsp-refresh.py | 120 +++++++++++++---------- 1 file changed, 70 insertions(+), 50 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 65f20be74a..bfbe3175d4 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -399,10 +399,15 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim # Explicitly use default SSL context with certificate verification enabled ssl_context = ssl.create_default_context() response = urlopen(req, timeout=timeout, context=ssl_context) - ocsp_response_data = response.read(102_400) # Cap at 100KB โ€” typical OCSP response is 1-4KB + # cap response size at 100KB to prevent memory exhaustion from malicious responder + ocsp_der = response.read(102400) + + if not ocsp_der: + LOG.warning("โš ๏ธ OCSP empty response from %s for %s", ocsp_url, cert_name) + return None, 0 - # Parse response and validate status - ocsp_response = x509_ocsp.load_der_ocsp_response(ocsp_response_data) + # Check if it's a valid OCSP response via cryptography + ocsp_response = x509_ocsp.load_der_ocsp_response(ocsp_der) if ocsp_response.response_status != x509_ocsp.OCSPResponseStatus.SUCCESSFUL: LOG.error("โŒ OCSP response status is not successful for %s: %s", cert_name, ocsp_response.response_status) return None, 0 @@ -422,7 +427,7 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim os.chmod(f_issuer.name, 0o600) os.chmod(f_leaf.name, 0o600) - f_der.write(ocsp_response_data) + f_der.write(ocsp_der) f_der.flush() f_issuer.write(issuer.public_bytes(Encoding.PEM).decode("utf-8")) @@ -458,7 +463,7 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim LOG.debug("โธ๏ธ OCSP delaying 5 seconds after fetch for %s to prevent rate limiting", cert_name) time.sleep(5) - return ocsp_response_data, ttl + return ocsp_der, ttl except HTTPError as e: # HTTP error code โ€” OCSP responder returned an error error_desc = f"HTTP {e.code}" @@ -639,24 +644,19 @@ def _get_cert_checksums(db: Any, cert_names: set) -> Dict[str, str]: return checksums try: - cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=False) + # Optimization: use with_data=True to fetch all checksums in one go + cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=True) for entry in cache_files: file_name = entry.get("file_name", "") if file_name.startswith("cert_checksum/"): cert_name = file_name[len("cert_checksum/"):] if cert_name in cert_names: - # Retrieve the actual checksum data - try: - checksum_data = db.get_job_cache_file( - job_name="ocsp-refresh", - file_name=file_name, - with_data=True, - with_info=False, - ) - if checksum_data: - checksums[cert_name] = checksum_data.decode("utf-8").strip() - except Exception: - pass # Checksum entry doesn't exist or couldn't be retrieved + data = entry.get("data") + if data: + try: + checksums[cert_name] = data.decode("utf-8").strip() + except Exception: + pass except Exception as e: LOG.debug("โš ๏ธ OCSP could not retrieve certificate checksums from database: %s", e) @@ -868,12 +868,12 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: LOG.debug("OCSP exception while attempting cache sync: %s", e) -def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None) -> Optional[Tuple[str, bytes, int, str, bytes]]: +def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None) -> Optional[Tuple[str, Optional[bytes], int, str, bytes]]: """ Process a single certificate for OCSP stapling. Works with in-memory PEM data. Returns a tuple of (cert_name, ocsp_der, ttl, checksum, pem_data) for batched database writes, - or None if the cert was skipped/failed/cached. + or None if the cert was skipped/failed entirely. """ if stats is None: stats = {} @@ -884,7 +884,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta if not _is_ocsp_enabled_for_service(service_name): LOG.info("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) cleanup_ocsp_cache(db, cert_name) - stats["le_certs_skipped"] = stats.get("le_certs_skipped", 0) + 1 + stats["le_certs_skipped"] = stats["le_certs_skipped"] + 1 return None LOG.info("๐Ÿ”„ OCSP processing certificate %s", cert_name) @@ -893,7 +893,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta ocsp_url = extract_ocsp_url(pem_data, cert_name) if not ocsp_url: LOG.info("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) - stats["le_certs_no_ocsp"] = stats.get("le_certs_no_ocsp", 0) + 1 + stats["le_certs_no_ocsp"] = stats["le_certs_no_ocsp"] + 1 return None LOG.info("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) @@ -915,7 +915,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta MIN_TTL, half_lifetime, ) - stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 + stats["ocsp_cached_responses"] = stats["ocsp_cached_responses"] + 1 return None if cached_ttl <= MIN_TTL: @@ -978,7 +978,7 @@ def process_custom_certs( lock_fd: Optional[int] = None, refresh_fn: Optional[Callable[[str], None]] = None, timeout_fn: Optional[Callable[[str], bool]] = None, -) -> List[Tuple[str, bytes, int, str, bytes]]: +) -> List[Tuple[str, Optional[bytes], int, str, bytes]]: """ Process OCSP for custom certificates using certificate data from the database. @@ -1268,7 +1268,7 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: LOG.debug("โ„น๏ธ OCSP no valid certs found, skipping orphan cleanup to avoid accidental deletion") return - orphaned_count = 0 + orphaned_count: int = 0 # Check database OCSP entries if db: @@ -1282,7 +1282,7 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: if cert_name not in valid_cert_names: LOG.info("๐Ÿงน OCSP removing orphaned entry for deleted service: %s", cert_name) cleanup_ocsp_cache(db, cert_name) - orphaned_count += 1 + orphaned_count = orphaned_count + 1 except Exception as e: LOG.warning("โš ๏ธ OCSP could not check database for orphaned entries: %s", e) @@ -1301,7 +1301,7 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: continue LOG.info("๐Ÿงน OCSP removing orphaned disk cache for deleted service: %s", cert_name) cleanup_ocsp_cache(db, cert_name) - orphaned_count += 1 + orphaned_count = orphaned_count + 1 if orphaned_count > 0: LOG.info("๐Ÿงน OCSP cleaned up %d orphaned cache entries for deleted services", orphaned_count) @@ -1537,7 +1537,7 @@ def refresh_job_lock(cert_name: str = "") -> None: restore_ocsp_from_database(db) # === Collect all OCSP data for batched database writes === - all_ocsp_results: List[Tuple[str, bytes, int, str, bytes]] = [] + all_ocsp_results: List[Tuple[str, Optional[bytes], int, str, bytes]] = [] # Process Let's Encrypt certificates from database tarball le_certs = _load_le_certs_from_db(db) @@ -1596,6 +1596,10 @@ def refresh_job_lock(cert_name: str = "") -> None: result = _process_cert(cert_name, pem_data, db, stats) if result: all_ocsp_results.append(result) + else: + # Still need to return the pem_data for checksum persistence if refresh was skipped + # We use a dummy result tuple (cert_name, None, 0, "", pem_data) to signal "just update checksum" + all_ocsp_results.append((cert_name, None, 0, "", pem_data)) # Process changed certs with OCSP refresh for cert_name, pem_data in sorted(changed_le_certs.items()): @@ -1610,6 +1614,15 @@ def refresh_job_lock(cert_name: str = "") -> None: result = _process_cert(cert_name, pem_data, db, stats) if result: all_ocsp_results.append(result) + else: + # Still need to return the pem_data for checksum persistence if refresh was skipped + # We use a dummy result tuple (cert_name, None, 0, "", pem_data) to signal "just update checksum" + all_ocsp_results.append((cert_name, None, 0, "", pem_data)) + + # For unchanged certs, we still need to return their pem_data for checksum persistence + for cert_name, pem_data in unchanged_le_certs.items(): + all_ocsp_results.append((cert_name, None, 0, "", pem_data)) + else: LOG.info("โ„น๏ธ OCSP no LE certificates found in database") @@ -1619,30 +1632,33 @@ def refresh_job_lock(cert_name: str = "") -> None: custom_results = process_custom_certs(db, stats, lock_fd=lock_fd_main, refresh_fn=refresh_job_lock, timeout_fn=check_job_timeout) all_ocsp_results.extend(custom_results) - # === Batch write all OCSP responses to database === + # === Batch write all OCSP responses and checksums to database === if db and all_ocsp_results: - LOG.info("๐Ÿ”„ OCSP batching %d OCSP response(s) to database", len(all_ocsp_results)) + LOG.info("๐Ÿ”„ OCSP batching database updates for %d certificate(s)", len(all_ocsp_results)) for cert_name, ocsp_der, ttl, checksum, pem_data in all_ocsp_results: - cache_key = f"ocsp/{cert_name}" - try: - err = db.upsert_job_cache( - service_id=None, # Global cache entry - file_name=cache_key, - data=ocsp_der, - job_name="ocsp-refresh", - checksum=checksum, - ) + # 1. Update OCSP response if we have a new one + if ocsp_der: + cache_key = f"ocsp/{cert_name}" + try: + err = db.upsert_job_cache( + service_id=None, # Global cache entry + file_name=cache_key, + data=ocsp_der, + job_name="ocsp-refresh", + checksum=checksum, + ) - if err: - LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) - stats["errors"] = stats.get("errors", 0) + 1 - else: - LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, checksum[:8]) - except Exception as e: - LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) - stats["errors"] = stats.get("errors", 0) + 1 + if err: + LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) + stats["errors"] = stats["errors"] + 1 + else: + LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, checksum[:8]) + except Exception as e: + LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) + stats["errors"] = stats["errors"] + 1 - # Store certificate content checksum for future differential checks + # 2. ALWAYS store/update certificate content checksum for future differential checks + # This ensures that even if OCSP refresh was skipped, we know we've seen this cert version. cert_checksum_key = f"cert_checksum/{cert_name}" cert_checksum = _calculate_cert_checksum(pem_data) try: @@ -1655,11 +1671,15 @@ def refresh_job_lock(cert_name: str = "") -> None: ) if err: LOG.debug("โš ๏ธ OCSP could not store cert checksum for %s: %s", cert_name, err) + else: + LOG.debug("โœ“ OCSP persisted checksum for %s", cert_name) except Exception as e: LOG.debug("โš ๏ธ OCSP exception while storing cert checksum for %s: %s", cert_name, e) # === Batch write all OCSP responses to disk === for cert_name, ocsp_der, ttl, checksum, pem_data in all_ocsp_results: + if not ocsp_der: + continue try: # Acquire lock to prevent race conditions with concurrent OCSP fetches lock_fd = _acquire_cert_lock(cert_name) @@ -1670,7 +1690,7 @@ def refresh_job_lock(cert_name: str = "") -> None: ocsp_cert_dir.mkdir(parents=True, exist_ok=True) except Exception as e: LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) - stats["errors"] = stats.get("errors", 0) + 1 + stats["errors"] = stats["errors"] + 1 continue # Write OCSP response to cache/ssl/{cert-name}/ocsp.der @@ -1679,7 +1699,7 @@ def refresh_job_lock(cert_name: str = "") -> None: ocsp_path.write_bytes(ocsp_der) ocsp_path.chmod(0o644) # Readable by nginx LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) - stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 + stats["ocsp_fetched_responses"] = stats["ocsp_fetched_responses"] + 1 # Create symlinks for each SAN so OCSP is found by any SNI name _create_san_symlinks(pem_data, ocsp_path, cert_name) From 7e933c19d68fae5c57e792dc30df2e5966eb01a1 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 19:03:30 +0100 Subject: [PATCH 29/59] optimizing execution time --- src/common/core/ssl/jobs/ocsp-refresh.py | 315 ++++++++--------------- 1 file changed, 113 insertions(+), 202 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index bfbe3175d4..30ccd6abaa 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -514,10 +514,10 @@ def get_cached_ocsp_ttl(cert_name: str) -> Tuple[Optional[int], Optional[int]]: Both values are None if they cannot be determined. """ ocsp_path = CONFIGS_SSL_BASE / cert_name / "ocsp.der" - LOG.info("โšก OCSP TTL check: checking cache for %s at %s", cert_name, ocsp_path) + LOG.debug("โšก OCSP TTL check: checking cache for %s at %s", cert_name, ocsp_path) if not ocsp_path.is_file(): - LOG.info("โšก OCSP TTL check: cache miss - file not found for %s", cert_name) + LOG.debug("โšก OCSP TTL check: cache miss - file not found for %s", cert_name) return None, None LOG.debug("โšก OCSP cached file found for %s, reading This/Next Update...", cert_name) @@ -527,7 +527,7 @@ def get_cached_ocsp_ttl(cert_name: str) -> Tuple[Optional[int], Optional[int]]: remaining, total_lifetime = _ocsp_response_lifetimes(ocsp_response) if remaining is None or total_lifetime is None: - LOG.info("๐Ÿ”„ OCSP could not determine precise lifetime for %s from cached response", cert_name) + LOG.debug("๐Ÿ”„ OCSP could not determine precise lifetime for %s from cached response", cert_name) return None, None # Type assertion: after None checks above, these are guaranteed to be int @@ -582,11 +582,11 @@ def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> No if target == expected_target: LOG.debug("๐Ÿ”— OCSP SAN symlink for %s already correct (%s -> %s)", san, san_ocsp, target) else: - LOG.warning("โš ๏ธ OCSP SAN symlink for %s points to unexpected target (got %s, expected %s)", san, target, expected_target) + LOG.debug("โ„น๏ธ OCSP SAN symlink for %s points to different target (got %s, expected %s) โ€” likely overlapping certs, skipping", san, target, expected_target) except Exception: - LOG.warning("โš ๏ธ OCSP SAN symlink for %s is broken or unreadable", san) + LOG.debug("โš ๏ธ OCSP SAN symlink for %s is broken or unreadable", san) elif san_ocsp.is_file(): - LOG.warning("โš ๏ธ OCSP SAN path %s is a regular file (not a symlink); OCSP lookup for %s may fail if not configured correctly", san_ocsp, san) + LOG.debug("โ„น๏ธ OCSP SAN path %s is a regular file (not a symlink) โ€” likely another real cert, skipping symlink", san_ocsp) continue try: @@ -666,14 +666,28 @@ def _get_cert_checksums(db: Any, cert_names: set) -> Dict[str, str]: def _calculate_cert_checksum(pem_data: bytes) -> str: """ Calculate SHA256 checksum of certificate PEM data. + """ + return hashlib.sha256(pem_data).hexdigest() - Args: - pem_data: Certificate PEM content - Returns: - Hex string of SHA256 hash +def _clean_pem(pem_data: bytes) -> bytes: """ - return hashlib.sha256(pem_data).hexdigest() + Strip private keys, comments, and noise before the first certificate block. + Ensures consistent checksums regardless of extra data in the database. + """ + # 1. Strip embedded private keys + if b"PRIVATE KEY" in pem_data: + pem_data = re.sub(rb"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----\s*", b"", pem_data) + + # 2. Strip comment lines + pem_data = b"\n".join(line for line in pem_data.split(b"\n") if not line.startswith(b"#")) + + # 3. Strip everything before the first -----BEGIN + pem_start = pem_data.find(b"-----BEGIN") + if pem_start > 0: + pem_data = pem_data[pem_start:] + + return pem_data def _load_le_certs_from_db(db: Any) -> Dict[str, bytes]: @@ -790,7 +804,7 @@ def _load_custom_certs_from_db(db: Any) -> Dict[str, bytes]: continue # Derive suffix from filename: cert-ecdsa.pem -> -ecdsa, cert.pem -> "" suffix = file_name.replace("cert", "").replace(".pem", "") # e.g. "-ecdsa", "-rsa", "" - key = f"{service_id}{suffix}" + key = f"customcert-{service_id}{suffix}" # Double-check derived key is safe (defensive validation) if not re.match(r"^[A-Za-z0-9_.*-]+$", key): LOG.warning("โš ๏ธ OCSP sanitization: skipping custom cert with unsafe derived key: %s", key) @@ -868,55 +882,61 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: LOG.debug("OCSP exception while attempting cache sync: %s", e) -def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None) -> Optional[Tuple[str, Optional[bytes], int, str, bytes]]: +def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None, force_fetch: bool = False) -> Tuple[str, Optional[bytes], int, str, bytes]: """ Process a single certificate for OCSP stapling. Works with in-memory PEM data. + If force_fetch is True, skips the cached TTL check and retrieves a new response. - Returns a tuple of (cert_name, ocsp_der, ttl, checksum, pem_data) for batched database writes, - or None if the cert was skipped/failed entirely. + Returns a tuple of (cert_name, ocsp_der, ttl, cert_checksum, pem_data) for batched database writes. + If ocsp_der is None, it means the fetch was skipped or failed. """ if stats is None: stats = {} service_name = _service_name_from_dir(cert_name) + # === PEM CLEANING & SECURITY STRIPPING === + # Clean it immediately so all subsequent logic (OCSP fetch, checksum, parsing) + # uses the normalized/safe version. + pem_data = _clean_pem(pem_data) + cert_checksum = _calculate_cert_checksum(pem_data) + # Check per-service OCSP stapling setting if not _is_ocsp_enabled_for_service(service_name): - LOG.info("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) + LOG.debug("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) cleanup_ocsp_cache(db, cert_name) stats["le_certs_skipped"] = stats["le_certs_skipped"] + 1 - return None + return (cert_name, None, 0, cert_checksum, pem_data) - LOG.info("๐Ÿ”„ OCSP processing certificate %s", cert_name) + LOG.debug("๐Ÿ”„ OCSP processing certificate %s", cert_name) try: + if not pem_data.startswith(b"-----BEGIN"): + LOG.warning("โš ๏ธ OCSP cert for %s has no PEM BEGIN marker or is invalid after cleaning, skipping", cert_name) + return (cert_name, None, 0, cert_checksum, pem_data) + + # Proceed with extracting OCSP URL using the CLEANED pem_data ocsp_url = extract_ocsp_url(pem_data, cert_name) if not ocsp_url: - LOG.info("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) + LOG.debug("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) stats["le_certs_no_ocsp"] = stats["le_certs_no_ocsp"] + 1 - return None + return (cert_name, None, 0, cert_checksum, pem_data) - LOG.info("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) + LOG.debug("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) # === Check if cached OCSP response is still fresh === - # Refresh when remaining TTL is at or below 50% of the original lifetime, - # OR if it's below the 75-minute threshold (MIN_TTL). cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name) if cached_ttl is not None: half_lifetime = (total_lifetime // 2) if (total_lifetime and total_lifetime > 0) else 0 - # Skip fetch ONLY if TTL is above MIN_TTL AND above 50% lifetime - if cached_ttl > MIN_TTL and cached_ttl > half_lifetime: - LOG.info( + # Skip fetch ONLY if not forced AND TTL is above MIN_TTL AND above 50% lifetime + if not force_fetch and cached_ttl > MIN_TTL and cached_ttl > half_lifetime: + LOG.debug( "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), above thresholds (MIN_TTL=%ds, 50%% threshold=%ds), skipping fetch", - cert_name, - cached_ttl, - cached_ttl / 86400.0, - MIN_TTL, - half_lifetime, + cert_name, cached_ttl, cached_ttl / 86400.0, MIN_TTL, half_lifetime, ) stats["ocsp_cached_responses"] = stats["ocsp_cached_responses"] + 1 - return None + return (cert_name, None, cached_ttl, cert_checksum, pem_data) if cached_ttl <= MIN_TTL: LOG.info("๐Ÿ”„ OCSP response for %s is near expiration (TTL=%ds < %ds), attempting aggressive refresh", cert_name, cached_ttl, MIN_TTL) @@ -929,6 +949,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta ocsp_der, ttl = fetch_ocsp_response(pem_data, ocsp_url, cert_name=cert_name, timeout=10) if ocsp_der: LOG.debug("โœ“ OCSP successfully fetched response for %s on attempt %d (TTL=%ds)", cert_name, attempt, ttl) + stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 break if attempt == 1: LOG.warning("โš ๏ธ OCSP fetch failed for %s, retrying once after 10 seconds ...", cert_name) @@ -937,16 +958,12 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta if not ocsp_der: LOG.error("โŒ OCSP failed to fetch response for %s after retries", cert_name) - # AGGRESSIVE CLEANUP: If we had a cached response that is now below 75 mins (MIN_TTL), - # and the refresh failed, we MUST remove it now to prevent stapling expired data - # soon (since the next job runs in 60 mins). if cached_ttl is not None: current_ttl = cast(int, cached_ttl) if current_ttl <= MIN_TTL: LOG.warning( "๐Ÿšจ๐Ÿšจ๐Ÿšจ OCSP CRITICAL WARNING: Cached response for %s is near expiration (TTL=%ds) " - "and refresh failed. Removing response from cache to prevent stapling expired data. " - "Security status: Stapling will be DISABLED for this certificate until refresh succeeds.", + "and refresh failed. Removing response from cache to prevent stapling expired data.", cert_name, current_ttl ) cleanup_ocsp_cache(db, cert_name) @@ -955,21 +972,19 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta cleanup_ocsp_cache(db, cert_name) stats["errors"] = stats.get("errors", 0) + 1 - return None - - LOG.debug("โšก OCSP final TTL for %s is %ds (from Next Update parsing)", cert_name, ttl) + return (cert_name, None, 0, cert_checksum, pem_data) # Calculate checksum for integrity verification - checksum = hashlib.sha256(ocsp_der).hexdigest() + ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() + LOG.info("โšก OCSP final TTL for %s is %ds (checksum=%s)", cert_name, ttl, ocsp_checksum[:8]) # === Return result for batched database writes === - # Database write will be done later in main() in a batch with all other certs - return (cert_name, ocsp_der, ttl, checksum, pem_data) + return (cert_name, ocsp_der, ttl, cert_checksum, pem_data) except Exception as e: LOG.error("โŒ OCSP exception while processing certificate %s: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 - return None + return (cert_name, None, 0, cert_checksum, pem_data) def process_custom_certs( @@ -994,7 +1009,7 @@ def process_custom_certs( if stats is None: stats = {} - results: List[Tuple[str, bytes, int, str, bytes]] = [] + results: List[Tuple[str, Optional[bytes], int, str, bytes]] = [] if not db: LOG.info("โ„น๏ธ OCSP database not available, cannot process custom certificates") @@ -1009,12 +1024,15 @@ def process_custom_certs( LOG.info("๐Ÿ”„ OCSP loaded %d custom certificate(s) from database", len(custom_certs)) # Check which custom certs have changed since last OCSP refresh + # The names from _load_custom_certs_from_db already include the 'customcert-' prefix previous_custom_checksums = _get_cert_checksums(db, set(custom_certs.keys())) changed_custom_certs = {} unchanged_custom_certs = {} for cert_name, pem_data in custom_certs.items(): - current_checksum = _calculate_cert_checksum(pem_data) + # Clean PEM before calculating checksum to ensure consistency + cleaned_pem = _clean_pem(pem_data) + current_checksum = _calculate_cert_checksum(cleaned_pem) previous_checksum = previous_custom_checksums.get(cert_name) if previous_checksum is None: @@ -1030,128 +1048,43 @@ def process_custom_certs( unchanged_custom_certs[cert_name] = pem_data if unchanged_custom_certs: - LOG.info("โœ“ OCSP found %d unchanged custom certificate(s) (skipping): %s", len(unchanged_custom_certs), ", ".join(sorted(unchanged_custom_certs.keys()))) - stats["custom_certs_unchanged"] = len(unchanged_custom_certs) + LOG.info("โœ“ OCSP checking TTL for %d unchanged custom certificate(s): %s", len(unchanged_custom_certs), ", ".join(sorted(unchanged_custom_certs.keys()))) + stats["custom_certs_unchanged"] = stats.get("custom_certs_unchanged", 0) + len(unchanged_custom_certs) - stats["custom_certs_processed"] = len(changed_custom_certs) + stats["custom_certs_processed"] = stats.get("custom_certs_processed", 0) + len(custom_certs) - for service_name, cert_pem in sorted(changed_custom_certs.items()): - # Check timeout before processing each certificate - if callable(timeout_fn) and timeout_fn(f"processing custom cert {service_name}"): + # 1. Process changed custom certificates (force refresh) + for cert_name, cert_pem in sorted(changed_custom_certs.items()): + if callable(timeout_fn) and timeout_fn(f"changed custom cert {cert_name}"): break - - # Refresh lock to prove we're still alive if callable(refresh_fn): - refresh_fn(service_name) - # service_name is e.g. "opdash.net-ecdsa" โ€” strip suffix to get actual service - svc = _service_name_from_dir(service_name) - if not _is_ocsp_enabled_for_service(svc): - LOG.info("๐Ÿงน OCSP stapling disabled for service %s, cleaning up custom cert cache for %s", svc, service_name) - cleanup_ocsp_cache(db, f"customcert-{service_name}") - stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + 1 - continue - - try: - # Warn and strip embedded private keys (should never be in cert file) - if b"-----BEGIN" in cert_pem and b"PRIVATE KEY" in cert_pem: - LOG.warning("โš ๏ธ OCSP custom cert for %s contains an embedded private key! This is a security risk โ€” private keys should not be stored in cert files.", service_name) - # Remove all private key PEM blocks to avoid logging secrets - cert_pem = re.sub(rb"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----\s*", b"", cert_pem) - - # Strip comment lines (lines starting with #) and content before first -----BEGIN - cert_pem = b"\n".join(line for line in cert_pem.split(b"\n") if not line.startswith(b"#")) - pem_start = cert_pem.find(b"-----BEGIN") - if pem_start > 0: - cert_pem = cert_pem[pem_start:] - elif pem_start < 0: - LOG.warning("โš ๏ธ OCSP custom cert for %s has no PEM BEGIN marker, skipping", service_name) - stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + 1 - continue - - # Validate it's a real certificate - try: - x509.load_pem_x509_certificate(cert_pem) - except Exception as e: - LOG.warning("โš ๏ธ OCSP custom cert for %s is not a valid PEM certificate (len=%d): %s", service_name, len(cert_pem), e) - cert_checksum = hashlib.sha256(cert_pem).hexdigest() - LOG.debug("๐Ÿ” OCSP custom cert %s checksum: %s", service_name, cert_checksum) - stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + 1 - continue - - LOG.info("๐Ÿ”„ OCSP processing custom certificate for service %s", service_name) + refresh_fn(cert_name) + results.append(_process_cert(cert_name, cert_pem, db, stats, force_fetch=True)) - # Use customcert- prefix for OCSP cache key - cert_name = f"customcert-{service_name}" - - # Extract OCSP URL from certificate - ocsp_url = extract_ocsp_url(cert_pem, cert_name) - if not ocsp_url: - LOG.info("โ„น๏ธ OCSP custom cert %s has no responder, skipping", cert_name) - stats["custom_certs_no_ocsp"] = stats.get("custom_certs_no_ocsp", 0) + 1 - continue - - LOG.info("๐ŸŒ OCSP responder for custom cert %s: %s", cert_name, ocsp_url) - - # Check if cached OCSP response is still fresh - cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name) - if cached_ttl is not None and total_lifetime is not None and total_lifetime > 0: - half_lifetime = total_lifetime // 2 - if cached_ttl > half_lifetime: - LOG.info( - "โœ“ OCSP cached response for custom cert %s still valid for %ds (%.1f days), above 50%% lifetime threshold=%ds (%.1f days), skipping fetch", - cert_name, - cached_ttl, - cached_ttl / 86400.0, - half_lifetime, - half_lifetime / 86400.0, - ) - stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 - continue - - ocsp_der: Optional[bytes] = None - ttl: int = 0 - - # Use a timeout of 10 seconds per attempt, retry once after 10 seconds - for attempt in (1, 2): - ocsp_der, ttl = fetch_ocsp_response(cert_pem, ocsp_url, cert_name=cert_name, timeout=10) - if ocsp_der: - LOG.debug("โœ“ OCSP successfully fetched response for custom cert %s on attempt %d (TTL=%ds)", cert_name, attempt, ttl) - break - if attempt == 1: - LOG.warning("โš ๏ธ OCSP fetch failed for custom cert %s, retrying once after 10 seconds ...", cert_name) - time.sleep(10) - - if not ocsp_der: - LOG.error("โŒ OCSP failed to fetch response for custom cert %s after retries", cert_name) - stats["errors"] = stats.get("errors", 0) + 1 - continue - - LOG.debug("โšก OCSP final TTL for custom cert %s is %ds", cert_name, ttl) - - # Calculate checksum for integrity verification - checksum = hashlib.sha256(ocsp_der).hexdigest() - - # === Add to results for batched database writes === - # Database write will be done later in main() in a batch with all other certs - results.append((cert_name, ocsp_der, ttl, checksum, cert_pem)) - - except Exception as e: - LOG.error("OCSP exception while processing custom certificate for %s: %s", service_name, e) - stats["errors"] = stats.get("errors", 0) + 1 + # 2. Process unchanged custom certificates (TTL check only) + for cert_name, cert_pem in sorted(unchanged_custom_certs.items()): + if callable(timeout_fn) and timeout_fn(f"unchanged custom cert {cert_name}"): + break + if callable(refresh_fn): + refresh_fn(cert_name) + results.append(_process_cert(cert_name, cert_pem, db, stats, force_fetch=False)) except Exception as e: LOG.error("OCSP exception while processing custom certificates: %s", e) - stats["errors"] = stats.get("errors", 0) + 1 + stats["errors"] = stats["errors"] + 1 return results def _service_name_from_dir(dir_name: str) -> str: - """Strip -rsa or -ecdsa suffix from a directory name to get the service name.""" + """Strip -rsa or -ecdsa suffix and customcert- prefix from a name to get the service name.""" + name = dir_name + if name.startswith("customcert-"): + name = name[len("customcert-"):] for suffix in ("-rsa", "-ecdsa"): - if dir_name.endswith(suffix): - return dir_name[: -len(suffix)] - return dir_name + if name.endswith(suffix): + return name[: -len(suffix)] + return name def _is_ocsp_enabled_for_service(service_name: str) -> bool: @@ -1254,12 +1187,13 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: # Build set of valid cert names from LE certs valid_cert_names: set = set(le_certs.keys()) - # Add custom cert names (with customcert- prefix) + # Add custom cert names if db: try: custom_certs = _load_custom_certs_from_db(db) for key in custom_certs: - valid_cert_names.add(f"customcert-{key}") + # key already starts with customcert- + valid_cert_names.add(key) except Exception as e: LOG.warning("โš ๏ธ OCSP could not load custom certs for orphan check: %s", e) return # Don't clean up if we can't verify what's valid @@ -1323,9 +1257,7 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic LOG.warning("โš ๏ธ OCSP cannot verify files without database connection") return - # Sleep briefly to avoid race conditions with concurrent writes to cache directories - LOG.debug("๐Ÿ’ค OCSP waiting 5s before verification (to allow concurrent writers to finish)...") - time.sleep(5) + # Verification: check if files exist on disk and match database checksums verify_count = 0 restored_count = 0 @@ -1554,7 +1486,8 @@ def refresh_job_lock(cert_name: str = "") -> None: unchanged_le_certs = {} for cert_name, pem_data in existing_le_certs.items(): - current_checksum = _calculate_cert_checksum(pem_data) + cleaned_pem = _clean_pem(pem_data) + current_checksum = _calculate_cert_checksum(cleaned_pem) previous_checksum = previous_le_checksums.get(cert_name) if previous_checksum is None: @@ -1578,50 +1511,28 @@ def refresh_job_lock(cert_name: str = "") -> None: stats["le_certs_changed"] = len(changed_le_certs) if unchanged_le_certs: - LOG.info("โœ“ OCSP found %d LE certificate(s) unchanged (skipping): %s", len(unchanged_le_certs), ", ".join(sorted(unchanged_le_certs.keys()))) - stats["le_certs_skipped"] = len(unchanged_le_certs) + LOG.info("โœ“ OCSP checking TTL for %d LE certificate(s) unchanged: %s", len(unchanged_le_certs), ", ".join(sorted(unchanged_le_certs.keys()))) + # stats["le_certs_skipped"] = len(unchanged_le_certs) <-- Removed, used for disabled status only stats["le_certs_processed"] = len(le_certs) - # Process new certs first with full OCSP refresh + # 1. Process newly issued certs (force refresh) for cert_name, pem_data in sorted(new_le_certs.items()): - # Check timeout before processing each certificate - if check_job_timeout(f"processing new LE cert {cert_name}"): - break - - # Refresh lock to prove we're still alive + if check_job_timeout(f"new LE cert {cert_name}"): break refresh_job_lock(cert_name) + all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=True)) - LOG.debug("๐Ÿ†• OCSP refreshing new LE certificate: %s", cert_name) - result = _process_cert(cert_name, pem_data, db, stats) - if result: - all_ocsp_results.append(result) - else: - # Still need to return the pem_data for checksum persistence if refresh was skipped - # We use a dummy result tuple (cert_name, None, 0, "", pem_data) to signal "just update checksum" - all_ocsp_results.append((cert_name, None, 0, "", pem_data)) - - # Process changed certs with OCSP refresh + # 2. Process changed certs (force refresh) for cert_name, pem_data in sorted(changed_le_certs.items()): - # Check timeout before processing each certificate - if check_job_timeout(f"processing changed LE cert {cert_name}"): - break - - # Refresh lock to prove we're still alive + if check_job_timeout(f"changed LE cert {cert_name}"): break refresh_job_lock(cert_name) + all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=True)) - LOG.debug("๐Ÿ”„ OCSP refreshing changed LE certificate: %s", cert_name) - result = _process_cert(cert_name, pem_data, db, stats) - if result: - all_ocsp_results.append(result) - else: - # Still need to return the pem_data for checksum persistence if refresh was skipped - # We use a dummy result tuple (cert_name, None, 0, "", pem_data) to signal "just update checksum" - all_ocsp_results.append((cert_name, None, 0, "", pem_data)) - - # For unchanged certs, we still need to return their pem_data for checksum persistence - for cert_name, pem_data in unchanged_le_certs.items(): - all_ocsp_results.append((cert_name, None, 0, "", pem_data)) + # 3. Process unchanged certs (TTL check only) + for cert_name, pem_data in sorted(unchanged_le_certs.items()): + if check_job_timeout(f"unchanged LE cert {cert_name}"): break + refresh_job_lock(cert_name) + all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=False)) else: LOG.info("โ„น๏ธ OCSP no LE certificates found in database") @@ -1635,32 +1546,31 @@ def refresh_job_lock(cert_name: str = "") -> None: # === Batch write all OCSP responses and checksums to database === if db and all_ocsp_results: LOG.info("๐Ÿ”„ OCSP batching database updates for %d certificate(s)", len(all_ocsp_results)) - for cert_name, ocsp_der, ttl, checksum, pem_data in all_ocsp_results: + for cert_name, ocsp_der, ttl, cert_checksum, pem_data in all_ocsp_results: # 1. Update OCSP response if we have a new one if ocsp_der: cache_key = f"ocsp/{cert_name}" try: + ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() err = db.upsert_job_cache( service_id=None, # Global cache entry file_name=cache_key, data=ocsp_der, job_name="ocsp-refresh", - checksum=checksum, + checksum=ocsp_checksum, ) if err: LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) stats["errors"] = stats["errors"] + 1 else: - LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, checksum[:8]) + LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, ocsp_checksum[:8]) except Exception as e: LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) stats["errors"] = stats["errors"] + 1 # 2. ALWAYS store/update certificate content checksum for future differential checks - # This ensures that even if OCSP refresh was skipped, we know we've seen this cert version. cert_checksum_key = f"cert_checksum/{cert_name}" - cert_checksum = _calculate_cert_checksum(pem_data) try: err = db.upsert_job_cache( service_id=None, @@ -1699,7 +1609,6 @@ def refresh_job_lock(cert_name: str = "") -> None: ocsp_path.write_bytes(ocsp_der) ocsp_path.chmod(0o644) # Readable by nginx LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) - stats["ocsp_fetched_responses"] = stats["ocsp_fetched_responses"] + 1 # Create symlinks for each SAN so OCSP is found by any SNI name _create_san_symlinks(pem_data, ocsp_path, cert_name) @@ -1734,12 +1643,14 @@ def refresh_job_lock(cert_name: str = "") -> None: elapsed = time.time() - job_start_time LOG.info( - "๐Ÿ“Š Statistics (completed in %.0fs): ๐Ÿ” LE certs=%d (skipped=%d, no OCSP=%d) | ๐Ÿ” Custom certs=%d (no OCSP=%d) | ๐Ÿ”„ Fetched=%d | โœ“ Cached=%d | ๐Ÿงน Orphaned=%d | ๐Ÿ” Verified=%d (restored=%d, corrected=%d) | โŒ Errors=%d", + "๐Ÿ“Š Statistics (completed in %.3fs): ๐Ÿ” LE certs=%d (skipped=%d, no OCSP=%d) | ๐Ÿ” Custom certs=%d (unchanged=%d, skipped=%d, no OCSP=%d) | ๐Ÿ”„ Fetched=%d | โœ“ Cached=%d | ๐Ÿงน Orphaned=%d | ๐Ÿ” Verified=%d (restored=%d, corrected=%d) | โŒ Errors=%d", elapsed, stats["le_certs_processed"], stats["le_certs_skipped"], stats["le_certs_no_ocsp"], stats["custom_certs_processed"], + stats.get("custom_certs_unchanged", 0), + stats["custom_certs_skipped"], stats["custom_certs_no_ocsp"], stats["ocsp_fetched_responses"], stats["ocsp_cached_responses"], From eac35118fb91daf591ed965eeda79057cef6b3a9 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 19:49:02 +0100 Subject: [PATCH 30/59] fix: Trigger OCSP stapling refresh for newly issued certificates - before: every service restart triggered an OCSP check - after: add a timer for OCSP full refresh. use --force to run a refresh --- .../core/customcert/jobs/custom-cert.py | 2 +- .../core/letsencrypt/jobs/certbot-new.py | 4 +- .../core/letsencrypt/jobs/certbot-renew.py | 20 ++- src/common/core/ssl/jobs/ocsp-refresh.py | 139 +++++++++++++----- 4 files changed, 118 insertions(+), 47 deletions(-) diff --git a/src/common/core/customcert/jobs/custom-cert.py b/src/common/core/customcert/jobs/custom-cert.py index 69d592b50f..a857fe9eee 100644 --- a/src/common/core/customcert/jobs/custom-cert.py +++ b/src/common/core/customcert/jobs/custom-cert.py @@ -229,7 +229,7 @@ def check_cert(cert_file: Union[Path, bytes], key_file: Union[Path, bytes], firs import sys ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") - result = run([sys.executable, ocsp_script], stdin=DEVNULL, capture_output=True, text=True, timeout=300) + result = run([sys.executable, ocsp_script, "--force"], stdin=DEVNULL, capture_output=True, text=True, timeout=300) if result.returncode == 0: LOGGER.info("โœ“ OCSP refresh completed successfully after cert change") else: diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index e8092e785f..b8def0b038 100644 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -1110,13 +1110,13 @@ def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, D # * Trigger OCSP stapling refresh for newly issued certificates (AFTER database save) # OCSP job will compare new certs with cached ones and process differential updates - if getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": + if status == 1 and getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": LOGGER.info("๐Ÿ”„ OCSP triggering refresh for newly issued certificates") try: import sys ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") - result = run([sys.executable, ocsp_script], stdin=DEVNULL, capture_output=True, text=True, timeout=300) + result = run([sys.executable, ocsp_script, "--force"], stdin=DEVNULL, capture_output=True, text=True, timeout=300) if result.returncode == 0: LOGGER.info("โœ“ OCSP refresh completed successfully after issuance") else: diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index 314c1a999b..80c478085b 100644 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -80,14 +80,22 @@ ] + (["-v"] if getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper() == "DEBUG" else []), stdin=DEVNULL, + stdout=PIPE, stderr=PIPE, universal_newlines=True, env=cmd_env, ) - while process.poll() is None: - if process.stderr: - for line in process.stderr: - LOGGER_CERTBOT.info(line.strip()) + stdout, stderr = process.communicate() + if stdout: + for line in stdout.splitlines(): + line_str = line.strip() + if line_str: + LOGGER_CERTBOT.info(line_str) + if "(success)" in line_str or "Congratulations" in line_str: + status = 1 + if stderr: + for line in stderr.splitlines(): + LOGGER_CERTBOT.info(line.strip()) if process.returncode != 0: status = 2 @@ -103,14 +111,14 @@ # Trigger OCSP refresh after successful renewal (AFTER database save) # OCSP job will compare new certs with cached ones and process differential updates - if process.returncode == 0 and getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": + if status == 1 and getenv("SSL_USE_OCSP_STAPLING", "yes").lower() == "yes": LOGGER.info("๐Ÿ”„ OCSP triggering refresh for renewed certificates") try: import sys ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") - result = run([sys.executable, ocsp_script], stdin=DEVNULL, capture_output=True, text=True, timeout=300) + result = run([sys.executable, ocsp_script, "--force"], stdin=DEVNULL, capture_output=True, text=True, timeout=300) if result.returncode == 0: LOGGER.info("โœ“ OCSP refresh completed successfully after renewal") else: diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 30ccd6abaa..52a6577126 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import fcntl import hashlib import io @@ -460,8 +461,8 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim ttl = 86400 # RFC standard fallback # Delay after successful fetch to prevent rate limiting - LOG.debug("โธ๏ธ OCSP delaying 5 seconds after fetch for %s to prevent rate limiting", cert_name) - time.sleep(5) + LOG.debug("โธ๏ธ OCSP delaying 2 seconds after fetch for %s to prevent rate limiting", cert_name) + time.sleep(2) return ocsp_der, ttl except HTTPError as e: @@ -952,8 +953,8 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 break if attempt == 1: - LOG.warning("โš ๏ธ OCSP fetch failed for %s, retrying once after 10 seconds ...", cert_name) - time.sleep(10) + LOG.warning("โš ๏ธ OCSP fetch failed for %s, retrying once after 2 seconds ...", cert_name) + time.sleep(2) if not ocsp_der: LOG.error("โŒ OCSP failed to fetch response for %s after retries", cert_name) @@ -976,7 +977,8 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta # Calculate checksum for integrity verification ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() - LOG.info("โšก OCSP final TTL for %s is %ds (checksum=%s)", cert_name, ttl, ocsp_checksum[:8]) + ttl_readable = f"{ttl / 86400.0:.1f} days" if ttl >= 86400 else f"{ttl / 3600.0:.1f} hours" + LOG.info("โšก OCSP final TTL for %s is %ds (%s) (checksum=%s)", cert_name, ttl, ttl_readable, ocsp_checksum[:8]) # === Return result for batched database writes === return (cert_name, ocsp_der, ttl, cert_checksum, pem_data) @@ -993,6 +995,7 @@ def process_custom_certs( lock_fd: Optional[int] = None, refresh_fn: Optional[Callable[[str], None]] = None, timeout_fn: Optional[Callable[[str], bool]] = None, + skip_unchanged_ttl_checks: bool = False, ) -> List[Tuple[str, Optional[bytes], int, str, bytes]]: """ Process OCSP for custom certificates using certificate data from the database. @@ -1048,8 +1051,12 @@ def process_custom_certs( unchanged_custom_certs[cert_name] = pem_data if unchanged_custom_certs: - LOG.info("โœ“ OCSP checking TTL for %d unchanged custom certificate(s): %s", len(unchanged_custom_certs), ", ".join(sorted(unchanged_custom_certs.keys()))) - stats["custom_certs_unchanged"] = stats.get("custom_certs_unchanged", 0) + len(unchanged_custom_certs) + if skip_unchanged_ttl_checks: + LOG.info("โœ“ OCSP skipping TTL checks for %d unchanged custom certificate(s) (recently run)", len(unchanged_custom_certs)) + stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + len(unchanged_custom_certs) + else: + LOG.info("โœ“ OCSP checking TTL for %d unchanged custom certificate(s): %s", len(unchanged_custom_certs), ", ".join(sorted(unchanged_custom_certs.keys()))) + stats["custom_certs_unchanged"] = stats.get("custom_certs_unchanged", 0) + len(unchanged_custom_certs) stats["custom_certs_processed"] = stats.get("custom_certs_processed", 0) + len(custom_certs) @@ -1062,12 +1069,13 @@ def process_custom_certs( results.append(_process_cert(cert_name, cert_pem, db, stats, force_fetch=True)) # 2. Process unchanged custom certificates (TTL check only) - for cert_name, cert_pem in sorted(unchanged_custom_certs.items()): - if callable(timeout_fn) and timeout_fn(f"unchanged custom cert {cert_name}"): - break - if callable(refresh_fn): - refresh_fn(cert_name) - results.append(_process_cert(cert_name, cert_pem, db, stats, force_fetch=False)) + if not skip_unchanged_ttl_checks: + for cert_name, cert_pem in sorted(unchanged_custom_certs.items()): + if callable(timeout_fn) and timeout_fn(f"unchanged custom cert {cert_name}"): + break + if callable(refresh_fn): + refresh_fn(cert_name) + results.append(_process_cert(cert_name, cert_pem, db, stats, force_fetch=False)) except Exception as e: LOG.error("OCSP exception while processing custom certificates: %s", e) @@ -1159,19 +1167,20 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None if db and purge_db: try: - # Remove all OCSP cache entries from database + # Remove all OCSP-related cache entries from database job_cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh") for cache_file in job_cache_files: - if cache_file.get("file_name", "").startswith("ocsp/"): + file_name = cache_file.get("file_name", "") + if file_name.startswith("ocsp/") or file_name.startswith("cert_checksum/") or file_name == "last_full_refresh": try: - db.delete_job_cache(file_name=cache_file["file_name"], job_name="ocsp-refresh") - LOG.debug("๐Ÿงน OCSP removed database entry %s", cache_file["file_name"]) + db.delete_job_cache(file_name=file_name, job_name="ocsp-refresh") + LOG.debug("๐Ÿงน OCSP removed database entry %s", file_name) except Exception as e: - LOG.debug("๐Ÿงน OCSP could not remove database entry %s: %s", cache_file["file_name"], e) + LOG.debug("๐Ÿงน OCSP could not remove database entry %s: %s", file_name, e) except Exception as e: LOG.debug("๐Ÿงน OCSP could not clean database entries: %s", e) elif not purge_db: - LOG.info("๐Ÿงน OCSP disk caches cleaned up (database entries preserved for quick restoration)") + LOG.info("๐Ÿงน OCSP disk caches cleaned up") LOG.info("๐Ÿงน OCSP all stapling caches cleaned up") @@ -1265,14 +1274,16 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic try: # Get all OCSP entries from database - cache_entries = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=True) - if not cache_entries: + # Get all entries and filter for actual OCSP responses + all_entries = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=True) + ocsp_entries = [e for e in all_entries if e.get("file_name", "").startswith("ocsp/")] + if not ocsp_entries: LOG.info("โ„น๏ธ OCSP no cache entries in database to verify") return + + LOG.info("๐Ÿ” OCSP verifying %d cache entry(ies) from database", len(ocsp_entries)) - LOG.info("๐Ÿ” OCSP verifying %d cache entry(ies) from database", len(cache_entries)) - - for entry in cache_entries: + for entry in ocsp_entries: file_name = entry.get("file_name", "") if not file_name.startswith("ocsp/"): continue @@ -1410,8 +1421,15 @@ def refresh_job_lock(cert_name: str = "") -> None: "orphaned_cleaned": 0, } + # Parse command line arguments + parser = argparse.ArgumentParser(description="OCSP refresh job for BunkerWeb") + parser.add_argument("--force", action="store_true", help="Force full OCSP refresh and TTL checks managed by the job") + args, unknown = parser.parse_known_args() + force_all = args.force + try: - LOG.info("๐Ÿ”„ OCSP refresh job started with differential update strategy (timeout in %d minutes)", JOB_TIMEOUT // 60) + LOG.info("๐Ÿ”„ OCSP refresh job started with differential update strategy (timeout in %d minutes%s)", + JOB_TIMEOUT // 60, " [FORCE]" if force_all else "") # Acquire main lock for the entire OCSP refresh operation lock_fd_main = _acquire_cert_lock("main", timeout=300, stale_threshold=1800) @@ -1455,8 +1473,8 @@ def refresh_job_lock(cert_name: str = "") -> None: # Check if OCSP stapling is globally disabled ocsp_enabled = os.getenv("SSL_USE_OCSP_STAPLING", "yes").lower() if ocsp_enabled != "yes": - LOG.info("๐Ÿงน OCSP stapling is globally disabled (SSL_USE_OCSP_STAPLING=%s), cleaning up disk caches (preserving database for quick re-enable)", ocsp_enabled) - cleanup_ocsp_cache(db, purge_db=False) + LOG.info("๐Ÿงน OCSP stapling is globally disabled (SSL_USE_OCSP_STAPLING=%s), cleaning up all caches", ocsp_enabled) + cleanup_ocsp_cache(db, purge_db=True) return 0 # Wait for scheduler's directory purge to finish after service restart, @@ -1464,10 +1482,26 @@ def refresh_job_lock(cert_name: str = "") -> None: # This handles ephemeral storage and post-restart cache directory cleanup. ocsp_files_exist = any((CONFIGS_SSL_BASE / d / "ocsp.der").is_file() for d in CONFIGS_SSL_BASE.iterdir()) if CONFIGS_SSL_BASE.is_dir() else False if not ocsp_files_exist: - LOG.info("๐Ÿ”„ OCSP no cached files on disk, waiting 10s for scheduler purge to finish before restoring from database...") - time.sleep(10) + LOG.info("๐Ÿ”„ OCSP no cached files on disk, waiting 2s for scheduler purge to finish before restoring from database...") + time.sleep(2) restore_ocsp_from_database(db) + # === Efficiency optimization: skip full refresh if run recently and no changes detected === + last_refresh_key = "last_full_refresh" + now_ts = int(time.time()) + skip_unchanged_ttl_checks = False + + if not force_all: + last_refresh_entry = db.get_job_cache_file(file_name=last_refresh_key, job_name="ocsp-refresh", with_info=True) + if last_refresh_entry and last_refresh_entry.get("data"): + try: + last_refresh_time = int(last_refresh_entry["data"].decode("utf-8")) + if now_ts - last_refresh_time < 1800: # 30 minutes window + skip_unchanged_ttl_checks = True + LOG.info("โ„น๏ธ OCSP full refresh was recently run (%ds ago), will only process new/changed certificates", now_ts - last_refresh_time) + except Exception: + pass + # === Collect all OCSP data for batched database writes === all_ocsp_results: List[Tuple[str, Optional[bytes], int, str, bytes]] = [] @@ -1511,8 +1545,11 @@ def refresh_job_lock(cert_name: str = "") -> None: stats["le_certs_changed"] = len(changed_le_certs) if unchanged_le_certs: - LOG.info("โœ“ OCSP checking TTL for %d LE certificate(s) unchanged: %s", len(unchanged_le_certs), ", ".join(sorted(unchanged_le_certs.keys()))) - # stats["le_certs_skipped"] = len(unchanged_le_certs) <-- Removed, used for disabled status only + if skip_unchanged_ttl_checks: + LOG.info("โœ“ OCSP skipping TTL checks for %d unchanged LE certificate(s) (recently run)", len(unchanged_le_certs)) + stats["le_certs_skipped"] = stats.get("le_certs_skipped", 0) + len(unchanged_le_certs) + else: + LOG.info("โœ“ OCSP checking TTL for %d LE certificate(s) unchanged: %s", len(unchanged_le_certs), ", ".join(sorted(unchanged_le_certs.keys()))) stats["le_certs_processed"] = len(le_certs) @@ -1529,10 +1566,11 @@ def refresh_job_lock(cert_name: str = "") -> None: all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=True)) # 3. Process unchanged certs (TTL check only) - for cert_name, pem_data in sorted(unchanged_le_certs.items()): - if check_job_timeout(f"unchanged LE cert {cert_name}"): break - refresh_job_lock(cert_name) - all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=False)) + if not skip_unchanged_ttl_checks: + for cert_name, pem_data in sorted(unchanged_le_certs.items()): + if check_job_timeout(f"unchanged LE cert {cert_name}"): break + refresh_job_lock(cert_name) + all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=False)) else: LOG.info("โ„น๏ธ OCSP no LE certificates found in database") @@ -1540,7 +1578,14 @@ def refresh_job_lock(cert_name: str = "") -> None: # Check timeout before processing custom certs if not check_job_timeout("before custom cert processing"): # Process custom certificates from database - custom_results = process_custom_certs(db, stats, lock_fd=lock_fd_main, refresh_fn=refresh_job_lock, timeout_fn=check_job_timeout) + custom_results = process_custom_certs( + db, + stats, + lock_fd=lock_fd_main, + refresh_fn=refresh_job_lock, + timeout_fn=check_job_timeout, + skip_unchanged_ttl_checks=skip_unchanged_ttl_checks + ) all_ocsp_results.extend(custom_results) # === Batch write all OCSP responses and checksums to database === @@ -1626,11 +1671,29 @@ def refresh_job_lock(cert_name: str = "") -> None: # Clean up orphaned OCSP entries for deleted services _cleanup_orphaned_ocsp(db, le_certs or {}, stats) + # Update last full refresh timestamp if we did a full run or found changes + if not skip_unchanged_ttl_checks or any(r[1] is not None for r in all_ocsp_results): + try: + db.upsert_job_cache( + service_id=None, + file_name=last_refresh_key, + data=str(now_ts).encode("utf-8"), + job_name="ocsp-refresh", + checksum=hashlib.sha256(str(now_ts).encode("utf-8")).hexdigest(), + ) + LOG.debug("โœ“ OCSP updated last full refresh timestamp to %d", now_ts) + except Exception as e: + LOG.debug("โš ๏ธ OCSP could not update last full refresh timestamp: %s", e) + # End-of-job verification: ensure OCSP files match database checksums # Restores missing files from database if not check_job_timeout("before final verification"): - LOG.info("๐Ÿ” OCSP running end-of-job verification and restoration...") - _verify_and_restore_ocsp_files(db, stats) + # Skip expensive verification if nothing changed and we were in "skip" mode + if skip_unchanged_ttl_checks and not any(r[1] is not None for r in all_ocsp_results): + LOG.info("๐Ÿ” OCSP skipping final verification (nothing changed and recently run)") + else: + LOG.info("๐Ÿ” OCSP running end-of-job verification and restoration...") + _verify_and_restore_ocsp_files(db, stats) # Decide exit status based on results if stats["errors"] > 0: From 3a793b5bbf96bdfea1c37c505556484dce8fafb4 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 20:01:38 +0100 Subject: [PATCH 31/59] add wildcard support - *.example.com may not be used for filenames - use _wildcard_.example.com for filename. _wildcard_ is not a permitted hostname so there will not be any lookup issues. --- .../server-http/ssl-certificate-lua.conf | 80 ++++++--- src/common/core/ssl/jobs/ocsp-refresh.py | 154 +++++++++++------- 2 files changed, 151 insertions(+), 83 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 02c55bb4b3..24328eb0a5 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -87,7 +87,11 @@ ssl_certificate_by_lua_block { local require_plugin = helpers.require_plugin local new_plugin = helpers.new_plugin local call_plugin = helpers.call_plugin - local tostring = tostring + local tostring = tostring + local insert = table.insert + local lower = string.lower + local match = string.match + local concat = table.concat -- Start ssl_certificate phase local logger = clogger:new("SSL-CERTIFICATE") @@ -121,6 +125,12 @@ ssl_certificate_by_lua_block { return true end + -- Helper: sanitize domain/cert name for filesystem (replace * with _wildcard_) + local function sanitize_name(name) + if not name then return nil end + return name:gsub("%*", "_wildcard_") + end + -- Helper: set OCSP stapling from cache (internalstore + ocsp.der disk files) local function set_ocsp_from_cache() logger:log(INFO, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) @@ -139,39 +149,63 @@ ssl_certificate_by_lua_block { local resp -- 1) Local shared dict lookup (per-worker cache) - if not resp then - local db_val, gerr = internalstore:get(cache_key, true) - if not db_val and gerr and gerr ~= "not found" then - logger:log(ERR, "OCSP error while getting response from internalstore: " .. gerr) - end - if db_val then - resp = db_val - end + local db_val, gerr = internalstore:get(cache_key, true) + if db_val then + resp = db_val + elseif gerr and gerr ~= "not found" then + logger:log(ERR, "OCSP error while getting response from internalstore: " .. gerr) end -- 2) Fallback to on-disk OCSP response (/var/cache/bunkerweb/ssl/{domain}/ocsp.der) if not resp then local base_path = "/var/cache/bunkerweb/ssl/" - local candidates = { - server_name, - server_name .. "-ecdsa", - server_name .. "-rsa", - "customcert-" .. server_name, - "customcert-" .. server_name .. "-ecdsa", - "customcert-" .. server_name .. "-rsa", - } + + -- Generate candidates for lookup + local candidates = {} + + -- Direct matches + insert(candidates, server_name) + insert(candidates, server_name .. "-ecdsa") + insert(candidates, server_name .. "-rsa") + insert(candidates, "customcert-" .. server_name) + insert(candidates, "customcert-" .. server_name .. "-ecdsa") + insert(candidates, "customcert-" .. server_name .. "-rsa") + + -- Wildcard matches (e.g., if server_name is foo.example.com, try *.example.com) + local labels = {} + for label in server_name:gmatch("[^.]+") do + insert(labels, label) + end + + -- Only try wildcard if we have at least foo.example.com (3+ parts if counting empty root, 2+ normally) + if #labels >= 2 then + -- Join labels back to create wildcard candidate: *.label2.label3... + for i = 2, #labels do + local suffix = table.concat(labels, ".", i) + if suffix and suffix ~= "" then + local wildcard_base = "*." .. suffix + insert(candidates, wildcard_base) + insert(candidates, wildcard_base .. "-ecdsa") + insert(candidates, wildcard_base .. "-rsa") + insert(candidates, "customcert-" .. wildcard_base) + insert(candidates, "customcert-" .. wildcard_base .. "-ecdsa") + insert(candidates, "customcert-" .. wildcard_base .. "-rsa") + end + end + end for _, name in ipairs(candidates) do if name and name ~= "" then - local path = base_path .. name .. "/ocsp.der" + local sanitized = sanitize_name(name) + local path = base_path .. sanitized .. "/ocsp.der" local f = io.open(path, "rb") if f then local data = f:read("*a") f:close() if data and #data > 0 then - logger:log(INFO, "OCSP loaded response (" .. #data .. " bytes) from " .. path) + logger:log(INFO, "OCSP loaded response (" .. #data .. " bytes) from " .. path .. " (matches " .. name .. ")") resp = data - -- Cache in shared memory to avoid disk I/O on every handshake + -- Cache in shared memory to avoid disk I/O on every handshake (TTL 300s) local ok_set, serr = internalstore:set(cache_key, resp, 300, true) if not ok_set then logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) @@ -179,12 +213,6 @@ ssl_certificate_by_lua_block { break end end - else - -- Check if file exists but couldn't be read (permission denied) - local file_exists = os.execute("test -f " .. path .. " 2>/dev/null") - if file_exists == 0 then - logger:log(ERR, "OCSP permission denied reading " .. path .. " (check file permissions)") - end end end end diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 52a6577126..19e0203d1d 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -73,6 +73,14 @@ MIN_TTL = 4500 # 75 minutes: aggressivly refresh/cleanup if TTL is below this +def _sanitize_filename(name: str) -> str: + """ + Replace characters that are invalid in filenames or could cause issues. + Specifically replaces '*' with '_wildcard_' for wildcard certificates. + """ + return name.replace("*", "_wildcard_") + + def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int = 600) -> Optional[int]: """ Acquire a file-based lock for a specific certificate to prevent race conditions @@ -96,7 +104,8 @@ def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int except Exception: return None - lock_file = lock_dir / f"ocsp-{cert_name}.lock" + sanitized_name = _sanitize_filename(cert_name) + lock_file = lock_dir / f"ocsp-{sanitized_name}.lock" start_time = time.time() while time.time() - start_time < timeout: @@ -148,14 +157,22 @@ def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int return None -def _release_cert_lock(fd: Optional[int]) -> None: - """Release a file-based lock.""" - if fd is not None: - try: - fcntl.flock(fd, fcntl.LOCK_UN) - os.close(fd) - except Exception: - pass +def _release_cert_lock(fd: Optional[int], cert_name: str = "undefined") -> None: + """ + Release and delete a certificate-specific lock file. + """ + if fd is None: + return + + sanitized_name = _sanitize_filename(cert_name) + lock_file = Path(os.sep, "tmp", "bunkerweb", f"ocsp-{sanitized_name}.lock") + try: + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + if lock_file.exists(): + lock_file.unlink() + except Exception: + pass def _refresh_cert_lock(fd: Optional[int], cert_name: str) -> None: @@ -272,7 +289,7 @@ def _fetch_issuer_from_aia(leaf: x509.Certificate, cert_name: str = "") -> Optio parsed = urlparse(issuer_url) if parsed.scheme not in ("http", "https"): continue - LOG.info("๐ŸŒ OCSP fetching issuer certificate from AIA: %s", issuer_url) + LOG.debug("๐ŸŒ OCSP fetching issuer certificate from AIA: %s", issuer_url) if not is_safe_url(issuer_url): LOG.error("โŒ OCSP unsafe AIA issuer URL detected: %s", issuer_url) @@ -334,6 +351,10 @@ def _parse_chain(pem_data: bytes, cert_name: str = "") -> Tuple[x509.Certificate raise RuntimeError(f"fullchain for {cert_name} does not contain issuer and AIA fetch failed") +# Default OCSP TTL fallback (1 day) if Next Update is missing +DEFAULT_OCSP_TTL = 86400 + + def _ocsp_response_lifetimes(ocsp_response: x509_ocsp.OCSPResponse) -> Tuple[Optional[int], Optional[int]]: """ Extract (remaining_ttl_seconds, total_lifetime_seconds) from a parsed OCSP response. @@ -514,7 +535,8 @@ def get_cached_ocsp_ttl(cert_name: str) -> Tuple[Optional[int], Optional[int]]: - total lifetime in seconds (Next Update - This Update) Both values are None if they cannot be determined. """ - ocsp_path = CONFIGS_SSL_BASE / cert_name / "ocsp.der" + sanitized_name = _sanitize_filename(cert_name) + ocsp_path = CONFIGS_SSL_BASE / sanitized_name / "ocsp.der" LOG.debug("โšก OCSP TTL check: checking cache for %s at %s", cert_name, ocsp_path) if not ocsp_path.is_file(): @@ -570,7 +592,8 @@ def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> No if san == base_name or san == cert_name: continue - san_dir = CONFIGS_SSL_BASE / san + sanitized_san = _sanitize_filename(san) + san_dir = CONFIGS_SSL_BASE / sanitized_san san_ocsp = san_dir / "ocsp.der" # Skip if target already exists (real file or valid symlink) @@ -579,7 +602,7 @@ def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> No if san_ocsp.is_symlink(): try: target = san_ocsp.readlink() - expected_target = Path("..") / cert_name / "ocsp.der" + expected_target = Path("..") / _sanitize_filename(cert_name) / "ocsp.der" if target == expected_target: LOG.debug("๐Ÿ”— OCSP SAN symlink for %s already correct (%s -> %s)", san, san_ocsp, target) else: @@ -593,7 +616,7 @@ def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> No try: san_dir.mkdir(parents=True, exist_ok=True) # Use relative symlink: ../cert_name/ocsp.der - rel_target = Path("..") / cert_name / "ocsp.der" + rel_target = Path("..") / _sanitize_filename(cert_name) / "ocsp.der" san_ocsp.symlink_to(rel_target) LOG.debug("๐Ÿ”— OCSP created SAN symlink %s -> %s", san_ocsp, rel_target) except Exception as e: @@ -617,10 +640,10 @@ def _get_cached_ocsp_certs(db: Any) -> set: for entry in cache_files: file_name = entry.get("file_name", "") if file_name.startswith("ocsp/"): - cert_name = file_name[len("ocsp/"):] + cert_name_raw = file_name[len("ocsp/"):] # Prevent path traversal from crafted database entries - if re.match(r"^[A-Za-z0-9_.*-]+$", cert_name): - cached_certs.add(cert_name) + if re.match(r"^[A-Za-z0-9_.*-]+$", cert_name_raw): + cached_certs.add(cert_name_raw) except Exception as e: LOG.warning("โš ๏ธ OCSP could not retrieve cached certificate list from database: %s", e) @@ -650,12 +673,12 @@ def _get_cert_checksums(db: Any, cert_names: set) -> Dict[str, str]: for entry in cache_files: file_name = entry.get("file_name", "") if file_name.startswith("cert_checksum/"): - cert_name = file_name[len("cert_checksum/"):] - if cert_name in cert_names: + cert_name_raw = file_name[len("cert_checksum/"):] + if cert_name_raw in cert_names: data = entry.get("data") if data: try: - checksums[cert_name] = data.decode("utf-8").strip() + checksums[cert_name_raw] = data.decode("utf-8").strip() except Exception: pass except Exception as e: @@ -841,16 +864,18 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: if not file_name.startswith("ocsp/") or not entry.get("data"): continue - cert_name = file_name[len("ocsp/"):] + cert_name_raw = file_name[len("ocsp/"):] # Prevent path traversal from crafted database entries - if not re.match(r"^[A-Za-z0-9_.*-]+$", cert_name): - LOG.warning("โš ๏ธ OCSP sanitization: skipping unsafe cert_name from database: %s", cert_name) + if not re.match(r"^[A-Za-z0-9_.*-]+$", cert_name_raw): + LOG.warning("โš ๏ธ OCSP sanitization: skipping unsafe cert_name from database: %s", cert_name_raw) continue + + sanitized_name = _sanitize_filename(cert_name_raw) db_data = entry["data"] db_checksum = entry.get("checksum") or hashlib.sha256(db_data).hexdigest() try: - ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + ocsp_cert_dir = CONFIGS_SSL_BASE / sanitized_name ocsp_path = ocsp_cert_dir / "ocsp.der" if ocsp_path.is_file(): @@ -858,10 +883,10 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: disk_checksum = hashlib.sha256(ocsp_path.read_bytes()).hexdigest() if disk_checksum == db_checksum: ok_count += 1 - LOG.debug("โœ“ OCSP disk file for %s matches database (checksum=%s)", cert_name, db_checksum[:8]) + LOG.debug("โœ“ OCSP disk file for %s matches database (checksum=%s)", cert_name_raw, db_checksum[:8]) continue # Checksum mismatch โ€” replace with database version - LOG.info("๐Ÿ”„ OCSP disk file for %s has wrong checksum (disk=%s, db=%s), replacing", cert_name, disk_checksum[:8], db_checksum[:8]) + LOG.info("๐Ÿ”„ OCSP disk file for %s has wrong checksum (disk=%s, db=%s), replacing", cert_name_raw, disk_checksum[:8], db_checksum[:8]) ocsp_path.write_bytes(db_data) ocsp_path.chmod(0o644) replaced_count += 1 @@ -871,9 +896,9 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: ocsp_path.write_bytes(db_data) ocsp_path.chmod(0o644) restored_count += 1 - LOG.debug("โœ“ OCSP restored cached response for %s from database", cert_name) + LOG.debug("โœ“ OCSP restored cached response for %s from database", cert_name_raw) except Exception as e: - LOG.debug("โš ๏ธ OCSP could not sync cache for %s: %s", cert_name, e) + LOG.debug("โš ๏ธ OCSP could not sync cache for %s: %s", cert_name_raw, e) if restored_count > 0 or replaced_count > 0: LOG.info("โœ“ OCSP sync complete: restored=%d, replaced=%d, unchanged=%d", restored_count, replaced_count, ok_count) @@ -895,6 +920,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta stats = {} service_name = _service_name_from_dir(cert_name) + sanitized_name = _sanitize_filename(cert_name) # === PEM CLEANING & SECURITY STRIPPING === # Clean it immediately so all subsequent logic (OCSP fetch, checksum, parsing) @@ -906,9 +932,19 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta if not _is_ocsp_enabled_for_service(service_name): LOG.debug("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) cleanup_ocsp_cache(db, cert_name) - stats["le_certs_skipped"] = stats["le_certs_skipped"] + 1 + stats["le_certs_skipped"] = stats.get("le_certs_skipped", 0) + 1 return (cert_name, None, 0, cert_checksum, pem_data) + # Check if certificate checksum matches what we have in database + cached_checksum = None + if db: + try: + checksum_data = db.get_job_cache_file(file_name=f"cert_checksum/{sanitized_name}", job_name="ocsp-refresh") + if checksum_data: + cached_checksum = checksum_data.decode("utf-8").strip() + except Exception as e: + LOG.debug("โš ๏ธ OCSP could not check cached checksum for %s: %s", cert_name, e) + LOG.debug("๐Ÿ”„ OCSP processing certificate %s", cert_name) try: @@ -1117,14 +1153,15 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None return # Clean up a single certificate's OCSP cache - ocsp_dir = CONFIGS_SSL_BASE / cert_name + sanitized_name = _sanitize_filename(cert_name) + ocsp_dir = CONFIGS_SSL_BASE / sanitized_name if ocsp_dir.is_dir(): shutil.rmtree(ocsp_dir, ignore_errors=True) LOG.info("๐Ÿงน OCSP removed cache directory for %s", cert_name) # Also remove any SAN symlinks that point to this cert's OCSP cache if CONFIGS_SSL_BASE.is_dir(): - target_rel = Path("..") / cert_name / "ocsp.der" + target_rel = Path("..") / sanitized_name / "ocsp.der" for entry in CONFIGS_SSL_BASE.iterdir(): if not entry.is_dir(): continue @@ -1144,9 +1181,10 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None if db and purge_db: try: - cache_key = f"ocsp/{cert_name}" - db.delete_job_cache(file_name=cache_key, job_name="ocsp-refresh") - LOG.debug("๐Ÿงน OCSP removed database entry for %s", cert_name) + # Delete both response and checksum from database + db.delete_job_cache(file_name=f"ocsp/{sanitized_name}", job_name="ocsp-refresh") + db.delete_job_cache(file_name=f"cert_checksum/{sanitized_name}", job_name="ocsp-refresh") + LOG.debug("๐Ÿงน OCSP database records removed for %s", cert_name) except Exception as e: LOG.debug("๐Ÿงน OCSP could not remove database entry for %s: %s", cert_name, e) else: @@ -1221,10 +1259,10 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: file_name = entry.get("file_name", "") if not file_name.startswith("ocsp/"): continue - cert_name = file_name[len("ocsp/"):] - if cert_name not in valid_cert_names: - LOG.info("๐Ÿงน OCSP removing orphaned entry for deleted service: %s", cert_name) - cleanup_ocsp_cache(db, cert_name) + cert_name_raw = file_name[len("ocsp/"):] + if cert_name_raw not in valid_cert_names: + LOG.info("๐Ÿงน OCSP removing orphaned entry for deleted service: %s", cert_name_raw) + cleanup_ocsp_cache(db, cert_name_raw) orphaned_count = orphaned_count + 1 except Exception as e: LOG.warning("โš ๏ธ OCSP could not check database for orphaned entries: %s", e) @@ -1237,13 +1275,13 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: ocsp_file = entry / "ocsp.der" if not ocsp_file.is_file() and not ocsp_file.is_symlink(): continue - cert_name = entry.name - if cert_name not in valid_cert_names: + cert_name_raw = entry.name + if cert_name_raw not in valid_cert_names: # Check if it's a SAN symlink (those are managed by _create_san_symlinks) if ocsp_file.is_symlink(): continue - LOG.info("๐Ÿงน OCSP removing orphaned disk cache for deleted service: %s", cert_name) - cleanup_ocsp_cache(db, cert_name) + LOG.info("๐Ÿงน OCSP removing orphaned disk cache for deleted service: %s", cert_name_raw) + cleanup_ocsp_cache(db, cert_name_raw) orphaned_count = orphaned_count + 1 if orphaned_count > 0: @@ -1288,18 +1326,19 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic if not file_name.startswith("ocsp/"): continue - cert_name = file_name[len("ocsp/"):] + cert_name_raw = file_name[len("ocsp/"):] + sanitized_name = _sanitize_filename(cert_name_raw) data = entry.get("data") db_checksum = entry.get("checksum", "") if not data or not db_checksum: - LOG.warning("โš ๏ธ OCSP database entry %s has no data or checksum, skipping verification", cert_name) + LOG.warning("โš ๏ธ OCSP database entry %s has no data or checksum, skipping verification", cert_name_raw) continue verify_count += 1 # Check if file exists on disk - ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + ocsp_cert_dir = CONFIGS_SSL_BASE / sanitized_name ocsp_path = ocsp_cert_dir / "ocsp.der" # Check if database response is already expired @@ -1307,7 +1346,7 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic ocsp_response = x509_ocsp.load_der_ocsp_response(data) remaining, _ = _ocsp_response_lifetimes(ocsp_response) if remaining is not None and remaining <= 0: - LOG.warning("๐Ÿงน OCSP response in database for %s is expired (remaining=%ds). Skipping restoration to disk.", cert_name, remaining) + LOG.warning("๐Ÿงน OCSP response in database for %s is expired (remaining=%ds). Skipping restoration to disk.", cert_name_raw, remaining) # Optionally remove from database to prevent future attempts if db: try: @@ -1317,20 +1356,20 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic pass continue except Exception as e: - LOG.warning("โš ๏ธ OCSP could not parse response from database for %s during verification: %s", cert_name, e) + LOG.warning("โš ๏ธ OCSP could not parse response from database for %s during verification: %s", cert_name_raw, e) try: if not ocsp_path.is_file(): # File missing: restore from database - LOG.warning("โš ๏ธ OCSP file missing for %s, restoring from database", cert_name) + LOG.warning("โš ๏ธ OCSP file missing for %s, restoring from database", cert_name_raw) try: ocsp_cert_dir.mkdir(parents=True, exist_ok=True) ocsp_path.write_bytes(data) ocsp_path.chmod(0o644) - LOG.info("โœ“ OCSP restored %s from database", cert_name) + LOG.info("โœ“ OCSP restored %s from database", cert_name_raw) restored_count += 1 except Exception as e: - LOG.error("โŒ OCSP could not restore %s from database: %s", cert_name, e) + LOG.error("โŒ OCSP could not restore %s from database: %s", cert_name_raw, e) stats["errors"] = stats.get("errors", 0) + 1 continue @@ -1343,21 +1382,21 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic LOG.warning( "โš ๏ธ OCSP checksum mismatch for %s (file=%s, db=%s). " "Restoring from database.", - cert_name, file_checksum[:8], db_checksum[:8] + cert_name_raw, file_checksum[:8], db_checksum[:8] ) try: ocsp_path.write_bytes(data) ocsp_path.chmod(0o644) - LOG.info("โœ“ OCSP restored correct version of %s", cert_name) + LOG.info("โœ“ OCSP restored correct version of %s", cert_name_raw) mismatch_count += 1 except Exception as e: - LOG.error("โŒ OCSP could not restore %s: %s", cert_name, e) + LOG.error("โŒ OCSP could not restore %s: %s", cert_name_raw, e) stats["errors"] = stats.get("errors", 0) + 1 else: - LOG.debug("โœ“ OCSP %s checksum verified (matches database)", cert_name) + LOG.debug("โœ“ OCSP %s checksum verified (matches database)", cert_name_raw) except Exception as e: - LOG.warning("โš ๏ธ OCSP error verifying %s: %s", cert_name, e) + LOG.warning("โš ๏ธ OCSP error verifying %s: %s", cert_name_raw, e) stats["errors"] = stats.get("errors", 0) + 1 if verify_count > 0: @@ -1592,9 +1631,10 @@ def refresh_job_lock(cert_name: str = "") -> None: if db and all_ocsp_results: LOG.info("๐Ÿ”„ OCSP batching database updates for %d certificate(s)", len(all_ocsp_results)) for cert_name, ocsp_der, ttl, cert_checksum, pem_data in all_ocsp_results: + sanitized_name = _sanitize_filename(cert_name) # 1. Update OCSP response if we have a new one - if ocsp_der: - cache_key = f"ocsp/{cert_name}" + if ocsp_der and ttl > 0: + cache_key = f"ocsp/{sanitized_name}" try: ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() err = db.upsert_job_cache( From 16758473bb28bb3eaf203bc95ec791235492e527 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 20:07:05 +0100 Subject: [PATCH 32/59] Update ssl-certificate-lua.conf --- src/common/confs/server-http/ssl-certificate-lua.conf | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 24328eb0a5..3298402c0b 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -210,7 +210,12 @@ ssl_certificate_by_lua_block { if not ok_set then logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) end - break + break + else + -- Check if file exists but couldn't be read (permission denied) + local file_exists = os.execute("test -f " .. path .. " 2>/dev/null") + if file_exists == 0 then + logger:log(ERR, "OCSP permission denied reading " .. path .. " (check file permissions)") end end end From 81fac6190581909c85b82465dd5ae4552638e007 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 20:07:17 +0100 Subject: [PATCH 33/59] Update ssl-certificate-lua.conf --- src/common/confs/server-http/ssl-certificate-lua.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 3298402c0b..decb499df1 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -210,7 +210,8 @@ ssl_certificate_by_lua_block { if not ok_set then logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) end - break + break + end else -- Check if file exists but couldn't be read (permission denied) local file_exists = os.execute("test -f " .. path .. " 2>/dev/null") From 00aeb9eb1669e104120ee519167f26b32f29eeeb Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 20:17:04 +0100 Subject: [PATCH 34/59] Update ocsp-refresh.py --- src/common/core/ssl/jobs/ocsp-refresh.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 19e0203d1d..cb5dfdd785 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -1655,7 +1655,7 @@ def refresh_job_lock(cert_name: str = "") -> None: stats["errors"] = stats["errors"] + 1 # 2. ALWAYS store/update certificate content checksum for future differential checks - cert_checksum_key = f"cert_checksum/{cert_name}" + cert_checksum_key = f"cert_checksum/{sanitized_name}" try: err = db.upsert_job_cache( service_id=None, @@ -1678,9 +1678,10 @@ def refresh_job_lock(cert_name: str = "") -> None: try: # Acquire lock to prevent race conditions with concurrent OCSP fetches lock_fd = _acquire_cert_lock(cert_name) + sanitized_name = _sanitize_filename(cert_name) try: # Create the SSL configs directory for this certificate - ocsp_cert_dir = CONFIGS_SSL_BASE / cert_name + ocsp_cert_dir = CONFIGS_SSL_BASE / sanitized_name try: ocsp_cert_dir.mkdir(parents=True, exist_ok=True) except Exception as e: From 2748e0eaae59623cd32c3ceebbb3afd2ba41f43c Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 20:52:07 +0100 Subject: [PATCH 35/59] Update ocsp-refresh.py - fallback to sha1 in case the PKI does not understand sha256 - stash failed request and retry after 120 seconds --- src/common/core/ssl/jobs/ocsp-refresh.py | 164 ++++++++++++++++------- 1 file changed, 115 insertions(+), 49 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index cb5dfdd785..dcf4ccc1b9 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -398,41 +398,76 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim return None, 0 try: - # Build OCSP request - builder = x509_ocsp.OCSPRequestBuilder() - builder = builder.add_certificate(leaf, issuer, hashes.SHA256()) - ocsp_request = builder.build() - ocsp_request_data = ocsp_request.public_bytes(Encoding.DER) - - # --- Build HTTP request --- - if not is_safe_url(ocsp_url): - LOG.error("โŒ OCSP unsafe responder URL detected: %s", ocsp_url) - return None, 0 + # Try SHA256 first, fallback to SHA1 if responder returns non-successful status (RFC 6960 compatibility) + ocsp_der = None + for hash_alg, alg_name in [(hashes.SHA256(), "SHA256"), (hashes.SHA1(), "SHA1")]: + try: + # Build OCSP request + builder = x509_ocsp.OCSPRequestBuilder() + builder = builder.add_certificate(leaf, issuer, hash_alg) + ocsp_request = builder.build() + ocsp_request_data = ocsp_request.public_bytes(Encoding.DER) - req = Request( - ocsp_url, - data=ocsp_request_data, - headers={ - "Content-Type": "application/ocsp-request", - "User-Agent": "BunkerWeb OCSP Fetcher (SSRF-Protected)", - }, - method="POST", - ) - # Explicitly use default SSL context with certificate verification enabled - ssl_context = ssl.create_default_context() - response = urlopen(req, timeout=timeout, context=ssl_context) - # cap response size at 100KB to prevent memory exhaustion from malicious responder - ocsp_der = response.read(102400) - - if not ocsp_der: - LOG.warning("โš ๏ธ OCSP empty response from %s for %s", ocsp_url, cert_name) + LOG.debug( + "๐ŸŒ OCSP fetching for %s (using %s): serial=%d, issuer=%s, responder=%s", + cert_name, alg_name, leaf.serial_number, issuer.subject.rfc4514_string(), ocsp_url + ) + + # --- Build HTTP request --- + if not is_safe_url(ocsp_url): + LOG.error("โŒ OCSP unsafe responder URL detected: %s", ocsp_url) + return None, 0 + + req = Request( + ocsp_url, + data=ocsp_request_data, + headers={ + "Content-Type": "application/ocsp-request", + "User-Agent": "BunkerWeb OCSP Fetcher (SSRF-Protected)", + }, + method="POST", + ) + # Explicitly use default SSL context with certificate verification enabled + ssl_context = ssl.create_default_context() + with urlopen(req, timeout=timeout, context=ssl_context) as response: + # cap response size at 100KB to prevent memory exhaustion from malicious responder + ocsp_der = response.read(102400) + + if not ocsp_der: + LOG.warning("โš ๏ธ OCSP empty response from %s for %s", ocsp_url, cert_name) + continue + + # Check if it's a valid OCSP response via cryptography + ocsp_response = x509_ocsp.load_der_ocsp_response(ocsp_der) + if ocsp_response.response_status != x509_ocsp.OCSPResponseStatus.SUCCESSFUL: + LOG.warning( + "โš ๏ธ OCSP responder returned %s for %s (using %s), retrying if fallback available...", + ocsp_response.response_status, cert_name, alg_name + ) + ocsp_der = None + continue + + # If we reached here, we have a SUCCESSFUL response + break + + except HTTPError as e: + LOG.warning("โš ๏ธ OCSP HTTP %d error for %s using %s: %s", e.code, cert_name, alg_name, e.reason) + ocsp_der = None + continue + except Exception as e: + LOG.warning("โš ๏ธ OCSP request failed for %s using %s: %s", cert_name, alg_name, e) + ocsp_der = None + continue + else: + # Loop finished without a break: all attempts failed + LOG.error("โŒ OCSP failed to fetch successful response for %s after trying both SHA256 and SHA1", cert_name) return None, 0 - # Check if it's a valid OCSP response via cryptography + # === SECURE OCSP RESPONSE VERIFICATION === + # Use OpenSSL to cryptographically verify the OCSP response signature. + # This prevents an attacker from MITM-ing the HTTP request and feeding a fake "SUCCESSFUL" payload. + # At this point, ocsp_der is guaranteed to be set and successful ocsp_response = x509_ocsp.load_der_ocsp_response(ocsp_der) - if ocsp_response.response_status != x509_ocsp.OCSPResponseStatus.SUCCESSFUL: - LOG.error("โŒ OCSP response status is not successful for %s: %s", cert_name, ocsp_response.response_status) - return None, 0 # === SECURE OCSP RESPONSE VERIFICATION === # Use OpenSSL to cryptographically verify the OCSP response signature. @@ -908,12 +943,12 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: LOG.debug("OCSP exception while attempting cache sync: %s", e) -def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None, force_fetch: bool = False) -> Tuple[str, Optional[bytes], int, str, bytes]: +def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None, force_fetch: bool = False) -> Tuple[str, Optional[bytes], int, str, bytes, Optional[str]]: """ Process a single certificate for OCSP stapling. Works with in-memory PEM data. If force_fetch is True, skips the cached TTL check and retrieves a new response. - Returns a tuple of (cert_name, ocsp_der, ttl, cert_checksum, pem_data) for batched database writes. + Returns a tuple of (cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url) for batched database writes. If ocsp_der is None, it means the fetch was skipped or failed. """ if stats is None: @@ -933,7 +968,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta LOG.debug("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) cleanup_ocsp_cache(db, cert_name) stats["le_certs_skipped"] = stats.get("le_certs_skipped", 0) + 1 - return (cert_name, None, 0, cert_checksum, pem_data) + return (cert_name, None, 0, cert_checksum, pem_data, None) # Check if certificate checksum matches what we have in database cached_checksum = None @@ -950,14 +985,14 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta try: if not pem_data.startswith(b"-----BEGIN"): LOG.warning("โš ๏ธ OCSP cert for %s has no PEM BEGIN marker or is invalid after cleaning, skipping", cert_name) - return (cert_name, None, 0, cert_checksum, pem_data) + return (cert_name, None, 0, cert_checksum, pem_data, None) # Proceed with extracting OCSP URL using the CLEANED pem_data ocsp_url = extract_ocsp_url(pem_data, cert_name) if not ocsp_url: LOG.debug("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) stats["le_certs_no_ocsp"] = stats["le_certs_no_ocsp"] + 1 - return (cert_name, None, 0, cert_checksum, pem_data) + return (cert_name, None, 0, cert_checksum, pem_data, None) LOG.debug("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) @@ -973,7 +1008,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta cert_name, cached_ttl, cached_ttl / 86400.0, MIN_TTL, half_lifetime, ) stats["ocsp_cached_responses"] = stats["ocsp_cached_responses"] + 1 - return (cert_name, None, cached_ttl, cert_checksum, pem_data) + return (cert_name, None, cached_ttl, cert_checksum, pem_data, ocsp_url) if cached_ttl <= MIN_TTL: LOG.info("๐Ÿ”„ OCSP response for %s is near expiration (TTL=%ds < %ds), attempting aggressive refresh", cert_name, cached_ttl, MIN_TTL) @@ -1009,7 +1044,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta cleanup_ocsp_cache(db, cert_name) stats["errors"] = stats.get("errors", 0) + 1 - return (cert_name, None, 0, cert_checksum, pem_data) + return (cert_name, None, 0, cert_checksum, pem_data, ocsp_url) # Calculate checksum for integrity verification ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() @@ -1017,12 +1052,12 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta LOG.info("โšก OCSP final TTL for %s is %ds (%s) (checksum=%s)", cert_name, ttl, ttl_readable, ocsp_checksum[:8]) # === Return result for batched database writes === - return (cert_name, ocsp_der, ttl, cert_checksum, pem_data) + return (cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url) except Exception as e: LOG.error("โŒ OCSP exception while processing certificate %s: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 - return (cert_name, None, 0, cert_checksum, pem_data) + return (cert_name, None, 0, cert_checksum, pem_data, None) def process_custom_certs( @@ -1032,11 +1067,11 @@ def process_custom_certs( refresh_fn: Optional[Callable[[str], None]] = None, timeout_fn: Optional[Callable[[str], bool]] = None, skip_unchanged_ttl_checks: bool = False, -) -> List[Tuple[str, Optional[bytes], int, str, bytes]]: +) -> List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str]]]: """ Process OCSP for custom certificates using certificate data from the database. - Returns a list of tuples (cert_name, ocsp_der, ttl, checksum, pem_data) for batched database writes. + Returns a list of tuples (cert_name, ocsp_der, ttl, checksum, pem_data, ocsp_url) for batched database writes. Args: db: Database connection @@ -1048,7 +1083,7 @@ def process_custom_certs( if stats is None: stats = {} - results: List[Tuple[str, Optional[bytes], int, str, bytes]] = [] + results: List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str]]] = [] if not db: LOG.info("โ„น๏ธ OCSP database not available, cannot process custom certificates") @@ -1542,7 +1577,9 @@ def refresh_job_lock(cert_name: str = "") -> None: pass # === Collect all OCSP data for batched database writes === - all_ocsp_results: List[Tuple[str, Optional[bytes], int, str, bytes]] = [] + all_ocsp_results: List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str]]] = [] + stashed_failures: List[Tuple[str, bytes]] = [] + # Process Let's Encrypt certificates from database tarball le_certs = _load_le_certs_from_db(db) @@ -1596,20 +1633,29 @@ def refresh_job_lock(cert_name: str = "") -> None: for cert_name, pem_data in sorted(new_le_certs.items()): if check_job_timeout(f"new LE cert {cert_name}"): break refresh_job_lock(cert_name) - all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=True)) + res = _process_cert(cert_name, pem_data, db, stats, force_fetch=True) + all_ocsp_results.append(res) + if res[1] is None and res[5]: # ocsp_der is None and ocsp_url is present + stashed_failures.append((cert_name, pem_data)) # 2. Process changed certs (force refresh) for cert_name, pem_data in sorted(changed_le_certs.items()): if check_job_timeout(f"changed LE cert {cert_name}"): break refresh_job_lock(cert_name) - all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=True)) + res = _process_cert(cert_name, pem_data, db, stats, force_fetch=True) + all_ocsp_results.append(res) + if res[1] is None and res[5]: + stashed_failures.append((cert_name, pem_data)) # 3. Process unchanged certs (TTL check only) if not skip_unchanged_ttl_checks: for cert_name, pem_data in sorted(unchanged_le_certs.items()): if check_job_timeout(f"unchanged LE cert {cert_name}"): break refresh_job_lock(cert_name) - all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=False)) + res = _process_cert(cert_name, pem_data, db, stats, force_fetch=False) + all_ocsp_results.append(res) + if res[1] is None and res[5]: + stashed_failures.append((cert_name, pem_data)) else: LOG.info("โ„น๏ธ OCSP no LE certificates found in database") @@ -1626,11 +1672,31 @@ def refresh_job_lock(cert_name: str = "") -> None: skip_unchanged_ttl_checks=skip_unchanged_ttl_checks ) all_ocsp_results.extend(custom_results) + for res in custom_results: + if res[1] is None and res[5]: + stashed_failures.append((res[0], res[4])) # cert_name, pem_data + + # === Final deferred retry for stashed failures === + if stashed_failures: + LOG.info("โธ๏ธ OCSP stashed %d failed fetch(es), waiting 120 seconds before final retry...", len(stashed_failures)) + + # Use smaller sleeps to stay responsive and allow timeout checks + for i in range(120): + if check_job_timeout("during stashed retry wait"): break + time.sleep(1) + + if not check_job_timeout("before starting stashed retries"): + for cert_name, pem_data in stashed_failures: + if check_job_timeout(f"stashed retry for {cert_name}"): break + LOG.info("๐Ÿ”„ OCSP retrying fetch for stashed failure: %s", cert_name) + refresh_job_lock(cert_name) + # Force fetch for the final retry attempt + all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=True)) # === Batch write all OCSP responses and checksums to database === if db and all_ocsp_results: LOG.info("๐Ÿ”„ OCSP batching database updates for %d certificate(s)", len(all_ocsp_results)) - for cert_name, ocsp_der, ttl, cert_checksum, pem_data in all_ocsp_results: + for cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url in all_ocsp_results: sanitized_name = _sanitize_filename(cert_name) # 1. Update OCSP response if we have a new one if ocsp_der and ttl > 0: @@ -1672,7 +1738,7 @@ def refresh_job_lock(cert_name: str = "") -> None: LOG.debug("โš ๏ธ OCSP exception while storing cert checksum for %s: %s", cert_name, e) # === Batch write all OCSP responses to disk === - for cert_name, ocsp_der, ttl, checksum, pem_data in all_ocsp_results: + for cert_name, ocsp_der, ttl, checksum, pem_data, ocsp_url in all_ocsp_results: if not ocsp_der: continue try: From 7ce0e7e416590e80f6faa617dfbf9c910f2f555f Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 20:59:52 +0100 Subject: [PATCH 36/59] Update ocsp-refresh.py --- src/common/core/ssl/jobs/ocsp-refresh.py | 64 ++++++++++++++---------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index dcf4ccc1b9..1c0d321591 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -943,12 +943,12 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: LOG.debug("OCSP exception while attempting cache sync: %s", e) -def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None, force_fetch: bool = False) -> Tuple[str, Optional[bytes], int, str, bytes, Optional[str]]: +def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None, force_fetch: bool = False) -> Tuple[str, Optional[bytes], int, str, bytes, Optional[str], bool]: """ Process a single certificate for OCSP stapling. Works with in-memory PEM data. If force_fetch is True, skips the cached TTL check and retrieves a new response. - Returns a tuple of (cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url) for batched database writes. + Returns a tuple of (cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url, was_attempted) for batched database writes. If ocsp_der is None, it means the fetch was skipped or failed. """ if stats is None: @@ -968,7 +968,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta LOG.debug("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) cleanup_ocsp_cache(db, cert_name) stats["le_certs_skipped"] = stats.get("le_certs_skipped", 0) + 1 - return (cert_name, None, 0, cert_checksum, pem_data, None) + return (cert_name, None, 0, cert_checksum, pem_data, None, False) # Check if certificate checksum matches what we have in database cached_checksum = None @@ -985,14 +985,14 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta try: if not pem_data.startswith(b"-----BEGIN"): LOG.warning("โš ๏ธ OCSP cert for %s has no PEM BEGIN marker or is invalid after cleaning, skipping", cert_name) - return (cert_name, None, 0, cert_checksum, pem_data, None) + return (cert_name, None, 0, cert_checksum, pem_data, None, False) # Proceed with extracting OCSP URL using the CLEANED pem_data ocsp_url = extract_ocsp_url(pem_data, cert_name) if not ocsp_url: LOG.debug("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) stats["le_certs_no_ocsp"] = stats["le_certs_no_ocsp"] + 1 - return (cert_name, None, 0, cert_checksum, pem_data, None) + return (cert_name, None, 0, cert_checksum, pem_data, None, False) LOG.debug("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) @@ -1008,7 +1008,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta cert_name, cached_ttl, cached_ttl / 86400.0, MIN_TTL, half_lifetime, ) stats["ocsp_cached_responses"] = stats["ocsp_cached_responses"] + 1 - return (cert_name, None, cached_ttl, cert_checksum, pem_data, ocsp_url) + return (cert_name, None, cached_ttl, cert_checksum, pem_data, ocsp_url, False) if cached_ttl <= MIN_TTL: LOG.info("๐Ÿ”„ OCSP response for %s is near expiration (TTL=%ds < %ds), attempting aggressive refresh", cert_name, cached_ttl, MIN_TTL) @@ -1029,22 +1029,32 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta if not ocsp_der: LOG.error("โŒ OCSP failed to fetch response for %s after retries", cert_name) - + if cached_ttl is not None: current_ttl = cast(int, cached_ttl) - if current_ttl <= MIN_TTL: + if current_ttl <= 0: LOG.warning( - "๐Ÿšจ๐Ÿšจ๐Ÿšจ OCSP CRITICAL WARNING: Cached response for %s is near expiration (TTL=%ds) " - "and refresh failed. Removing response from cache to prevent stapling expired data.", - cert_name, current_ttl + "๐Ÿงน OCSP cached response for %s is already expired and refresh failed. Cleaning up invalid cache.", + cert_name, ) cleanup_ocsp_cache(db, cert_name) - elif current_ttl <= 0: - LOG.warning("๐Ÿงน OCSP cached response for %s is already expired and refresh failed. Cleaning up invalid cache.", cert_name) + elif current_ttl <= MIN_TTL: + LOG.warning( + "๐Ÿšจ OCSP CRITICAL: Cached response for %s is near expiration (TTL=%ds) and refresh failed. " + "Removing from cache to prevent stapling expired data.", + cert_name, + current_ttl, + ) cleanup_ocsp_cache(db, cert_name) + else: + LOG.warning( + "โš ๏ธ OCSP could NOT refresh response for %s, keeping existing cache (TTL=%ds) for now", + cert_name, + current_ttl, + ) + return (cert_name, None, current_ttl, cert_checksum, pem_data, ocsp_url, True) - stats["errors"] = stats.get("errors", 0) + 1 - return (cert_name, None, 0, cert_checksum, pem_data, ocsp_url) + return (cert_name, None, 0, cert_checksum, pem_data, ocsp_url, True) # Calculate checksum for integrity verification ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() @@ -1052,12 +1062,12 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta LOG.info("โšก OCSP final TTL for %s is %ds (%s) (checksum=%s)", cert_name, ttl, ttl_readable, ocsp_checksum[:8]) # === Return result for batched database writes === - return (cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url) + return (cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url, True) except Exception as e: LOG.error("โŒ OCSP exception while processing certificate %s: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 - return (cert_name, None, 0, cert_checksum, pem_data, None) + return (cert_name, None, 0, cert_checksum, pem_data, None, False) def process_custom_certs( @@ -1067,11 +1077,11 @@ def process_custom_certs( refresh_fn: Optional[Callable[[str], None]] = None, timeout_fn: Optional[Callable[[str], bool]] = None, skip_unchanged_ttl_checks: bool = False, -) -> List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str]]]: +) -> List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str], bool]]: """ Process OCSP for custom certificates using certificate data from the database. - Returns a list of tuples (cert_name, ocsp_der, ttl, checksum, pem_data, ocsp_url) for batched database writes. + Returns a list of tuples (cert_name, ocsp_der, ttl, checksum, pem_data, ocsp_url, was_attempted) for batched database writes. Args: db: Database connection @@ -1083,7 +1093,7 @@ def process_custom_certs( if stats is None: stats = {} - results: List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str]]] = [] + results: List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str], bool]] = [] if not db: LOG.info("โ„น๏ธ OCSP database not available, cannot process custom certificates") @@ -1577,7 +1587,7 @@ def refresh_job_lock(cert_name: str = "") -> None: pass # === Collect all OCSP data for batched database writes === - all_ocsp_results: List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str]]] = [] + all_ocsp_results: List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str], bool]] = [] stashed_failures: List[Tuple[str, bytes]] = [] @@ -1635,7 +1645,7 @@ def refresh_job_lock(cert_name: str = "") -> None: refresh_job_lock(cert_name) res = _process_cert(cert_name, pem_data, db, stats, force_fetch=True) all_ocsp_results.append(res) - if res[1] is None and res[5]: # ocsp_der is None and ocsp_url is present + if res[1] is None and res[5] and res[6]: # ocsp_der is None AND ocsp_url is present AND was_attempted is True stashed_failures.append((cert_name, pem_data)) # 2. Process changed certs (force refresh) @@ -1644,7 +1654,7 @@ def refresh_job_lock(cert_name: str = "") -> None: refresh_job_lock(cert_name) res = _process_cert(cert_name, pem_data, db, stats, force_fetch=True) all_ocsp_results.append(res) - if res[1] is None and res[5]: + if res[1] is None and res[5] and res[6]: stashed_failures.append((cert_name, pem_data)) # 3. Process unchanged certs (TTL check only) @@ -1654,7 +1664,7 @@ def refresh_job_lock(cert_name: str = "") -> None: refresh_job_lock(cert_name) res = _process_cert(cert_name, pem_data, db, stats, force_fetch=False) all_ocsp_results.append(res) - if res[1] is None and res[5]: + if res[1] is None and res[5] and res[6]: stashed_failures.append((cert_name, pem_data)) else: @@ -1673,7 +1683,7 @@ def refresh_job_lock(cert_name: str = "") -> None: ) all_ocsp_results.extend(custom_results) for res in custom_results: - if res[1] is None and res[5]: + if res[1] is None and res[5] and res[6]: # ocsp_der is None AND ocsp_url is present AND was_attempted is True stashed_failures.append((res[0], res[4])) # cert_name, pem_data # === Final deferred retry for stashed failures === @@ -1696,7 +1706,7 @@ def refresh_job_lock(cert_name: str = "") -> None: # === Batch write all OCSP responses and checksums to database === if db and all_ocsp_results: LOG.info("๐Ÿ”„ OCSP batching database updates for %d certificate(s)", len(all_ocsp_results)) - for cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url in all_ocsp_results: + for cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url, was_attempted in all_ocsp_results: sanitized_name = _sanitize_filename(cert_name) # 1. Update OCSP response if we have a new one if ocsp_der and ttl > 0: @@ -1738,7 +1748,7 @@ def refresh_job_lock(cert_name: str = "") -> None: LOG.debug("โš ๏ธ OCSP exception while storing cert checksum for %s: %s", cert_name, e) # === Batch write all OCSP responses to disk === - for cert_name, ocsp_der, ttl, checksum, pem_data, ocsp_url in all_ocsp_results: + for cert_name, ocsp_der, ttl, checksum, pem_data, ocsp_url, was_attempted in all_ocsp_results: if not ocsp_der: continue try: From 273279c3b02f2bb1f17092391e5ddfb39238505e Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 21:25:11 +0100 Subject: [PATCH 37/59] fix wildcard error --- .../confs/server-http/ssl-certificate-lua.conf | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index decb499df1..ecfba83ea0 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -171,18 +171,25 @@ ssl_certificate_by_lua_block { insert(candidates, "customcert-" .. server_name .. "-ecdsa") insert(candidates, "customcert-" .. server_name .. "-rsa") - -- Wildcard matches (e.g., if server_name is foo.example.com, try *.example.com) + -- Wildcard and apex matches (e.g., if server_name is www.example.com, try *.example.com and example.com) local labels = {} for label in server_name:gmatch("[^.]+") do insert(labels, label) end - -- Only try wildcard if we have at least foo.example.com (3+ parts if counting empty root, 2+ normally) + -- Only try wildcard/apex if we have at least subdomain.example.com (2+ labels) if #labels >= 2 then - -- Join labels back to create wildcard candidate: *.label2.label3... for i = 2, #labels do local suffix = table.concat(labels, ".", i) if suffix and suffix ~= "" then + -- Apex domain (e.g. sysangels.net) โ€” wildcard certs are often stored under apex name + insert(candidates, suffix) + insert(candidates, suffix .. "-ecdsa") + insert(candidates, suffix .. "-rsa") + insert(candidates, "customcert-" .. suffix) + insert(candidates, "customcert-" .. suffix .. "-ecdsa") + insert(candidates, "customcert-" .. suffix .. "-rsa") + -- Wildcard form (e.g. *.sysangels.net) local wildcard_base = "*." .. suffix insert(candidates, wildcard_base) insert(candidates, wildcard_base .. "-ecdsa") From 4e9af3493320c2c95682c39b4446f6ac496f8824 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 13 Mar 2026 21:50:18 +0100 Subject: [PATCH 38/59] fix merge issue --- src/common/core/letsencrypt/jobs/certbot-renew.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index 80c478085b..4fffc3d08b 100644 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -2,7 +2,7 @@ from os import getenv, sep from os.path import join -from subprocess import DEVNULL, PIPE, Popen, run +from subprocess import DEVNULL, PIPE, Popen, TimeoutExpired, run from sys import exit as sys_exit, path as sys_path from traceback import format_exc @@ -28,6 +28,7 @@ LOGGER = getLogger("LETS-ENCRYPT.RENEW") LOGGER_CERTBOT = getLogger("LETS-ENCRYPT.RENEW.CERTBOT") +CERTBOT_TIMEOUT = 600 # seconds status = 0 @@ -85,7 +86,13 @@ universal_newlines=True, env=cmd_env, ) - stdout, stderr = process.communicate() + try: + stdout, stderr = process.communicate(timeout=CERTBOT_TIMEOUT) + except TimeoutExpired: + LOGGER.error(f"certbot renew timed out after {CERTBOT_TIMEOUT}s, killing process.") + process.kill() + stdout, stderr = process.communicate() + status = 2 if stdout: for line in stdout.splitlines(): line_str = line.strip() @@ -97,7 +104,7 @@ for line in stderr.splitlines(): LOGGER_CERTBOT.info(line.strip()) - if process.returncode != 0: + if process.returncode and process.returncode != 0: status = 2 LOGGER.error("Certificates renewal failed") From 184f7bb279e193d00388ff585ec6850c946a6596 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Sat, 14 Mar 2026 09:05:41 +0100 Subject: [PATCH 39/59] Update ssl-certificate-lua.conf --- src/common/confs/server-http/ssl-certificate-lua.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index ecfba83ea0..27e101c980 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -182,14 +182,14 @@ ssl_certificate_by_lua_block { for i = 2, #labels do local suffix = table.concat(labels, ".", i) if suffix and suffix ~= "" then - -- Apex domain (e.g. sysangels.net) โ€” wildcard certs are often stored under apex name + -- Apex domain (e.g. example.com) โ€” wildcard certs are often stored under apex name insert(candidates, suffix) insert(candidates, suffix .. "-ecdsa") insert(candidates, suffix .. "-rsa") insert(candidates, "customcert-" .. suffix) insert(candidates, "customcert-" .. suffix .. "-ecdsa") insert(candidates, "customcert-" .. suffix .. "-rsa") - -- Wildcard form (e.g. *.sysangels.net) + -- Wildcard form (e.g. *.example.com) local wildcard_base = "*." .. suffix insert(candidates, wildcard_base) insert(candidates, wildcard_base .. "-ecdsa") From 8b8c5463eab505f8cec194ede55aba33865585c6 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Sun, 15 Mar 2026 10:48:45 +0100 Subject: [PATCH 40/59] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/common/core/letsencrypt/jobs/certbot-renew.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index 787a34e2fc..178470cb9c 100644 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -29,8 +29,7 @@ LOGGER = getLogger("LETS-ENCRYPT.RENEW") LOGGER_CERTBOT = getLogger("LETS-ENCRYPT.RENEW.CERTBOT") -CERTBOT_TIMEOUT = 600 # seconds -CERTBOT_TIMEOUT = 900 # 15 minutes max for a single certbot invocation +CERTBOT_TIMEOUT = 900 # 900 seconds (15 minutes) max for a single certbot invocation status = 0 From b735de9bbdf553460fb38908ca008485c97a5ec8 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Sun, 15 Mar 2026 12:58:24 +0100 Subject: [PATCH 41/59] Address Copilot review: OCSP timeouts, SSL_USE_OCSP_STAPLING, stats safety, dead code --- .../server-http/ssl-certificate-lua.conf | 35 ++- .../core/customcert/jobs/custom-cert.py | 11 +- .../core/letsencrypt/jobs/certbot-new.py | 9 +- .../core/letsencrypt/jobs/certbot-renew.py | 19 +- src/common/core/ssl/jobs/ocsp-refresh.py | 225 +++++++++++------- 5 files changed, 193 insertions(+), 106 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 27e101c980..df17a12246 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -120,8 +120,39 @@ ssl_certificate_by_lua_block { -- Helper: check if OCSP stapling is enabled for this site local function is_ocsp_stapling_enabled() - -- TODO: Check SSL_USE_OCSP_STAPLING variable when variable loading is available in ssl_certificate phase - -- For now, always attempt OCSP stapling if files exist (graceful fallback if not) + -- Default to enabled (backward compatible behavior) if variables are not available + local vars, err = internalstore:get("variables", true) + if not vars then + return true + end + + -- Resolve effective SSL_USE_OCSP_STAPLING value with per-site override + local value + if vars.per_site and server_name and vars.per_site[server_name] then + value = vars.per_site[server_name]["SSL_USE_OCSP_STAPLING"] + end + if value == nil and vars.global then + value = vars.global["SSL_USE_OCSP_STAPLING"] + end + if value == nil then + value = vars["SSL_USE_OCSP_STAPLING"] + end + + -- If still nil, keep default behavior (enabled) + if value == nil then + return true + end + + -- Normalize and interpret common falsy/disabled representations + if type(value) == "boolean" then + return value + end + local str = tostring(value):lower() + if str == "0" or str == "false" or str == "off" or str == "no" then + return false + end + + -- Any other value is treated as enabled return true end diff --git a/src/common/core/customcert/jobs/custom-cert.py b/src/common/core/customcert/jobs/custom-cert.py index a857fe9eee..8e876ca5a9 100644 --- a/src/common/core/customcert/jobs/custom-cert.py +++ b/src/common/core/customcert/jobs/custom-cert.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -from os import environ, getenv, sep +from os import getenv, sep from os.path import join from pathlib import Path -from subprocess import DEVNULL, run +from subprocess import DEVNULL, TimeoutExpired, run from sys import exit as sys_exit, path as sys_path from base64 import b64decode from tempfile import NamedTemporaryFile @@ -20,6 +20,7 @@ LOGGER = getLogger("CUSTOM-CERT") JOB = Job(LOGGER, __file__) +OCSP_REFRESH_TIMEOUT = 2100 # 35 minutes max to cover ocsp-refresh job worst-case duration def process_ssl_data(data: str, file_path: Optional[str], data_type: Literal["cert", "key"], server_name: str) -> Union[bytes, Path, None]: @@ -229,7 +230,7 @@ def check_cert(cert_file: Union[Path, bytes], key_file: Union[Path, bytes], firs import sys ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") - result = run([sys.executable, ocsp_script, "--force"], stdin=DEVNULL, capture_output=True, text=True, timeout=300) + result = run([sys.executable, ocsp_script, "--force"], stdin=DEVNULL, capture_output=True, text=True, timeout=OCSP_REFRESH_TIMEOUT) if result.returncode == 0: LOGGER.info("โœ“ OCSP refresh completed successfully after cert change") else: @@ -237,6 +238,10 @@ def check_cert(cert_file: Union[Path, bytes], key_file: Union[Path, bytes], firs if result.stderr: for line in result.stderr.strip().splitlines(): LOGGER.debug(f"OCSP: {line}") + except TimeoutExpired as e: + LOGGER.warning( + f"โš ๏ธ OCSP post-change refresh timed out after {OCSP_REFRESH_TIMEOUT}s (non-fatal): {e}" + ) except Exception as e: LOGGER.warning(f"โš ๏ธ OCSP post-change refresh failed (non-fatal): {e}") except SystemExit as e: diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index 2f41705b4c..fceedd4ff8 100644 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -10,7 +10,7 @@ from pathlib import Path from re import MULTILINE, match, search from select import select -from subprocess import DEVNULL, PIPE, STDOUT, Popen, run +from subprocess import DEVNULL, PIPE, STDOUT, Popen, TimeoutExpired, run from sys import exit as sys_exit, path as sys_path from time import monotonic, sleep from threading import Event, Lock, Thread @@ -127,6 +127,7 @@ def stop_progress_monitor() -> None: ACME_SERVER_TYPES = ("letsencrypt", "zerossl") DNS_PROPAGATION_DEFAULT = "default" CERTBOT_TIMEOUT = 900 # 15 minutes max for a single certbot invocation +OCSP_REFRESH_TIMEOUT = 2100 # 35 minutes max to cover ocsp-refresh job worst-case duration def normalize_server_names(server_names: str) -> Set[str]: @@ -1157,7 +1158,7 @@ def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, D import sys ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") - result = run([sys.executable, ocsp_script, "--force"], stdin=DEVNULL, capture_output=True, text=True, timeout=300) + result = run([sys.executable, ocsp_script, "--force"], stdin=DEVNULL, capture_output=True, text=True, timeout=OCSP_REFRESH_TIMEOUT) if result.returncode == 0: LOGGER.info("โœ“ OCSP refresh completed successfully after issuance") else: @@ -1165,6 +1166,10 @@ def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, D if result.stderr: for line in result.stderr.strip().splitlines(): LOGGER.debug(f"OCSP: {line}") + except TimeoutExpired as e: + LOGGER.warning( + f"โš ๏ธ OCSP post-issuance refresh timed out after {OCSP_REFRESH_TIMEOUT}s (non-fatal): {e}" + ) except Exception as e: LOGGER.warning(f"โš ๏ธ OCSP post-issuance refresh failed (non-fatal): {e}") except SystemExit as e: diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index 178470cb9c..f678178b36 100644 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -4,7 +4,6 @@ from os.path import join from subprocess import DEVNULL, PIPE, Popen, TimeoutExpired, run from sys import exit as sys_exit, path as sys_path -from time import monotonic from traceback import format_exc for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: @@ -30,6 +29,7 @@ LOGGER_CERTBOT = getLogger("LETS-ENCRYPT.RENEW.CERTBOT") CERTBOT_TIMEOUT = 900 # 900 seconds (15 minutes) max for a single certbot invocation +OCSP_REFRESH_TIMEOUT = 2100 # 2100 seconds (35 minutes) max to cover ocsp-refresh job worst-case duration status = 0 @@ -104,17 +104,6 @@ if stderr: for line in stderr.splitlines(): LOGGER_CERTBOT.info(line.strip()) - deadline = monotonic() + CERTBOT_TIMEOUT - while process.poll() is None: - if monotonic() > deadline: - LOGGER.error(f"certbot renew timed out after {CERTBOT_TIMEOUT}s, killing process.") - process.kill() - process.wait() - status = 2 - break - if process.stderr: - for line in process.stderr: - LOGGER_CERTBOT.info(line.strip()) if process.returncode and process.returncode != 0: status = 2 @@ -137,7 +126,7 @@ import sys ocsp_script = join(sep, "usr", "share", "bunkerweb", "core", "ssl", "jobs", "ocsp-refresh.py") - result = run([sys.executable, ocsp_script, "--force"], stdin=DEVNULL, capture_output=True, text=True, timeout=300) + result = run([sys.executable, ocsp_script, "--force"], stdin=DEVNULL, capture_output=True, text=True, timeout=OCSP_REFRESH_TIMEOUT) if result.returncode == 0: LOGGER.info("โœ“ OCSP refresh completed successfully after renewal") else: @@ -145,6 +134,10 @@ if result.stderr: for line in result.stderr.strip().splitlines(): LOGGER.debug(f"OCSP: {line}") + except TimeoutExpired as e: + LOGGER.warning( + f"โš ๏ธ OCSP post-renewal refresh timed out after {OCSP_REFRESH_TIMEOUT}s (non-fatal): {e}" + ) except Exception as e: LOGGER.warning(f"โš ๏ธ OCSP post-renewal refresh failed (non-fatal): {e}") except SystemExit as e: diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 1c0d321591..764faf917b 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -503,8 +503,17 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim "-CAfile", f_issuer.name, "-partial_chain" ] - - p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + try: + # Add a timeout so a stuck openssl process cannot hang the whole job + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=30) + except subprocess.TimeoutExpired as e: + LOG.error( + "โŒ OCSP response cryptographic signature verification timed out for %s after %s seconds. Discarding response.", + cert_name, + e.timeout, + ) + return None, 0 if p.returncode != 0: LOG.error("โŒ OCSP response cryptographic signature verification failed for %s. Discarding forged/invalid response. OpenSSL Error: %s", cert_name, p.stderr.strip() or p.stdout.strip()) return None, 0 @@ -991,7 +1000,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta ocsp_url = extract_ocsp_url(pem_data, cert_name) if not ocsp_url: LOG.debug("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) - stats["le_certs_no_ocsp"] = stats["le_certs_no_ocsp"] + 1 + stats["le_certs_no_ocsp"] = stats.get("le_certs_no_ocsp", 0) + 1 return (cert_name, None, 0, cert_checksum, pem_data, None, False) LOG.debug("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) @@ -1007,7 +1016,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), above thresholds (MIN_TTL=%ds, 50%% threshold=%ds), skipping fetch", cert_name, cached_ttl, cached_ttl / 86400.0, MIN_TTL, half_lifetime, ) - stats["ocsp_cached_responses"] = stats["ocsp_cached_responses"] + 1 + stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 return (cert_name, None, cached_ttl, cert_checksum, pem_data, ocsp_url, False) if cached_ttl <= MIN_TTL: @@ -1160,7 +1169,7 @@ def process_custom_certs( except Exception as e: LOG.error("OCSP exception while processing custom certificates: %s", e) - stats["errors"] = stats["errors"] + 1 + stats["errors"] = stats.get("errors", 0) + 1 return results @@ -1459,6 +1468,116 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic stats["errors"] = stats.get("errors", 0) + 1 +def _persist_ocsp_results_to_db( + db: Optional[Any], + all_ocsp_results: List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str], bool]], + stats: Optional[Dict[str, int]] = None, +) -> None: + """ + Persist OCSP responses and certificate checksums to database. + Called both at normal completion and when timeout occurs. + """ + if db is None or not all_ocsp_results: + return + + if stats is None: + stats = {} + + LOG.info("๐Ÿ”„ OCSP batching database updates for %d certificate(s)", len(all_ocsp_results)) + + for cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url, was_attempted in all_ocsp_results: + sanitized_name = _sanitize_filename(cert_name) + + # 1. Update OCSP response if we have a new one + if ocsp_der and ttl > 0: + cache_key = f"ocsp/{sanitized_name}" + try: + ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() + err = db.upsert_job_cache( + service_id=None, # Global cache entry + file_name=cache_key, + data=ocsp_der, + job_name="ocsp-refresh", + checksum=ocsp_checksum, + ) + + if err: + LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) + stats["errors"] = stats.get("errors", 0) + 1 + else: + LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, ocsp_checksum[:8]) + except Exception as e: + LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + + # 2. ALWAYS store/update certificate content checksum for future differential checks + cert_checksum_key = f"cert_checksum/{sanitized_name}" + try: + err = db.upsert_job_cache( + service_id=None, + file_name=cert_checksum_key, + data=cert_checksum.encode("utf-8"), + job_name="ocsp-refresh", + checksum=hashlib.sha256(cert_checksum.encode("utf-8")).hexdigest(), + ) + if err: + LOG.debug("โš ๏ธ OCSP could not store cert checksum for %s: %s", cert_name, err) + else: + LOG.debug("โœ“ OCSP persisted checksum for %s", cert_name) + except Exception as e: + LOG.debug("โš ๏ธ OCSP exception while storing cert checksum for %s: %s", cert_name, e) + + +def _persist_ocsp_results_to_disk( + all_ocsp_results: List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str], bool]], + stats: Optional[Dict[str, int]] = None, +) -> None: + """ + Write OCSP responses to disk cache files. + Called both at normal completion and when timeout occurs. + """ + if not all_ocsp_results: + return + + if stats is None: + stats = {} + + for cert_name, ocsp_der, ttl, checksum, pem_data, ocsp_url, was_attempted in all_ocsp_results: + if not ocsp_der: + continue + try: + # Acquire lock to prevent race conditions with concurrent OCSP fetches + lock_fd = _acquire_cert_lock(cert_name) + sanitized_name = _sanitize_filename(cert_name) + try: + # Create the SSL configs directory for this certificate + ocsp_cert_dir = CONFIGS_SSL_BASE / sanitized_name + try: + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + except Exception as e: + LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) + stats["errors"] = stats.get("errors", 0) + 1 + continue + + # Write OCSP response to cache/ssl/{cert-name}/ocsp.der + ocsp_path = ocsp_cert_dir / "ocsp.der" + try: + ocsp_path.write_bytes(ocsp_der) + ocsp_path.chmod(0o644) # Readable by nginx + LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) + + # Create symlinks for each SAN so OCSP is found by any SNI name + _create_san_symlinks(pem_data, ocsp_path, cert_name) + except Exception as e: + LOG.error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + finally: + _release_cert_lock(lock_fd) + except Exception as e: + LOG.error("โŒ OCSP exception while writing response for %s to disk: %s", cert_name, e) + stats["errors"] = stats.get("errors", 0) + 1 + + def main() -> int: global status db: Optional[Any] = None @@ -1689,12 +1808,12 @@ def refresh_job_lock(cert_name: str = "") -> None: # === Final deferred retry for stashed failures === if stashed_failures: LOG.info("โธ๏ธ OCSP stashed %d failed fetch(es), waiting 120 seconds before final retry...", len(stashed_failures)) - + # Use smaller sleeps to stay responsive and allow timeout checks for i in range(120): if check_job_timeout("during stashed retry wait"): break time.sleep(1) - + if not check_job_timeout("before starting stashed retries"): for cert_name, pem_data in stashed_failures: if check_job_timeout(f"stashed retry for {cert_name}"): break @@ -1703,85 +1822,19 @@ def refresh_job_lock(cert_name: str = "") -> None: # Force fetch for the final retry attempt all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=True)) - # === Batch write all OCSP responses and checksums to database === - if db and all_ocsp_results: - LOG.info("๐Ÿ”„ OCSP batching database updates for %d certificate(s)", len(all_ocsp_results)) - for cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url, was_attempted in all_ocsp_results: - sanitized_name = _sanitize_filename(cert_name) - # 1. Update OCSP response if we have a new one - if ocsp_der and ttl > 0: - cache_key = f"ocsp/{sanitized_name}" - try: - ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() - err = db.upsert_job_cache( - service_id=None, # Global cache entry - file_name=cache_key, - data=ocsp_der, - job_name="ocsp-refresh", - checksum=ocsp_checksum, - ) + # === Check if timeout has been reached and save partial results === + if check_job_timeout("after processing phase"): + LOG.warning("โฑ๏ธ OCSP job timeout during processing. Saving partial results (%d cert(s)) to database and disk.", len(all_ocsp_results)) + _persist_ocsp_results_to_db(db, all_ocsp_results, stats) + _persist_ocsp_results_to_disk(all_ocsp_results, stats) + # Return early with partial results saved + elapsed = time.time() - job_start_time + LOG.warning("๐Ÿ“Š OCSP partial job completed in %.3fs with %d results saved", elapsed, len(all_ocsp_results)) + return status - if err: - LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) - stats["errors"] = stats["errors"] + 1 - else: - LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, ocsp_checksum[:8]) - except Exception as e: - LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) - stats["errors"] = stats["errors"] + 1 - - # 2. ALWAYS store/update certificate content checksum for future differential checks - cert_checksum_key = f"cert_checksum/{sanitized_name}" - try: - err = db.upsert_job_cache( - service_id=None, - file_name=cert_checksum_key, - data=cert_checksum.encode("utf-8"), - job_name="ocsp-refresh", - checksum=hashlib.sha256(cert_checksum.encode("utf-8")).hexdigest(), - ) - if err: - LOG.debug("โš ๏ธ OCSP could not store cert checksum for %s: %s", cert_name, err) - else: - LOG.debug("โœ“ OCSP persisted checksum for %s", cert_name) - except Exception as e: - LOG.debug("โš ๏ธ OCSP exception while storing cert checksum for %s: %s", cert_name, e) - - # === Batch write all OCSP responses to disk === - for cert_name, ocsp_der, ttl, checksum, pem_data, ocsp_url, was_attempted in all_ocsp_results: - if not ocsp_der: - continue - try: - # Acquire lock to prevent race conditions with concurrent OCSP fetches - lock_fd = _acquire_cert_lock(cert_name) - sanitized_name = _sanitize_filename(cert_name) - try: - # Create the SSL configs directory for this certificate - ocsp_cert_dir = CONFIGS_SSL_BASE / sanitized_name - try: - ocsp_cert_dir.mkdir(parents=True, exist_ok=True) - except Exception as e: - LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) - stats["errors"] = stats["errors"] + 1 - continue - - # Write OCSP response to cache/ssl/{cert-name}/ocsp.der - ocsp_path = ocsp_cert_dir / "ocsp.der" - try: - ocsp_path.write_bytes(ocsp_der) - ocsp_path.chmod(0o644) # Readable by nginx - LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) - - # Create symlinks for each SAN so OCSP is found by any SNI name - _create_san_symlinks(pem_data, ocsp_path, cert_name) - except Exception as e: - LOG.error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) - stats["errors"] = stats.get("errors", 0) + 1 - finally: - _release_cert_lock(lock_fd) - except Exception as e: - LOG.error("โŒ OCSP exception while writing response for %s to disk: %s", cert_name, e) - stats["errors"] = stats.get("errors", 0) + 1 + # === Persist all OCSP responses to database and disk === + _persist_ocsp_results_to_db(db, all_ocsp_results, stats) + _persist_ocsp_results_to_disk(all_ocsp_results, stats) # Check timeout before cleanup if not check_job_timeout("before orphaned cleanup"): @@ -1789,7 +1842,7 @@ def refresh_job_lock(cert_name: str = "") -> None: _cleanup_orphaned_ocsp(db, le_certs or {}, stats) # Update last full refresh timestamp if we did a full run or found changes - if not skip_unchanged_ttl_checks or any(r[1] is not None for r in all_ocsp_results): + if db is not None and (not skip_unchanged_ttl_checks or any(r[1] is not None for r in all_ocsp_results)): try: db.upsert_job_cache( service_id=None, From 1841a32f66e647c329da42bc1f4934b1832ef988 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Sun, 15 Mar 2026 13:02:00 +0100 Subject: [PATCH 42/59] Update ssl-certificate-lua.conf --- .../server-http/ssl-certificate-lua.conf | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index df17a12246..6c359f0a08 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -81,6 +81,7 @@ ssl_certificate_by_lua_block { local is_internal = ngx_req.is_internal local ERR = ngx.ERR local INFO = ngx.INFO + local DEBUG = ngx.DEBUG local clear_certs = ssl.clear_certs local set_cert = ssl.set_cert local set_priv_key = ssl.set_priv_key @@ -164,10 +165,10 @@ ssl_certificate_by_lua_block { -- Helper: set OCSP stapling from cache (internalstore + ocsp.der disk files) local function set_ocsp_from_cache() - logger:log(INFO, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) + logger:log(DEBUG, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) if not is_ocsp_stapling_enabled() then - logger:log(INFO, "OCSP stapling disabled via SSL_USE_OCSP_STAPLING setting") + logger:log(DEBUG, "OCSP stapling disabled via SSL_USE_OCSP_STAPLING setting") return end @@ -208,9 +209,9 @@ ssl_certificate_by_lua_block { insert(labels, label) end - -- Only try wildcard/apex if we have at least subdomain.example.com (2+ labels) - if #labels >= 2 then - for i = 2, #labels do + -- Only try wildcard/apex if we have at least subdomain.example.com (3+ labels), and avoid TLD-only suffixes + if #labels >= 3 then + for i = 2, #labels - 1 do local suffix = table.concat(labels, ".", i) if suffix and suffix ~= "" then -- Apex domain (e.g. example.com) โ€” wildcard certs are often stored under apex name @@ -236,7 +237,7 @@ ssl_certificate_by_lua_block { if name and name ~= "" then local sanitized = sanitize_name(name) local path = base_path .. sanitized .. "/ocsp.der" - local f = io.open(path, "rb") + local f, err = io.open(path, "rb") if f then local data = f:read("*a") f:close() @@ -251,9 +252,8 @@ ssl_certificate_by_lua_block { break end else - -- Check if file exists but couldn't be read (permission denied) - local file_exists = os.execute("test -f " .. path .. " 2>/dev/null") - if file_exists == 0 then + -- Log only on permission denied (avoid shell; use io.open error) + if err and lower(err):find("permission denied", 1, true) then logger:log(ERR, "OCSP permission denied reading " .. path .. " (check file permissions)") end end @@ -313,7 +313,7 @@ ssl_certificate_by_lua_block { logger:log(ERR, "error while setting private key : " .. err) else -- Try to set OCSP stapling from cache (if enabled) - logger:log(INFO, "DEBUG: About to call set_ocsp_from_cache() for " .. server_name) + logger:log(DEBUG, "About to call set_ocsp_from_cache() for " .. server_name) set_ocsp_from_cache() logger:log(INFO, "certificate set by " .. plugin_id) return true From bee29ca9e2fe41ee17f329baec11ac1f1aa7eee2 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Sun, 15 Mar 2026 14:53:58 +0100 Subject: [PATCH 43/59] fix OCSP certificate verification + optimize log level we need to verify that the OCSP file is for the current certificate in case the certificate changes but the old OCSP is still cached. use resty.openssl with fallback to openssl set debug level to messages so we don't fill up logs --- .../server-http/ssl-certificate-lua.conf | 179 ++++++++++++++++-- 1 file changed, 160 insertions(+), 19 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 6c359f0a08..f32215cd96 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -94,10 +94,18 @@ ssl_certificate_by_lua_block { local match = string.match local concat = table.concat + -- Try to load resty.openssl for native certificate parsing + local resty_x509 = nil + local has_resty_ssl = false + pcall(function() + resty_x509 = require "resty.openssl.x509" + has_resty_ssl = true + end) + -- Start ssl_certificate phase local logger = clogger:new("SSL-CERTIFICATE") local internalstore = cdatastore:new(ngx.shared.internalstore) - logger:log(INFO, "ssl_certificate phase started") + logger:log(DEBUG, "ssl_certificate phase started") -- Get plugins order local order, err = internalstore:get("plugins_order", true) @@ -163,8 +171,132 @@ ssl_certificate_by_lua_block { return name:gsub("%*", "_wildcard_") end + -- Helper: extract certificate serial number from PEM + local function get_cert_serial(cert_pem) + if not cert_pem or #cert_pem == 0 then + return nil + end + + -- Try native resty.openssl first (fast, no shell call) + if has_resty_ssl then + local cert, err = resty_x509.new(cert_pem) + if cert then + local serial_num = cert:get_serial_number() + if serial_num then + return tostring(serial_num) + end + else + logger:log(DEBUG, "OCSP resty.openssl parse error: " .. tostring(err)) + end + end + + -- Fallback to openssl command (slower, but more compatible) + local tmp_cert = "/tmp/ocsp_cert_" .. ngx.var.connection .. ".pem" + local f = io.open(tmp_cert, "w") + if not f then + return nil + end + f:write(cert_pem) + f:close() + + local handle = io.popen("openssl x509 -serial -noout -in " .. tmp_cert .. " 2>/dev/null", "r") + if not handle then + os.remove(tmp_cert) + return nil + end + + local result = handle:read("*a") + handle:close() + os.remove(tmp_cert) + + local serial = result:match("serial=([A-F0-9]+)") + return serial + end + + -- Helper: extract certificate serial number from OCSP response DER + local function get_ocsp_serial(ocsp_der) + if not ocsp_der or #ocsp_der == 0 then + return nil + end + + -- Try native resty.openssl first (fast, direct DER parsing) + if has_resty_ssl then + local resty_ocsp = nil + pcall(function() + resty_ocsp = require "resty.openssl.ocsp" + end) + + if resty_ocsp then + local resp, err = resty_ocsp.new(ocsp_der) + if resp then + local serial_num = resp:get_serial() + if serial_num then + return tostring(serial_num) + end + else + logger:log(DEBUG, "OCSP resty parsing error: " .. tostring(err)) + end + end + end + + -- Fallback: write to temp file and use openssl + local tmp_ocsp = "/tmp/ocsp_resp_" .. ngx.var.connection .. ".der" + local f = io.open(tmp_ocsp, "wb") + if not f then + return nil + end + f:write(ocsp_der) + f:close() + + local handle = io.popen("openssl ocsp -respin " .. tmp_ocsp .. " -text -noout 2>/dev/null | grep 'Serial Number'", "r") + if not handle then + os.remove(tmp_ocsp) + return nil + end + + local result = handle:read("*a") + handle:close() + os.remove(tmp_ocsp) + + -- Parse "Serial Number: XXXXX" format + local serial = result:match("Serial Number:%s*([A-F0-9]+)") + return serial + end + + -- Helper: verify OCSP response matches current certificate (by comparing serials) + local function verify_ocsp_cert_match(cert_pem, ocsp_der) + if not cert_pem or not ocsp_der then + logger:log(DEBUG, "OCSP verification skipped (missing cert or OCSP data)") + return true -- Allow OCSP if can't verify + end + + -- Get serial from certificate + local cert_serial = get_cert_serial(cert_pem) + if not cert_serial then + logger:log(DEBUG, "OCSP could not extract certificate serial") + return false + end + + -- Get serial from OCSP response (direct DER parsing - no files needed) + local ocsp_serial = get_ocsp_serial(ocsp_der) + if not ocsp_serial then + logger:log(DEBUG, "OCSP could not extract OCSP response serial") + return false + end + + -- Compare serials + if cert_serial == ocsp_serial then + logger:log(DEBUG, "OCSP verified: certificate serial matches OCSP response serial: " .. cert_serial) + return true + else + logger:log(DEBUG, "OCSP mismatch: cert serial=" .. cert_serial .. " vs ocsp serial=" .. ocsp_serial) + return false + end + end + -- Helper: set OCSP stapling from cache (internalstore + ocsp.der disk files) - local function set_ocsp_from_cache() + -- cert_pem: optional certificate PEM data for verification + local function set_ocsp_from_cache(cert_pem) logger:log(DEBUG, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) if not is_ocsp_stapling_enabled() then @@ -242,14 +374,22 @@ ssl_certificate_by_lua_block { local data = f:read("*a") f:close() if data and #data > 0 then - logger:log(INFO, "OCSP loaded response (" .. #data .. " bytes) from " .. path .. " (matches " .. name .. ")") - resp = data - -- Cache in shared memory to avoid disk I/O on every handshake (TTL 300s) - local ok_set, serr = internalstore:set(cache_key, resp, 300, true) - if not ok_set then - logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) + -- Verify OCSP response matches current certificate + -- Parse DER response directly to extract and compare serials + local cert_matches = verify_ocsp_cert_match(cert_pem, data) + + if cert_matches then + logger:log(DEBUG, "OCSP loaded response (" .. #data .. " bytes) from " .. path .. " (verified against certificate)") + resp = data + -- Cache in shared memory to avoid disk I/O on every handshake (TTL 300s) + local ok_set, serr = internalstore:set(cache_key, resp, 300, true) + if not ok_set then + logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) + end + break + else + logger:log(DEBUG, "OCSP skipping response for " .. name .. " (certificate mismatch - certificate may have been updated)") end - break end else -- Log only on permission denied (avoid shell; use io.open error) @@ -270,19 +410,19 @@ ssl_certificate_by_lua_block { if not ok_set then logger:log(ERR, "OCSP failed to set stapling: " .. oerr) else - logger:log(INFO, "OCSP stapling set from cache for " .. server_name) + logger:log(DEBUG, "OCSP stapling set from cache for " .. server_name) end end -- Call ssl_certificate() methods - logger:log(INFO, "calling ssl_certificate() methods of plugins ...") + logger:log(DEBUG, "calling ssl_certificate() methods of plugins ...") for i, plugin_id in ipairs(phase_order) do -- Require call local plugin_lua, err = require_plugin(plugin_id) if plugin_lua == false then logger:log(ERR, err) elseif plugin_lua == nil then - logger:log(INFO, err) + logger:log(DEBUG, err) else -- Check if plugin has ssl_certificate method if plugin_lua.ssl_certificate ~= nil then @@ -297,9 +437,9 @@ ssl_certificate_by_lua_block { elseif not ret.ret then logger:log(ERR, plugin_id .. ":ssl_certificate() call failed : " .. ret.msg) else - logger:log(INFO, plugin_id .. ":ssl_certificate() call successful : " .. ret.msg) + logger:log(DEBUG, plugin_id .. ":ssl_certificate() call successful : " .. ret.msg) if ret.status then - logger:log(INFO, plugin_id .. " is setting certificate/key : " .. ret.msg) + logger:log(DEBUG, plugin_id .. " is setting certificate/key : " .. ret.msg) local ok, err = clear_certs() if not ok then logger:log(ERR, "error while clearing certificates : " .. err) @@ -313,9 +453,10 @@ ssl_certificate_by_lua_block { logger:log(ERR, "error while setting private key : " .. err) else -- Try to set OCSP stapling from cache (if enabled) + -- Pass certificate PEM data for verification logger:log(DEBUG, "About to call set_ocsp_from_cache() for " .. server_name) - set_ocsp_from_cache() - logger:log(INFO, "certificate set by " .. plugin_id) + set_ocsp_from_cache(ret.status[1]) + logger:log(DEBUG, "certificate set by " .. plugin_id) return true end end @@ -323,13 +464,13 @@ ssl_certificate_by_lua_block { end end else - logger:log(INFO, "skipped execution of " .. plugin_id .. " because method ssl_certificate() is not defined") + logger:log(DEBUG, "skipped execution of " .. plugin_id .. " because method ssl_certificate() is not defined") end end end - logger:log(INFO, "called ssl_certificate() methods of plugins") + logger:log(DEBUG, "called ssl_certificate() methods of plugins") - logger:log(INFO, "ssl_certificate phase ended") + logger:log(DEBUG, "ssl_certificate phase ended") return true } From 3b3ac2f9008b22676de7a8107b3b99ee4542c33d Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Sun, 15 Mar 2026 20:30:16 +0100 Subject: [PATCH 44/59] fallback to tls on error --- .../server-http/ssl-certificate-lua.conf | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index f32215cd96..845b4b0324 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -406,11 +406,20 @@ ssl_certificate_by_lua_block { return end - local ok_set, oerr = ocsp.set_ocsp_status_resp(resp) - if not ok_set then - logger:log(ERR, "OCSP failed to set stapling: " .. oerr) + -- Safely set OCSP stapling with exception handling + local ok_pcall, ocsp_result = pcall(function() + return ocsp.set_ocsp_status_resp(resp) + end) + + if not ok_pcall then + logger:log(ERR, "OCSP exception while setting stapling: " .. tostring(ocsp_result)) else - logger:log(DEBUG, "OCSP stapling set from cache for " .. server_name) + local ok_set, oerr = ocsp_result + if not ok_set then + logger:log(ERR, "OCSP failed to set stapling: " .. oerr) + else + logger:log(DEBUG, "OCSP stapling set from cache for " .. server_name) + end end end @@ -455,7 +464,12 @@ ssl_certificate_by_lua_block { -- Try to set OCSP stapling from cache (if enabled) -- Pass certificate PEM data for verification logger:log(DEBUG, "About to call set_ocsp_from_cache() for " .. server_name) - set_ocsp_from_cache(ret.status[1]) + local ok_ocsp, ocsp_err = pcall(function() + set_ocsp_from_cache(ret.status[1]) + end) + if not ok_ocsp then + logger:log(ERR, "OCSP function error (certificate still set): " .. tostring(ocsp_err)) + end logger:log(DEBUG, "certificate set by " .. plugin_id) return true end From 94c4d52972930360baa4aa398e6a8d0401e380b0 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Sun, 15 Mar 2026 21:37:23 +0100 Subject: [PATCH 45/59] fix ocsp loading error --- src/common/confs/server-http/ssl-certificate-lua.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 845b4b0324..24cce2e726 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -402,7 +402,7 @@ ssl_certificate_by_lua_block { end if not resp then - logger:log(ngx.DEBUG, "OCSP not found for " .. server_name) + logger:log(DEBUG, "OCSP not found for " .. server_name) return end @@ -462,10 +462,10 @@ ssl_certificate_by_lua_block { logger:log(ERR, "error while setting private key : " .. err) else -- Try to set OCSP stapling from cache (if enabled) - -- Pass certificate PEM data for verification + -- Try to set OCSP stapling (certificate verification optional) logger:log(DEBUG, "About to call set_ocsp_from_cache() for " .. server_name) local ok_ocsp, ocsp_err = pcall(function() - set_ocsp_from_cache(ret.status[1]) + set_ocsp_from_cache() end) if not ok_ocsp then logger:log(ERR, "OCSP function error (certificate still set): " .. tostring(ocsp_err)) From 7616e4b1ae67a3cd3572721a40a61a83527f0889 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Mon, 16 Mar 2026 07:22:29 +0100 Subject: [PATCH 46/59] fix SN comparison, check OCSP existence early, fallback to native tls on OCSP errors - Serial number uppercase comparison - Both certificate and OCSP response serials are converted to uppercase before comparison to ensure consistent matching - Early certificate reading - Certificate is read immediately using ngx.ssl.cert_pem() at the start of set_ocsp_from_cache() - OCSP responder URL extraction - New get_ocsp_responder_url() function that: -- Tries native resty.openssl first (fast) -- Falls back to openssl x509 -ocsp_uri command -- Returns the OCSP responder URL if found -- Skip OCSP if no responder URL - If certificate doesn't have an OCSP responder URL, function returns early without loading OCSP files - Any error at any stage is logged but doesn't break the TLS handshake. The connection always continues with native TLS if OCSP fails for any reason. The certificate is already served before OCSP is even attempted. --- .../server-http/ssl-certificate-lua.conf | 153 +++++++++++++++--- 1 file changed, 134 insertions(+), 19 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 24cce2e726..0681e46eeb 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -284,19 +284,82 @@ ssl_certificate_by_lua_block { return false end - -- Compare serials - if cert_serial == ocsp_serial then - logger:log(DEBUG, "OCSP verified: certificate serial matches OCSP response serial: " .. cert_serial) + -- Compare serials (uppercase for consistent comparison) + local cert_serial_upper = cert_serial:upper() + local ocsp_serial_upper = ocsp_serial:upper() + if cert_serial_upper == ocsp_serial_upper then + logger:log(DEBUG, "OCSP verified: certificate serial matches OCSP response serial: " .. cert_serial_upper) return true else - logger:log(DEBUG, "OCSP mismatch: cert serial=" .. cert_serial .. " vs ocsp serial=" .. ocsp_serial) + logger:log(DEBUG, "OCSP mismatch: cert serial=" .. cert_serial_upper .. " vs ocsp serial=" .. ocsp_serial_upper) return false end end + -- Helper: extract OCSP responder URL from certificate using OID 1.3.6.1.5.5.7.48.1 + local function get_ocsp_responder_url(cert_pem) + if not cert_pem or #cert_pem == 0 then + return nil + end + + -- Try native resty.openssl first (fast, OID-based) + if has_resty_ssl then + local cert, err = resty_x509.new(cert_pem) + if cert then + -- Get AIA extension (OID 1.3.6.1.5.5.7.1.1) + local aia_ext = cert:get_extension("authorityInfoAccess") + if aia_ext then + local aia_text = aia_ext:text() + -- Look for OCSP OID 1.3.6.1.5.5.7.48.1 with URI format + -- Text format: "OCSP - URI:http://..." or just extract any http/https URL after OCSP + local ocsp_url = aia_text:match("1%.3%.6%.1%.5%.5%.7%.48%.1%s*=%s*URI:([%w:/.-]+)") + if not ocsp_url then + -- Fallback: text-based pattern in case formatting differs + ocsp_url = aia_text:match("OCSP%s*-%s*URI:([%w:/.-]+)") + end + if ocsp_url then + logger:log(DEBUG, "OCSP found responder URL (OID 1.3.6.1.5.5.7.48.1) in certificate: " .. ocsp_url) + return ocsp_url + end + end + else + logger:log(DEBUG, "OCSP resty.openssl parse error: " .. tostring(err)) + end + end + + -- Fallback to openssl command (also OID-based) + local tmp_cert = "/tmp/ocsp_cert_" .. ngx.var.connection .. ".pem" + local f = io.open(tmp_cert, "w") + if not f then + return nil + end + f:write(cert_pem) + f:close() + + -- Use openssl to extract OCSP URI (searches for OID 1.3.6.1.5.5.7.48.1 internally) + local handle = io.popen("openssl x509 -ocsp_uri -noout -in " .. tmp_cert .. " 2>/dev/null", "r") + if not handle then + os.remove(tmp_cert) + return nil + end + + local result = handle:read("*a") + handle:close() + os.remove(tmp_cert) + + -- Extract URL (openssl x509 -ocsp_uri returns the URL or empty string) + local ocsp_url = result:match("^([%w:/.-]+)") + if ocsp_url then + logger:log(DEBUG, "OCSP found responder URL (OID 1.3.6.1.5.5.7.48.1) in certificate: " .. ocsp_url) + return ocsp_url + end + + logger:log(DEBUG, "OCSP no responder URL (OID 1.3.6.1.5.5.7.48.1) found in certificate") + return nil + end + -- Helper: set OCSP stapling from cache (internalstore + ocsp.der disk files) - -- cert_pem: optional certificate PEM data for verification - local function set_ocsp_from_cache(cert_pem) + local function set_ocsp_from_cache() logger:log(DEBUG, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) if not is_ocsp_stapling_enabled() then @@ -309,15 +372,52 @@ ssl_certificate_by_lua_block { return end + -- Get the current certificate early (with error handling) + local cert_pem, err + local ok_cert, cert_result = pcall(function() + return ngx.ssl.cert_pem() + end) + if not ok_cert then + logger:log(ERR, "OCSP exception reading certificate: " .. tostring(cert_result)) + return + end + cert_pem, err = cert_result, nil + if not cert_pem then + logger:log(DEBUG, "OCSP could not read current certificate: " .. (err or "unknown")) + return + end + + -- Check if certificate has OCSP responder URL (with error handling) + local ocsp_responder_url + local ok_url, url_result = pcall(function() + return get_ocsp_responder_url(cert_pem) + end) + if not ok_url then + logger:log(ERR, "OCSP exception extracting responder URL: " .. tostring(url_result)) + return + end + ocsp_responder_url = url_result + if not ocsp_responder_url then + logger:log(DEBUG, "OCSP skipping: certificate does not have OCSP responder URL") + return + end + local cache_key = "SSL:ocsp_status:" .. server_name local resp - -- 1) Local shared dict lookup (per-worker cache) - local db_val, gerr = internalstore:get(cache_key, true) - if db_val then - resp = db_val - elseif gerr and gerr ~= "not found" then - logger:log(ERR, "OCSP error while getting response from internalstore: " .. gerr) + -- 1) Local shared dict lookup (per-worker cache) - with error handling + local ok_cache, cache_result = pcall(function() + return internalstore:get(cache_key, true) + end) + if ok_cache then + local db_val, gerr = cache_result, nil + if db_val then + resp = db_val + elseif gerr and gerr ~= "not found" then + logger:log(ERR, "OCSP error while getting response from internalstore: " .. gerr) + end + else + logger:log(ERR, "OCSP exception reading from cache: " .. tostring(cache_result)) end -- 2) Fallback to on-disk OCSP response (/var/cache/bunkerweb/ssl/{domain}/ocsp.der) @@ -374,17 +474,31 @@ ssl_certificate_by_lua_block { local data = f:read("*a") f:close() if data and #data > 0 then - -- Verify OCSP response matches current certificate - -- Parse DER response directly to extract and compare serials - local cert_matches = verify_ocsp_cert_match(cert_pem, data) + -- Verify OCSP response matches current certificate (with error handling) + local cert_matches + local ok_verify, verify_result = pcall(function() + return verify_ocsp_cert_match(cert_pem, data) + end) + if not ok_verify then + logger:log(ERR, "OCSP exception verifying response: " .. tostring(verify_result)) + goto next_candidate + end + cert_matches = verify_result if cert_matches then logger:log(DEBUG, "OCSP loaded response (" .. #data .. " bytes) from " .. path .. " (verified against certificate)") resp = data - -- Cache in shared memory to avoid disk I/O on every handshake (TTL 300s) - local ok_set, serr = internalstore:set(cache_key, resp, 300, true) - if not ok_set then - logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) + -- Cache in shared memory to avoid disk I/O on every handshake (TTL 300s) - with error handling + local ok_set_cache, set_result = pcall(function() + return internalstore:set(cache_key, resp, 300, true) + end) + if ok_set_cache then + local ok_set, serr = set_result, nil + if not ok_set then + logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) + end + else + logger:log(ERR, "OCSP exception caching response: " .. tostring(set_result)) end break else @@ -397,6 +511,7 @@ ssl_certificate_by_lua_block { logger:log(ERR, "OCSP permission denied reading " .. path .. " (check file permissions)") end end + ::next_candidate:: end end end From edd608c5d61336833b72b8a2c950ec28a9fc3548 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Mon, 16 Mar 2026 08:16:15 +0100 Subject: [PATCH 47/59] add redis backend --- .../server-http/ssl-certificate-lua.conf | 93 +++++++++++++++++-- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 0681e46eeb..9aea41af46 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -72,6 +72,7 @@ ssl_certificate_by_lua_block { local helpers = require "bunkerweb.helpers" local utils = require "bunkerweb.utils" local cdatastore = require "bunkerweb.datastore" + local cclusterstore = require "bunkerweb.clusterstore" local cjson = require "cjson" local ssl = require "ngx.ssl" local ocsp = require "ngx.ocsp" @@ -105,7 +106,6 @@ ssl_certificate_by_lua_block { -- Start ssl_certificate phase local logger = clogger:new("SSL-CERTIFICATE") local internalstore = cdatastore:new(ngx.shared.internalstore) - logger:log(DEBUG, "ssl_certificate phase started") -- Get plugins order local order, err = internalstore:get("plugins_order", true) @@ -127,6 +127,28 @@ ssl_certificate_by_lua_block { local server_name = ssl.server_name() local phase_order = get_phase_order(order, "ssl_certificate", server_name) + logger:log(INFO, "ssl_certificate phase started") + + -- Helper: check if Redis is enabled globally + local function is_redis_enabled() + local vars, err = internalstore:get("variables", true) + if not vars or not vars.global then + return false -- Default to disabled if not configured + end + + local value = vars.global["USE_REDIS"] + if value == nil then + return false + end + + -- Normalize and interpret common truthy representations + if type(value) == "boolean" then + return value + end + local str = tostring(value):lower() + return str == "1" or str == "true" or str == "on" or str == "yes" + end + -- Helper: check if OCSP stapling is enabled for this site local function is_ocsp_stapling_enabled() -- Default to enabled (backward compatible behavior) if variables are not available @@ -402,7 +424,7 @@ ssl_certificate_by_lua_block { return end - local cache_key = "SSL:ocsp_status:" .. server_name + local cache_key = "TLS:SSL:ocsp:" .. server_name local resp -- 1) Local shared dict lookup (per-worker cache) - with error handling @@ -413,6 +435,7 @@ ssl_certificate_by_lua_block { local db_val, gerr = cache_result, nil if db_val then resp = db_val + logger:log(DEBUG, "OCSP found response in internalstore for " .. server_name) elseif gerr and gerr ~= "not found" then logger:log(ERR, "OCSP error while getting response from internalstore: " .. gerr) end @@ -420,7 +443,42 @@ ssl_certificate_by_lua_block { logger:log(ERR, "OCSP exception reading from cache: " .. tostring(cache_result)) end - -- 2) Fallback to on-disk OCSP response (/var/cache/bunkerweb/ssl/{domain}/ocsp.der) + -- 2) Redis/cluster lookup - with 0.05s timeout and error handling (skip if Redis not enabled) + if not resp and is_redis_enabled() then + local ok_cluster_read, cluster_read_result = pcall(function() + local cluster = cclusterstore:new() + -- Set timeout to 0.05 seconds (50ms) for fast read fail - cache hits should be instant + cluster:set_timeout(50) + local ok_connect, rerr = cluster:connect(true) + if not ok_connect then + logger:log(DEBUG, "OCSP cluster read connection failed (non-critical): " .. (rerr or "unknown")) + return nil + end + local ok_get, val, gerr = cluster:get(cache_key, true) + if not ok_get then + logger:log(DEBUG, "OCSP error while reading response from cluster: " .. (gerr or "unknown")) + return nil + end + return val + end) + if ok_cluster_read then + resp = cluster_read_result + if resp then + logger:log(DEBUG, "OCSP found response in cluster for " .. server_name) + -- Backfill to internalstore for faster future access + local ok_backfill, backfill_result = pcall(function() + return internalstore:set(cache_key, resp, 300, true) + end) + if not ok_backfill then + logger:log(DEBUG, "OCSP could not backfill to internalstore: " .. tostring(backfill_result)) + end + end + else + logger:log(DEBUG, "OCSP exception reading from cluster (non-critical): " .. tostring(cluster_read_result)) + end + end + + -- 3) Fallback to on-disk OCSP response (/var/cache/bunkerweb/ssl/{domain}/ocsp.der) if not resp then local base_path = "/var/cache/bunkerweb/ssl/" @@ -536,10 +594,33 @@ ssl_certificate_by_lua_block { logger:log(DEBUG, "OCSP stapling set from cache for " .. server_name) end end + + -- Backfill Redis AFTER certificate and OCSP are loaded and set (non-blocking, lazy write, skip if Redis not enabled) + if resp and is_redis_enabled() then + local ok_cluster, cluster_result = pcall(function() + local cluster = cclusterstore:new() + -- Set timeout to 0.1 seconds - this write should not block + cluster:set_timeout(100) + local ok_connect, rerr = cluster:connect(true) + if not ok_connect then + logger:log(DEBUG, "OCSP cluster backfill connection failed (non-critical): " .. (rerr or "unknown")) + return + end + local ok_set, serr = cluster:set(cache_key, resp, 300, true) + if not ok_set then + logger:log(DEBUG, "OCSP error while backfilling response to cluster: " .. (serr or "unknown")) + else + logger:log(DEBUG, "OCSP backfilled response to cluster for " .. server_name) + end + end) + if not ok_cluster then + logger:log(DEBUG, "OCSP exception backfilling to cluster (non-critical): " .. tostring(cluster_result)) + end + end end -- Call ssl_certificate() methods - logger:log(DEBUG, "calling ssl_certificate() methods of plugins ...") + logger:log(INFO, "calling ssl_certificate() methods of plugins ...") for i, plugin_id in ipairs(phase_order) do -- Require call local plugin_lua, err = require_plugin(plugin_id) @@ -597,9 +678,9 @@ ssl_certificate_by_lua_block { end end end - logger:log(DEBUG, "called ssl_certificate() methods of plugins") + logger:log(INFO, "called ssl_certificate() methods of plugins") - logger:log(DEBUG, "ssl_certificate phase ended") + logger:log(INFO, "ssl_certificate phase ended") return true } From 2fba405641f5771915bfef8aeb55b4adeb74ed0b Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Mon, 16 Mar 2026 08:24:21 +0100 Subject: [PATCH 48/59] fix openssl file race condition on fast concurrent connections the filenames could create a race condition --- .../server-http/ssl-certificate-lua.conf | 78 ++++++++++++++++--- 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 9aea41af46..c8b4952c03 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -213,25 +213,44 @@ ssl_certificate_by_lua_block { end -- Fallback to openssl command (slower, but more compatible) - local tmp_cert = "/tmp/ocsp_cert_" .. ngx.var.connection .. ".pem" + local worker_id = ngx.worker.id() or "0" + local tmp_cert = "/tmp/ocsp_cert_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" local f = io.open(tmp_cert, "w") if not f then + logger:log(ERR, "OCSP cert serial: could not open temp file for writing: " .. tmp_cert) return nil end - f:write(cert_pem) + + -- Write with error checking + local ok_write, write_err = f:write(cert_pem) f:close() + if not ok_write then + logger:log(ERR, "OCSP cert serial: write failed: " .. (write_err or "unknown")) + pcall(function() os.remove(tmp_cert) end) + return nil + end + -- Run openssl command and ensure cleanup on all paths local handle = io.popen("openssl x509 -serial -noout -in " .. tmp_cert .. " 2>/dev/null", "r") if not handle then - os.remove(tmp_cert) + logger:log(DEBUG, "OCSP cert serial: openssl command failed") + pcall(function() os.remove(tmp_cert) end) return nil end local result = handle:read("*a") handle:close() - os.remove(tmp_cert) + pcall(function() os.remove(tmp_cert) end) + + if not result or #result == 0 then + logger:log(DEBUG, "OCSP cert serial: openssl returned no output") + return nil + end local serial = result:match("serial=([A-F0-9]+)") + if not serial then + logger:log(DEBUG, "OCSP cert serial: could not parse serial from output: " .. result) + end return serial end @@ -262,26 +281,45 @@ ssl_certificate_by_lua_block { end -- Fallback: write to temp file and use openssl - local tmp_ocsp = "/tmp/ocsp_resp_" .. ngx.var.connection .. ".der" + local worker_id = ngx.worker.id() or "0" + local tmp_ocsp = "/tmp/ocsp_resp_w" .. worker_id .. "_c" .. ngx.var.connection .. ".der" local f = io.open(tmp_ocsp, "wb") if not f then + logger:log(ERR, "OCSP serial: could not open temp file for writing: " .. tmp_ocsp) return nil end - f:write(ocsp_der) + + -- Write with error checking + local ok_write, write_err = f:write(ocsp_der) f:close() + if not ok_write then + logger:log(ERR, "OCSP serial: write failed: " .. (write_err or "unknown")) + pcall(function() os.remove(tmp_ocsp) end) + return nil + end + -- Run openssl command and ensure cleanup on all paths local handle = io.popen("openssl ocsp -respin " .. tmp_ocsp .. " -text -noout 2>/dev/null | grep 'Serial Number'", "r") if not handle then - os.remove(tmp_ocsp) + logger:log(DEBUG, "OCSP serial: openssl command failed") + pcall(function() os.remove(tmp_ocsp) end) return nil end local result = handle:read("*a") handle:close() - os.remove(tmp_ocsp) + pcall(function() os.remove(tmp_ocsp) end) + + if not result or #result == 0 then + logger:log(DEBUG, "OCSP serial: openssl returned no output") + return nil + end -- Parse "Serial Number: XXXXX" format local serial = result:match("Serial Number:%s*([A-F0-9]+)") + if not serial then + logger:log(DEBUG, "OCSP serial: could not parse serial from output: " .. result) + end return serial end @@ -350,24 +388,40 @@ ssl_certificate_by_lua_block { end -- Fallback to openssl command (also OID-based) - local tmp_cert = "/tmp/ocsp_cert_" .. ngx.var.connection .. ".pem" + local worker_id = ngx.worker.id() or "0" + local tmp_cert = "/tmp/ocsp_cert_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" local f = io.open(tmp_cert, "w") if not f then + logger:log(ERR, "OCSP responder URL: could not open temp file for writing: " .. tmp_cert) return nil end - f:write(cert_pem) + + -- Write with error checking + local ok_write, write_err = f:write(cert_pem) f:close() + if not ok_write then + logger:log(ERR, "OCSP responder URL: write failed: " .. (write_err or "unknown")) + pcall(function() os.remove(tmp_cert) end) + return nil + end -- Use openssl to extract OCSP URI (searches for OID 1.3.6.1.5.5.7.48.1 internally) + -- Ensure cleanup on all paths local handle = io.popen("openssl x509 -ocsp_uri -noout -in " .. tmp_cert .. " 2>/dev/null", "r") if not handle then - os.remove(tmp_cert) + logger:log(DEBUG, "OCSP responder URL: openssl command failed") + pcall(function() os.remove(tmp_cert) end) return nil end local result = handle:read("*a") handle:close() - os.remove(tmp_cert) + pcall(function() os.remove(tmp_cert) end) + + if not result or #result == 0 then + logger:log(DEBUG, "OCSP no responder URL (OID 1.3.6.1.5.5.7.48.1) found in certificate") + return nil + end -- Extract URL (openssl x509 -ocsp_uri returns the URL or empty string) local ocsp_url = result:match("^([%w:/.-]+)") From c196df23efba57a702e78581e0e1e9eb08bd1b52 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Mon, 16 Mar 2026 08:27:55 +0100 Subject: [PATCH 49/59] add fail safe --- .../server-http/ssl-certificate-lua.conf | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index c8b4952c03..ea85edfac8 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -129,9 +129,19 @@ ssl_certificate_by_lua_block { logger:log(INFO, "ssl_certificate phase started") - -- Helper: check if Redis is enabled globally + -- Helper: check if Redis is enabled globally (with exception handling) local function is_redis_enabled() - local vars, err = internalstore:get("variables", true) + -- Get variables with exception handling + local ok_get, get_result = pcall(function() + return internalstore:get("variables", true) + end) + + if not ok_get then + logger:log(DEBUG, "Redis enabled check: exception reading variables: " .. tostring(get_result)) + return false -- Fail safely to disabled + end + + local vars, err = get_result, nil if not vars or not vars.global then return false -- Default to disabled if not configured end @@ -149,12 +159,21 @@ ssl_certificate_by_lua_block { return str == "1" or str == "true" or str == "on" or str == "yes" end - -- Helper: check if OCSP stapling is enabled for this site + -- Helper: check if OCSP stapling is enabled for this site (with exception handling) local function is_ocsp_stapling_enabled() - -- Default to enabled (backward compatible behavior) if variables are not available - local vars, err = internalstore:get("variables", true) + -- Get variables with exception handling + local ok_get, get_result = pcall(function() + return internalstore:get("variables", true) + end) + + if not ok_get then + logger:log(DEBUG, "OCSP stapling enabled check: exception reading variables: " .. tostring(get_result)) + return true -- Fail safely to enabled (backward compatible) + end + + local vars = get_result if not vars then - return true + return true -- Default to enabled if not available end -- Resolve effective SSL_USE_OCSP_STAPLING value with per-site override From eb5fe1f0f5b557325f79d2b414897cf40dc3fe1e Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Mon, 16 Mar 2026 11:32:08 +0100 Subject: [PATCH 50/59] use safe module loading + implement OCSP-must-stapple + remove redis - load modules with fail safe - check if certificate has OCSP Must-Staple extension - remove redis for a simple config --- .../server-http/ssl-certificate-lua.conf | 1418 ++++++++++------- src/common/core/ssl/jobs/ocsp-refresh.py | 507 +++--- 2 files changed, 1171 insertions(+), 754 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index ea85edfac8..583a203a3e 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -67,693 +67,983 @@ add_header Alt-Svc 'h3=":{{ HTTP3_ALT_SVC_PORT }}"; ma=86400'; {% endif %} ssl_certificate_by_lua_block { - local class = require "middleclass" - local clogger = require "bunkerweb.logger" - local helpers = require "bunkerweb.helpers" - local utils = require "bunkerweb.utils" - local cdatastore = require "bunkerweb.datastore" - local cclusterstore = require "bunkerweb.clusterstore" - local cjson = require "cjson" - local ssl = require "ngx.ssl" - local ocsp = require "ngx.ocsp" - - local ngx = ngx - local ngx_req = ngx.req - local is_internal = ngx_req.is_internal - local ERR = ngx.ERR - local INFO = ngx.INFO - local DEBUG = ngx.DEBUG - local clear_certs = ssl.clear_certs - local set_cert = ssl.set_cert - local set_priv_key = ssl.set_priv_key - local require_plugin = helpers.require_plugin - local new_plugin = helpers.new_plugin - local call_plugin = helpers.call_plugin - local tostring = tostring - local insert = table.insert - local lower = string.lower - local match = string.match - local concat = table.concat - - -- Try to load resty.openssl for native certificate parsing - local resty_x509 = nil - local has_resty_ssl = false - pcall(function() - resty_x509 = require "resty.openssl.x509" - has_resty_ssl = true - end) - - -- Start ssl_certificate phase - local logger = clogger:new("SSL-CERTIFICATE") - local internalstore = cdatastore:new(ngx.shared.internalstore) - - -- Get plugins order - local order, err = internalstore:get("plugins_order", true) - if not order then - logger:log(ERR, "can't get plugins order from internalstore : " .. err) - return - end - - -- Resolve per-site plugin order - local function get_phase_order(ord, phase, server_name) - if ord.per_site and server_name and ord.per_site[server_name] and ord.per_site[server_name][phase] then - return ord.per_site[server_name][phase] - elseif ord.global and ord.global[phase] then - return ord.global[phase] - end - return ord[phase] - end - - local server_name = ssl.server_name() - local phase_order = get_phase_order(order, "ssl_certificate", server_name) - - logger:log(INFO, "ssl_certificate phase started") - - -- Helper: check if Redis is enabled globally (with exception handling) - local function is_redis_enabled() - -- Get variables with exception handling - local ok_get, get_result = pcall(function() - return internalstore:get("variables", true) + -- Top-level pcall to ensure any unexpected Lua error does not abort the handshake + local ok, err = pcall(function() + -- 1. Load logger first (critical for diagnostics) + local clogger + local ok_logger, err_logger = pcall(function() + return require "bunkerweb.logger" end) - - if not ok_get then - logger:log(DEBUG, "Redis enabled check: exception reading variables: " .. tostring(get_result)) - return false -- Fail safely to disabled + + if not ok_logger then + local err_msg = tostring(err_logger):sub(1, 1024) + ngx.log(ngx.ERR, ("SSL-CERTIFICATE critical error: failed to load bunkerweb.logger: " .. err_msg):sub(1, 2048)) + return end + clogger = err_logger + local logger = clogger:new("SSL-CERTIFICATE") - local vars, err = get_result, nil - if not vars or not vars.global then - return false -- Default to disabled if not configured + local function safe_log(level, msg) + if msg then + logger:log(level, tostring(msg):sub(1, 2048)) + end end - local value = vars.global["USE_REDIS"] - if value == nil then - return false - end + safe_log(ngx.DEBUG, "bunkerweb.logger loaded successfully") - -- Normalize and interpret common truthy representations - if type(value) == "boolean" then - return value + -- 2. Load other core modules with logging + local function safe_require(name) + local ok_mod, mod = pcall(require, name) + if ok_mod then + safe_log(ngx.DEBUG, "Module " .. name .. " loaded successfully") + return mod + else + safe_log(ngx.ERR, "Failed to load core module " .. name .. ": " .. tostring(mod)) + return nil + end end - local str = tostring(value):lower() - return str == "1" or str == "true" or str == "on" or str == "yes" - end - -- Helper: check if OCSP stapling is enabled for this site (with exception handling) - local function is_ocsp_stapling_enabled() - -- Get variables with exception handling - local ok_get, get_result = pcall(function() - return internalstore:get("variables", true) - end) + local class = safe_require "middleclass" + local helpers = safe_require "bunkerweb.helpers" + local utils = safe_require "bunkerweb.utils" + local cdatastore = safe_require "bunkerweb.datastore" + local cjson = safe_require "cjson" + local ssl = safe_require "ngx.ssl" + + -- Check critical modules (ssl, helpers, cdatastore are essential) + if not ssl then + safe_log(ngx.ERR, "Critical module ngx.ssl failed to load - cannot set custom certificates, nginx will use default certificate") + return -- Let nginx use default certificate configured in http block + end - if not ok_get then - logger:log(DEBUG, "OCSP stapling enabled check: exception reading variables: " .. tostring(get_result)) - return true -- Fail safely to enabled (backward compatible) + if not helpers then + safe_log(ngx.ERR, "Critical module bunkerweb.helpers failed to load - plugins cannot be loaded, nginx will use default certificate") + return -- Let nginx use default certificate end - local vars = get_result - if not vars then - return true -- Default to enabled if not available + if not cdatastore then + safe_log(ngx.ERR, "Critical module bunkerweb.datastore failed to load - cannot access configuration, nginx will use default certificate") + return -- Let nginx use default certificate end - -- Resolve effective SSL_USE_OCSP_STAPLING value with per-site override - local value - if vars.per_site and server_name and vars.per_site[server_name] then - value = vars.per_site[server_name]["SSL_USE_OCSP_STAPLING"] + -- Log non-critical module failures but continue + if not class then + safe_log(ngx.ERR, "Non-critical module middleclass failed to load - some features may not work") end - if value == nil and vars.global then - value = vars.global["SSL_USE_OCSP_STAPLING"] + if not utils then + safe_log(ngx.ERR, "Non-critical module bunkerweb.utils failed to load - some features may not work") end - if value == nil then - value = vars["SSL_USE_OCSP_STAPLING"] + if not cjson then + safe_log(ngx.ERR, "Non-critical module cjson failed to load - JSON operations may not work") end - -- If still nil, keep default behavior (enabled) - if value == nil then - return true + local ngx = ngx + local ngx_req = ngx.req + local is_internal = ngx_req.is_internal + local ERR = ngx.ERR + local INFO = ngx.INFO + local NOTICE = ngx.NOTICE + local DEBUG = ngx.DEBUG + + -- 3. Optional modules (require with pcall for safety) + local function optional_require(name, report_error) + local ok_mod, mod = pcall(require, name) + if ok_mod then + safe_log(DEBUG, "Optional module " .. name .. " loaded successfully") + return mod + else + local log_level = report_error and ERR or DEBUG + safe_log(log_level, "Optional module " .. name .. " not found or failed to load: " .. tostring(mod)) + return nil + end end - -- Normalize and interpret common falsy/disabled representations - if type(value) == "boolean" then - return value + local ocsp = optional_require("ngx.ocsp", true) -- Report failure (OCSP stapling feature) + local cclusterstore = optional_require("bunkerweb.clusterstore", false) -- Silent failure (cache optimization) + + -- Try to load resty.openssl for native certificate parsing (report failure - performance optimization) + local resty_x509 = optional_require("resty.openssl.x509", true) + local has_resty_ssl = (resty_x509 ~= nil) + + local clear_certs = ssl and ssl.clear_certs + local set_cert = ssl and ssl.set_cert + local set_priv_key = ssl and ssl.set_priv_key + local require_plugin = helpers and helpers.require_plugin + local new_plugin = helpers and helpers.new_plugin + local call_plugin = helpers and helpers.call_plugin + + local tostring = tostring + local insert = table.insert + local lower = string.lower + local match = string.match + local concat = table.concat + + -- Start ssl_certificate phase + local internalstore = cdatastore and cdatastore:new(ngx.shared.internalstore) + if not internalstore then + safe_log(ERR, "Failed to initialize internalstore") + return end - local str = tostring(value):lower() - if str == "0" or str == "false" or str == "off" or str == "no" then - return false + + -- Get plugins order + local order, err = internalstore:get("plugins_order", true) + if not order then + safe_log(ERR, "cannot get plugins order from internalstore : " .. (err or "unknown")) + return end - -- Any other value is treated as enabled - return true - end + -- Resolve per-site plugin order + local function get_phase_order(ord, phase, server_name) + if ord.per_site and server_name and ord.per_site[server_name] and ord.per_site[server_name][phase] then + return ord.per_site[server_name][phase] + elseif ord.global and ord.global[phase] then + return ord.global[phase] + end + return ord[phase] + end - -- Helper: sanitize domain/cert name for filesystem (replace * with _wildcard_) - local function sanitize_name(name) - if not name then return nil end - return name:gsub("%*", "_wildcard_") - end + local server_name = ssl and ssl.server_name() + local phase_order = get_phase_order(order, "ssl_certificate", server_name) - -- Helper: extract certificate serial number from PEM - local function get_cert_serial(cert_pem) - if not cert_pem or #cert_pem == 0 then - return nil - end + safe_log(INFO, "ssl_certificate phase started for server_name=" .. (server_name or "nil")) - -- Try native resty.openssl first (fast, no shell call) - if has_resty_ssl then - local cert, err = resty_x509.new(cert_pem) - if cert then - local serial_num = cert:get_serial_number() - if serial_num then - return tostring(serial_num) - end - else - logger:log(DEBUG, "OCSP resty.openssl parse error: " .. tostring(err)) + -- Helper: check if Redis is enabled globally (with exception handling) + local function is_redis_enabled() + -- Get variables with exception handling + local ok_get, get_result = pcall(function() + return internalstore:get("variables", true) + end) + + if not ok_get then + safe_log(DEBUG, "Redis enabled check: exception reading variables: " .. tostring(get_result)) + return false -- Fail safely to disabled end - end - -- Fallback to openssl command (slower, but more compatible) - local worker_id = ngx.worker.id() or "0" - local tmp_cert = "/tmp/ocsp_cert_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" - local f = io.open(tmp_cert, "w") - if not f then - logger:log(ERR, "OCSP cert serial: could not open temp file for writing: " .. tmp_cert) - return nil - end + local vars, err = get_result, nil + if not vars or not vars.global then + return false -- Default to disabled if not configured + end - -- Write with error checking - local ok_write, write_err = f:write(cert_pem) - f:close() - if not ok_write then - logger:log(ERR, "OCSP cert serial: write failed: " .. (write_err or "unknown")) - pcall(function() os.remove(tmp_cert) end) - return nil - end + local value = vars.global["USE_REDIS"] + if value == nil then + return false + end - -- Run openssl command and ensure cleanup on all paths - local handle = io.popen("openssl x509 -serial -noout -in " .. tmp_cert .. " 2>/dev/null", "r") - if not handle then - logger:log(DEBUG, "OCSP cert serial: openssl command failed") - pcall(function() os.remove(tmp_cert) end) - return nil + -- Normalize and interpret common truthy representations + if type(value) == "boolean" then + return value + end + local str = tostring(value):lower() + return str == "1" or str == "true" or str == "on" or str == "yes" end - local result = handle:read("*a") - handle:close() - pcall(function() os.remove(tmp_cert) end) + -- Helper: check if OCSP stapling is enabled for this site (with exception handling) + local function is_ocsp_stapling_enabled() + -- Get variables with exception handling + local ok_get, get_result = pcall(function() + return internalstore:get("variables", true) + end) - if not result or #result == 0 then - logger:log(DEBUG, "OCSP cert serial: openssl returned no output") - return nil - end + if not ok_get then + safe_log(DEBUG, "OCSP stapling enabled check: exception reading variables: " .. tostring(get_result)) + return true -- Fail safely to enabled (backward compatible) + end - local serial = result:match("serial=([A-F0-9]+)") - if not serial then - logger:log(DEBUG, "OCSP cert serial: could not parse serial from output: " .. result) - end - return serial - end + local vars = get_result + if not vars then + return true -- Default to enabled if not available + end + + -- Resolve effective SSL_USE_OCSP_STAPLING value with per-site override + local value + if vars.per_site and server_name and vars.per_site[server_name] then + value = vars.per_site[server_name]["SSL_USE_OCSP_STAPLING"] + end + if value == nil and vars.global then + value = vars.global["SSL_USE_OCSP_STAPLING"] + end + if value == nil then + value = vars["SSL_USE_OCSP_STAPLING"] + end + + -- If still nil, keep default behavior (enabled) + if value == nil then + return true + end - -- Helper: extract certificate serial number from OCSP response DER - local function get_ocsp_serial(ocsp_der) - if not ocsp_der or #ocsp_der == 0 then - return nil + -- Normalize and interpret common falsy/disabled representations + if type(value) == "boolean" then + return value + end + local str = tostring(value):lower() + if str == "0" or str == "false" or str == "off" or str == "no" then + return false + end + + -- Any other value is treated as enabled + return true end - -- Try native resty.openssl first (fast, direct DER parsing) - if has_resty_ssl then - local resty_ocsp = nil + -- Helper: cleanup old temp files from /tmp (older than 5 minutes) + local function cleanup_temp_files() pcall(function() - resty_ocsp = require "resty.openssl.ocsp" + -- Safely execute find to delete old ocsp related temp files + -- Pattern matches files created by this script: ocsp_cert_*, ocsp_resp_*, ocsp_must_staple_* + os.execute("find /tmp -name 'ocsp_*' -mmin +5 -delete 2>/dev/null") end) + end + + -- Helper: sanitize domain/cert name for filesystem (replace * with _wildcard_) + local function sanitize_name(name) + if not name then return nil end + return name:gsub("%*", "_wildcard_") + end + + -- Helper: extract certificate serial number from PEM + local function get_cert_serial(cert_pem) + if not cert_pem or #cert_pem == 0 then + return nil + end - if resty_ocsp then - local resp, err = resty_ocsp.new(ocsp_der) - if resp then - local serial_num = resp:get_serial() + -- Try native resty.openssl (fast, no shell call) + if has_resty_ssl then + local cert, err = resty_x509.new(cert_pem) + if cert then + local serial_num = cert:get_serial_number() if serial_num then return tostring(serial_num) end else - logger:log(DEBUG, "OCSP resty parsing error: " .. tostring(err)) + safe_log(DEBUG, "OCSP resty.openssl parse error: " .. tostring(err)) end end - end - -- Fallback: write to temp file and use openssl - local worker_id = ngx.worker.id() or "0" - local tmp_ocsp = "/tmp/ocsp_resp_w" .. worker_id .. "_c" .. ngx.var.connection .. ".der" - local f = io.open(tmp_ocsp, "wb") - if not f then - logger:log(ERR, "OCSP serial: could not open temp file for writing: " .. tmp_ocsp) - return nil - end + -- Fallback to openssl command (slower, but more compatible) - wrap in pcall to avoid block crash + local serial = nil + pcall(function() + local worker_id = ngx.worker.id() or "0" + local tmp_cert = "/tmp/ocsp_cert_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" + local f = io.open(tmp_cert, "w") + if not f then + safe_log(ERR, "OCSP cert serial: could not open temp file for writing: " .. tmp_cert) + return + end - -- Write with error checking - local ok_write, write_err = f:write(ocsp_der) - f:close() - if not ok_write then - logger:log(ERR, "OCSP serial: write failed: " .. (write_err or "unknown")) - pcall(function() os.remove(tmp_ocsp) end) - return nil - end + -- Write with error checking + local ok_write, write_err = f:write(cert_pem) + f:close() + if not ok_write then + safe_log(ERR, "OCSP cert serial: write failed: " .. (write_err or "unknown")) + pcall(function() os.remove(tmp_cert) end) + return + end - -- Run openssl command and ensure cleanup on all paths - local handle = io.popen("openssl ocsp -respin " .. tmp_ocsp .. " -text -noout 2>/dev/null | grep 'Serial Number'", "r") - if not handle then - logger:log(DEBUG, "OCSP serial: openssl command failed") - pcall(function() os.remove(tmp_ocsp) end) - return nil - end + -- Run openssl command and ensure cleanup on all paths + local handle = io.popen("openssl x509 -serial -noout -in " .. tmp_cert .. " 2>/dev/null", "r") + if not handle then + safe_log(DEBUG, "OCSP cert serial: openssl command failed") + pcall(function() os.remove(tmp_cert) end) + return + end - local result = handle:read("*a") - handle:close() - pcall(function() os.remove(tmp_ocsp) end) + local result = handle:read("*a") + handle:close() + pcall(function() os.remove(tmp_cert) end) - if not result or #result == 0 then - logger:log(DEBUG, "OCSP serial: openssl returned no output") - return nil - end + if not result or #result == 0 then + safe_log(DEBUG, "OCSP cert serial: openssl returned no output") + return + end - -- Parse "Serial Number: XXXXX" format - local serial = result:match("Serial Number:%s*([A-F0-9]+)") - if not serial then - logger:log(DEBUG, "OCSP serial: could not parse serial from output: " .. result) - end - return serial - end + serial = result:match("serial=([A-F0-9]+)") + if not serial then + safe_log(DEBUG, "OCSP cert serial: could not parse serial from output: " .. result) + end + end) - -- Helper: verify OCSP response matches current certificate (by comparing serials) - local function verify_ocsp_cert_match(cert_pem, ocsp_der) - if not cert_pem or not ocsp_der then - logger:log(DEBUG, "OCSP verification skipped (missing cert or OCSP data)") - return true -- Allow OCSP if can't verify + return serial end - -- Get serial from certificate - local cert_serial = get_cert_serial(cert_pem) - if not cert_serial then - logger:log(DEBUG, "OCSP could not extract certificate serial") - return false - end + -- Helper: extract certificate serial number from OCSP response DER + local function get_ocsp_serial(ocsp_der) + if not ocsp_der or #ocsp_der == 0 then + return nil + end - -- Get serial from OCSP response (direct DER parsing - no files needed) - local ocsp_serial = get_ocsp_serial(ocsp_der) - if not ocsp_serial then - logger:log(DEBUG, "OCSP could not extract OCSP response serial") - return false - end + -- Try native resty.openssl first (fast, direct DER parsing) + if has_resty_ssl then + local resty_ocsp = nil + pcall(function() + resty_ocsp = require "resty.openssl.ocsp" + end) + + if resty_ocsp then + local resp, err = resty_ocsp.new(ocsp_der) + if resp then + local serial_num = resp:get_serial() + if serial_num then + return tostring(serial_num) + end + else + safe_log(DEBUG, "OCSP resty parsing error: " .. tostring(err)) + end + end + end - -- Compare serials (uppercase for consistent comparison) - local cert_serial_upper = cert_serial:upper() - local ocsp_serial_upper = ocsp_serial:upper() - if cert_serial_upper == ocsp_serial_upper then - logger:log(DEBUG, "OCSP verified: certificate serial matches OCSP response serial: " .. cert_serial_upper) - return true - else - logger:log(DEBUG, "OCSP mismatch: cert serial=" .. cert_serial_upper .. " vs ocsp serial=" .. ocsp_serial_upper) - return false + -- Fallback: use openssl command (wrap in pcall for safety) + local serial = nil + pcall(function() + local worker_id = ngx.worker.id() or "0" + local tmp_ocsp = "/tmp/ocsp_resp_w" .. worker_id .. "_c" .. ngx.var.connection .. ".der" + local f = io.open(tmp_ocsp, "wb") + if not f then + safe_log(ERR, "OCSP serial: could not open temp file for writing: " .. tmp_ocsp) + return + end + + -- Write with error checking + local ok_write, write_err = f:write(ocsp_der) + f:close() + if not ok_write then + safe_log(ERR, "OCSP serial: write failed: " .. (write_err or "unknown")) + pcall(function() os.remove(tmp_ocsp) end) + return + end + + -- Run openssl command and ensure cleanup on all paths + local handle = io.popen("openssl ocsp -respin " .. tmp_ocsp .. " -text -noout 2>/dev/null | grep 'Serial Number'", "r") + if not handle then + safe_log(DEBUG, "OCSP serial: openssl command failed") + pcall(function() os.remove(tmp_ocsp) end) + return + end + + local result = handle:read("*a") + handle:close() + pcall(function() os.remove(tmp_ocsp) end) + + if not result or #result == 0 then + safe_log(DEBUG, "OCSP serial: openssl returned no output") + return + end + + -- Parse "Serial Number: XXXXX" format + serial = result:match("Serial Number:%s*([A-F0-9]+)") + if not serial then + safe_log(DEBUG, "OCSP serial: could not parse serial from output: " .. result) + end + end) + + return serial end - end - -- Helper: extract OCSP responder URL from certificate using OID 1.3.6.1.5.5.7.48.1 - local function get_ocsp_responder_url(cert_pem) - if not cert_pem or #cert_pem == 0 then - return nil + -- Helper: verify OCSP response matches current certificate (by comparing serials) + local function verify_ocsp_cert_match(cert_pem, ocsp_der) + if not cert_pem or not ocsp_der then + safe_log(DEBUG, "OCSP verification skipped (missing cert or OCSP data)") + return true -- Allow OCSP if cannot verify + end + + -- If cert_pem is not a PEM string (e.g., parsed object from ssl.parse_pem_cert), we cannot re-verify it here + -- OCSP responses are pre-verified and written by ocsp-refresh.py, so in this runtime path we trust the cached response + if type(cert_pem) ~= "string" then + safe_log(DEBUG, "OCSP verification skipped in ssl_certificate phase because certificate is not a PEM string (parsed object from ngx.ssl); using pre-validated cached OCSP response from ocsp-refresh job") + return true + end + + + -- Get serial from certificate + local cert_serial = get_cert_serial(cert_pem) + if not cert_serial then + safe_log(DEBUG, "OCSP could not extract certificate serial") + return false + end + + -- Get serial from OCSP response (direct DER parsing - no files needed) + local ocsp_serial = get_ocsp_serial(ocsp_der) + if not ocsp_serial then + safe_log(DEBUG, "OCSP could not extract OCSP response serial") + return false + end + + -- Compare serials (uppercase for consistent comparison) + local cert_serial_upper = cert_serial:upper() + local ocsp_serial_upper = ocsp_serial:upper() + if cert_serial_upper == ocsp_serial_upper then + safe_log(DEBUG, "OCSP verified: certificate serial matches OCSP response serial: " .. cert_serial_upper) + return true + else + safe_log(DEBUG, "OCSP mismatch: cert serial=" .. cert_serial_upper .. " vs ocsp serial=" .. ocsp_serial_upper) + return false + end end - -- Try native resty.openssl first (fast, OID-based) - if has_resty_ssl then - local cert, err = resty_x509.new(cert_pem) - if cert then - -- Get AIA extension (OID 1.3.6.1.5.5.7.1.1) - local aia_ext = cert:get_extension("authorityInfoAccess") - if aia_ext then - local aia_text = aia_ext:text() - -- Look for OCSP OID 1.3.6.1.5.5.7.48.1 with URI format - -- Text format: "OCSP - URI:http://..." or just extract any http/https URL after OCSP - local ocsp_url = aia_text:match("1%.3%.6%.1%.5%.5%.7%.48%.1%s*=%s*URI:([%w:/.-]+)") - if not ocsp_url then - -- Fallback: text-based pattern in case formatting differs - ocsp_url = aia_text:match("OCSP%s*-%s*URI:([%w:/.-]+)") + -- Helper: read and parse certificate once, extract Must-Staple + responder URL + -- Input: cert_pem - certificate in PEM format (passed from plugin, NOT read via ssl.cert_pem()) + -- Returns: { cert_parsed = resty cert object or nil, must_staple = bool, responder_url = string or nil } + local function read_certificate_metadata(cert_pem) + local result = { + cert_parsed = nil, + must_staple = false, + responder_url = nil + } + + if not cert_pem or #cert_pem == 0 then + return result + end + + -- Try to parse with resty.openssl (parse once, extract all) + if has_resty_ssl then + local cert, err = resty_x509.new(cert_pem) + if cert then + result.cert_parsed = cert + + -- Extract TLS Feature extension for Must-Staple (OID 1.3.6.1.5.5.7.1.24) + local tls_feature_ext = cert:get_extension("tlsfeature") + if tls_feature_ext then + local tls_feature_text = tls_feature_ext:text() + if tls_feature_text:find("OCSP status request", 1, true) or tls_feature_text:find("5", 1, true) then + result.must_staple = true + safe_log(DEBUG, "OCSP-Must-Staple extension found (OID 1.3.6.1.5.5.7.1.24)") + end end - if ocsp_url then - logger:log(DEBUG, "OCSP found responder URL (OID 1.3.6.1.5.5.7.48.1) in certificate: " .. ocsp_url) - return ocsp_url + + -- Extract AIA extension for OCSP responder URL (OID 1.3.6.1.5.5.7.48.1) + local aia_ext = cert:get_extension("authorityInfoAccess") + if aia_ext then + local aia_text = aia_ext:text() + -- Look for OCSP OID with URI format + local ocsp_url = aia_text:match("1%%.3%%.6%%.1%%.5%%.5%%.7%%.48%%.1%%s*=%%s*URI:([%%w:/.-]+)") + if not ocsp_url then + -- Fallback: text-based pattern + ocsp_url = aia_text:match("OCSP%%s*-%%s*URI:([%%w:/.-]+)") + end + if ocsp_url then + result.responder_url = ocsp_url + safe_log(DEBUG, "OCSP responder URL found (OID 1.3.6.1.5.5.7.48.1): " .. ocsp_url) + end end + + return result + else + safe_log(DEBUG, "Certificate parse error with resty.openssl: " .. tostring(err)) end - else - logger:log(DEBUG, "OCSP resty.openssl parse error: " .. tostring(err)) end - end - -- Fallback to openssl command (also OID-based) - local worker_id = ngx.worker.id() or "0" - local tmp_cert = "/tmp/ocsp_cert_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" - local f = io.open(tmp_cert, "w") - if not f then - logger:log(ERR, "OCSP responder URL: could not open temp file for writing: " .. tmp_cert) - return nil - end + -- Fallback: use openssl command to extract both Must-Staple and responder URL + pcall(function() + local worker_id = ngx.worker.id() or "0" + local tmp_cert = "/tmp/ocsp_cert_meta_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" + local f = io.open(tmp_cert, "w") + if not f then + return + end - -- Write with error checking - local ok_write, write_err = f:write(cert_pem) - f:close() - if not ok_write then - logger:log(ERR, "OCSP responder URL: write failed: " .. (write_err or "unknown")) - pcall(function() os.remove(tmp_cert) end) - return nil - end + local ok_write = f:write(cert_pem) + f:close() + if not ok_write then + pcall(function() os.remove(tmp_cert) end) + return + end - -- Use openssl to extract OCSP URI (searches for OID 1.3.6.1.5.5.7.48.1 internally) - -- Ensure cleanup on all paths - local handle = io.popen("openssl x509 -ocsp_uri -noout -in " .. tmp_cert .. " 2>/dev/null", "r") - if not handle then - logger:log(DEBUG, "OCSP responder URL: openssl command failed") - pcall(function() os.remove(tmp_cert) end) - return nil - end + -- Extract both TLS Feature and OCSP URI in one openssl call + local handle = io.popen("openssl x509 -text -noout -in " .. tmp_cert .. " 2>/dev/null", "r") + if not handle then + pcall(function() os.remove(tmp_cert) end) + return + end - local result = handle:read("*a") - handle:close() - pcall(function() os.remove(tmp_cert) end) + local output = handle:read("*a") + handle:close() + pcall(function() os.remove(tmp_cert) end) - if not result or #result == 0 then - logger:log(DEBUG, "OCSP no responder URL (OID 1.3.6.1.5.5.7.48.1) found in certificate") - return nil + if not output or #output == 0 then + return + end + + -- Check for TLS Feature (Must-Staple) + if output:find("TLS Feature", 1, true) then + if output:find("OCSP status request", 1, true) or output:find("5", 1, true) then + result.must_staple = true + safe_log(DEBUG, "OCSP-Must-Staple extension found via openssl") + end + end + + -- Extract OCSP responder URL + local ocsp_url = output:match("OCSP%s*%-%s*URI:([%w:/.-]+)") + if ocsp_url then + result.responder_url = ocsp_url + safe_log(DEBUG, "OCSP responder URL found via openssl: " .. ocsp_url) + end + end) + + return result end - -- Extract URL (openssl x509 -ocsp_uri returns the URL or empty string) - local ocsp_url = result:match("^([%w:/.-]+)") - if ocsp_url then - logger:log(DEBUG, "OCSP found responder URL (OID 1.3.6.1.5.5.7.48.1) in certificate: " .. ocsp_url) + -- Helper: extract OCSP responder URL from certificate using OID 1.3.6.1.5.5.7.48.1 + local function get_ocsp_responder_url(cert_pem) + if not cert_pem or #cert_pem == 0 then + return nil + end + + -- Try native resty.openssl first (fast, OID-based) + if has_resty_ssl then + local cert, err = resty_x509.new(cert_pem) + if cert then + -- Get AIA extension (OID 1.3.6.1.5.5.7.1.1) + local aia_ext = cert:get_extension("authorityInfoAccess") + if aia_ext then + local aia_text = aia_ext:text() + -- Look for OCSP OID 1.3.6.1.5.5.7.48.1 with URI format + -- Text format: "OCSP - URI:http://..." or just extract any http/https URL after OCSP + local ocsp_url = aia_text:match("1%%.3%%.6%%.1%%.5%%.5%%.7%%.48%%.1%%s*=%%s*URI:([%%w:/.-]+)") + if not ocsp_url then + -- Fallback: text-based pattern in case formatting differs + ocsp_url = aia_text:match("OCSP%%s*-%%s*URI:([%%w:/.-]+)") + end + if ocsp_url then + safe_log(DEBUG, "OCSP found responder URL (OID 1.3.6.1.5.5.7.48.1) in certificate: " .. ocsp_url) + return ocsp_url + end + end + else + safe_log(DEBUG, "OCSP resty.openssl parse error: " .. tostring(err)) + end + end + + -- Fallback to openssl command (wrap in pcall for safety) + local ocsp_url = nil + pcall(function() + local worker_id = ngx.worker.id() or "0" + local tmp_cert = "/tmp/ocsp_cert_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" + local f = io.open(tmp_cert, "w") + if not f then + safe_log(ERR, "OCSP responder URL: could not open temp file for writing: " .. tmp_cert) + return + end + + -- Write with error checking + local ok_write, write_err = f:write(cert_pem) + f:close() + if not ok_write then + safe_log(ERR, "OCSP responder URL: write failed: " .. (write_err or "unknown")) + pcall(function() os.remove(tmp_cert) end) + return + end + + -- Use openssl to extract OCSP URI (searches for OID 1.3.6.1.5.5.7.48.1 internally) + -- Ensure cleanup on all paths + local handle = io.popen("openssl x509 -ocsp_uri -noout -in " .. tmp_cert .. " 2>/dev/null", "r") + if not handle then + safe_log(DEBUG, "OCSP responder URL: openssl command failed") + pcall(function() os.remove(tmp_cert) end) + return + end + + local result = handle:read("*a") + handle:close() + pcall(function() os.remove(tmp_cert) end) + + if not result or #result == 0 then + safe_log(DEBUG, "OCSP no responder URL (OID 1.3.6.1.5.5.7.48.1) found in certificate") + return + end + + -- Extract URL (openssl x509 -ocsp_uri returns the URL or empty string) + ocsp_url = result:match("^([%%w:/.-]+)") + if ocsp_url then + safe_log(DEBUG, "OCSP found responder URL (OID 1.3.6.1.5.5.7.48.1) in certificate: " .. ocsp_url) + end + end) + return ocsp_url end - logger:log(DEBUG, "OCSP no responder URL (OID 1.3.6.1.5.5.7.48.1) found in certificate") - return nil - end + -- Helper: check if certificate has OCSP Must-Staple extension (OID 1.3.6.1.5.5.7.1.24) + local function has_ocsp_must_staple(cert_pem) + if not cert_pem or #cert_pem == 0 then + return false + end - -- Helper: set OCSP stapling from cache (internalstore + ocsp.der disk files) - local function set_ocsp_from_cache() - logger:log(DEBUG, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) + -- Try native resty.openssl first (fast, OID-based) + if has_resty_ssl then + local cert, err = resty_x509.new(cert_pem) + if cert then + -- Get TLS Feature extension (OID 1.3.6.1.5.5.7.1.24) + -- This extension contains the OCSP-must-staple feature + local tls_feature_ext = cert:get_extension("tlsfeature") + if tls_feature_ext then + local tls_feature_text = tls_feature_ext:text() + -- Look for OCSP status request (5) in the text + -- openssl reports it as "OCSP status request" or feature code "5" + if tls_feature_text:find("OCSP status request", 1, true) or tls_feature_text:find("5", 1, true) then + safe_log(DEBUG, "OCSP-Must-Staple extension found in certificate (OID 1.3.6.1.5.5.7.1.24)") + return true + end + end + else + safe_log(DEBUG, "OCSP-Must-Staple check: resty.openssl parse error: " .. tostring(err)) + end + end - if not is_ocsp_stapling_enabled() then - logger:log(DEBUG, "OCSP stapling disabled via SSL_USE_OCSP_STAPLING setting") - return - end + -- Fallback to openssl command (wrap in pcall for safety) + local must_staple = false + pcall(function() + local worker_id = ngx.worker.id() or "0" + local tmp_cert = "/tmp/ocsp_must_staple_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" + local f = io.open(tmp_cert, "w") + if not f then + safe_log(DEBUG, "OCSP-Must-Staple check: could not open temp file") + return + end - if not server_name or server_name == "" then - logger:log(ERR, "OCSP no server_name available") - return - end + local ok_write, write_err = f:write(cert_pem) + f:close() + if not ok_write then + pcall(function() os.remove(tmp_cert) end) + return + end - -- Get the current certificate early (with error handling) - local cert_pem, err - local ok_cert, cert_result = pcall(function() - return ngx.ssl.cert_pem() - end) - if not ok_cert then - logger:log(ERR, "OCSP exception reading certificate: " .. tostring(cert_result)) - return - end - cert_pem, err = cert_result, nil - if not cert_pem then - logger:log(DEBUG, "OCSP could not read current certificate: " .. (err or "unknown")) - return - end + -- Extract TLS Feature extension (OID 1.3.6.1.5.5.7.1.24) + local handle = io.popen("openssl x509 -text -noout -in " .. tmp_cert .. " 2>/dev/null | grep -A1 'TLS Feature' | head -2", "r") + if not handle then + pcall(function() os.remove(tmp_cert) end) + return + end - -- Check if certificate has OCSP responder URL (with error handling) - local ocsp_responder_url - local ok_url, url_result = pcall(function() - return get_ocsp_responder_url(cert_pem) - end) - if not ok_url then - logger:log(ERR, "OCSP exception extracting responder URL: " .. tostring(url_result)) - return - end - ocsp_responder_url = url_result - if not ocsp_responder_url then - logger:log(DEBUG, "OCSP skipping: certificate does not have OCSP responder URL") - return - end + local result = handle:read("*a") + handle:close() + pcall(function() os.remove(tmp_cert) end) - local cache_key = "TLS:SSL:ocsp:" .. server_name - local resp + if result and (result:find("OCSP status request", 1, true) or result:find("5", 1, true)) then + safe_log(DEBUG, "OCSP-Must-Staple extension found in certificate (OID 1.3.6.1.5.5.7.1.24)") + must_staple = true + end + end) - -- 1) Local shared dict lookup (per-worker cache) - with error handling - local ok_cache, cache_result = pcall(function() - return internalstore:get(cache_key, true) - end) - if ok_cache then - local db_val, gerr = cache_result, nil - if db_val then - resp = db_val - logger:log(DEBUG, "OCSP found response in internalstore for " .. server_name) - elseif gerr and gerr ~= "not found" then - logger:log(ERR, "OCSP error while getting response from internalstore: " .. gerr) - end - else - logger:log(ERR, "OCSP exception reading from cache: " .. tostring(cache_result)) + return must_staple end - -- 2) Redis/cluster lookup - with 0.05s timeout and error handling (skip if Redis not enabled) - if not resp and is_redis_enabled() then - local ok_cluster_read, cluster_read_result = pcall(function() - local cluster = cclusterstore:new() - -- Set timeout to 0.05 seconds (50ms) for fast read fail - cache hits should be instant - cluster:set_timeout(50) - local ok_connect, rerr = cluster:connect(true) - if not ok_connect then - logger:log(DEBUG, "OCSP cluster read connection failed (non-critical): " .. (rerr or "unknown")) - return nil - end - local ok_get, val, gerr = cluster:get(cache_key, true) - if not ok_get then - logger:log(DEBUG, "OCSP error while reading response from cluster: " .. (gerr or "unknown")) - return nil - end - return val - end) - if ok_cluster_read then - resp = cluster_read_result - if resp then - logger:log(DEBUG, "OCSP found response in cluster for " .. server_name) - -- Backfill to internalstore for faster future access - local ok_backfill, backfill_result = pcall(function() - return internalstore:set(cache_key, resp, 300, true) - end) - if not ok_backfill then - logger:log(DEBUG, "OCSP could not backfill to internalstore: " .. tostring(backfill_result)) + -- Helper: set OCSP stapling from cache (internalstore + ocsp.der disk files) + -- Input: cert_pem - certificate in PEM format string (or nil if from parsed plugin object) + -- Returns: true if cert is acceptable, false if OCSP-Must-Staple requirement not met + local function set_ocsp_from_cache(cert_pem) + safe_log(DEBUG, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) + + if not is_ocsp_stapling_enabled() then + safe_log(DEBUG, "OCSP stapling disabled via SSL_USE_OCSP_STAPLING setting") + return true -- Certificate acceptable even without OCSP + end + + if not server_name or server_name == "" then + safe_log(ERR, "OCSP no server_name available") + return true -- Accept cert even without server_name check + end + + -- Extract certificate metadata (Must-Staple + responder URL) + -- OCSP responses are ALWAYS loaded from pre-cached files only, never downloaded on worker + local has_must_staple = false + local ocsp_responder_url = nil + + -- Read and parse certificate for Must-Staple + responder URL extraction + -- cert_pem is a parsed certificate object from ssl.parse_pem_cert(), not a string + -- We can only validate Must-Staple if we have the actual PEM string + if cert_pem and type(cert_pem) == "string" and #cert_pem > 0 then + -- Read certificate and extract both Must-Staple and responder URL in one pass + local ok_read, cert_meta = pcall(read_certificate_metadata, cert_pem) + if ok_read and cert_meta then + has_must_staple = cert_meta.must_staple + ocsp_responder_url = cert_meta.responder_url + if has_must_staple then + safe_log(INFO, "OCSP-Must-Staple extension detected in certificate for " .. server_name) end + else + safe_log(DEBUG, "Certificate metadata extraction failed: " .. tostring(cert_meta)) end else - logger:log(DEBUG, "OCSP exception reading from cluster (non-critical): " .. tostring(cluster_read_result)) + -- cert_pem is a parsed certificate object (from ssl.parse_pem_cert), not a PEM string + -- Must-Staple validation requires the original PEM, which is only available at OCSP generation time + -- In this runtime path we rely on ocsp-refresh.py to have enforced Must-Staple when creating ocsp.der + safe_log(DEBUG, "OCSP-Must-Staple check skipped in ssl_certificate phase because certificate is not a PEM string (parsed object from ngx.ssl); relying on ocsp-refresh job's validation when ocsp.der was generated") end - end - -- 3) Fallback to on-disk OCSP response (/var/cache/bunkerweb/ssl/{domain}/ocsp.der) - if not resp then - local base_path = "/var/cache/bunkerweb/ssl/" - - -- Generate candidates for lookup - local candidates = {} - - -- Direct matches - insert(candidates, server_name) - insert(candidates, server_name .. "-ecdsa") - insert(candidates, server_name .. "-rsa") - insert(candidates, "customcert-" .. server_name) - insert(candidates, "customcert-" .. server_name .. "-ecdsa") - insert(candidates, "customcert-" .. server_name .. "-rsa") - - -- Wildcard and apex matches (e.g., if server_name is www.example.com, try *.example.com and example.com) - local labels = {} - for label in server_name:gmatch("[^.]+") do - insert(labels, label) - end - - -- Only try wildcard/apex if we have at least subdomain.example.com (3+ labels), and avoid TLD-only suffixes - if #labels >= 3 then - for i = 2, #labels - 1 do - local suffix = table.concat(labels, ".", i) - if suffix and suffix ~= "" then - -- Apex domain (e.g. example.com) โ€” wildcard certs are often stored under apex name - insert(candidates, suffix) - insert(candidates, suffix .. "-ecdsa") - insert(candidates, suffix .. "-rsa") - insert(candidates, "customcert-" .. suffix) - insert(candidates, "customcert-" .. suffix .. "-ecdsa") - insert(candidates, "customcert-" .. suffix .. "-rsa") - -- Wildcard form (e.g. *.example.com) - local wildcard_base = "*." .. suffix - insert(candidates, wildcard_base) - insert(candidates, wildcard_base .. "-ecdsa") - insert(candidates, wildcard_base .. "-rsa") - insert(candidates, "customcert-" .. wildcard_base) - insert(candidates, "customcert-" .. wildcard_base .. "-ecdsa") - insert(candidates, "customcert-" .. wildcard_base .. "-rsa") - end + -- Validate: if certificate has OCSP-Must-Staple extension, it MUST have an OCSP responder URL + if has_must_staple then + if not ocsp_responder_url then + safe_log(ERR, "OCSP-Must-Staple detected but no responder URL (OID 1.3.6.1.5.5.7.48.1) found in certificate - malformed certificate for " .. server_name) + else + safe_log(INFO, "OCSP-Must-Staple validated: certificate has both Must-Staple extension and responder URL for " .. server_name) + end + end + + local cache_key = "TLS:SSL:ocsp:" .. server_name + local resp + local checked_paths = {} + + -- 1) Local shared dict lookup (per-worker cache) - with error handling + local ok_cache, cache_result = pcall(function() + return internalstore:get(cache_key, true) + end) + if ok_cache then + local db_val, gerr = cache_result, nil + if db_val then + resp = db_val + safe_log(DEBUG, "OCSP found response in internalstore for " .. server_name) + elseif gerr and gerr ~= "not found" then + safe_log(ERR, "OCSP error while getting response from internalstore: " .. gerr) end + else + safe_log(ERR, "OCSP exception reading from cache: " .. tostring(cache_result)) end - for _, name in ipairs(candidates) do - if name and name ~= "" then - local sanitized = sanitize_name(name) - local path = base_path .. sanitized .. "/ocsp.der" - local f, err = io.open(path, "rb") - if f then - local data = f:read("*a") - f:close() + + -- 2) Fallback to on-disk OCSP response files + -- Looks for variations in /var/cache/bunkerweb/ssl/ like: + -- - {domain}/ocsp.der + -- - {domain}-ecdsa/ocsp.der + -- - {domain}-rsa/ocsp.der + -- - _wildcard_.{domain}/ocsp.der (for *.domain) + -- - _wildcard_.{domain}-ecdsa/ocsp.der + -- - customcert-{domain}/ocsp.der + -- - and other variants + if not resp then + local base_path = "/var/cache/bunkerweb/ssl/" + safe_log(DEBUG, "OCSP: Searching for cached responses for server_name=" .. server_name .. " in " .. base_path) + + -- Generate candidates for lookup + local candidates = {} + + -- Direct matches + insert(candidates, server_name) + insert(candidates, server_name .. "-ecdsa") + insert(candidates, server_name .. "-rsa") + insert(candidates, "customcert-" .. server_name) + insert(candidates, "customcert-" .. server_name .. "-ecdsa") + insert(candidates, "customcert-" .. server_name .. "-rsa") + + -- Wildcard and apex matches (e.g., if server_name is www.example.com, try *.example.com and example.com) + local labels = {} + for label in server_name:gmatch("[^.]+") do + insert(labels, label) + end + + -- Only try wildcard/apex if we have at least subdomain.example.com (3+ labels), and avoid TLD-only suffixes + if #labels >= 3 then + for i = 2, #labels - 1 do + local suffix = table.concat(labels, ".", i) + if suffix and suffix ~= "" then + -- Apex domain (e.g. example.com) โ€” wildcard certs are often stored under apex name + insert(candidates, suffix) + insert(candidates, suffix .. "-ecdsa") + insert(candidates, suffix .. "-rsa") + insert(candidates, "customcert-" .. suffix) + insert(candidates, "customcert-" .. suffix .. "-ecdsa") + insert(candidates, "customcert-" .. suffix .. "-rsa") + -- Wildcard form (e.g. *.example.com) + local wildcard_base = "*." .. suffix + insert(candidates, wildcard_base) + insert(candidates, wildcard_base .. "-ecdsa") + insert(candidates, wildcard_base .. "-rsa") + insert(candidates, "customcert-" .. wildcard_base) + insert(candidates, "customcert-" .. wildcard_base .. "-ecdsa") + insert(candidates, "customcert-" .. wildcard_base .. "-rsa") + end + end + end + + for _, name in ipairs(candidates) do + if name and name ~= "" then + local sanitized = sanitize_name(name) + local path = base_path .. sanitized .. "/ocsp.der" + insert(checked_paths, path) + safe_log(DEBUG, "OCSP: Checking candidate " .. name .. " -> " .. path) + local f, err = io.open(path, "rb") + if f then + local data = f:read("*a") + f:close() if data and #data > 0 then + -- Load optional OCSP metadata written by ocsp-refresh.py (serial, ocsp_url, must_staple) + local meta = nil + local meta_path = base_path .. sanitized .. "/ocsp.json" + local fmeta = io.open(meta_path, "r") + if fmeta then + local ok_read, meta_raw = pcall(function() + return fmeta:read("*a") + end) + fmeta:close() + if ok_read and meta_raw and #meta_raw > 0 then + local ok_decode, decoded = pcall(cjson.decode, meta_raw) + if ok_decode and type(decoded) == "table" then + meta = decoded + else + safe_log(DEBUG, "OCSP metadata decode failed for " .. meta_path) + end + else + safe_log(DEBUG, "OCSP metadata read failed or empty for " .. meta_path) + end + end + + if meta and meta.must_staple then + safe_log(INFO, "OCSP metadata indicates Must-Staple=on for " .. server_name .. " (serial=" .. tostring(meta.serial or "unknown") .. ")") + end + -- Verify OCSP response matches current certificate (with error handling) local cert_matches local ok_verify, verify_result = pcall(function() return verify_ocsp_cert_match(cert_pem, data) end) if not ok_verify then - logger:log(ERR, "OCSP exception verifying response: " .. tostring(verify_result)) + safe_log(ERR, "OCSP exception verifying response: " .. tostring(verify_result)) goto next_candidate end cert_matches = verify_result if cert_matches then - logger:log(DEBUG, "OCSP loaded response (" .. #data .. " bytes) from " .. path .. " (verified against certificate)") - resp = data - -- Cache in shared memory to avoid disk I/O on every handshake (TTL 300s) - with error handling - local ok_set_cache, set_result = pcall(function() - return internalstore:set(cache_key, resp, 300, true) - end) - if ok_set_cache then - local ok_set, serr = set_result, nil - if not ok_set then - logger:log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) + safe_log(DEBUG, "OCSP loaded response (" .. #data .. " bytes) from " .. path .. " (verified against certificate or trusted via ocsp-refresh metadata)") + resp = data + -- Cache in shared memory to avoid disk I/O on every handshake (TTL 300s) - with error handling + local ok_set_cache, set_result = pcall(function() + return internalstore:set(cache_key, resp, 300, true) + end) + if ok_set_cache then + local ok_set, serr = set_result, nil + if not ok_set then + safe_log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) + end + else + safe_log(ERR, "OCSP exception caching response: " .. tostring(set_result)) end + break else - logger:log(ERR, "OCSP exception caching response: " .. tostring(set_result)) + safe_log(DEBUG, "OCSP skipping response for " .. name .. " (certificate mismatch - certificate may have been updated)") end - break - else - logger:log(DEBUG, "OCSP skipping response for " .. name .. " (certificate mismatch - certificate may have been updated)") + end + else + -- Log only on permission denied (avoid shell; use io.open error) + if err and lower(err):find("permission denied", 1, true) then + safe_log(ERR, "OCSP permission denied reading " .. path .. " (check file permissions)") end end - else - -- Log only on permission denied (avoid shell; use io.open error) - if err and lower(err):find("permission denied", 1, true) then - logger:log(ERR, "OCSP permission denied reading " .. path .. " (check file permissions)") - end + ::next_candidate:: end - ::next_candidate:: end end - end - if not resp then - logger:log(DEBUG, "OCSP not found for " .. server_name) - return - end - - -- Safely set OCSP stapling with exception handling - local ok_pcall, ocsp_result = pcall(function() - return ocsp.set_ocsp_status_resp(resp) - end) + if not resp then + safe_log(DEBUG, "OCSP not found for " .. server_name) + -- Debug: log all paths that were checked + if #checked_paths > 0 then + local paths_str = table.concat(checked_paths, " | ") + if #paths_str > 2048 then + paths_str = paths_str:sub(1, 2045) .. "..." + end + safe_log(DEBUG, "OCSP: Checked " .. #checked_paths .. " paths: " .. paths_str) + end + if has_must_staple then + safe_log(ERR, "OCSP-Must-Staple required but OCSP response not found - will attempt TLS anyway") + end + return true + end - if not ok_pcall then - logger:log(ERR, "OCSP exception while setting stapling: " .. tostring(ocsp_result)) - else - local ok_set, oerr = ocsp_result - if not ok_set then - logger:log(ERR, "OCSP failed to set stapling: " .. oerr) - else - logger:log(DEBUG, "OCSP stapling set from cache for " .. server_name) + -- Safely set OCSP stapling with exception handling (only if ocsp module is available) + if not ocsp then + safe_log(DEBUG, "OCSP not available (ngx.ocsp module not loaded)") + return true end - end + local ok_pcall, ok_set, oerr = pcall(function() + return ocsp.set_ocsp_status_resp(resp) + end) - -- Backfill Redis AFTER certificate and OCSP are loaded and set (non-blocking, lazy write, skip if Redis not enabled) - if resp and is_redis_enabled() then - local ok_cluster, cluster_result = pcall(function() - local cluster = cclusterstore:new() - -- Set timeout to 0.1 seconds - this write should not block - cluster:set_timeout(100) - local ok_connect, rerr = cluster:connect(true) - if not ok_connect then - logger:log(DEBUG, "OCSP cluster backfill connection failed (non-critical): " .. (rerr or "unknown")) - return - end - local ok_set, serr = cluster:set(cache_key, resp, 300, true) + if not ok_pcall then + safe_log(ERR, "OCSP exception while setting stapling: " .. tostring(ok_set)) + else if not ok_set then - logger:log(DEBUG, "OCSP error while backfilling response to cluster: " .. (serr or "unknown")) + safe_log(ERR, "OCSP failed to set stapling: " .. (oerr or "unknown")) else - logger:log(DEBUG, "OCSP backfilled response to cluster for " .. server_name) + safe_log(DEBUG, "OCSP stapling set from cache for " .. server_name) end - end) - if not ok_cluster then - logger:log(DEBUG, "OCSP exception backfilling to cluster (non-critical): " .. tostring(cluster_result)) end + + -- If we reach here without returning earlier, OCSP failed but is not required (or succeeded) + return true end - end - -- Call ssl_certificate() methods - logger:log(INFO, "calling ssl_certificate() methods of plugins ...") - for i, plugin_id in ipairs(phase_order) do - -- Require call - local plugin_lua, err = require_plugin(plugin_id) - if plugin_lua == false then - logger:log(ERR, err) - elseif plugin_lua == nil then - logger:log(DEBUG, err) - else - -- Check if plugin has ssl_certificate method - if plugin_lua.ssl_certificate ~= nil then - -- New call - local ok, plugin_obj = new_plugin(plugin_lua) - if not ok then - logger:log(ERR, plugin_obj) - else - local ok, ret = call_plugin(plugin_obj, "ssl_certificate") - if not ok then - logger:log(ERR, ret) - elseif not ret.ret then - logger:log(ERR, plugin_id .. ":ssl_certificate() call failed : " .. ret.msg) + -- Call ssl_certificate() methods + safe_log(INFO, "calling ssl_certificate() methods of plugins ...") + for _, plugin_id in ipairs(phase_order) do + -- Require call + safe_log(DEBUG, "Loading plugin: " .. plugin_id) + local plugin_lua, err = require_plugin(plugin_id) + if plugin_lua == false then + safe_log(ERR, "Failed to load plugin " .. plugin_id .. " (not found): " .. (err or "unknown")) + elseif plugin_lua == nil then + safe_log(DEBUG, "Plugin " .. plugin_id .. " module not found or failed to load: " .. (err or "unknown")) + else + safe_log(DEBUG, "Plugin " .. plugin_id .. " loaded successfully") + -- Check if plugin has ssl_certificate method + if plugin_lua.ssl_certificate ~= nil then + -- New call + local ok_p, plugin_obj = new_plugin(plugin_lua) + if not ok_p then + safe_log(ERR, plugin_obj) else - logger:log(DEBUG, plugin_id .. ":ssl_certificate() call successful : " .. ret.msg) - if ret.status then - logger:log(DEBUG, plugin_id .. " is setting certificate/key : " .. ret.msg) - local ok, err = clear_certs() - if not ok then - logger:log(ERR, "error while clearing certificates : " .. err) - end - ok, err = set_cert(ret.status[1]) - if not ok then - logger:log(ERR, "error while setting certificate : " .. err) - else - local ok, err = set_priv_key(ret.status[2]) - if not ok then - logger:log(ERR, "error while setting private key : " .. err) + local ok_c, ret = call_plugin(plugin_obj, "ssl_certificate") + if not ok_c then + safe_log(ERR, ret) + elseif not ret.ret then + safe_log(ERR, plugin_id .. ":ssl_certificate() call failed : " .. ret.msg) + else + safe_log(DEBUG, plugin_id .. ":ssl_certificate() call successful : " .. ret.msg) + if ret.status then + safe_log(DEBUG, plugin_id .. " is setting certificate/key : " .. ret.msg) + + -- Clear old certificates before setting new ones (fail-safe) + pcall(clear_certs) + + -- Try to set the new certificate + local ok_cert, err_cert = set_cert(ret.status[1]) + if not ok_cert then + safe_log(ERR, "error while setting certificate from " .. plugin_id .. ": " .. (err_cert or "unknown")) else - -- Try to set OCSP stapling from cache (if enabled) - -- Try to set OCSP stapling (certificate verification optional) - logger:log(DEBUG, "About to call set_ocsp_from_cache() for " .. server_name) - local ok_ocsp, ocsp_err = pcall(function() - set_ocsp_from_cache() - end) - if not ok_ocsp then - logger:log(ERR, "OCSP function error (certificate still set): " .. tostring(ocsp_err)) - end - logger:log(DEBUG, "certificate set by " .. plugin_id) - return true - end - end - end - end - end - else - logger:log(DEBUG, "skipped execution of " .. plugin_id .. " because method ssl_certificate() is not defined") - end - end - end - logger:log(INFO, "called ssl_certificate() methods of plugins") + -- Certificate set successfully, now set private key + local ok_key, err_key = set_priv_key(ret.status[2]) + if not ok_key then + safe_log(ERR, "error while setting private key from " .. plugin_id .. ": " .. (err_key or "unknown")) + else + -- Certificate and key both set successfully + -- Try to set OCSP stapling from cache (fail-safe) + -- Pass the certificate PEM directly (from ret.status[1]) + safe_log(DEBUG, "About to call set_ocsp_from_cache() for " .. server_name) + local ok_ocsp, ocsp_cert_acceptable = pcall(set_ocsp_from_cache, ret.status[1]) + if not ok_ocsp then + safe_log(ERR, "OCSP function error: " .. tostring(ocsp_cert_acceptable)) + elseif ocsp_cert_acceptable == false then + -- OCSP-Must-Staple requirement not met - but user requested to always run TLS + safe_log(ERR, "OCSP-Must-Staple requirement not met for " .. server_name .. " - proceeding anyway as per failsafe policy") + end + safe_log(INFO, "certificate and key set by " .. plugin_id) + return true + end -- end ok_key + end -- end ok_cert + end -- end ret.status + end -- end call_plugin result + end -- end new_plugin result + else + safe_log(DEBUG, "skipped execution of " .. plugin_id .. " because method ssl_certificate() is not defined") + end -- end ssl_certificate defined check + end -- end plugin_lua result check + end -- end for loop + + safe_log(INFO, "ssl_certificate phase ended") - logger:log(INFO, "ssl_certificate phase ended") + -- Cleanup old temp files at the very end to prioritize speed + cleanup_temp_files() - return true + return true + end) + + if not ok then + ngx.log(ngx.ERR, ("SSL-CERTIFICATE critical error in Lua block: " .. tostring(err)):sub(1, 2048)) + end + return true -- Always return true so nginx uses default certificate on any error } diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 764faf917b..065a311142 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -4,6 +4,7 @@ import fcntl import hashlib import io +import json import os import re import shutil @@ -25,7 +26,7 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.serialization import Encoding from cryptography.x509 import ocsp as x509_ocsp -from cryptography.x509 import AuthorityInformationAccess, SubjectAlternativeName +from cryptography.x509 import AuthorityInformationAccess, SubjectAlternativeName, TLSFeature, TLSFeatureType from cryptography.x509.oid import ExtensionOID, AuthorityInformationAccessOID # Add BunkerWeb Python deps (Job, logger, Database) to path @@ -66,6 +67,45 @@ print(f"FATAL: Could not initialize logger: {e}", file=_sys.stderr) _sys.exit(1) + +def _truncate_log(msg: str, max_len: int = 2048) -> str: + """Truncate log message to max_len characters, adding ellipsis if truncated.""" + if isinstance(msg, str) and len(msg) > max_len: + return msg[:max_len - 3] + "..." + return str(msg) if msg is not None else "" + + +# Wrapper functions to enforce 2048 character limit on all log messages +def log_debug(msg: str, *args: Any, **kwargs: Any) -> None: + """Log at DEBUG level with 2048 char limit.""" + formatted = (msg % args) if args else msg + LOG.debug(_truncate_log(formatted), **kwargs) + + +def log_info(msg: str, *args: Any, **kwargs: Any) -> None: + """Log at INFO level with 2048 char limit.""" + formatted = (msg % args) if args else msg + LOG.info(_truncate_log(formatted), **kwargs) + + +def log_warning(msg: str, *args: Any, **kwargs: Any) -> None: + """Log at WARNING level with 2048 char limit.""" + formatted = (msg % args) if args else msg + LOG.warning(_truncate_log(formatted), **kwargs) + + +def log_error(msg: str, *args: Any, **kwargs: Any) -> None: + """Log at ERROR level with 2048 char limit.""" + formatted = (msg % args) if args else msg + LOG.error(_truncate_log(formatted), **kwargs) + + +def log_critical(msg: str, *args: Any, **kwargs: Any) -> None: + """Log at CRITICAL level with 2048 char limit.""" + formatted = (msg % args) if args else msg + LOG.critical(_truncate_log(formatted), **kwargs) + + status = 0 # Use scheduler-managed cache directory (automatically synced from database on restart) @@ -115,7 +155,7 @@ def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int mtime = lock_file.stat().st_mtime age = time.time() - mtime if age > stale_threshold: - LOG.warning( + log_warning( "โš ๏ธ OCSP detected stale lock for %s (age %.0fs > threshold %ds). " "Previous process may have crashed. Removing stale lock.", cert_name, age, stale_threshold @@ -144,11 +184,11 @@ def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int time.sleep(0.5) continue except Exception as e: - LOG.debug("โš ๏ธ OCSP lock acquisition attempt failed for %s: %s", cert_name, e) + log_debug("โš ๏ธ OCSP lock acquisition attempt failed for %s: %s", cert_name, e) time.sleep(0.5) # Timeout reached: log warning and proceed without lock - LOG.warning( + log_warning( "โš ๏ธ OCSP could not acquire lock for %s after %ds. " "If concurrent OCSP fetches happen, cryptographic verification protects data integrity. " "Consider increasing scheduler interval or timeout if jobs consistently exceed %ds.", @@ -195,9 +235,9 @@ def _refresh_cert_lock(fd: Optional[int], cert_name: str) -> None: timestamp = str(int(time.time())).encode() os.write(fd, timestamp) os.fsync(fd) # Ensure write is persisted - LOG.debug("๐Ÿ”’ OCSP refreshed lock timestamp for %s", cert_name) + log_debug("๐Ÿ”’ OCSP refreshed lock timestamp for %s", cert_name) except Exception as e: - LOG.debug("โš ๏ธ OCSP could not refresh lock timestamp for %s: %s", cert_name, e) + log_debug("โš ๏ธ OCSP could not refresh lock timestamp for %s: %s", cert_name, e) def is_safe_url(url: str) -> bool: @@ -216,30 +256,30 @@ def is_safe_url(url: str) -> bool: try: parsed = urlparse(url) if parsed.scheme not in ("http", "https"): - LOG.warning("โš ๏ธ SSRF protection: blocked URL with disallowed scheme: %s", url) + log_warning("โš ๏ธ SSRF protection: blocked URL with disallowed scheme: %s", url) return False hostname = parsed.hostname if not hostname: - LOG.warning("โš ๏ธ SSRF protection: blocked URL with no hostname: %s", url) + log_warning("โš ๏ธ SSRF protection: blocked URL with no hostname: %s", url) return False try: addr_info = socket.getaddrinfo(hostname, None) except socket.gaierror: - LOG.warning("โš ๏ธ SSRF protection: DNS resolution failed for %s", hostname) + log_warning("โš ๏ธ SSRF protection: DNS resolution failed for %s", hostname) return False for info in addr_info: ip_str = info[4][0] ip_obj = ipaddress.ip_address(ip_str) if ip_obj.is_private or ip_obj.is_loopback or ip_obj.is_link_local or ip_obj.is_multicast or ip_obj.is_reserved: - LOG.warning("โš ๏ธ SSRF protection: blocked request to %s (resolves to internal IP %s)", hostname, ip_str) + log_warning("โš ๏ธ SSRF protection: blocked request to %s (resolves to internal IP %s)", hostname, ip_str) return False return True except Exception as e: - LOG.debug("SSRF protection: error validating URL %s: %s", url, e) + log_debug("SSRF protection: error validating URL %s: %s", url, e) return False @@ -249,7 +289,7 @@ def extract_ocsp_url(pem_data: bytes, cert_name: str = "") -> Optional[str]: Validates the URL scheme is http:// or https://. Returns OCSP responder URL if present and valid, else None. """ - LOG.debug("๐Ÿ”’ OCSP checking support for certificate %s", cert_name) + log_debug("๐Ÿ”’ OCSP checking support for certificate %s", cert_name) try: cert = x509.load_pem_x509_certificate(pem_data) @@ -260,18 +300,18 @@ def extract_ocsp_url(pem_data: bytes, cert_name: str = "") -> Optional[str]: url = access_description.access_location.value parsed = urlparse(url) if parsed.scheme not in ("http", "https"): - LOG.warning("โš ๏ธ OCSP URL has invalid scheme for %s: %s", cert_name, url) + log_warning("โš ๏ธ OCSP URL has invalid scheme for %s: %s", cert_name, url) return None - LOG.debug("๐ŸŒ OCSP found responder URL for %s: %s", cert_name, url) + log_debug("๐ŸŒ OCSP found responder URL for %s: %s", cert_name, url) return url - LOG.debug("๐Ÿ”’ OCSP no responder URL advertised in %s", cert_name) + log_debug("๐Ÿ”’ OCSP no responder URL advertised in %s", cert_name) return None except x509.ExtensionNotFound: - LOG.debug("๐Ÿ”’ OCSP no AIA extension found in %s", cert_name) + log_debug("๐Ÿ”’ OCSP no AIA extension found in %s", cert_name) return None except Exception as e: - LOG.debug("๐Ÿ”’ OCSP failed to extract OCSP URL from %s: %s", cert_name, e) + log_debug("๐Ÿ”’ OCSP failed to extract OCSP URL from %s: %s", cert_name, e) return None @@ -289,10 +329,10 @@ def _fetch_issuer_from_aia(leaf: x509.Certificate, cert_name: str = "") -> Optio parsed = urlparse(issuer_url) if parsed.scheme not in ("http", "https"): continue - LOG.debug("๐ŸŒ OCSP fetching issuer certificate from AIA: %s", issuer_url) + log_debug("๐ŸŒ OCSP fetching issuer certificate from AIA: %s", issuer_url) if not is_safe_url(issuer_url): - LOG.error("โŒ OCSP unsafe AIA issuer URL detected: %s", issuer_url) + log_error("โŒ OCSP unsafe AIA issuer URL detected: %s", issuer_url) return None req = Request(issuer_url, headers={"User-Agent": "BunkerWeb OCSP Fetcher (SSRF-Protected)"}) @@ -306,18 +346,75 @@ def _fetch_issuer_from_aia(leaf: x509.Certificate, cert_name: str = "") -> Optio except Exception: return x509.load_pem_x509_certificate(issuer_der) except HTTPError as e: - LOG.warning("โš ๏ธ OCSP HTTP %d error fetching issuer from AIA for %s from %s: %s", e.code, cert_name, issuer_url, e.reason) + log_warning("โš ๏ธ OCSP HTTP %d error fetching issuer from AIA for %s from %s: %s", e.code, cert_name, issuer_url, e.reason) except URLError as e: - LOG.warning("โš ๏ธ OCSP network error fetching issuer from AIA for %s from %s: %s", cert_name, issuer_url, e) + log_warning("โš ๏ธ OCSP network error fetching issuer from AIA for %s from %s: %s", cert_name, issuer_url, e) except Exception as e: - LOG.warning("โš ๏ธ OCSP failed to fetch issuer from AIA for %s from %s: %s", cert_name, issuer_url, e) + log_warning("โš ๏ธ OCSP failed to fetch issuer from AIA for %s from %s: %s", cert_name, issuer_url, e) except x509.ExtensionNotFound: - LOG.debug("๐Ÿ”’ OCSP no AIA extension in leaf cert for %s", cert_name) + log_debug("๐Ÿ”’ OCSP no AIA extension in leaf cert for %s", cert_name) except Exception as e: - LOG.warning("โš ๏ธ OCSP failed to fetch issuer from AIA for %s: %s", cert_name, e) + log_warning("โš ๏ธ OCSP failed to fetch issuer from AIA for %s: %s", cert_name, e) return None +def _extract_cert_metadata(pem_data: bytes, cert_name: str = "") -> Dict[str, Any]: + """ + Extract OCSP-related metadata from a certificate PEM: + - serial (hex string) + - ocsp_url (if present) + - must_staple (bool via TLS Feature extension, OID 1.3.6.1.5.5.7.1.24) + + This metadata is stored alongside ocsp.der so NGINX Lua can make + Must-Staple and OCSP decisions even when only a parsed certificate + object is available at runtime. + """ + meta: Dict[str, Any] = { + "serial": None, + "ocsp_url": None, + "must_staple": False, + } + + try: + cert = x509.load_pem_x509_certificate(pem_data) + except Exception as e: + log_debug("๐Ÿ”’ OCSP metadata: failed to parse PEM for %s: %s", cert_name, e) + return meta + + try: + serial_int = cert.serial_number + meta["serial"] = format(serial_int, "X") + except Exception as e: + log_debug("๐Ÿ”’ OCSP metadata: failed to extract serial for %s: %s", cert_name, e) + + try: + aia = cert.extensions.get_extension_for_oid(ExtensionOID.AUTHORITY_INFORMATION_ACCESS) + aia_value = cast(AuthorityInformationAccess, aia.value) + for access_description in aia_value: + if access_description.access_method == AuthorityInformationAccessOID.OCSP: + meta["ocsp_url"] = access_description.access_location.value + break + except x509.ExtensionNotFound: + log_debug("๐Ÿ”’ OCSP metadata: no AIA/OCSP URL for %s", cert_name) + except Exception as e: + log_debug("๐Ÿ”’ OCSP metadata: failed to extract OCSP URL for %s: %s", cert_name, e) + + try: + tls_feature_ext = cert.extensions.get_extension_for_oid(ExtensionOID.TLS_FEATURE) + tls_features = cast(TLSFeature, tls_feature_ext.value) + for feature in tls_features: + if feature == TLSFeatureType.status_request: + meta["must_staple"] = True + break + except x509.ExtensionNotFound: + # No TLS Feature extension -> no Must-Staple + pass + except Exception as e: + log_debug("๐Ÿ”’ OCSP metadata: failed to inspect TLS Feature extension for %s: %s", cert_name, e) + + return meta + + def _parse_chain(pem_data: bytes, cert_name: str = "") -> Tuple[x509.Certificate, x509.Certificate]: """ Parse fullchain PEM data and return (leaf_cert, issuer_cert). @@ -332,18 +429,18 @@ def _parse_chain(pem_data: bytes, cert_name: str = "") -> Tuple[x509.Certificate # Find the issuer by matching leaf.issuer to candidate.subject for idx, candidate in enumerate(certs[1:], start=1): if leaf.issuer == candidate.subject: - LOG.debug("โœ“ OCSP selected issuer certificate index %d for %s", idx, cert_name) + log_debug("โœ“ OCSP selected issuer certificate index %d for %s", idx, cert_name) return leaf, candidate # Fallback: use the second certificate - LOG.warning( + log_warning( "โš ๏ธ OCSP could not verify issuer chain for %s by DN match, falling back to second certificate", cert_name, ) return leaf, certs[1] # Single cert (no chain): try to fetch issuer from AIA caIssuers URL - LOG.debug("๐Ÿ”„ OCSP single cert for %s, attempting to fetch issuer from AIA caIssuers", cert_name) + log_debug("๐Ÿ”„ OCSP single cert for %s, attempting to fetch issuer from AIA caIssuers", cert_name) issuer = _fetch_issuer_from_aia(leaf, cert_name) if issuer: return leaf, issuer @@ -371,14 +468,14 @@ def _ocsp_response_lifetimes(ocsp_response: x509_ocsp.OCSPResponse) -> Tuple[Opt next_update = getattr(ocsp_response, "next_update_utc", None) or ocsp_response.next_update if next_update is None: # No Next Update: treat lifetime as 24 hours from This Update (RFC standard fallback) - LOG.debug("โšก OCSP Next Update missing, using default lifetime of 24h from This Update") + log_debug("โšก OCSP Next Update missing, using default lifetime of 24h from This Update") next_update = this_update + timedelta(hours=24) elif next_update.tzinfo is None: next_update = next_update.replace(tzinfo=timezone.utc) total_lifetime = int((next_update - this_update).total_seconds()) if total_lifetime <= 0: - LOG.debug("โšก OCSP invalid lifetime: Next Update is not after This Update") + log_debug("โšก OCSP invalid lifetime: Next Update is not after This Update") return None, None now = datetime.now(timezone.utc) @@ -394,7 +491,7 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim try: leaf, issuer = _parse_chain(pem_data, cert_name) except Exception as e: - LOG.error("โŒ OCSP failed to parse chain for %s: %s", cert_name, e) + log_error("โŒ OCSP failed to parse chain for %s: %s", cert_name, e) return None, 0 try: @@ -408,14 +505,14 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim ocsp_request = builder.build() ocsp_request_data = ocsp_request.public_bytes(Encoding.DER) - LOG.debug( + log_debug( "๐ŸŒ OCSP fetching for %s (using %s): serial=%d, issuer=%s, responder=%s", cert_name, alg_name, leaf.serial_number, issuer.subject.rfc4514_string(), ocsp_url ) # --- Build HTTP request --- if not is_safe_url(ocsp_url): - LOG.error("โŒ OCSP unsafe responder URL detected: %s", ocsp_url) + log_error("โŒ OCSP unsafe responder URL detected: %s", ocsp_url) return None, 0 req = Request( @@ -434,13 +531,13 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim ocsp_der = response.read(102400) if not ocsp_der: - LOG.warning("โš ๏ธ OCSP empty response from %s for %s", ocsp_url, cert_name) + log_warning("โš ๏ธ OCSP empty response from %s for %s", ocsp_url, cert_name) continue # Check if it's a valid OCSP response via cryptography ocsp_response = x509_ocsp.load_der_ocsp_response(ocsp_der) if ocsp_response.response_status != x509_ocsp.OCSPResponseStatus.SUCCESSFUL: - LOG.warning( + log_warning( "โš ๏ธ OCSP responder returned %s for %s (using %s), retrying if fallback available...", ocsp_response.response_status, cert_name, alg_name ) @@ -451,16 +548,16 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim break except HTTPError as e: - LOG.warning("โš ๏ธ OCSP HTTP %d error for %s using %s: %s", e.code, cert_name, alg_name, e.reason) + log_warning("โš ๏ธ OCSP HTTP %d error for %s using %s: %s", e.code, cert_name, alg_name, e.reason) ocsp_der = None continue except Exception as e: - LOG.warning("โš ๏ธ OCSP request failed for %s using %s: %s", cert_name, alg_name, e) + log_warning("โš ๏ธ OCSP request failed for %s using %s: %s", cert_name, alg_name, e) ocsp_der = None continue else: # Loop finished without a break: all attempts failed - LOG.error("โŒ OCSP failed to fetch successful response for %s after trying both SHA256 and SHA1", cert_name) + log_error("โŒ OCSP failed to fetch successful response for %s after trying both SHA256 and SHA1", cert_name) return None, 0 # === SECURE OCSP RESPONSE VERIFICATION === @@ -508,14 +605,14 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim # Add a timeout so a stuck openssl process cannot hang the whole job p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=30) except subprocess.TimeoutExpired as e: - LOG.error( + log_error( "โŒ OCSP response cryptographic signature verification timed out for %s after %s seconds. Discarding response.", cert_name, e.timeout, ) return None, 0 if p.returncode != 0: - LOG.error("โŒ OCSP response cryptographic signature verification failed for %s. Discarding forged/invalid response. OpenSSL Error: %s", cert_name, p.stderr.strip() or p.stdout.strip()) + log_error("โŒ OCSP response cryptographic signature verification failed for %s. Discarding forged/invalid response. OpenSSL Error: %s", cert_name, p.stderr.strip() or p.stdout.strip()) return None, 0 # Extract TTL @@ -526,7 +623,7 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim ttl = 86400 # RFC standard fallback # Delay after successful fetch to prevent rate limiting - LOG.debug("โธ๏ธ OCSP delaying 2 seconds after fetch for %s to prevent rate limiting", cert_name) + log_debug("โธ๏ธ OCSP delaying 2 seconds after fetch for %s to prevent rate limiting", cert_name) time.sleep(2) return ocsp_der, ttl @@ -543,14 +640,14 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim error_desc += f" (Client Error - {e.reason})" elif e.code >= 500: error_desc += f" (Server Error - {e.reason})" - LOG.error("โŒ OCSP %s from responder for %s at %s", error_desc, cert_name, ocsp_url) + log_error("โŒ OCSP %s from responder for %s at %s", error_desc, cert_name, ocsp_url) return None, 0 except URLError as e: # Network error โ€” DNS, connection refused, timeout, SSL error, etc. - LOG.error("โŒ OCSP network error fetching response for %s from %s: %s", cert_name, ocsp_url, e) + log_error("โŒ OCSP network error fetching response for %s from %s: %s", cert_name, ocsp_url, e) return None, 0 except Exception as e: - LOG.error("โŒ OCSP failed to fetch response for %s: %s", cert_name, e) + log_error("โŒ OCSP failed to fetch response for %s: %s", cert_name, e) return None, 0 @@ -568,7 +665,7 @@ def extract_san_dns(pem_data: bytes, cert_name: str = "") -> List[str]: except x509.ExtensionNotFound: return [] except Exception as e: - LOG.debug("OCSP failed to extract SAN from %s: %s", cert_name, e) + log_debug("OCSP failed to extract SAN from %s: %s", cert_name, e) return [] @@ -581,27 +678,27 @@ def get_cached_ocsp_ttl(cert_name: str) -> Tuple[Optional[int], Optional[int]]: """ sanitized_name = _sanitize_filename(cert_name) ocsp_path = CONFIGS_SSL_BASE / sanitized_name / "ocsp.der" - LOG.debug("โšก OCSP TTL check: checking cache for %s at %s", cert_name, ocsp_path) + log_debug("โšก OCSP TTL check: checking cache for %s at %s", cert_name, ocsp_path) if not ocsp_path.is_file(): - LOG.debug("โšก OCSP TTL check: cache miss - file not found for %s", cert_name) + log_debug("โšก OCSP TTL check: cache miss - file not found for %s", cert_name) return None, None - LOG.debug("โšก OCSP cached file found for %s, reading This/Next Update...", cert_name) + log_debug("โšก OCSP cached file found for %s, reading This/Next Update...", cert_name) try: ocsp_data = ocsp_path.read_bytes() ocsp_response = x509_ocsp.load_der_ocsp_response(ocsp_data) remaining, total_lifetime = _ocsp_response_lifetimes(ocsp_response) if remaining is None or total_lifetime is None: - LOG.debug("๐Ÿ”„ OCSP could not determine precise lifetime for %s from cached response", cert_name) + log_debug("๐Ÿ”„ OCSP could not determine precise lifetime for %s from cached response", cert_name) return None, None # Type assertion: after None checks above, these are guaranteed to be int assert remaining is not None assert total_lifetime is not None - LOG.info( + log_info( "โšก OCSP cached response for %s: remaining=%ds (%.1f days), total_lifetime=%ds (%.1f days)", cert_name, remaining, @@ -611,7 +708,7 @@ def get_cached_ocsp_ttl(cert_name: str) -> Tuple[Optional[int], Optional[int]]: ) return remaining, total_lifetime except Exception as e: - LOG.warning("โš ๏ธ OCSP exception while reading cached TTL for %s: %s", cert_name, e) + log_warning("โš ๏ธ OCSP exception while reading cached TTL for %s: %s", cert_name, e) return None, None @@ -630,7 +727,7 @@ def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> No # Prevent Path Traversal by validating SAN format # Allow alphanumeric, hyphens, dots, and wildcards (e.g., *.example.com) if not re.match(r"^[A-Za-z0-9_.*-]+$", san): - LOG.warning("โš ๏ธ OCSP sanitization: skipping invalid/unsafe SAN %s", san) + log_warning("โš ๏ธ OCSP sanitization: skipping invalid/unsafe SAN %s", san) continue if san == base_name or san == cert_name: @@ -648,13 +745,13 @@ def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> No target = san_ocsp.readlink() expected_target = Path("..") / _sanitize_filename(cert_name) / "ocsp.der" if target == expected_target: - LOG.debug("๐Ÿ”— OCSP SAN symlink for %s already correct (%s -> %s)", san, san_ocsp, target) + log_debug("๐Ÿ”— OCSP SAN symlink for %s already correct (%s -> %s)", san, san_ocsp, target) else: - LOG.debug("โ„น๏ธ OCSP SAN symlink for %s points to different target (got %s, expected %s) โ€” likely overlapping certs, skipping", san, target, expected_target) + log_debug("โ„น๏ธ OCSP SAN symlink for %s points to different target (got %s, expected %s) โ€” likely overlapping certs, skipping", san, target, expected_target) except Exception: - LOG.debug("โš ๏ธ OCSP SAN symlink for %s is broken or unreadable", san) + log_debug("โš ๏ธ OCSP SAN symlink for %s is broken or unreadable", san) elif san_ocsp.is_file(): - LOG.debug("โ„น๏ธ OCSP SAN path %s is a regular file (not a symlink) โ€” likely another real cert, skipping symlink", san_ocsp) + log_debug("โ„น๏ธ OCSP SAN path %s is a regular file (not a symlink) โ€” likely another real cert, skipping symlink", san_ocsp) continue try: @@ -662,9 +759,9 @@ def _create_san_symlinks(pem_data: bytes, ocsp_path: Path, cert_name: str) -> No # Use relative symlink: ../cert_name/ocsp.der rel_target = Path("..") / _sanitize_filename(cert_name) / "ocsp.der" san_ocsp.symlink_to(rel_target) - LOG.debug("๐Ÿ”— OCSP created SAN symlink %s -> %s", san_ocsp, rel_target) + log_debug("๐Ÿ”— OCSP created SAN symlink %s -> %s", san_ocsp, rel_target) except Exception as e: - LOG.debug("โš ๏ธ OCSP could not create SAN symlink for %s: %s", san, e) + log_debug("โš ๏ธ OCSP could not create SAN symlink for %s: %s", san, e) def _get_cached_ocsp_certs(db: Any) -> set: @@ -689,7 +786,7 @@ def _get_cached_ocsp_certs(db: Any) -> set: if re.match(r"^[A-Za-z0-9_.*-]+$", cert_name_raw): cached_certs.add(cert_name_raw) except Exception as e: - LOG.warning("โš ๏ธ OCSP could not retrieve cached certificate list from database: %s", e) + log_warning("โš ๏ธ OCSP could not retrieve cached certificate list from database: %s", e) return cached_certs @@ -726,7 +823,7 @@ def _get_cert_checksums(db: Any, cert_names: set) -> Dict[str, str]: except Exception: pass except Exception as e: - LOG.debug("โš ๏ธ OCSP could not retrieve certificate checksums from database: %s", e) + log_debug("โš ๏ธ OCSP could not retrieve certificate checksums from database: %s", e) return checksums @@ -782,14 +879,14 @@ def _load_le_certs_from_db(db: Any) -> Dict[str, bytes]: with_info=False, ) except Exception as e: - LOG.warning("โš ๏ธ OCSP failed to query database for LE tarball (job=%s): %s", job_name, e) + log_warning("โš ๏ธ OCSP failed to query database for LE tarball (job=%s): %s", job_name, e) continue if tgz_data: - LOG.debug("โœ“ OCSP found LE tarball from %s job", job_name) + log_debug("โœ“ OCSP found LE tarball from %s job", job_name) break if tgz_data is None: - LOG.debug("โ„น๏ธ OCSP no LE tarball found in database (certbot-new cache)") + log_debug("โ„น๏ธ OCSP no LE tarball found in database (certbot-new cache)") return result try: @@ -808,27 +905,27 @@ def _load_le_certs_from_db(db: Any) -> Dict[str, bytes]: cert_name = parts[idx + 1] # Prevent path traversal from crafted tarball entries if not re.match(r"^[A-Za-z0-9_.*-]+$", cert_name): - LOG.warning("โš ๏ธ OCSP sanitization: skipping unsafe cert_name from tarball: %s", cert_name) + log_warning("โš ๏ธ OCSP sanitization: skipping unsafe cert_name from tarball: %s", cert_name) continue try: f = tar.extractfile(member) if f: pem_data = f.read() if not pem_data or b"-----BEGIN" not in pem_data: - LOG.warning("โš ๏ธ OCSP LE fullchain for %s is empty or not valid PEM, skipping", cert_name) + log_warning("โš ๏ธ OCSP LE fullchain for %s is empty or not valid PEM, skipping", cert_name) continue result[cert_name] = pem_data - LOG.debug("โœ“ OCSP extracted LE fullchain for %s from database tarball", cert_name) + log_debug("โœ“ OCSP extracted LE fullchain for %s from database tarball", cert_name) else: - LOG.warning("โš ๏ธ OCSP could not read LE fullchain for %s from tarball (extractfile returned None)", cert_name) + log_warning("โš ๏ธ OCSP could not read LE fullchain for %s from tarball (extractfile returned None)", cert_name) except KeyError: - LOG.warning("โš ๏ธ OCSP symlink target missing in tarball for %s", cert_name) + log_warning("โš ๏ธ OCSP symlink target missing in tarball for %s", cert_name) except Exception as e: - LOG.warning("โš ๏ธ OCSP failed to extract LE fullchain for %s: %s", cert_name, e) + log_warning("โš ๏ธ OCSP failed to extract LE fullchain for %s: %s", cert_name, e) except tarfile.TarError as e: - LOG.error("โŒ OCSP LE tarball is corrupted or invalid: %s", e) + log_error("โŒ OCSP LE tarball is corrupted or invalid: %s", e) except Exception as e: - LOG.error("โŒ OCSP failed to extract LE certificates from database tarball: %s", e) + log_error("โŒ OCSP failed to extract LE certificates from database tarball: %s", e) return result @@ -845,11 +942,11 @@ def _load_custom_certs_from_db(db: Any) -> Dict[str, bytes]: try: cache_files = db.get_jobs_cache_files(job_name="custom-cert", with_data=True) except Exception as e: - LOG.error("โŒ OCSP failed to query database for custom certificates: %s", e) + log_error("โŒ OCSP failed to query database for custom certificates: %s", e) return result if not cache_files: - LOG.debug("โ„น๏ธ OCSP no custom-cert cache entries found in database") + log_debug("โ„น๏ธ OCSP no custom-cert cache entries found in database") return result for entry in cache_files: @@ -860,27 +957,27 @@ def _load_custom_certs_from_db(db: Any) -> Dict[str, bytes]: if not (file_name.startswith("cert") and file_name.endswith(".pem")): continue if not service_id: - LOG.debug("โš ๏ธ OCSP custom cert entry %s has no service_id, skipping", file_name) + log_debug("โš ๏ธ OCSP custom cert entry %s has no service_id, skipping", file_name) continue # Prevent path traversal: validate service_id format if not re.match(r"^[A-Za-z0-9_.*-]+$", service_id): - LOG.warning("โš ๏ธ OCSP sanitization: skipping custom cert with invalid service_id: %s", service_id) + log_warning("โš ๏ธ OCSP sanitization: skipping custom cert with invalid service_id: %s", service_id) continue data = entry.get("data") if not data: - LOG.warning("โš ๏ธ OCSP custom cert %s for service %s has no data, skipping", file_name, service_id) + log_warning("โš ๏ธ OCSP custom cert %s for service %s has no data, skipping", file_name, service_id) continue # Derive suffix from filename: cert-ecdsa.pem -> -ecdsa, cert.pem -> "" suffix = file_name.replace("cert", "").replace(".pem", "") # e.g. "-ecdsa", "-rsa", "" key = f"customcert-{service_id}{suffix}" # Double-check derived key is safe (defensive validation) if not re.match(r"^[A-Za-z0-9_.*-]+$", key): - LOG.warning("โš ๏ธ OCSP sanitization: skipping custom cert with unsafe derived key: %s", key) + log_warning("โš ๏ธ OCSP sanitization: skipping custom cert with unsafe derived key: %s", key) continue result[key] = data - LOG.debug("โœ“ OCSP loaded custom cert for %s from database (file=%s, size=%d)", key, file_name, len(data)) + log_debug("โœ“ OCSP loaded custom cert for %s from database (file=%s, size=%d)", key, file_name, len(data)) except Exception as e: - LOG.warning("โš ๏ธ OCSP failed to process custom cert entry %s: %s", entry.get("file_name", "?"), e) + log_warning("โš ๏ธ OCSP failed to process custom cert entry %s: %s", entry.get("file_name", "?"), e) return result @@ -892,10 +989,10 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: This handles ephemeral storage (tmpfs, etc.) by restoring files on each run. """ if not db: - LOG.debug("โ„น๏ธ OCSP database not available, skipping cache restoration") + log_debug("โ„น๏ธ OCSP database not available, skipping cache restoration") return - LOG.info("๐Ÿ”„ OCSP syncing cached responses from database to disk...") + log_info("๐Ÿ”„ OCSP syncing cached responses from database to disk...") try: restored_count = 0 replaced_count = 0 @@ -911,7 +1008,7 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: cert_name_raw = file_name[len("ocsp/"):] # Prevent path traversal from crafted database entries if not re.match(r"^[A-Za-z0-9_.*-]+$", cert_name_raw): - LOG.warning("โš ๏ธ OCSP sanitization: skipping unsafe cert_name from database: %s", cert_name_raw) + log_warning("โš ๏ธ OCSP sanitization: skipping unsafe cert_name from database: %s", cert_name_raw) continue sanitized_name = _sanitize_filename(cert_name_raw) @@ -927,10 +1024,10 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: disk_checksum = hashlib.sha256(ocsp_path.read_bytes()).hexdigest() if disk_checksum == db_checksum: ok_count += 1 - LOG.debug("โœ“ OCSP disk file for %s matches database (checksum=%s)", cert_name_raw, db_checksum[:8]) + log_debug("โœ“ OCSP disk file for %s matches database (checksum=%s)", cert_name_raw, db_checksum[:8]) continue # Checksum mismatch โ€” replace with database version - LOG.info("๐Ÿ”„ OCSP disk file for %s has wrong checksum (disk=%s, db=%s), replacing", cert_name_raw, disk_checksum[:8], db_checksum[:8]) + log_info("๐Ÿ”„ OCSP disk file for %s has wrong checksum (disk=%s, db=%s), replacing", cert_name_raw, disk_checksum[:8], db_checksum[:8]) ocsp_path.write_bytes(db_data) ocsp_path.chmod(0o644) replaced_count += 1 @@ -940,16 +1037,16 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: ocsp_path.write_bytes(db_data) ocsp_path.chmod(0o644) restored_count += 1 - LOG.debug("โœ“ OCSP restored cached response for %s from database", cert_name_raw) + log_debug("โœ“ OCSP restored cached response for %s from database", cert_name_raw) except Exception as e: - LOG.debug("โš ๏ธ OCSP could not sync cache for %s: %s", cert_name_raw, e) + log_debug("โš ๏ธ OCSP could not sync cache for %s: %s", cert_name_raw, e) if restored_count > 0 or replaced_count > 0: - LOG.info("โœ“ OCSP sync complete: restored=%d, replaced=%d, unchanged=%d", restored_count, replaced_count, ok_count) + log_info("โœ“ OCSP sync complete: restored=%d, replaced=%d, unchanged=%d", restored_count, replaced_count, ok_count) else: - LOG.debug("โ„น๏ธ OCSP sync complete: all %d disk files match database (restored=0, replaced=0)", ok_count) + log_debug("โ„น๏ธ OCSP sync complete: all %d disk files match database (restored=0, replaced=0)", ok_count) except Exception as e: - LOG.debug("OCSP exception while attempting cache sync: %s", e) + log_debug("OCSP exception while attempting cache sync: %s", e) def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None, force_fetch: bool = False) -> Tuple[str, Optional[bytes], int, str, bytes, Optional[str], bool]: @@ -974,7 +1071,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta # Check per-service OCSP stapling setting if not _is_ocsp_enabled_for_service(service_name): - LOG.debug("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) + log_debug("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) cleanup_ocsp_cache(db, cert_name) stats["le_certs_skipped"] = stats.get("le_certs_skipped", 0) + 1 return (cert_name, None, 0, cert_checksum, pem_data, None, False) @@ -987,23 +1084,23 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta if checksum_data: cached_checksum = checksum_data.decode("utf-8").strip() except Exception as e: - LOG.debug("โš ๏ธ OCSP could not check cached checksum for %s: %s", cert_name, e) + log_debug("โš ๏ธ OCSP could not check cached checksum for %s: %s", cert_name, e) - LOG.debug("๐Ÿ”„ OCSP processing certificate %s", cert_name) + log_debug("๐Ÿ”„ OCSP processing certificate %s", cert_name) try: if not pem_data.startswith(b"-----BEGIN"): - LOG.warning("โš ๏ธ OCSP cert for %s has no PEM BEGIN marker or is invalid after cleaning, skipping", cert_name) + log_warning("โš ๏ธ OCSP cert for %s has no PEM BEGIN marker or is invalid after cleaning, skipping", cert_name) return (cert_name, None, 0, cert_checksum, pem_data, None, False) # Proceed with extracting OCSP URL using the CLEANED pem_data ocsp_url = extract_ocsp_url(pem_data, cert_name) if not ocsp_url: - LOG.debug("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) + log_debug("โ„น๏ธ OCSP certificate %s has no responder, skipping fetch", cert_name) stats["le_certs_no_ocsp"] = stats.get("le_certs_no_ocsp", 0) + 1 return (cert_name, None, 0, cert_checksum, pem_data, None, False) - LOG.debug("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) + log_debug("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) # === Check if cached OCSP response is still fresh === cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name) @@ -1012,7 +1109,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta # Skip fetch ONLY if not forced AND TTL is above MIN_TTL AND above 50% lifetime if not force_fetch and cached_ttl > MIN_TTL and cached_ttl > half_lifetime: - LOG.debug( + log_debug( "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), above thresholds (MIN_TTL=%ds, 50%% threshold=%ds), skipping fetch", cert_name, cached_ttl, cached_ttl / 86400.0, MIN_TTL, half_lifetime, ) @@ -1020,7 +1117,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta return (cert_name, None, cached_ttl, cert_checksum, pem_data, ocsp_url, False) if cached_ttl <= MIN_TTL: - LOG.info("๐Ÿ”„ OCSP response for %s is near expiration (TTL=%ds < %ds), attempting aggressive refresh", cert_name, cached_ttl, MIN_TTL) + log_info("๐Ÿ”„ OCSP response for %s is near expiration (TTL=%ds < %ds), attempting aggressive refresh", cert_name, cached_ttl, MIN_TTL) ocsp_der: Optional[bytes] = None ttl: int = 0 @@ -1029,26 +1126,26 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta for attempt in (1, 2): ocsp_der, ttl = fetch_ocsp_response(pem_data, ocsp_url, cert_name=cert_name, timeout=10) if ocsp_der: - LOG.debug("โœ“ OCSP successfully fetched response for %s on attempt %d (TTL=%ds)", cert_name, attempt, ttl) + log_debug("โœ“ OCSP successfully fetched response for %s on attempt %d (TTL=%ds)", cert_name, attempt, ttl) stats["ocsp_fetched_responses"] = stats.get("ocsp_fetched_responses", 0) + 1 break if attempt == 1: - LOG.warning("โš ๏ธ OCSP fetch failed for %s, retrying once after 2 seconds ...", cert_name) + log_warning("โš ๏ธ OCSP fetch failed for %s, retrying once after 2 seconds ...", cert_name) time.sleep(2) if not ocsp_der: - LOG.error("โŒ OCSP failed to fetch response for %s after retries", cert_name) + log_error("โŒ OCSP failed to fetch response for %s after retries", cert_name) if cached_ttl is not None: current_ttl = cast(int, cached_ttl) if current_ttl <= 0: - LOG.warning( + log_warning( "๐Ÿงน OCSP cached response for %s is already expired and refresh failed. Cleaning up invalid cache.", cert_name, ) cleanup_ocsp_cache(db, cert_name) elif current_ttl <= MIN_TTL: - LOG.warning( + log_warning( "๐Ÿšจ OCSP CRITICAL: Cached response for %s is near expiration (TTL=%ds) and refresh failed. " "Removing from cache to prevent stapling expired data.", cert_name, @@ -1056,7 +1153,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta ) cleanup_ocsp_cache(db, cert_name) else: - LOG.warning( + log_warning( "โš ๏ธ OCSP could NOT refresh response for %s, keeping existing cache (TTL=%ds) for now", cert_name, current_ttl, @@ -1068,13 +1165,13 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta # Calculate checksum for integrity verification ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() ttl_readable = f"{ttl / 86400.0:.1f} days" if ttl >= 86400 else f"{ttl / 3600.0:.1f} hours" - LOG.info("โšก OCSP final TTL for %s is %ds (%s) (checksum=%s)", cert_name, ttl, ttl_readable, ocsp_checksum[:8]) + log_info("โšก OCSP final TTL for %s is %ds (%s) (checksum=%s)", cert_name, ttl, ttl_readable, ocsp_checksum[:8]) # === Return result for batched database writes === return (cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url, True) except Exception as e: - LOG.error("โŒ OCSP exception while processing certificate %s: %s", cert_name, e) + log_error("โŒ OCSP exception while processing certificate %s: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 return (cert_name, None, 0, cert_checksum, pem_data, None, False) @@ -1105,16 +1202,16 @@ def process_custom_certs( results: List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str], bool]] = [] if not db: - LOG.info("โ„น๏ธ OCSP database not available, cannot process custom certificates") + log_info("โ„น๏ธ OCSP database not available, cannot process custom certificates") return results try: custom_certs = _load_custom_certs_from_db(db) if not custom_certs: - LOG.info("โ„น๏ธ OCSP no custom certificates found in database") + log_info("โ„น๏ธ OCSP no custom certificates found in database") return results - LOG.info("๐Ÿ”„ OCSP loaded %d custom certificate(s) from database", len(custom_certs)) + log_info("๐Ÿ”„ OCSP loaded %d custom certificate(s) from database", len(custom_certs)) # Check which custom certs have changed since last OCSP refresh # The names from _load_custom_certs_from_db already include the 'customcert-' prefix @@ -1131,21 +1228,21 @@ def process_custom_certs( if previous_checksum is None: # No previous checksum, treat as changed changed_custom_certs[cert_name] = pem_data - LOG.debug("โš ๏ธ OCSP no previous checksum found for custom cert %s, will refresh", cert_name) + log_debug("โš ๏ธ OCSP no previous checksum found for custom cert %s, will refresh", cert_name) elif current_checksum != previous_checksum: # Certificate content has changed changed_custom_certs[cert_name] = pem_data - LOG.debug("๐Ÿ”„ OCSP custom certificate content changed for %s", cert_name) + log_debug("๐Ÿ”„ OCSP custom certificate content changed for %s", cert_name) else: # Certificate content unchanged unchanged_custom_certs[cert_name] = pem_data if unchanged_custom_certs: if skip_unchanged_ttl_checks: - LOG.info("โœ“ OCSP skipping TTL checks for %d unchanged custom certificate(s) (recently run)", len(unchanged_custom_certs)) + log_info("โœ“ OCSP skipping TTL checks for %d unchanged custom certificate(s) (recently run)", len(unchanged_custom_certs)) stats["custom_certs_skipped"] = stats.get("custom_certs_skipped", 0) + len(unchanged_custom_certs) else: - LOG.info("โœ“ OCSP checking TTL for %d unchanged custom certificate(s): %s", len(unchanged_custom_certs), ", ".join(sorted(unchanged_custom_certs.keys()))) + log_info("โœ“ OCSP checking TTL for %d unchanged custom certificate(s): %s", len(unchanged_custom_certs), ", ".join(sorted(unchanged_custom_certs.keys()))) stats["custom_certs_unchanged"] = stats.get("custom_certs_unchanged", 0) + len(unchanged_custom_certs) stats["custom_certs_processed"] = stats.get("custom_certs_processed", 0) + len(custom_certs) @@ -1168,7 +1265,7 @@ def process_custom_certs( results.append(_process_cert(cert_name, cert_pem, db, stats, force_fetch=False)) except Exception as e: - LOG.error("OCSP exception while processing custom certificates: %s", e) + log_error("OCSP exception while processing custom certificates: %s", e) stats["errors"] = stats.get("errors", 0) + 1 return results @@ -1203,7 +1300,7 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None if cert_name: # Prevent Path Traversal during cleanup if not re.match(r"^[A-Za-z0-9_.*-]+$", cert_name): - LOG.error("โŒ OCSP sanitization: refusing to clean up invalid/unsafe cert_name %s", cert_name) + log_error("โŒ OCSP sanitization: refusing to clean up invalid/unsafe cert_name %s", cert_name) return # Clean up a single certificate's OCSP cache @@ -1211,7 +1308,7 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None ocsp_dir = CONFIGS_SSL_BASE / sanitized_name if ocsp_dir.is_dir(): shutil.rmtree(ocsp_dir, ignore_errors=True) - LOG.info("๐Ÿงน OCSP removed cache directory for %s", cert_name) + log_info("๐Ÿงน OCSP removed cache directory for %s", cert_name) # Also remove any SAN symlinks that point to this cert's OCSP cache if CONFIGS_SSL_BASE.is_dir(): @@ -1224,7 +1321,7 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None try: if san_ocsp.readlink() == target_rel: san_ocsp.unlink() - LOG.debug("๐Ÿงน OCSP removed SAN symlink %s", san_ocsp) + log_debug("๐Ÿงน OCSP removed SAN symlink %s", san_ocsp) # Remove empty directory try: entry.rmdir() @@ -1238,9 +1335,9 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None # Delete both response and checksum from database db.delete_job_cache(file_name=f"ocsp/{sanitized_name}", job_name="ocsp-refresh") db.delete_job_cache(file_name=f"cert_checksum/{sanitized_name}", job_name="ocsp-refresh") - LOG.debug("๐Ÿงน OCSP database records removed for %s", cert_name) + log_debug("๐Ÿงน OCSP database records removed for %s", cert_name) except Exception as e: - LOG.debug("๐Ÿงน OCSP could not remove database entry for %s: %s", cert_name, e) + log_debug("๐Ÿงน OCSP could not remove database entry for %s: %s", cert_name, e) else: # Clean up ALL OCSP caches (including symlinks) if CONFIGS_SSL_BASE.is_dir(): @@ -1250,7 +1347,7 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None ocsp_file = entry / "ocsp.der" if ocsp_file.is_symlink() or ocsp_file.is_file(): ocsp_file.unlink(missing_ok=True) - LOG.info("๐Ÿงน OCSP removed cached response %s", ocsp_file) + log_info("๐Ÿงน OCSP removed cached response %s", ocsp_file) # Remove the directory if it's now empty try: entry.rmdir() @@ -1266,15 +1363,15 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None if file_name.startswith("ocsp/") or file_name.startswith("cert_checksum/") or file_name == "last_full_refresh": try: db.delete_job_cache(file_name=file_name, job_name="ocsp-refresh") - LOG.debug("๐Ÿงน OCSP removed database entry %s", file_name) + log_debug("๐Ÿงน OCSP removed database entry %s", file_name) except Exception as e: - LOG.debug("๐Ÿงน OCSP could not remove database entry %s: %s", file_name, e) + log_debug("๐Ÿงน OCSP could not remove database entry %s: %s", file_name, e) except Exception as e: - LOG.debug("๐Ÿงน OCSP could not clean database entries: %s", e) + log_debug("๐Ÿงน OCSP could not clean database entries: %s", e) elif not purge_db: - LOG.info("๐Ÿงน OCSP disk caches cleaned up") + log_info("๐Ÿงน OCSP disk caches cleaned up") - LOG.info("๐Ÿงน OCSP all stapling caches cleaned up") + log_info("๐Ÿงน OCSP all stapling caches cleaned up") def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: Optional[dict] = None) -> None: @@ -1296,11 +1393,11 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: # key already starts with customcert- valid_cert_names.add(key) except Exception as e: - LOG.warning("โš ๏ธ OCSP could not load custom certs for orphan check: %s", e) + log_warning("โš ๏ธ OCSP could not load custom certs for orphan check: %s", e) return # Don't clean up if we can't verify what's valid if not valid_cert_names: - LOG.debug("โ„น๏ธ OCSP no valid certs found, skipping orphan cleanup to avoid accidental deletion") + log_debug("โ„น๏ธ OCSP no valid certs found, skipping orphan cleanup to avoid accidental deletion") return orphaned_count: int = 0 @@ -1315,11 +1412,11 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: continue cert_name_raw = file_name[len("ocsp/"):] if cert_name_raw not in valid_cert_names: - LOG.info("๐Ÿงน OCSP removing orphaned entry for deleted service: %s", cert_name_raw) + log_info("๐Ÿงน OCSP removing orphaned entry for deleted service: %s", cert_name_raw) cleanup_ocsp_cache(db, cert_name_raw) orphaned_count = orphaned_count + 1 except Exception as e: - LOG.warning("โš ๏ธ OCSP could not check database for orphaned entries: %s", e) + log_warning("โš ๏ธ OCSP could not check database for orphaned entries: %s", e) # Check disk directories for orphaned OCSP files if CONFIGS_SSL_BASE.is_dir(): @@ -1334,12 +1431,12 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: # Check if it's a SAN symlink (those are managed by _create_san_symlinks) if ocsp_file.is_symlink(): continue - LOG.info("๐Ÿงน OCSP removing orphaned disk cache for deleted service: %s", cert_name_raw) + log_info("๐Ÿงน OCSP removing orphaned disk cache for deleted service: %s", cert_name_raw) cleanup_ocsp_cache(db, cert_name_raw) orphaned_count = orphaned_count + 1 if orphaned_count > 0: - LOG.info("๐Ÿงน OCSP cleaned up %d orphaned cache entries for deleted services", orphaned_count) + log_info("๐Ÿงน OCSP cleaned up %d orphaned cache entries for deleted services", orphaned_count) stats["orphaned_cleaned"] = orphaned_count @@ -1355,7 +1452,7 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic stats = {} if not db: - LOG.warning("โš ๏ธ OCSP cannot verify files without database connection") + log_warning("โš ๏ธ OCSP cannot verify files without database connection") return # Verification: check if files exist on disk and match database checksums @@ -1370,10 +1467,10 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic all_entries = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=True) ocsp_entries = [e for e in all_entries if e.get("file_name", "").startswith("ocsp/")] if not ocsp_entries: - LOG.info("โ„น๏ธ OCSP no cache entries in database to verify") + log_info("โ„น๏ธ OCSP no cache entries in database to verify") return - LOG.info("๐Ÿ” OCSP verifying %d cache entry(ies) from database", len(ocsp_entries)) + log_info("๐Ÿ” OCSP verifying %d cache entry(ies) from database", len(ocsp_entries)) for entry in ocsp_entries: file_name = entry.get("file_name", "") @@ -1386,7 +1483,7 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic db_checksum = entry.get("checksum", "") if not data or not db_checksum: - LOG.warning("โš ๏ธ OCSP database entry %s has no data or checksum, skipping verification", cert_name_raw) + log_warning("โš ๏ธ OCSP database entry %s has no data or checksum, skipping verification", cert_name_raw) continue verify_count += 1 @@ -1400,30 +1497,30 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic ocsp_response = x509_ocsp.load_der_ocsp_response(data) remaining, _ = _ocsp_response_lifetimes(ocsp_response) if remaining is not None and remaining <= 0: - LOG.warning("๐Ÿงน OCSP response in database for %s is expired (remaining=%ds). Skipping restoration to disk.", cert_name_raw, remaining) + log_warning("๐Ÿงน OCSP response in database for %s is expired (remaining=%ds). Skipping restoration to disk.", cert_name_raw, remaining) # Optionally remove from database to prevent future attempts if db: try: db.delete_job_cache(file_name=file_name, job_name="ocsp-refresh") - LOG.debug("๐Ÿงน OCSP removed expired database entry %s", file_name) + log_debug("๐Ÿงน OCSP removed expired database entry %s", file_name) except Exception: pass continue except Exception as e: - LOG.warning("โš ๏ธ OCSP could not parse response from database for %s during verification: %s", cert_name_raw, e) + log_warning("โš ๏ธ OCSP could not parse response from database for %s during verification: %s", cert_name_raw, e) try: if not ocsp_path.is_file(): # File missing: restore from database - LOG.warning("โš ๏ธ OCSP file missing for %s, restoring from database", cert_name_raw) + log_warning("โš ๏ธ OCSP file missing for %s, restoring from database", cert_name_raw) try: ocsp_cert_dir.mkdir(parents=True, exist_ok=True) ocsp_path.write_bytes(data) ocsp_path.chmod(0o644) - LOG.info("โœ“ OCSP restored %s from database", cert_name_raw) + log_info("โœ“ OCSP restored %s from database", cert_name_raw) restored_count += 1 except Exception as e: - LOG.error("โŒ OCSP could not restore %s from database: %s", cert_name_raw, e) + log_error("โŒ OCSP could not restore %s from database: %s", cert_name_raw, e) stats["errors"] = stats.get("errors", 0) + 1 continue @@ -1433,7 +1530,7 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic file_checksum = hashlib.sha256(file_data).hexdigest() if file_checksum != db_checksum: - LOG.warning( + log_warning( "โš ๏ธ OCSP checksum mismatch for %s (file=%s, db=%s). " "Restoring from database.", cert_name_raw, file_checksum[:8], db_checksum[:8] @@ -1441,20 +1538,20 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic try: ocsp_path.write_bytes(data) ocsp_path.chmod(0o644) - LOG.info("โœ“ OCSP restored correct version of %s", cert_name_raw) + log_info("โœ“ OCSP restored correct version of %s", cert_name_raw) mismatch_count += 1 except Exception as e: - LOG.error("โŒ OCSP could not restore %s: %s", cert_name_raw, e) + log_error("โŒ OCSP could not restore %s: %s", cert_name_raw, e) stats["errors"] = stats.get("errors", 0) + 1 else: - LOG.debug("โœ“ OCSP %s checksum verified (matches database)", cert_name_raw) + log_debug("โœ“ OCSP %s checksum verified (matches database)", cert_name_raw) except Exception as e: - LOG.warning("โš ๏ธ OCSP error verifying %s: %s", cert_name_raw, e) + log_warning("โš ๏ธ OCSP error verifying %s: %s", cert_name_raw, e) stats["errors"] = stats.get("errors", 0) + 1 if verify_count > 0: - LOG.info( + log_info( "๐Ÿ” OCSP verification complete: %d checked | โœ“ %d restored (missing) | ๐Ÿ”„ %d corrected (mismatch)", verify_count, restored_count, mismatch_count ) @@ -1464,7 +1561,7 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic stats["ocsp_corrected"] = stats.get("ocsp_corrected", 0) + mismatch_count except Exception as e: - LOG.warning("โš ๏ธ OCSP verification failed: %s", e) + log_warning("โš ๏ธ OCSP verification failed: %s", e) stats["errors"] = stats.get("errors", 0) + 1 @@ -1483,7 +1580,7 @@ def _persist_ocsp_results_to_db( if stats is None: stats = {} - LOG.info("๐Ÿ”„ OCSP batching database updates for %d certificate(s)", len(all_ocsp_results)) + log_info("๐Ÿ”„ OCSP batching database updates for %d certificate(s)", len(all_ocsp_results)) for cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url, was_attempted in all_ocsp_results: sanitized_name = _sanitize_filename(cert_name) @@ -1502,12 +1599,12 @@ def _persist_ocsp_results_to_db( ) if err: - LOG.error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) + log_error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) stats["errors"] = stats.get("errors", 0) + 1 else: - LOG.info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, ocsp_checksum[:8]) + log_info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, ocsp_checksum[:8]) except Exception as e: - LOG.error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) + log_error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 # 2. ALWAYS store/update certificate content checksum for future differential checks @@ -1521,11 +1618,11 @@ def _persist_ocsp_results_to_db( checksum=hashlib.sha256(cert_checksum.encode("utf-8")).hexdigest(), ) if err: - LOG.debug("โš ๏ธ OCSP could not store cert checksum for %s: %s", cert_name, err) + log_debug("โš ๏ธ OCSP could not store cert checksum for %s: %s", cert_name, err) else: - LOG.debug("โœ“ OCSP persisted checksum for %s", cert_name) + log_debug("โœ“ OCSP persisted checksum for %s", cert_name) except Exception as e: - LOG.debug("โš ๏ธ OCSP exception while storing cert checksum for %s: %s", cert_name, e) + log_debug("โš ๏ธ OCSP exception while storing cert checksum for %s: %s", cert_name, e) def _persist_ocsp_results_to_disk( @@ -1555,7 +1652,7 @@ def _persist_ocsp_results_to_disk( try: ocsp_cert_dir.mkdir(parents=True, exist_ok=True) except Exception as e: - LOG.error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) + log_error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) stats["errors"] = stats.get("errors", 0) + 1 continue @@ -1564,17 +1661,47 @@ def _persist_ocsp_results_to_disk( try: ocsp_path.write_bytes(ocsp_der) ocsp_path.chmod(0o644) # Readable by nginx - LOG.info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) + log_info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) + + # Write OCSP metadata to cache/ssl/{cert-name}/ocsp.json + meta_path = ocsp_cert_dir / "ocsp.json" + try: + meta = _extract_cert_metadata(pem_data, cert_name) + meta_path.write_text(json.dumps(meta, separators=(",", ":")), encoding="utf-8") + meta_path.chmod(0o644) + log_debug( + "โœ“ OCSP saved metadata for %s (serial=%s, must_staple=%s)", + cert_name, + meta.get("serial") or "unknown", + meta.get("must_staple"), + ) + except Exception as e: + log_debug("โš ๏ธ OCSP metadata write failed for %s: %s", cert_name, e) + + # If metadata file is missing, recreate it + if not meta_path.exists(): + try: + meta = _extract_cert_metadata(pem_data, cert_name) + meta_path.write_text(json.dumps(meta, separators=(",", ":")), encoding="utf-8") + meta_path.chmod(0o644) + log_info( + "โœ“ OCSP recreated missing metadata for %s (serial=%s, must_staple=%s)", + cert_name, + meta.get("serial") or "unknown", + meta.get("must_staple"), + ) + except Exception as e: + log_warning("โš ๏ธ OCSP failed to recreate metadata for %s: %s", cert_name, e) # Create symlinks for each SAN so OCSP is found by any SNI name _create_san_symlinks(pem_data, ocsp_path, cert_name) except Exception as e: - LOG.error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) + log_error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 finally: _release_cert_lock(lock_fd) except Exception as e: - LOG.error("โŒ OCSP exception while writing response for %s to disk: %s", cert_name, e) + log_error("โŒ OCSP exception while writing response for %s to disk: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 @@ -1592,7 +1719,7 @@ def check_job_timeout(phase: str = "") -> bool: """Check if job has exceeded timeout. Returns True if timeout exceeded.""" elapsed = time.time() - job_start_time if elapsed > JOB_TIMEOUT: - LOG.warning( + log_warning( "โฑ๏ธ OCSP job timeout after %.0fs (%.1f minutes) %s. " "Saved responses fetched so far; next run will continue processing remaining certificates.", elapsed, elapsed / 60.0, phase @@ -1631,7 +1758,7 @@ def refresh_job_lock(cert_name: str = "") -> None: force_all = args.force try: - LOG.info("๐Ÿ”„ OCSP refresh job started with differential update strategy (timeout in %d minutes%s)", + log_info("๐Ÿ”„ OCSP refresh job started with differential update strategy (timeout in %d minutes%s)", JOB_TIMEOUT // 60, " [FORCE]" if force_all else "") # Acquire main lock for the entire OCSP refresh operation @@ -1642,17 +1769,17 @@ def refresh_job_lock(cert_name: str = "") -> None: if Database is not None: try: db = Database(LOG) - LOG.debug("โœ“ OCSP database connection established") + log_debug("โœ“ OCSP database connection established") except Exception as e: - LOG.error("โŒ OCSP could not establish database connection: %s", e) + log_error("โŒ OCSP could not establish database connection: %s", e) return 2 else: - LOG.error("โŒ OCSP Database module not available, cannot proceed") + log_error("โŒ OCSP Database module not available, cannot proceed") return 2 # === Check for previously cached OCSP responses === previous_ocsp_certs = _get_cached_ocsp_certs(db) - LOG.info("โ„น๏ธ OCSP found %d previously cached certificate(s)", len(previous_ocsp_certs)) + log_info("โ„น๏ธ OCSP found %d previously cached certificate(s)", len(previous_ocsp_certs)) # === Early validation: check cache directory permissions === try: @@ -1662,21 +1789,21 @@ def refresh_job_lock(cert_name: str = "") -> None: try: test_file.touch() test_file.unlink() - LOG.debug("โœ“ OCSP cache directory %s is readable and writable", CONFIGS_SSL_BASE) + log_debug("โœ“ OCSP cache directory %s is readable and writable", CONFIGS_SSL_BASE) except PermissionError: - LOG.error("โŒ OCSP cache directory %s is not writable (permission denied). Check directory ownership and permissions.", CONFIGS_SSL_BASE) + log_error("โŒ OCSP cache directory %s is not writable (permission denied). Check directory ownership and permissions.", CONFIGS_SSL_BASE) return 2 except Exception as e: - LOG.error("โŒ OCSP could not verify write access to cache directory %s: %s", CONFIGS_SSL_BASE, e) + log_error("โŒ OCSP could not verify write access to cache directory %s: %s", CONFIGS_SSL_BASE, e) return 2 except Exception as e: - LOG.error("โŒ OCSP could not create cache directory %s: %s", CONFIGS_SSL_BASE, e) + log_error("โŒ OCSP could not create cache directory %s: %s", CONFIGS_SSL_BASE, e) return 2 # Check if OCSP stapling is globally disabled ocsp_enabled = os.getenv("SSL_USE_OCSP_STAPLING", "yes").lower() if ocsp_enabled != "yes": - LOG.info("๐Ÿงน OCSP stapling is globally disabled (SSL_USE_OCSP_STAPLING=%s), cleaning up all caches", ocsp_enabled) + log_info("๐Ÿงน OCSP stapling is globally disabled (SSL_USE_OCSP_STAPLING=%s), cleaning up all caches", ocsp_enabled) cleanup_ocsp_cache(db, purge_db=True) return 0 @@ -1685,7 +1812,7 @@ def refresh_job_lock(cert_name: str = "") -> None: # This handles ephemeral storage and post-restart cache directory cleanup. ocsp_files_exist = any((CONFIGS_SSL_BASE / d / "ocsp.der").is_file() for d in CONFIGS_SSL_BASE.iterdir()) if CONFIGS_SSL_BASE.is_dir() else False if not ocsp_files_exist: - LOG.info("๐Ÿ”„ OCSP no cached files on disk, waiting 2s for scheduler purge to finish before restoring from database...") + log_info("๐Ÿ”„ OCSP no cached files on disk, waiting 2s for scheduler purge to finish before restoring from database...") time.sleep(2) restore_ocsp_from_database(db) @@ -1701,7 +1828,7 @@ def refresh_job_lock(cert_name: str = "") -> None: last_refresh_time = int(last_refresh_entry["data"].decode("utf-8")) if now_ts - last_refresh_time < 1800: # 30 minutes window skip_unchanged_ttl_checks = True - LOG.info("โ„น๏ธ OCSP full refresh was recently run (%ds ago), will only process new/changed certificates", now_ts - last_refresh_time) + log_info("โ„น๏ธ OCSP full refresh was recently run (%ds ago), will only process new/changed certificates", now_ts - last_refresh_time) except Exception: pass @@ -1713,7 +1840,7 @@ def refresh_job_lock(cert_name: str = "") -> None: # Process Let's Encrypt certificates from database tarball le_certs = _load_le_certs_from_db(db) if le_certs: - LOG.info("โ„น๏ธ OCSP loaded %d LE certificate(s) from database", len(le_certs)) + log_info("โ„น๏ธ OCSP loaded %d LE certificate(s) from database", len(le_certs)) # Separate new certs from existing ones new_le_certs = {k: v for k, v in le_certs.items() if k not in previous_ocsp_certs} @@ -1732,29 +1859,29 @@ def refresh_job_lock(cert_name: str = "") -> None: if previous_checksum is None: # No previous checksum found, treat as changed (data loss or first refresh after upgrade) changed_le_certs[cert_name] = pem_data - LOG.debug("โš ๏ธ OCSP no previous checksum found for %s, will refresh", cert_name) + log_debug("โš ๏ธ OCSP no previous checksum found for %s, will refresh", cert_name) elif current_checksum != previous_checksum: # Certificate content has changed changed_le_certs[cert_name] = pem_data - LOG.debug("๐Ÿ”„ OCSP certificate content changed for %s", cert_name) + log_debug("๐Ÿ”„ OCSP certificate content changed for %s", cert_name) else: # Certificate content unchanged unchanged_le_certs[cert_name] = pem_data if new_le_certs: - LOG.info("๐Ÿ†• OCSP found %d newly issued LE certificate(s): %s", len(new_le_certs), ", ".join(sorted(new_le_certs.keys()))) + log_info("๐Ÿ†• OCSP found %d newly issued LE certificate(s): %s", len(new_le_certs), ", ".join(sorted(new_le_certs.keys()))) stats["le_certs_new"] = len(new_le_certs) if changed_le_certs: - LOG.info("๐Ÿ”„ OCSP found %d LE certificate(s) with changed content: %s", len(changed_le_certs), ", ".join(sorted(changed_le_certs.keys()))) + log_info("๐Ÿ”„ OCSP found %d LE certificate(s) with changed content: %s", len(changed_le_certs), ", ".join(sorted(changed_le_certs.keys()))) stats["le_certs_changed"] = len(changed_le_certs) if unchanged_le_certs: if skip_unchanged_ttl_checks: - LOG.info("โœ“ OCSP skipping TTL checks for %d unchanged LE certificate(s) (recently run)", len(unchanged_le_certs)) + log_info("โœ“ OCSP skipping TTL checks for %d unchanged LE certificate(s) (recently run)", len(unchanged_le_certs)) stats["le_certs_skipped"] = stats.get("le_certs_skipped", 0) + len(unchanged_le_certs) else: - LOG.info("โœ“ OCSP checking TTL for %d LE certificate(s) unchanged: %s", len(unchanged_le_certs), ", ".join(sorted(unchanged_le_certs.keys()))) + log_info("โœ“ OCSP checking TTL for %d LE certificate(s) unchanged: %s", len(unchanged_le_certs), ", ".join(sorted(unchanged_le_certs.keys()))) stats["le_certs_processed"] = len(le_certs) @@ -1787,7 +1914,7 @@ def refresh_job_lock(cert_name: str = "") -> None: stashed_failures.append((cert_name, pem_data)) else: - LOG.info("โ„น๏ธ OCSP no LE certificates found in database") + log_info("โ„น๏ธ OCSP no LE certificates found in database") # Check timeout before processing custom certs if not check_job_timeout("before custom cert processing"): @@ -1807,7 +1934,7 @@ def refresh_job_lock(cert_name: str = "") -> None: # === Final deferred retry for stashed failures === if stashed_failures: - LOG.info("โธ๏ธ OCSP stashed %d failed fetch(es), waiting 120 seconds before final retry...", len(stashed_failures)) + log_info("โธ๏ธ OCSP stashed %d failed fetch(es), waiting 120 seconds before final retry...", len(stashed_failures)) # Use smaller sleeps to stay responsive and allow timeout checks for i in range(120): @@ -1817,19 +1944,19 @@ def refresh_job_lock(cert_name: str = "") -> None: if not check_job_timeout("before starting stashed retries"): for cert_name, pem_data in stashed_failures: if check_job_timeout(f"stashed retry for {cert_name}"): break - LOG.info("๐Ÿ”„ OCSP retrying fetch for stashed failure: %s", cert_name) + log_info("๐Ÿ”„ OCSP retrying fetch for stashed failure: %s", cert_name) refresh_job_lock(cert_name) # Force fetch for the final retry attempt all_ocsp_results.append(_process_cert(cert_name, pem_data, db, stats, force_fetch=True)) # === Check if timeout has been reached and save partial results === if check_job_timeout("after processing phase"): - LOG.warning("โฑ๏ธ OCSP job timeout during processing. Saving partial results (%d cert(s)) to database and disk.", len(all_ocsp_results)) + log_warning("โฑ๏ธ OCSP job timeout during processing. Saving partial results (%d cert(s)) to database and disk.", len(all_ocsp_results)) _persist_ocsp_results_to_db(db, all_ocsp_results, stats) _persist_ocsp_results_to_disk(all_ocsp_results, stats) # Return early with partial results saved elapsed = time.time() - job_start_time - LOG.warning("๐Ÿ“Š OCSP partial job completed in %.3fs with %d results saved", elapsed, len(all_ocsp_results)) + log_warning("๐Ÿ“Š OCSP partial job completed in %.3fs with %d results saved", elapsed, len(all_ocsp_results)) return status # === Persist all OCSP responses to database and disk === @@ -1851,18 +1978,18 @@ def refresh_job_lock(cert_name: str = "") -> None: job_name="ocsp-refresh", checksum=hashlib.sha256(str(now_ts).encode("utf-8")).hexdigest(), ) - LOG.debug("โœ“ OCSP updated last full refresh timestamp to %d", now_ts) + log_debug("โœ“ OCSP updated last full refresh timestamp to %d", now_ts) except Exception as e: - LOG.debug("โš ๏ธ OCSP could not update last full refresh timestamp: %s", e) + log_debug("โš ๏ธ OCSP could not update last full refresh timestamp: %s", e) # End-of-job verification: ensure OCSP files match database checksums # Restores missing files from database if not check_job_timeout("before final verification"): # Skip expensive verification if nothing changed and we were in "skip" mode if skip_unchanged_ttl_checks and not any(r[1] is not None for r in all_ocsp_results): - LOG.info("๐Ÿ” OCSP skipping final verification (nothing changed and recently run)") + log_info("๐Ÿ” OCSP skipping final verification (nothing changed and recently run)") else: - LOG.info("๐Ÿ” OCSP running end-of-job verification and restoration...") + log_info("๐Ÿ” OCSP running end-of-job verification and restoration...") _verify_and_restore_ocsp_files(db, stats) # Decide exit status based on results @@ -1870,12 +1997,12 @@ def refresh_job_lock(cert_name: str = "") -> None: status = 2 if stats["errors"] == 0: - LOG.info("โœ“ OCSP refresh job completed successfully") + log_info("โœ“ OCSP refresh job completed successfully") else: - LOG.warning("โš ๏ธ OCSP refresh job completed with %d error(s)", stats["errors"]) + log_warning("โš ๏ธ OCSP refresh job completed with %d error(s)", stats["errors"]) elapsed = time.time() - job_start_time - LOG.info( + log_info( "๐Ÿ“Š Statistics (completed in %.3fs): ๐Ÿ” LE certs=%d (skipped=%d, no OCSP=%d) | ๐Ÿ” Custom certs=%d (unchanged=%d, skipped=%d, no OCSP=%d) | ๐Ÿ”„ Fetched=%d | โœ“ Cached=%d | ๐Ÿงน Orphaned=%d | ๐Ÿ” Verified=%d (restored=%d, corrected=%d) | โŒ Errors=%d", elapsed, stats["le_certs_processed"], @@ -1896,7 +2023,7 @@ def refresh_job_lock(cert_name: str = "") -> None: return status except BaseException as e: LOG.exception("โŒ OCSP exception in ocsp-refresh.py") - LOG.error("โŒ OCSP exception while running ocsp-refresh.py: %s", e) + log_error("โŒ OCSP exception while running ocsp-refresh.py: %s", e) return 2 finally: # Always release the main lock From 4a1b226dd24158fcd89686b4c4081f5577ef731d Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Mon, 16 Mar 2026 11:56:18 +0100 Subject: [PATCH 51/59] add --force-fetch + cleanup - add option to force a fresh fetch of the OCSP status files. - cleanup old files in case of a previous script run error --- src/common/core/ssl/jobs/ocsp-refresh.py | 85 +++++++++++++++++++----- 1 file changed, 68 insertions(+), 17 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 065a311142..a7d7f4a27c 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -1052,10 +1052,11 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, stats: Optional[dict] = None, force_fetch: bool = False) -> Tuple[str, Optional[bytes], int, str, bytes, Optional[str], bool]: """ Process a single certificate for OCSP stapling. Works with in-memory PEM data. - If force_fetch is True, skips the cached TTL check and retrieves a new response. + If force_fetch is True, skips the cached TTL check and retrieves a new response from upstream PKI. + On error with force_fetch, returns ocsp_der=None, and disk files are NOT replaced (existing files kept intact). Returns a tuple of (cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url, was_attempted) for batched database writes. - If ocsp_der is None, it means the fetch was skipped or failed. + If ocsp_der is None, it means the fetch was skipped or failed (disk files remain untouched). """ if stats is None: stats = {} @@ -1183,6 +1184,7 @@ def process_custom_certs( refresh_fn: Optional[Callable[[str], None]] = None, timeout_fn: Optional[Callable[[str], bool]] = None, skip_unchanged_ttl_checks: bool = False, + force_fetch: bool = False, ) -> List[Tuple[str, Optional[bytes], int, str, bytes, Optional[str], bool]]: """ Process OCSP for custom certificates using certificate data from the database. @@ -1195,6 +1197,7 @@ def process_custom_certs( lock_fd: Optional file descriptor for lock refresh (for long operations) refresh_fn: Optional callable to refresh lock (prevents stale detection) timeout_fn: Optional callable to check job timeout + force_fetch: If True, force refetch all OCSP responses from upstream PKI """ if stats is None: stats = {} @@ -1255,14 +1258,14 @@ def process_custom_certs( refresh_fn(cert_name) results.append(_process_cert(cert_name, cert_pem, db, stats, force_fetch=True)) - # 2. Process unchanged custom certificates (TTL check only) + # 2. Process unchanged custom certificates (TTL check only or force-fetch if requested) if not skip_unchanged_ttl_checks: for cert_name, cert_pem in sorted(unchanged_custom_certs.items()): if callable(timeout_fn) and timeout_fn(f"unchanged custom cert {cert_name}"): break if callable(refresh_fn): refresh_fn(cert_name) - results.append(_process_cert(cert_name, cert_pem, db, stats, force_fetch=False)) + results.append(_process_cert(cert_name, cert_pem, db, stats, force_fetch=force_fetch)) except Exception as e: log_error("OCSP exception while processing custom certificates: %s", e) @@ -1630,7 +1633,9 @@ def _persist_ocsp_results_to_disk( stats: Optional[Dict[str, int]] = None, ) -> None: """ - Write OCSP responses to disk cache files. + Write OCSP responses to disk cache files (atomic writes with temporary files). + Ensures existing files are only replaced when new fetches complete successfully. + On any write error, keeps existing OCSP files intact with remaining TTL. Called both at normal completion and when timeout occurs. """ if not all_ocsp_results: @@ -1656,11 +1661,22 @@ def _persist_ocsp_results_to_disk( stats["errors"] = stats.get("errors", 0) + 1 continue - # Write OCSP response to cache/ssl/{cert-name}/ocsp.der + # Write OCSP response to cache/ssl/{cert-name}/ocsp.der (atomic write with temp file) ocsp_path = ocsp_cert_dir / "ocsp.der" try: - ocsp_path.write_bytes(ocsp_der) - ocsp_path.chmod(0o644) # Readable by nginx + # Write to temporary file first, then atomically move to final location + # This ensures existing file is only replaced if write succeeds completely + with tempfile.NamedTemporaryFile(dir=ocsp_cert_dir, delete=False, prefix=".ocsp_", suffix=".tmp") as tmp_file: + tmp_file.write(ocsp_der) + tmp_file.flush() + os.fsync(tmp_file.fileno()) # Ensure write is persisted + tmp_path = Path(tmp_file.name) + + # Set permissions on temp file before moving + tmp_path.chmod(0o644) # Readable by nginx + + # Atomically move temp file to final location (overwrites only on success) + tmp_path.replace(ocsp_path) log_info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) # Write OCSP metadata to cache/ssl/{cert-name}/ocsp.json @@ -1696,7 +1712,14 @@ def _persist_ocsp_results_to_disk( # Create symlinks for each SAN so OCSP is found by any SNI name _create_san_symlinks(pem_data, ocsp_path, cert_name) except Exception as e: + # Clean up any leftover temp files on error (existing ocsp.der remains intact) + try: + for tmp_file in ocsp_cert_dir.glob(".ocsp_*.tmp"): + tmp_file.unlink() + except Exception: + pass # Ignore cleanup errors log_error("โŒ OCSP error while writing response for %s to disk: %s", cert_name, e) + log_info("โ„น๏ธ OCSP kept existing OCSP response file for %s (new fetch failed)", cert_name) stats["errors"] = stats.get("errors", 0) + 1 finally: _release_cert_lock(lock_fd) @@ -1754,12 +1777,19 @@ def refresh_job_lock(cert_name: str = "") -> None: # Parse command line arguments parser = argparse.ArgumentParser(description="OCSP refresh job for BunkerWeb") parser.add_argument("--force", action="store_true", help="Force full OCSP refresh and TTL checks managed by the job") + parser.add_argument("--force-fetch", action="store_true", help="Force refetch all OCSP responses from upstream PKI, do not replace existing files on error") args, unknown = parser.parse_known_args() force_all = args.force + force_fetch = args.force_fetch try: - log_info("๐Ÿ”„ OCSP refresh job started with differential update strategy (timeout in %d minutes%s)", - JOB_TIMEOUT // 60, " [FORCE]" if force_all else "") + force_flags = "" + if force_all: + force_flags += " [FORCE]" + if force_fetch: + force_flags += " [FORCE-FETCH]" + log_info("๐Ÿ”„ OCSP refresh job started with differential update strategy (timeout in %d minutes%s)", + JOB_TIMEOUT // 60, force_flags) # Acquire main lock for the entire OCSP refresh operation lock_fd_main = _acquire_cert_lock("main", timeout=300, stale_threshold=1800) @@ -1800,6 +1830,26 @@ def refresh_job_lock(cert_name: str = "") -> None: log_error("โŒ OCSP could not create cache directory %s: %s", CONFIGS_SSL_BASE, e) return 2 + # === Clean up old temporary OCSP files (older than 5 minutes) === + # Removes stale temp files from failed script runs before processing begins + try: + current_time = time.time() + temp_cutoff = current_time - (5 * 60) # 5 minutes ago + cleanup_count = 0 + + for tmp_file in CONFIGS_SSL_BASE.glob("**/.ocsp_*.tmp"): + try: + if tmp_file.stat().st_mtime < temp_cutoff: + tmp_file.unlink() + cleanup_count += 1 + except Exception: + pass # Ignore individual file cleanup errors + + if cleanup_count > 0: + log_debug("๐Ÿงน OCSP cleaned up %d stale temporary file(s) (older than 5 minutes)", cleanup_count) + except Exception as e: + log_debug("โš ๏ธ OCSP temporary file cleanup failed: %s", e) + # Check if OCSP stapling is globally disabled ocsp_enabled = os.getenv("SSL_USE_OCSP_STAPLING", "yes").lower() if ocsp_enabled != "yes": @@ -1903,12 +1953,12 @@ def refresh_job_lock(cert_name: str = "") -> None: if res[1] is None and res[5] and res[6]: stashed_failures.append((cert_name, pem_data)) - # 3. Process unchanged certs (TTL check only) + # 3. Process unchanged certs (TTL check only or force-fetch if requested) if not skip_unchanged_ttl_checks: for cert_name, pem_data in sorted(unchanged_le_certs.items()): if check_job_timeout(f"unchanged LE cert {cert_name}"): break refresh_job_lock(cert_name) - res = _process_cert(cert_name, pem_data, db, stats, force_fetch=False) + res = _process_cert(cert_name, pem_data, db, stats, force_fetch=force_fetch) all_ocsp_results.append(res) if res[1] is None and res[5] and res[6]: stashed_failures.append((cert_name, pem_data)) @@ -1920,12 +1970,13 @@ def refresh_job_lock(cert_name: str = "") -> None: if not check_job_timeout("before custom cert processing"): # Process custom certificates from database custom_results = process_custom_certs( - db, - stats, - lock_fd=lock_fd_main, - refresh_fn=refresh_job_lock, + db, + stats, + lock_fd=lock_fd_main, + refresh_fn=refresh_job_lock, timeout_fn=check_job_timeout, - skip_unchanged_ttl_checks=skip_unchanged_ttl_checks + skip_unchanged_ttl_checks=skip_unchanged_ttl_checks, + force_fetch=force_fetch ) all_ocsp_results.extend(custom_results) for res in custom_results: From c7de21a76b933271df082bf1614193749d183302 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 19 Mar 2026 15:07:27 +0100 Subject: [PATCH 52/59] optimize lookup mechanism use cert fingerprint for storing and loading use native lua optimizations optimize TTL for OCSP responses --- .../server-http/ssl-certificate-lua.conf | 576 ++++++++++++------ src/common/core/ssl/jobs/ocsp-refresh.py | 568 +++++++++++++---- src/common/core/ssl/plugin.json | 2 +- 3 files changed, 845 insertions(+), 301 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 583a203a3e..8dd513f494 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -178,6 +178,254 @@ ssl_certificate_by_lua_block { local match = string.match local concat = table.concat + -- Helper: parse multiple PEM certificates from a single string + -- Returns array of individual PEM certificate strings + local function parse_pem_certificates(pem_data) + if not pem_data or #pem_data == 0 then + return {} + end + + local certs = {} + local current_cert = nil + + for line in pem_data:gmatch("[^\n]+") do + if current_cert then + current_cert = current_cert .. "\n" .. line + end + + if line:find("-----BEGIN CERTIFICATE-----", 1, true) then + current_cert = line + elseif line:find("-----END CERTIFICATE-----", 1, true) then + if current_cert then + insert(certs, current_cert) + current_cert = nil + end + end + end + + return certs + end + + -- Helper: parse multiple PEM private keys from a single string + -- Returns array of individual PEM key strings + -- Matches RSA, EC, EdDSA, and other key types + local function parse_pem_keys(pem_data) + if not pem_data or #pem_data == 0 then + return {} + end + + local keys = {} + local current_key = nil + local in_key = false + + for line in pem_data:gmatch("[^\n]+") do + if line:find("-----BEGIN", 1, true) and line:find("PRIVATE KEY", 1, true) then + in_key = true + current_key = line + elseif in_key and current_key then + current_key = current_key .. "\n" .. line + if line:find("-----END", 1, true) and line:find("PRIVATE KEY", 1, true) then + insert(keys, current_key) + current_key = nil + in_key = false + end + end + end + + return keys + end + + -- Helper: extract certificate serial number for identification (normalized to uppercase) + local function get_cert_identifier(cert_pem) + if not cert_pem or #cert_pem == 0 then + return nil + end + + -- Try resty.openssl first (fast) + if has_resty_ssl then + pcall(function() + local cert, err = resty_x509.new(cert_pem) + if cert then + local serial = cert:get_serial_number() + if serial then + return tostring(serial):upper() + end + end + end) + end + + -- Fallback: extract via openssl command + local identifier = nil + pcall(function() + local worker_id = ngx.worker.id() or "0" + local tmp_cert = "/tmp/pair_cert_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" + local f = io.open(tmp_cert, "w") + if not f then return end + + f:write(cert_pem) + f:close() + + local handle = io.popen("openssl x509 -serial -noout -in " .. tmp_cert .. " 2>/dev/null", "r") + if handle then + local result = handle:read("*a") + handle:close() + if result then + local serial = result:match("serial=([A-Fa-f0-9]+)") + if serial then + identifier = serial:upper() + end + end + end + + pcall(function() os.remove(tmp_cert) end) + end) + + return identifier + end + + -- Helper: extract public key fingerprint (SHA256) from certificate or private key (supports PEM and DER) + -- Uses native lua-resty-openssl for zero-shell-overhead fingerprinting + local function get_pubkey_fingerprint(cert_data, is_key) + if not cert_data or #cert_data == 0 then + return nil + end + + local fingerprint = nil + pcall(function() + local x509 = require("resty.openssl.x509") + local pkey = require("resty.openssl.pkey") + local digest_lib = require("resty.openssl.digest") + + if is_key then + -- Load as private key (auto-detects PEM/DER) + local key_obj = pkey.new(cert_data) + if not key_obj then + safe_log(DEBUG, "OCSP failed to load private key for fingerprinting") + return + end + -- Export public key to DER format + local pubkey_der, err = key_obj:tostring(false, 'DER') -- false = public key only + if not pubkey_der then + safe_log(DEBUG, "OCSP failed to export public key to DER: " .. (err or "unknown")) + return + end + -- Compute SHA256 of the public key DER + local digest_ctx = digest_lib.new('sha256') + if digest_ctx then + digest_ctx:update(pubkey_der) + fingerprint = ngx.encode_base16(digest_ctx:final()):lower() + end + else + -- Load as certificate (auto-detects PEM/DER) + local cert_obj = x509.new(cert_data) + if not cert_obj then + safe_log(DEBUG, "OCSP failed to load certificate for fingerprinting") + return + end + -- Use built-in pubkey_digest method to compute SHA256 of public key + local digest_bytes, err = cert_obj:pubkey_digest('sha256') + if digest_bytes then + fingerprint = ngx.encode_base16(digest_bytes):lower() + else + safe_log(DEBUG, "OCSP pubkey_digest failed: " .. (err or "unknown")) + end + end + end) + + return fingerprint + end + + -- Helper: pair certificates with their corresponding keys by matching public key fingerprints + -- Returns array of {cert = "...", key = "...", cert_id = "...", matched = true/false} tables + local function pair_certs_and_keys(certs, keys) + local pairs = {} + local num_certs = #certs + local num_keys = #keys + + safe_log(DEBUG, "Matching " .. num_certs .. " certificate(s) with " .. num_keys .. " key(s) by public key fingerprint") + + -- Compute fingerprints for all keys (expensive, do once) + local key_fingerprints = {} + for i, key in ipairs(keys) do + local fp = get_pubkey_fingerprint(key, true) -- is_key = true + key_fingerprints[i] = fp + if fp then + safe_log(DEBUG, "Key #" .. i .. " fingerprint: " .. fp:sub(1, 16) .. "...") + else + safe_log(NOTICE, "Could not extract fingerprint from key #" .. i) + end + end + + -- Try to match each certificate with a key + for cert_idx, cert_pem in ipairs(certs) do + local cert_id = get_cert_identifier(cert_pem) or ("cert_" .. cert_idx) + local cert_fp = get_pubkey_fingerprint(cert_pem, false) -- is_key = false + + if not cert_fp then + safe_log(NOTICE, "Certificate #" .. cert_idx .. " (" .. cert_id .. "): could not extract public key fingerprint - skipping") + insert(pairs, { + cert = cert_pem, + key = nil, + cert_id = cert_id, + matched = false + }) + else + safe_log(DEBUG, "Certificate #" .. cert_idx .. " (" .. cert_id .. ") fingerprint: " .. cert_fp:sub(1, 16) .. "...") + + -- Find matching key by fingerprint + local matched_key = nil + local matched_key_idx = nil + + for key_idx, key_fp in ipairs(key_fingerprints) do + if key_fp and cert_fp == key_fp then + matched_key = keys[key_idx] + matched_key_idx = key_idx + break + end + end + + if matched_key then + safe_log(INFO, "Certificate #" .. cert_idx .. " (" .. cert_id .. ") matched with key #" .. matched_key_idx) + insert(pairs, { + cert = cert_pem, + key = matched_key, + cert_id = cert_id, + matched = true + }) + else + safe_log(NOTICE, "Certificate #" .. cert_idx .. " (" .. cert_id .. ") has no matching key - will attempt to use without key") + insert(pairs, { + cert = cert_pem, + key = nil, + cert_id = cert_id, + matched = false + }) + end + end + end + + -- Warn about unused keys + local keys_used = {} + for _, pair in ipairs(pairs) do + if pair.matched then + for key_idx, key in ipairs(keys) do + if key == pair.key then + keys_used[key_idx] = true + break + end + end + end + end + + for key_idx = 1, num_keys do + if not keys_used[key_idx] then + safe_log(NOTICE, "Key #" .. key_idx .. " was not matched to any certificate") + end + end + + return pairs + end + -- Start ssl_certificate phase local internalstore = cdatastore and cdatastore:new(ngx.shared.internalstore) if not internalstore then @@ -724,7 +972,7 @@ ssl_certificate_by_lua_block { return must_staple end - -- Helper: set OCSP stapling from cache (internalstore + ocsp.der disk files) + -- Helper: set OCSP stapling from cache (fingerprint-based lookup) -- Input: cert_pem - certificate in PEM format string (or nil if from parsed plugin object) -- Returns: true if cert is acceptable, false if OCSP-Must-Staple requirement not met local function set_ocsp_from_cache(cert_pem) @@ -735,210 +983,110 @@ ssl_certificate_by_lua_block { return true -- Certificate acceptable even without OCSP end - if not server_name or server_name == "" then - safe_log(ERR, "OCSP no server_name available") - return true -- Accept cert even without server_name check + if not cert_pem or type(cert_pem) ~= "string" or #cert_pem == 0 then + safe_log(DEBUG, "OCSP no certificate PEM available (parsed object from ngx.ssl), skipping fingerprint-based lookup") + return true -- Cannot compute fingerprint without PEM string end - -- Extract certificate metadata (Must-Staple + responder URL) - -- OCSP responses are ALWAYS loaded from pre-cached files only, never downloaded on worker + -- Extract certificate metadata (Must-Staple) local has_must_staple = false - local ocsp_responder_url = nil - - -- Read and parse certificate for Must-Staple + responder URL extraction - -- cert_pem is a parsed certificate object from ssl.parse_pem_cert(), not a string - -- We can only validate Must-Staple if we have the actual PEM string - if cert_pem and type(cert_pem) == "string" and #cert_pem > 0 then - -- Read certificate and extract both Must-Staple and responder URL in one pass - local ok_read, cert_meta = pcall(read_certificate_metadata, cert_pem) - if ok_read and cert_meta then - has_must_staple = cert_meta.must_staple - ocsp_responder_url = cert_meta.responder_url - if has_must_staple then - safe_log(INFO, "OCSP-Must-Staple extension detected in certificate for " .. server_name) - end - else - safe_log(DEBUG, "Certificate metadata extraction failed: " .. tostring(cert_meta)) + local ok_read, cert_meta = pcall(read_certificate_metadata, cert_pem) + if ok_read and cert_meta then + has_must_staple = cert_meta.must_staple + if has_must_staple then + safe_log(INFO, "OCSP-Must-Staple extension detected in certificate for " .. (server_name or "unknown")) end - else - -- cert_pem is a parsed certificate object (from ssl.parse_pem_cert), not a PEM string - -- Must-Staple validation requires the original PEM, which is only available at OCSP generation time - -- In this runtime path we rely on ocsp-refresh.py to have enforced Must-Staple when creating ocsp.der - safe_log(DEBUG, "OCSP-Must-Staple check skipped in ssl_certificate phase because certificate is not a PEM string (parsed object from ngx.ssl); relying on ocsp-refresh job's validation when ocsp.der was generated") end - -- Validate: if certificate has OCSP-Must-Staple extension, it MUST have an OCSP responder URL - if has_must_staple then - if not ocsp_responder_url then - safe_log(ERR, "OCSP-Must-Staple detected but no responder URL (OID 1.3.6.1.5.5.7.48.1) found in certificate - malformed certificate for " .. server_name) - else - safe_log(INFO, "OCSP-Must-Staple validated: certificate has both Must-Staple extension and responder URL for " .. server_name) + -- Compute certificate public key fingerprint (unique identifier) + local cert_fp = get_pubkey_fingerprint(cert_pem, false) + if not cert_fp then + safe_log(NOTICE, "OCSP could not compute certificate fingerprint for " .. (server_name or "unknown")) + if has_must_staple then + safe_log(ERR, "OCSP-Must-Staple required but cannot verify certificate - will attempt TLS anyway") end + return true end - local cache_key = "TLS:SSL:ocsp:" .. server_name - local resp - local checked_paths = {} + safe_log(DEBUG, "OCSP certificate fingerprint: " .. cert_fp:sub(1, 16) .. "...") + + -- Use tree-structured sharded path: /var/cache/bunkerweb/ssl/{hex1}/{hex2}/{full_fingerprint}/ocsp.der + -- hex1 and hex2 are individual hex digits creating 16ร—16 = 256 directory tree + local hex1 = cert_fp:sub(1, 1) + local hex2 = cert_fp:sub(2, 2) + local ocsp_path = "/var/cache/bunkerweb/ssl/" .. hex1 .. "/" .. hex2 .. "/" .. cert_fp .. "/ocsp.der" + local resp = nil - -- 1) Local shared dict lookup (per-worker cache) - with error handling + -- 1) Try local shared dict lookup first (per-worker cache) + local cache_key = "TLS:SSL:ocsp:" .. cert_fp local ok_cache, cache_result = pcall(function() return internalstore:get(cache_key, true) end) if ok_cache then - local db_val, gerr = cache_result, nil - if db_val then - resp = db_val - safe_log(DEBUG, "OCSP found response in internalstore for " .. server_name) - elseif gerr and gerr ~= "not found" then - safe_log(ERR, "OCSP error while getting response from internalstore: " .. gerr) + if cache_result then + resp = cache_result + safe_log(DEBUG, "OCSP found response in shared memory cache") end else - safe_log(ERR, "OCSP exception reading from cache: " .. tostring(cache_result)) + safe_log(DEBUG, "OCSP shared memory lookup skipped: " .. tostring(cache_result)) end - - -- 2) Fallback to on-disk OCSP response files - -- Looks for variations in /var/cache/bunkerweb/ssl/ like: - -- - {domain}/ocsp.der - -- - {domain}-ecdsa/ocsp.der - -- - {domain}-rsa/ocsp.der - -- - _wildcard_.{domain}/ocsp.der (for *.domain) - -- - _wildcard_.{domain}-ecdsa/ocsp.der - -- - customcert-{domain}/ocsp.der - -- - and other variants + -- 2) Fallback to disk file (fingerprint-based direct lookup) if not resp then - local base_path = "/var/cache/bunkerweb/ssl/" - safe_log(DEBUG, "OCSP: Searching for cached responses for server_name=" .. server_name .. " in " .. base_path) - - -- Generate candidates for lookup - local candidates = {} - - -- Direct matches - insert(candidates, server_name) - insert(candidates, server_name .. "-ecdsa") - insert(candidates, server_name .. "-rsa") - insert(candidates, "customcert-" .. server_name) - insert(candidates, "customcert-" .. server_name .. "-ecdsa") - insert(candidates, "customcert-" .. server_name .. "-rsa") - - -- Wildcard and apex matches (e.g., if server_name is www.example.com, try *.example.com and example.com) - local labels = {} - for label in server_name:gmatch("[^.]+") do - insert(labels, label) - end - - -- Only try wildcard/apex if we have at least subdomain.example.com (3+ labels), and avoid TLD-only suffixes - if #labels >= 3 then - for i = 2, #labels - 1 do - local suffix = table.concat(labels, ".", i) - if suffix and suffix ~= "" then - -- Apex domain (e.g. example.com) โ€” wildcard certs are often stored under apex name - insert(candidates, suffix) - insert(candidates, suffix .. "-ecdsa") - insert(candidates, suffix .. "-rsa") - insert(candidates, "customcert-" .. suffix) - insert(candidates, "customcert-" .. suffix .. "-ecdsa") - insert(candidates, "customcert-" .. suffix .. "-rsa") - -- Wildcard form (e.g. *.example.com) - local wildcard_base = "*." .. suffix - insert(candidates, wildcard_base) - insert(candidates, wildcard_base .. "-ecdsa") - insert(candidates, wildcard_base .. "-rsa") - insert(candidates, "customcert-" .. wildcard_base) - insert(candidates, "customcert-" .. wildcard_base .. "-ecdsa") - insert(candidates, "customcert-" .. wildcard_base .. "-rsa") - end - end - end + safe_log(DEBUG, "OCSP looking for cached response at: " .. ocsp_path) + pcall(function() + local f, err = io.open(ocsp_path, "rb") + if f then + local data = f:read("*a") + f:close() - for _, name in ipairs(candidates) do - if name and name ~= "" then - local sanitized = sanitize_name(name) - local path = base_path .. sanitized .. "/ocsp.der" - insert(checked_paths, path) - safe_log(DEBUG, "OCSP: Checking candidate " .. name .. " -> " .. path) - local f, err = io.open(path, "rb") - if f then - local data = f:read("*a") - f:close() if data and #data > 0 then - -- Load optional OCSP metadata written by ocsp-refresh.py (serial, ocsp_url, must_staple) + safe_log(DEBUG, "OCSP found response file (" .. #data .. " bytes)") + + -- Load optional metadata written by ocsp-refresh.py + local meta_path = base_path .. cert_fp .. "/ocsp.json" local meta = nil - local meta_path = base_path .. sanitized .. "/ocsp.json" - local fmeta = io.open(meta_path, "r") - if fmeta then - local ok_read, meta_raw = pcall(function() - return fmeta:read("*a") - end) - fmeta:close() - if ok_read and meta_raw and #meta_raw > 0 then - local ok_decode, decoded = pcall(cjson.decode, meta_raw) - if ok_decode and type(decoded) == "table" then - meta = decoded - else - safe_log(DEBUG, "OCSP metadata decode failed for " .. meta_path) + pcall(function() + local fmeta = io.open(meta_path, "r") + if fmeta then + local meta_raw = fmeta:read("*a") + fmeta:close() + if meta_raw and #meta_raw > 0 then + local ok_decode, decoded = pcall(cjson.decode, meta_raw) + if ok_decode and type(decoded) == "table" then + meta = decoded + safe_log(DEBUG, "OCSP metadata loaded (expires: " .. tostring(meta.expires or "unknown") .. ")") + end end - else - safe_log(DEBUG, "OCSP metadata read failed or empty for " .. meta_path) end - end + end) - if meta and meta.must_staple then - safe_log(INFO, "OCSP metadata indicates Must-Staple=on for " .. server_name .. " (serial=" .. tostring(meta.serial or "unknown") .. ")") - end + -- Since we matched by fingerprint, the OCSP response is guaranteed to match + resp = data + safe_log(INFO, "OCSP loaded response from: " .. ocsp_path) - -- Verify OCSP response matches current certificate (with error handling) - local cert_matches - local ok_verify, verify_result = pcall(function() - return verify_ocsp_cert_match(cert_pem, data) - end) - if not ok_verify then - safe_log(ERR, "OCSP exception verifying response: " .. tostring(verify_result)) - goto next_candidate - end - cert_matches = verify_result - - if cert_matches then - safe_log(DEBUG, "OCSP loaded response (" .. #data .. " bytes) from " .. path .. " (verified against certificate or trusted via ocsp-refresh metadata)") - resp = data - -- Cache in shared memory to avoid disk I/O on every handshake (TTL 300s) - with error handling - local ok_set_cache, set_result = pcall(function() - return internalstore:set(cache_key, resp, 300, true) - end) - if ok_set_cache then - local ok_set, serr = set_result, nil - if not ok_set then - safe_log(ERR, "OCSP error while caching file response into internalstore: " .. (serr or "unknown")) - end - else - safe_log(ERR, "OCSP exception caching response: " .. tostring(set_result)) - end - break - else - safe_log(DEBUG, "OCSP skipping response for " .. name .. " (certificate mismatch - certificate may have been updated)") + -- Cache in shared memory (TTL 300s) + pcall(function() + local ok_cache_set = internalstore:set(cache_key, resp, 300, true) + if not ok_cache_set then + safe_log(DEBUG, "OCSP failed to cache response in shared memory") end - end + end) + else + safe_log(DEBUG, "OCSP response file is empty: " .. ocsp_path) + end + else + if err and lower(err):find("permission denied", 1, true) then + safe_log(ERR, "OCSP permission denied reading " .. ocsp_path) else - -- Log only on permission denied (avoid shell; use io.open error) - if err and lower(err):find("permission denied", 1, true) then - safe_log(ERR, "OCSP permission denied reading " .. path .. " (check file permissions)") - end + safe_log(DEBUG, "OCSP response file not found: " .. ocsp_path) end - ::next_candidate:: end - end + end) end if not resp then - safe_log(DEBUG, "OCSP not found for " .. server_name) - -- Debug: log all paths that were checked - if #checked_paths > 0 then - local paths_str = table.concat(checked_paths, " | ") - if #paths_str > 2048 then - paths_str = paths_str:sub(1, 2045) .. "..." - end - safe_log(DEBUG, "OCSP: Checked " .. #checked_paths .. " paths: " .. paths_str) - end + safe_log(DEBUG, "OCSP response not found for certificate with fingerprint: " .. cert_fp:sub(1, 16) .. "...") if has_must_staple then safe_log(ERR, "OCSP-Must-Staple required but OCSP response not found - will attempt TLS anyway") end @@ -994,38 +1142,76 @@ ssl_certificate_by_lua_block { safe_log(ERR, plugin_id .. ":ssl_certificate() call failed : " .. ret.msg) else safe_log(DEBUG, plugin_id .. ":ssl_certificate() call successful : " .. ret.msg) - if ret.status then - safe_log(DEBUG, plugin_id .. " is setting certificate/key : " .. ret.msg) + if ret.status then + safe_log(DEBUG, plugin_id .. " is setting certificate/key(s) : " .. ret.msg) + + -- Clear old certificates before setting new ones (fail-safe) + pcall(clear_certs) + + -- Parse multiple certificates and keys (support RSA, ECDSA, PQC, etc.) + local cert_pem = ret.status[1] or "" + local key_pem = ret.status[2] or "" + + local certs = parse_pem_certificates(cert_pem) + local keys = parse_pem_keys(key_pem) + local cert_key_pairs = pair_certs_and_keys(certs, keys) - -- Clear old certificates before setting new ones (fail-safe) - pcall(clear_certs) + if #cert_key_pairs == 0 then + safe_log(ERR, "No valid certificate/key pairs extracted from " .. plugin_id) + else + safe_log(INFO, "Found " .. #cert_key_pairs .. " certificate/key pair(s) from " .. plugin_id) - -- Try to set the new certificate - local ok_cert, err_cert = set_cert(ret.status[1]) + local all_certs_set = true + + -- Loop through each certificate/key pair (supports RSA, ECDSA, PQC, etc.) + for idx, pair in ipairs(cert_key_pairs) do + safe_log(DEBUG, "Setting certificate #" .. idx .. " (" .. pair.cert_id .. ") from " .. plugin_id .. (pair.matched and " [matched key]" or " [unmatched - no key found]")) + + -- Try to set the certificate + local ok_cert, err_cert = set_cert(pair.cert) if not ok_cert then - safe_log(ERR, "error while setting certificate from " .. plugin_id .. ": " .. (err_cert or "unknown")) + safe_log(ERR, "error while setting certificate #" .. idx .. " (" .. pair.cert_id .. ") from " .. plugin_id .. ": " .. (err_cert or "unknown")) + all_certs_set = false else - -- Certificate set successfully, now set private key - local ok_key, err_key = set_priv_key(ret.status[2]) - if not ok_key then - safe_log(ERR, "error while setting private key from " .. plugin_id .. ": " .. (err_key or "unknown")) + -- Certificate set successfully, now try to set private key (if available) + local ok_key = true + local err_key = nil + + if pair.key then + ok_key, err_key = set_priv_key(pair.key) + if not ok_key then + safe_log(ERR, "error while setting private key #" .. idx .. " (" .. pair.cert_id .. ") from " .. plugin_id .. ": " .. (err_key or "unknown")) + all_certs_set = false + end else - -- Certificate and key both set successfully - -- Try to set OCSP stapling from cache (fail-safe) - -- Pass the certificate PEM directly (from ret.status[1]) - safe_log(DEBUG, "About to call set_ocsp_from_cache() for " .. server_name) - local ok_ocsp, ocsp_cert_acceptable = pcall(set_ocsp_from_cache, ret.status[1]) + safe_log(NOTICE, "Certificate #" .. idx .. " (" .. pair.cert_id .. ") has no matched private key - continuing without key (may fail during TLS handshake)") + end + + -- Only set OCSP if key was successfully set (or not needed) + if ok_key then + -- Certificate and key both set successfully (or cert-only is acceptable) + -- Try to set OCSP stapling from cache for this specific certificate (fail-safe) + safe_log(DEBUG, "Setting OCSP for certificate #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ")") + local ok_ocsp, ocsp_cert_acceptable = pcall(set_ocsp_from_cache, pair.cert) if not ok_ocsp then - safe_log(ERR, "OCSP function error: " .. tostring(ocsp_cert_acceptable)) + safe_log(ERR, "OCSP function error for cert #" .. idx .. " (" .. pair.cert_id .. "): " .. tostring(ocsp_cert_acceptable)) elseif ocsp_cert_acceptable == false then - -- OCSP-Must-Staple requirement not met - but user requested to always run TLS - safe_log(ERR, "OCSP-Must-Staple requirement not met for " .. server_name .. " - proceeding anyway as per failsafe policy") + -- OCSP-Must-Staple requirement not met - log but continue (failsafe) + safe_log(ERR, "OCSP-Must-Staple requirement not met for cert #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ") - proceeding anyway") + else + safe_log(DEBUG, "OCSP loaded successfully for certificate #" .. idx .. " (" .. pair.cert_id .. ")") end - safe_log(INFO, "certificate and key set by " .. plugin_id) - return true - end -- end ok_key + end -- end if ok_key end -- end ok_cert - end -- end ret.status + end -- end for each cert/key pair + + if all_certs_set then + safe_log(INFO, "all " .. #cert_key_pairs .. " certificate(s) and key(s) set by " .. plugin_id) + return true + else + safe_log(NOTICE, "some certificates from " .. plugin_id .. " failed to set, continuing to next plugin") + end + end -- end cert_key_pairs check end -- end call_plugin result end -- end new_plugin result else diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index a7d7f4a27c..7c32595389 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -110,7 +110,71 @@ def log_critical(msg: str, *args: Any, **kwargs: Any) -> None: # Use scheduler-managed cache directory (automatically synced from database on restart) CONFIGS_SSL_BASE = Path(os.sep, "var", "cache", "bunkerweb", "ssl") -MIN_TTL = 4500 # 75 minutes: aggressivly refresh/cleanup if TTL is below this +MIN_TTL = 4500 # 75 minutes: minimum safety threshold. Smart refresh uses max(MIN_TTL, 20% of response lifetime) + + +def _get_sharded_ocsp_path(fingerprint: str) -> Path: + """ + Get sharded directory path for OCSP response storage using two-level hex tree sharding. + Creates a tree structure: {hex1}/{hex2}/{fingerprint}/ where hex1 and hex2 are individual + hex digits from the fingerprint, distributing 256 subdirectories across 16 first-level dirs. + + Args: + fingerprint: SHA256 fingerprint (lowercase hex string, 64 chars) + + Returns: + Path: /var/cache/bunkerweb/ssl/{hex1}/{hex2}/{full_fingerprint}/ + + Example: + fingerprint: 089433bd22ca2b9536b597a9fc7ca86cdd1d1df0193caaa205043b40f1ea435b + returns: /var/cache/bunkerweb/ssl/0/8/089433bd22ca2b9536b597a9fc7ca86cdd1d1df0193caaa205043b40f1ea435b/ + """ + if not fingerprint or len(fingerprint) < 2: + return CONFIGS_SSL_BASE / "unknown" + # Use first hex digit (0-f) and second hex digit (0-f) as tree levels + hex1 = fingerprint[0].lower() + hex2 = fingerprint[1].lower() + return CONFIGS_SSL_BASE / hex1 / hex2 / fingerprint + + +def _init_sharded_ocsp_directories() -> bool: + """ + Initialize tree-structured sharded subdirectories for OCSP response distribution. + Creates 16 first-level (hex1: 0-f) ร— 16 second-level (hex2: 0-f) = 256 total directories. + Structure: /var/cache/bunkerweb/ssl/{hex1}/{hex2}/ + Called at job startup to ensure directory structure exists. + + Returns: + True if all directories initialized successfully, False on error + """ + try: + hex_digits = "0123456789abcdef" + failed_dirs = [] + + # Create tree structure: 16 first-level dirs ร— 16 second-level dirs = 256 total + for hex1 in hex_digits: + for hex2 in hex_digits: + dir_path = CONFIGS_SSL_BASE / hex1 / hex2 + try: + dir_path.mkdir(parents=True, exist_ok=True) + except Exception as e: + log_error("โŒ OCSP failed to create directory %s: %s", dir_path, e) + failed_dirs.append((f"{hex1}/{hex2}", str(e))) + + if failed_dirs: + log_warning( + "โš ๏ธ OCSP failed to create %d director(ies): %s", + len(failed_dirs), + ", ".join([f"{d[0]}({d[1]})" for d in failed_dirs[:3]]) # Show first 3 failures + ) + return False + + log_debug("โœ“ OCSP initialized tree-structured directories (16ร—16) for OCSP caching") + return True + + except Exception as e: + log_error("โŒ OCSP exception during sharded directory initialization: %s", e) + return False def _sanitize_filename(name: str) -> str: @@ -121,6 +185,54 @@ def _sanitize_filename(name: str) -> str: return name.replace("*", "_wildcard_") +def _get_cert_pubkey_fingerprint(cert_data: bytes) -> Optional[str]: + """ + Compute SHA256 fingerprint of certificate's public key (RFC 7469 format). + Used for fingerprint-based OCSP response storage and matching. + + Supports both PEM and DER certificate formats. + + Args: + cert_data: Certificate in PEM or DER format (bytes) + + Returns: + Lowercase hex string (64 chars) of SHA256 fingerprint, or None on error + """ + if not cert_data: + return None + + cert = None + try: + # Try PEM format first (most common) + try: + cert = x509.load_pem_x509_certificate(cert_data) + log_debug("โœ“ OCSP parsed certificate as PEM format") + except Exception as pem_err: + # Fallback to DER format + try: + cert = x509.load_der_x509_certificate(cert_data) + log_debug("โœ“ OCSP parsed certificate as DER format") + except Exception as der_err: + log_warning("โš ๏ธ OCSP certificate is neither PEM nor DER format: PEM error: %s, DER error: %s", + str(pem_err)[:100], str(der_err)[:100]) + return None + + # Extract public key + public_key = cert.public_key() + # Serialize public key to DER format (standard for fingerprinting) + pubkey_der = public_key.public_bytes( + encoding=Encoding.DER, + format=serialization.PublicFormat.SubjectPublicKeyInfo + ) + # Compute SHA256 fingerprint (lowercase for consistency) + fingerprint = hashlib.sha256(pubkey_der).hexdigest().lower() + log_debug("โœ“ OCSP computed certificate fingerprint: %s", fingerprint[:16] + "...") + return fingerprint + except Exception as e: + log_warning("โš ๏ธ OCSP could not compute certificate fingerprint: %s", e) + return None + + def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int = 600) -> Optional[int]: """ Acquire a file-based lock for a specific certificate to prevent race conditions @@ -179,21 +291,45 @@ def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int pass # Non-critical timestamp write return fd except BlockingIOError: - # Lock held by another process, close fd and retry after brief delay + # Lock held by another process, close fd and exit or retry os.close(fd) - time.sleep(0.5) - continue + + if cert_name == "main": + # For main lock, fail fast: another OCSP job is already running + log_error( + "โŒ OCSP job is already running (another instance holds the lock). " + "Exiting to avoid concurrent operations." + ) + return None + + # For cert-specific locks, retry with timeout + elapsed = time.time() - start_time + remaining_time = timeout - elapsed + if remaining_time > 0: + log_debug("โณ OCSP waiting for lock on %s (%.0fs remaining)...", cert_name, remaining_time) + time.sleep(0.5) + continue + else: + break except Exception as e: log_debug("โš ๏ธ OCSP lock acquisition attempt failed for %s: %s", cert_name, e) time.sleep(0.5) - # Timeout reached: log warning and proceed without lock - log_warning( - "โš ๏ธ OCSP could not acquire lock for %s after %ds. " - "If concurrent OCSP fetches happen, cryptographic verification protects data integrity. " - "Consider increasing scheduler interval or timeout if jobs consistently exceed %ds.", - cert_name, timeout, timeout - ) + # Timeout reached: log clear message based on lock type + if cert_name == "main": + log_error( + "โŒ OCSP job timed out waiting for the main lock (%ds timeout). " + "Another OCSP job is still running. This may indicate a hung or very slow job. " + "Check for stale lock files: rm -f /tmp/bunkerweb/ocsp*.lock", + timeout + ) + else: + log_warning( + "โš ๏ธ OCSP could not acquire lock for %s after %ds. " + "If concurrent OCSP fetches happen, cryptographic verification protects data integrity. " + "Consider increasing scheduler interval or timeout if jobs consistently exceed %ds.", + cert_name, timeout, timeout + ) return None @@ -240,6 +376,36 @@ def _refresh_cert_lock(fd: Optional[int], cert_name: str) -> None: log_debug("โš ๏ธ OCSP could not refresh lock timestamp for %s: %s", cert_name, e) +def _cleanup_stale_locks(stale_threshold: int = 300) -> None: + """ + Clean up stale lock files from previous crashed/aborted jobs at startup. + This is a defensive measure to prevent lock accumulation from crashed processes. + + Args: + stale_threshold: Lock age (seconds) to consider stale (default 300 = 5 minutes) + """ + lock_dir = Path(os.sep, "tmp", "bunkerweb") + if not lock_dir.exists(): + return + + current_time = time.time() + cleaned = 0 + + for lock_file in lock_dir.glob("ocsp-*.lock"): + try: + mtime = lock_file.stat().st_mtime + age = current_time - mtime + if age > stale_threshold: + lock_file.unlink() + cleaned += 1 + log_debug("๐Ÿงน OCSP removed stale lock file %s (age %.0fs)", lock_file.name, age) + except Exception as e: + log_debug("โš ๏ธ OCSP could not clean lock file %s: %s", lock_file.name, e) + + if cleaned > 0: + log_info("๐Ÿงน OCSP cleaned up %d stale lock file(s) from previous runs", cleaned) + + def is_safe_url(url: str) -> bool: """ Validate that a URL is safe to fetch (HTTP/HTTPS only, no internal/private IPs). @@ -669,16 +835,31 @@ def extract_san_dns(pem_data: bytes, cert_name: str = "") -> List[str]: return [] -def get_cached_ocsp_ttl(cert_name: str) -> Tuple[Optional[int], Optional[int]]: +def get_cached_ocsp_ttl(cert_name: str, cert_pem: Optional[bytes] = None, fingerprint: Optional[str] = None) -> Tuple[Optional[int], Optional[int]]: """ - Check if cached OCSP DER file exists and return: + Check if cached OCSP DER file exists (using fingerprint-based sharded path) and return: - remaining TTL in seconds (until Next Update) - total lifetime in seconds (Next Update - This Update) Both values are None if they cannot be determined. + + Args: + cert_name: Certificate identifier (for logging only) + cert_pem: Optional certificate PEM data to compute fingerprint if not provided + fingerprint: Optional pre-computed fingerprint. If not provided, computes from cert_pem. """ - sanitized_name = _sanitize_filename(cert_name) - ocsp_path = CONFIGS_SSL_BASE / sanitized_name / "ocsp.der" - log_debug("โšก OCSP TTL check: checking cache for %s at %s", cert_name, ocsp_path) + # Compute fingerprint if not provided + if fingerprint is None: + if cert_pem is None: + log_debug("โšก OCSP TTL check: no fingerprint or cert_pem provided for %s", cert_name) + return None, None + fingerprint = _get_cert_pubkey_fingerprint(cert_pem) + if fingerprint is None: + log_debug("โšก OCSP TTL check: could not compute fingerprint for %s", cert_name) + return None, None + + # Use sharded fingerprint-based path + ocsp_path = _get_sharded_ocsp_path(fingerprint) / "ocsp.der" + log_debug("โšก OCSP TTL check: checking cache for %s at sharded path %s (fingerprint: %s...)", cert_name, ocsp_path, fingerprint[:16]) if not ocsp_path.is_file(): log_debug("โšก OCSP TTL check: cache miss - file not found for %s", cert_name) @@ -791,35 +972,49 @@ def _get_cached_ocsp_certs(db: Any) -> set: return cached_certs -def _get_cert_checksums(db: Any, cert_names: set) -> Dict[str, str]: +def _get_cert_checksums(db: Any, cert_data: Dict[str, bytes]) -> Dict[str, str]: """ Get previously stored checksums for certificates from the database. - Checksums are stored in cache entries like 'cert_checksum/cert_name'. + Checksums are stored in cache entries like 'cert_checksum/{fingerprint}'. Args: db: Database connection - cert_names: Set of certificate names to retrieve checksums for + cert_data: Dict of {cert_name: pem_data} to retrieve checksums for Returns: Dict of {cert_name: checksum_hex} """ checksums = {} - if db is None or not cert_names: + if db is None or not cert_data: return checksums try: + # Build mapping of fingerprints to cert_names for lookup + fingerprint_to_name = {} + for cert_name, pem_data in cert_data.items(): + # Clean PEM before fingerprinting (custom certs may have private keys/noise) + cleaned_pem = _clean_pem(pem_data) + cert_fp = _get_cert_pubkey_fingerprint(cleaned_pem) + if cert_fp: + fingerprint_to_name[cert_fp] = cert_name + + if not fingerprint_to_name: + log_debug("โš ๏ธ OCSP could not compute fingerprints for %d cert(s) when retrieving checksums", len(cert_data)) + return checksums + # Optimization: use with_data=True to fetch all checksums in one go cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=True) for entry in cache_files: file_name = entry.get("file_name", "") if file_name.startswith("cert_checksum/"): - cert_name_raw = file_name[len("cert_checksum/"):] - if cert_name_raw in cert_names: + fingerprint = file_name[len("cert_checksum/"):] + if fingerprint in fingerprint_to_name: + cert_name = fingerprint_to_name[fingerprint] data = entry.get("data") if data: try: - checksums[cert_name_raw] = data.decode("utf-8").strip() + checksums[cert_name] = data.decode("utf-8").strip() except Exception: pass except Exception as e: @@ -832,7 +1027,7 @@ def _calculate_cert_checksum(pem_data: bytes) -> str: """ Calculate SHA256 checksum of certificate PEM data. """ - return hashlib.sha256(pem_data).hexdigest() + return hashlib.sha256(pem_data).hexdigest().lower() def _clean_pem(pem_data: bytes) -> bytes: @@ -1013,7 +1208,7 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: sanitized_name = _sanitize_filename(cert_name_raw) db_data = entry["data"] - db_checksum = entry.get("checksum") or hashlib.sha256(db_data).hexdigest() + db_checksum = (entry.get("checksum") or hashlib.sha256(db_data).hexdigest()).lower() try: ocsp_cert_dir = CONFIGS_SSL_BASE / sanitized_name @@ -1021,7 +1216,7 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: if ocsp_path.is_file(): # File exists โ€” compare checksum - disk_checksum = hashlib.sha256(ocsp_path.read_bytes()).hexdigest() + disk_checksum = hashlib.sha256(ocsp_path.read_bytes()).hexdigest().lower() if disk_checksum == db_checksum: ok_count += 1 log_debug("โœ“ OCSP disk file for %s matches database (checksum=%s)", cert_name_raw, db_checksum[:8]) @@ -1078,12 +1273,16 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta return (cert_name, None, 0, cert_checksum, pem_data, None, False) # Check if certificate checksum matches what we have in database + # Checksums are stored by fingerprint (cert_checksum/{fingerprint}), so compute it first cached_checksum = None - if db: + fingerprint_for_checksum = _get_cert_pubkey_fingerprint(pem_data) + if db and fingerprint_for_checksum: try: - checksum_data = db.get_job_cache_file(file_name=f"cert_checksum/{sanitized_name}", job_name="ocsp-refresh") + checksum_key = f"cert_checksum/{fingerprint_for_checksum}" + checksum_data = db.get_job_cache_file(file_name=checksum_key, job_name="ocsp-refresh") if checksum_data: cached_checksum = checksum_data.decode("utf-8").strip() + log_debug("โœ“ OCSP found cached checksum for %s (key=%s)", cert_name, checksum_key[:30]) except Exception as e: log_debug("โš ๏ธ OCSP could not check cached checksum for %s: %s", cert_name, e) @@ -1103,22 +1302,32 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta log_debug("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) - # === Check if cached OCSP response is still fresh === - cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name) + # === Compute fingerprint for sharded path lookup === + fingerprint = _get_cert_pubkey_fingerprint(pem_data) + if not fingerprint: + log_warning("โš ๏ธ OCSP could not compute fingerprint for %s, treating as new fetch", cert_name) + + # === Check if cached OCSP response is still fresh (disk + database) === + # This two-tier check handles aborted downloads, database inconsistencies, and ephemeral storage + cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name, pem_data, fingerprint) + + # If disk check found valid response, use it (even if database is out of sync) if cached_ttl is not None: half_lifetime = (total_lifetime // 2) if (total_lifetime and total_lifetime > 0) else 0 - - # Skip fetch ONLY if not forced AND TTL is above MIN_TTL AND above 50% lifetime - if not force_fetch and cached_ttl > MIN_TTL and cached_ttl > half_lifetime: - log_debug( - "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), above thresholds (MIN_TTL=%ds, 50%% threshold=%ds), skipping fetch", - cert_name, cached_ttl, cached_ttl / 86400.0, MIN_TTL, half_lifetime, + # Smart TTL calculation: use 20% of total lifetime as refresh threshold (or MIN_TTL if larger for safety) + refresh_threshold = max(MIN_TTL, int(total_lifetime * 0.20)) if total_lifetime and total_lifetime > 0 else MIN_TTL + + # Skip fetch ONLY if not forced AND TTL is above refresh_threshold AND above 50% lifetime + if not force_fetch and cached_ttl > refresh_threshold and cached_ttl > half_lifetime: + log_info( + "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), above thresholds (refresh_threshold=%ds [20%% of %ds], 50%% threshold=%ds), skipping fetch", + cert_name, cached_ttl, cached_ttl / 86400.0, refresh_threshold, total_lifetime, half_lifetime, ) stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 return (cert_name, None, cached_ttl, cert_checksum, pem_data, ocsp_url, False) - - if cached_ttl <= MIN_TTL: - log_info("๐Ÿ”„ OCSP response for %s is near expiration (TTL=%ds < %ds), attempting aggressive refresh", cert_name, cached_ttl, MIN_TTL) + + if cached_ttl <= refresh_threshold: + log_info("๐Ÿ”„ OCSP response for %s is near expiration (TTL=%ds <= refresh_threshold=%ds [20%% of %ds]), attempting aggressive refresh", cert_name, cached_ttl, refresh_threshold, total_lifetime) ocsp_der: Optional[bytes] = None ttl: int = 0 @@ -1139,32 +1348,38 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta if cached_ttl is not None: current_ttl = cast(int, cached_ttl) + # Recalculate refresh_threshold for failure handling (same logic as above) + current_refresh_threshold = max(MIN_TTL, int(total_lifetime * 0.20)) if total_lifetime and total_lifetime > 0 else MIN_TTL + if current_ttl <= 0: log_warning( "๐Ÿงน OCSP cached response for %s is already expired and refresh failed. Cleaning up invalid cache.", cert_name, ) cleanup_ocsp_cache(db, cert_name) - elif current_ttl <= MIN_TTL: + elif current_ttl <= current_refresh_threshold: log_warning( - "๐Ÿšจ OCSP CRITICAL: Cached response for %s is near expiration (TTL=%ds) and refresh failed. " + "๐Ÿšจ OCSP CRITICAL: Cached response for %s is near expiration (TTL=%ds <= refresh_threshold=%ds [20%% of %ds]) and refresh failed. " "Removing from cache to prevent stapling expired data.", cert_name, current_ttl, + current_refresh_threshold, + total_lifetime, ) cleanup_ocsp_cache(db, cert_name) else: log_warning( - "โš ๏ธ OCSP could NOT refresh response for %s, keeping existing cache (TTL=%ds) for now", + "โš ๏ธ OCSP could NOT refresh response for %s, keeping existing cache (TTL=%ds, above threshold %ds) for now", cert_name, current_ttl, + current_refresh_threshold, ) return (cert_name, None, current_ttl, cert_checksum, pem_data, ocsp_url, True) return (cert_name, None, 0, cert_checksum, pem_data, ocsp_url, True) - # Calculate checksum for integrity verification - ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() + # Calculate checksum for integrity verification (lowercase for consistency) + ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest().lower() ttl_readable = f"{ttl / 86400.0:.1f} days" if ttl >= 86400 else f"{ttl / 3600.0:.1f} hours" log_info("โšก OCSP final TTL for %s is %ds (%s) (checksum=%s)", cert_name, ttl, ttl_readable, ocsp_checksum[:8]) @@ -1218,7 +1433,7 @@ def process_custom_certs( # Check which custom certs have changed since last OCSP refresh # The names from _load_custom_certs_from_db already include the 'customcert-' prefix - previous_custom_checksums = _get_cert_checksums(db, set(custom_certs.keys())) + previous_custom_checksums = _get_cert_checksums(db, custom_certs) changed_custom_certs = {} unchanged_custom_certs = {} @@ -1229,9 +1444,9 @@ def process_custom_certs( previous_checksum = previous_custom_checksums.get(cert_name) if previous_checksum is None: - # No previous checksum, treat as changed + # No previous checksum, treat as changed (will be recategorized by robustness check if valid disk cache exists) changed_custom_certs[cert_name] = pem_data - log_debug("โš ๏ธ OCSP no previous checksum found for custom cert %s, will refresh", cert_name) + log_debug("โ„น๏ธ OCSP no previous checksum found for custom cert %s (will check disk cache)", cert_name) elif current_checksum != previous_checksum: # Certificate content has changed changed_custom_certs[cert_name] = pem_data @@ -1250,7 +1465,33 @@ def process_custom_certs( stats["custom_certs_processed"] = stats.get("custom_certs_processed", 0) + len(custom_certs) - # 1. Process changed custom certificates (force refresh) + # 1. Process changed custom certificates (robustness: skip force-fetch if valid OCSP cached on disk) + recategorized_changed_custom = {} + for cert_name, cert_pem in list(changed_custom_certs.items()): + # Clean PEM before fingerprinting (custom certs may have private keys/noise) + cleaned_pem = _clean_pem(cert_pem) + fingerprint = _get_cert_pubkey_fingerprint(cleaned_pem) + if fingerprint: + cached_ttl, _ = get_cached_ocsp_ttl(cert_name, cert_pem, fingerprint) + if cached_ttl is not None and cached_ttl > 0: + # Valid OCSP cached on disk - skip force refresh + log_info("โ„น๏ธ OCSP disk file exists for %s (marked changed due to missing checksum): TTL=%ds, skipping force-fetch", cert_name, cached_ttl) + recategorized_changed_custom[cert_name] = cert_pem + del changed_custom_certs[cert_name] + unchanged_custom_certs[cert_name] = cert_pem + + if recategorized_changed_custom: + log_info("โ„น๏ธ OCSP recategorized %d custom cert(s) from changedโ†’unchanged due to valid disk cache", len(recategorized_changed_custom)) + # Add recategorized certs to results so their checksums get persisted to database + # (even though we didn't fetch new OCSP responses, we need to record their checksums for future runs) + for cert_name, cert_pem in sorted(recategorized_changed_custom.items()): + pem_checksum = _calculate_cert_checksum(cert_pem) + # Tuple: (cert_name, ocsp_der=None, ttl=0, checksum, pem_data, ocsp_url=None, was_attempted=False) + # We're not fetching, just recording the cert's checksum for differential tracking + results.append((cert_name, None, 0, pem_checksum, cert_pem, None, False)) + log_debug("โœ“ OCSP added recategorized custom cert %s to database persist list (checksum=%s)", cert_name, pem_checksum[:8]) + + # Process remaining changed custom certificates with force refresh for cert_name, cert_pem in sorted(changed_custom_certs.items()): if callable(timeout_fn) and timeout_fn(f"changed custom cert {cert_name}"): break @@ -1342,20 +1583,33 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None except Exception as e: log_debug("๐Ÿงน OCSP could not remove database entry for %s: %s", cert_name, e) else: - # Clean up ALL OCSP caches (including symlinks) + # Clean up ALL OCSP caches (handles both old flat and new tree structures) if CONFIGS_SSL_BASE.is_dir(): - for entry in sorted(CONFIGS_SSL_BASE.iterdir()): - if not entry.is_dir(): - continue - ocsp_file = entry / "ocsp.der" + # Recursively find and remove all ocsp.der files + for root, dirs, files in os.walk(CONFIGS_SSL_BASE, topdown=False): + # Try to remove ocsp.der if it exists + ocsp_file = Path(root) / "ocsp.der" if ocsp_file.is_symlink() or ocsp_file.is_file(): - ocsp_file.unlink(missing_ok=True) - log_info("๐Ÿงน OCSP removed cached response %s", ocsp_file) - # Remove the directory if it's now empty + try: + ocsp_file.unlink() + log_info("๐Ÿงน OCSP removed cached response %s", ocsp_file) + except Exception: + pass + + # Try to remove ocsp.json metadata if it exists + meta_file = Path(root) / "ocsp.json" + if meta_file.is_file(): + try: + meta_file.unlink() + except Exception: + pass + + # Remove empty directories (walk in reverse order ensures we clean up bottom-up) try: - entry.rmdir() + if root != str(CONFIGS_SSL_BASE): # Don't remove the base directory + Path(root).rmdir() except OSError: - pass # Directory not empty, leave it + pass # Directory not empty or other error, skip if db and purge_db: try: @@ -1481,7 +1735,6 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic continue cert_name_raw = file_name[len("ocsp/"):] - sanitized_name = _sanitize_filename(cert_name_raw) data = entry.get("data") db_checksum = entry.get("checksum", "") @@ -1489,6 +1742,25 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic log_warning("โš ๏ธ OCSP database entry %s has no data or checksum, skipping verification", cert_name_raw) continue + # Check if this is a marker entry (fingerprint reference) vs actual OCSP response + # Marker entries: UTF-8 encoded fingerprints (~64 bytes) + # Real OCSP responses: DER-encoded binary (typically 500+ bytes) + is_marker = False + try: + # Try to decode as UTF-8 fingerprint string (marker entries are ~64 hex chars) + decoded = data.decode("utf-8", errors="strict") + if len(decoded) == 64 and all(c in "0123456789abcdef" for c in decoded): + # This is a marker entry - skip it during verification + log_debug("โ„น๏ธ OCSP skipping marker entry %s (fingerprint reference)", cert_name_raw) + is_marker = True + except (UnicodeDecodeError, Exception): + pass # Not a marker, try to parse as OCSP response + + if is_marker: + continue + + # At this point, data should be a DER-encoded OCSP response + sanitized_name = _sanitize_filename(cert_name_raw) verify_count += 1 # Check if file exists on disk @@ -1520,7 +1792,13 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic ocsp_cert_dir.mkdir(parents=True, exist_ok=True) ocsp_path.write_bytes(data) ocsp_path.chmod(0o644) - log_info("โœ“ OCSP restored %s from database", cert_name_raw) + # Verify checksum after restoration + written_checksum = hashlib.sha256(ocsp_path.read_bytes()).hexdigest().lower() + if written_checksum != db_checksum: + log_error("โŒ OCSP checksum mismatch after restoring %s (expected=%s, got=%s)", cert_name_raw, db_checksum[:8], written_checksum[:8]) + stats["errors"] = stats.get("errors", 0) + 1 + continue + log_info("โœ“ OCSP restored %s from database (verified)", cert_name_raw) restored_count += 1 except Exception as e: log_error("โŒ OCSP could not restore %s from database: %s", cert_name_raw, e) @@ -1528,9 +1806,9 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic continue else: - # File exists: verify checksum + # File exists: verify checksum (lowercase for consistency) file_data = ocsp_path.read_bytes() - file_checksum = hashlib.sha256(file_data).hexdigest() + file_checksum = hashlib.sha256(file_data).hexdigest().lower() if file_checksum != db_checksum: log_warning( @@ -1541,7 +1819,13 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic try: ocsp_path.write_bytes(data) ocsp_path.chmod(0o644) - log_info("โœ“ OCSP restored correct version of %s", cert_name_raw) + # Verify checksum after restoration + written_checksum = hashlib.sha256(ocsp_path.read_bytes()).hexdigest().lower() + if written_checksum != db_checksum: + log_error("โŒ OCSP checksum mismatch after restoring %s (expected=%s, got=%s)", cert_name_raw, db_checksum[:8], written_checksum[:8]) + stats["errors"] = stats.get("errors", 0) + 1 + continue + log_info("โœ“ OCSP restored correct version of %s (verified)", cert_name_raw) mismatch_count += 1 except Exception as e: log_error("โŒ OCSP could not restore %s: %s", cert_name_raw, e) @@ -1586,13 +1870,19 @@ def _persist_ocsp_results_to_db( log_info("๐Ÿ”„ OCSP batching database updates for %d certificate(s)", len(all_ocsp_results)) for cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url, was_attempted in all_ocsp_results: - sanitized_name = _sanitize_filename(cert_name) + # Clean PEM before fingerprinting (custom certs may have private keys/noise) + cleaned_pem = _clean_pem(pem_data) + # Compute certificate fingerprint for storage key + cert_fp = _get_cert_pubkey_fingerprint(cleaned_pem) + if not cert_fp: + log_warning("โš ๏ธ OCSP cannot store database cache for %s: failed to compute fingerprint", cert_name) + continue - # 1. Update OCSP response if we have a new one + # 1. Update OCSP response if we have a new one (using fingerprint-based key) if ocsp_der and ttl > 0: - cache_key = f"ocsp/{sanitized_name}" + cache_key = f"ocsp/{cert_fp}" # Use fingerprint instead of cert_name try: - ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest() + ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest().lower() err = db.upsert_job_cache( service_id=None, # Global cache entry file_name=cache_key, @@ -1602,23 +1892,38 @@ def _persist_ocsp_results_to_db( ) if err: - log_error("โŒ OCSP error while storing response for %s in database: %s", cert_name, err) + log_error("โŒ OCSP error while storing response for %s (fingerprint: %s) in database: %s", cert_name, cert_fp[:16] + "...", err) stats["errors"] = stats.get("errors", 0) + 1 else: - log_info("โœ“ OCSP stored response for %s in database (TTL=%ds, checksum=%s)", cert_name, ttl, ocsp_checksum[:8]) + log_info("โœ“ OCSP stored response for %s in database (fingerprint: %s, TTL=%ds)", cert_name, cert_fp[:16] + "...", ttl) + + # Also store a marker entry with cert_name for differential tracking + # This allows _get_cached_ocsp_certs() to identify cached certificates + sanitized_name = _sanitize_filename(cert_name) + try: + db.upsert_job_cache( + service_id=None, + file_name=f"ocsp/{sanitized_name}", + data=cert_fp.encode("utf-8"), # Store fingerprint reference + job_name="ocsp-refresh", + checksum=hashlib.sha256(cert_fp.encode("utf-8")).hexdigest().lower(), + ) + log_debug("โœ“ OCSP stored cert_name marker for %s (fingerprint: %s)", cert_name, cert_fp[:16] + "...") + except Exception as e: + log_debug("โš ๏ธ OCSP could not store cert_name marker for %s: %s", cert_name, e) except Exception as e: log_error("โŒ OCSP exception while storing response for %s in database: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 - # 2. ALWAYS store/update certificate content checksum for future differential checks - cert_checksum_key = f"cert_checksum/{sanitized_name}" + # 2. ALWAYS store/update certificate content checksum for future differential checks (using fingerprint) + cert_checksum_key = f"cert_checksum/{cert_fp}" try: err = db.upsert_job_cache( service_id=None, file_name=cert_checksum_key, data=cert_checksum.encode("utf-8"), job_name="ocsp-refresh", - checksum=hashlib.sha256(cert_checksum.encode("utf-8")).hexdigest(), + checksum=hashlib.sha256(cert_checksum.encode("utf-8")).hexdigest().lower(), ) if err: log_debug("โš ๏ธ OCSP could not store cert checksum for %s: %s", cert_name, err) @@ -1648,20 +1953,27 @@ def _persist_ocsp_results_to_disk( if not ocsp_der: continue try: + # Compute certificate public key fingerprint for storage location + cert_fp = _get_cert_pubkey_fingerprint(pem_data) + if not cert_fp: + log_error("โŒ OCSP cannot store response for %s: failed to compute fingerprint", cert_name) + stats["errors"] = stats.get("errors", 0) + 1 + continue + # Acquire lock to prevent race conditions with concurrent OCSP fetches lock_fd = _acquire_cert_lock(cert_name) - sanitized_name = _sanitize_filename(cert_name) try: - # Create the SSL configs directory for this certificate - ocsp_cert_dir = CONFIGS_SSL_BASE / sanitized_name + # Create sharded directory using fingerprint (0-f distribution) + # Example: /var/cache/bunkerweb/ssl/0/089433bd22ca2b9536b597a9fc7ca86cdd1d1df0193caaa205043b40f1ea435b/ocsp.der + ocsp_cert_dir = _get_sharded_ocsp_path(cert_fp) try: ocsp_cert_dir.mkdir(parents=True, exist_ok=True) except Exception as e: - log_error("โŒ OCSP error while creating directory %s: %s", ocsp_cert_dir, e) + log_error("โŒ OCSP error while creating sharded directory %s: %s", ocsp_cert_dir, e) stats["errors"] = stats.get("errors", 0) + 1 - continue + raise # Let finally block release the lock before skipping this cert - # Write OCSP response to cache/ssl/{cert-name}/ocsp.der (atomic write with temp file) + # Write OCSP response to sharded cache location (atomic write with temp file) ocsp_path = ocsp_cert_dir / "ocsp.der" try: # Write to temporary file first, then atomically move to final location @@ -1677,40 +1989,31 @@ def _persist_ocsp_results_to_disk( # Atomically move temp file to final location (overwrites only on success) tmp_path.replace(ocsp_path) - log_info("โœ“ OCSP saved response for %s to disk at %s", cert_name, ocsp_path) + log_info("โœ“ OCSP saved response for %s to disk at %s (fingerprint: %s)", cert_name, ocsp_path, cert_fp[:16] + "...") - # Write OCSP metadata to cache/ssl/{cert-name}/ocsp.json + # Write OCSP metadata to cache/ssl/{fingerprint}/ocsp.json + # Metadata includes cert_name, serial, fingerprint, expires, and must_staple flag meta_path = ocsp_cert_dir / "ocsp.json" try: meta = _extract_cert_metadata(pem_data, cert_name) + # Add fingerprint reference for debugging/verification + meta["fingerprint"] = cert_fp + meta["expires"] = datetime.now(timezone.utc).isoformat() + f" + {ttl}s" if ttl else "unknown" meta_path.write_text(json.dumps(meta, separators=(",", ":")), encoding="utf-8") meta_path.chmod(0o644) log_debug( - "โœ“ OCSP saved metadata for %s (serial=%s, must_staple=%s)", + "โœ“ OCSP saved metadata for %s (fingerprint: %s, serial=%s, must_staple=%s)", cert_name, + cert_fp[:16] + "...", meta.get("serial") or "unknown", meta.get("must_staple"), ) except Exception as e: log_debug("โš ๏ธ OCSP metadata write failed for %s: %s", cert_name, e) - # If metadata file is missing, recreate it - if not meta_path.exists(): - try: - meta = _extract_cert_metadata(pem_data, cert_name) - meta_path.write_text(json.dumps(meta, separators=(",", ":")), encoding="utf-8") - meta_path.chmod(0o644) - log_info( - "โœ“ OCSP recreated missing metadata for %s (serial=%s, must_staple=%s)", - cert_name, - meta.get("serial") or "unknown", - meta.get("must_staple"), - ) - except Exception as e: - log_warning("โš ๏ธ OCSP failed to recreate metadata for %s: %s", cert_name, e) - - # Create symlinks for each SAN so OCSP is found by any SNI name - _create_san_symlinks(pem_data, ocsp_path, cert_name) + # Note: With fingerprint-based storage, SAN symlinks are no longer needed + # The NGINX code directly looks up OCSP responses by certificate fingerprint + # (previous implementation left here for reference but not executed) except Exception as e: # Clean up any leftover temp files on error (existing ocsp.der remains intact) try: @@ -1791,6 +2094,9 @@ def refresh_job_lock(cert_name: str = "") -> None: log_info("๐Ÿ”„ OCSP refresh job started with differential update strategy (timeout in %d minutes%s)", JOB_TIMEOUT // 60, force_flags) + # Clean up stale lock files from previous crashed runs (defensive measure) + _cleanup_stale_locks() + # Acquire main lock for the entire OCSP refresh operation lock_fd_main = _acquire_cert_lock("main", timeout=300, stale_threshold=1800) @@ -1830,6 +2136,11 @@ def refresh_job_lock(cert_name: str = "") -> None: log_error("โŒ OCSP could not create cache directory %s: %s", CONFIGS_SSL_BASE, e) return 2 + # === Initialize sharded directory structure (16 subdirectories: 0-9, a-f) === + # This distributes OCSP responses across 16 directories for filesystem performance + if not _init_sharded_ocsp_directories(): + log_warning("โš ๏ธ OCSP sharded directory initialization had issues, will attempt to continue") + # === Clean up old temporary OCSP files (older than 5 minutes) === # Removes stale temp files from failed script runs before processing begins try: @@ -1896,8 +2207,25 @@ def refresh_job_lock(cert_name: str = "") -> None: new_le_certs = {k: v for k, v in le_certs.items() if k not in previous_ocsp_certs} existing_le_certs = {k: v for k, v in le_certs.items() if k in previous_ocsp_certs} + # ROBUSTNESS: Check disk files for "new" certs (database may be out of sync, cache cleared, or download aborted) + # If a "new" cert has valid cached OCSP on disk, move it to "existing" to avoid re-fetching + recategorized_certs = {} + for cert_name, pem_data in list(new_le_certs.items()): + fingerprint = _get_cert_pubkey_fingerprint(pem_data) + if fingerprint: + cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name, pem_data, fingerprint) + if cached_ttl is not None: + # Disk file exists with valid TTL - treat as unchanged, not new + log_info("โ„น๏ธ OCSP disk file exists for %s (marked new in DB): TTL=%ds, recategorizing to existing", cert_name, cached_ttl) + recategorized_certs[cert_name] = pem_data + del new_le_certs[cert_name] + existing_le_certs[cert_name] = pem_data + + if recategorized_certs: + log_info("โ„น๏ธ OCSP recategorized %d cert(s) from newโ†’existing due to valid disk cache", len(recategorized_certs)) + # For existing certs, check if certificate content has changed - previous_le_checksums = _get_cert_checksums(db, set(existing_le_certs.keys())) + previous_le_checksums = _get_cert_checksums(db, existing_le_certs) changed_le_certs = {} unchanged_le_certs = {} @@ -1907,9 +2235,9 @@ def refresh_job_lock(cert_name: str = "") -> None: previous_checksum = previous_le_checksums.get(cert_name) if previous_checksum is None: - # No previous checksum found, treat as changed (data loss or first refresh after upgrade) + # No previous checksum found, treat as changed (will be recategorized by robustness check if valid disk cache exists) changed_le_certs[cert_name] = pem_data - log_debug("โš ๏ธ OCSP no previous checksum found for %s, will refresh", cert_name) + log_debug("โ„น๏ธ OCSP no previous checksum found for %s (will check disk cache)", cert_name) elif current_checksum != previous_checksum: # Certificate content has changed changed_le_certs[cert_name] = pem_data @@ -1944,7 +2272,34 @@ def refresh_job_lock(cert_name: str = "") -> None: if res[1] is None and res[5] and res[6]: # ocsp_der is None AND ocsp_url is present AND was_attempted is True stashed_failures.append((cert_name, pem_data)) - # 2. Process changed certs (force refresh) + # 2. Process changed certs (robustness: skip force-fetch if valid OCSP cached on disk) + # This handles cases where checksums are missing but OCSP responses exist with fresh TTL + recategorized_changed = {} + for cert_name, pem_data in list(changed_le_certs.items()): + # Clean PEM before fingerprinting (custom certs may have private keys/noise) + cleaned_pem = _clean_pem(pem_data) + fingerprint = _get_cert_pubkey_fingerprint(cleaned_pem) + if fingerprint: + cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name, pem_data, fingerprint) + if cached_ttl is not None and cached_ttl > 0: + # Valid OCSP cached on disk - skip force refresh + log_info("โ„น๏ธ OCSP disk file exists for %s (marked changed due to missing checksum): TTL=%ds, skipping force-fetch", cert_name, cached_ttl) + recategorized_changed[cert_name] = pem_data + del changed_le_certs[cert_name] + unchanged_le_certs[cert_name] = pem_data + + if recategorized_changed: + log_info("โ„น๏ธ OCSP recategorized %d cert(s) from changedโ†’unchanged due to valid disk cache", len(recategorized_changed)) + # Add recategorized certs to all_ocsp_results so their checksums get persisted to database + # (even though we didn't fetch new OCSP responses, we need to record their checksums for future runs) + for cert_name, pem_data in sorted(recategorized_changed.items()): + pem_checksum = _calculate_cert_checksum(pem_data) + # Tuple: (cert_name, ocsp_der=None, ttl=0, checksum, pem_data, ocsp_url=None, was_attempted=False) + # We're not fetching, just recording the cert's checksum for differential tracking + all_ocsp_results.append((cert_name, None, 0, pem_checksum, pem_data, None, False)) + log_debug("โœ“ OCSP added recategorized cert %s to database persist list (checksum=%s)", cert_name, pem_checksum[:8]) + + # Process remaining changed certs with force refresh for cert_name, pem_data in sorted(changed_le_certs.items()): if check_job_timeout(f"changed LE cert {cert_name}"): break refresh_job_lock(cert_name) @@ -1988,7 +2343,7 @@ def refresh_job_lock(cert_name: str = "") -> None: log_info("โธ๏ธ OCSP stashed %d failed fetch(es), waiting 120 seconds before final retry...", len(stashed_failures)) # Use smaller sleeps to stay responsive and allow timeout checks - for i in range(120): + for _ in range(120): if check_job_timeout("during stashed retry wait"): break time.sleep(1) @@ -2027,7 +2382,7 @@ def refresh_job_lock(cert_name: str = "") -> None: file_name=last_refresh_key, data=str(now_ts).encode("utf-8"), job_name="ocsp-refresh", - checksum=hashlib.sha256(str(now_ts).encode("utf-8")).hexdigest(), + checksum=hashlib.sha256(str(now_ts).encode("utf-8")).hexdigest().lower(), ) log_debug("โœ“ OCSP updated last full refresh timestamp to %d", now_ts) except Exception as e: @@ -2049,6 +2404,9 @@ def refresh_job_lock(cert_name: str = "") -> None: if stats["errors"] == 0: log_info("โœ“ OCSP refresh job completed successfully") + # Check if all certificates are up-to-date (no fetches needed) + if stats["ocsp_fetched_responses"] == 0 and (stats["le_certs_processed"] + stats["custom_certs_processed"]) > 0: + log_info("โœ… All OCSP responses are current and valid - no updates needed") else: log_warning("โš ๏ธ OCSP refresh job completed with %d error(s)", stats["errors"]) @@ -2078,7 +2436,7 @@ def refresh_job_lock(cert_name: str = "") -> None: return 2 finally: # Always release the main lock - _release_cert_lock(lock_fd_main) + _release_cert_lock(lock_fd_main, "main") # run it diff --git a/src/common/core/ssl/plugin.json b/src/common/core/ssl/plugin.json index 9891b21160..8b9ff3506b 100644 --- a/src/common/core/ssl/plugin.json +++ b/src/common/core/ssl/plugin.json @@ -98,7 +98,7 @@ }, "SSL_USE_OCSP_STAPLING": { "context": "multisite", - "default": "yes", + "default": "no", "help": "Enable OCSP stapling for this site when certificates advertise an OCSP responder. Applies to Let's Encrypt and custom/self-signed certificates that support OCSP.", "id": "ssl-use-ocsp-stapling", "label": "Use OCSP stapling", From b916f354ea7c086f6f2d8f8b932954dca2f4dafc Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 19 Mar 2026 16:35:04 +0100 Subject: [PATCH 53/59] code fixes --- src/common/core/ssl/jobs/ocsp-refresh.py | 1165 +++++++++++++++++++--- 1 file changed, 1038 insertions(+), 127 deletions(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 7c32595389..a5d5ca67d1 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -16,6 +16,7 @@ import time from datetime import datetime, timezone, timedelta from pathlib import Path +from collections import OrderedDict from sys import exit as sys_exit, path as sys_path from typing import Any, Callable, Dict, List, Optional, Tuple, cast from urllib.error import HTTPError, URLError @@ -78,31 +79,46 @@ def _truncate_log(msg: str, max_len: int = 2048) -> str: # Wrapper functions to enforce 2048 character limit on all log messages def log_debug(msg: str, *args: Any, **kwargs: Any) -> None: """Log at DEBUG level with 2048 char limit.""" - formatted = (msg % args) if args else msg + try: + formatted = (msg % args) if args else msg + except Exception: + formatted = f"{msg} {args}" LOG.debug(_truncate_log(formatted), **kwargs) def log_info(msg: str, *args: Any, **kwargs: Any) -> None: """Log at INFO level with 2048 char limit.""" - formatted = (msg % args) if args else msg + try: + formatted = (msg % args) if args else msg + except Exception: + formatted = f"{msg} {args}" LOG.info(_truncate_log(formatted), **kwargs) def log_warning(msg: str, *args: Any, **kwargs: Any) -> None: """Log at WARNING level with 2048 char limit.""" - formatted = (msg % args) if args else msg + try: + formatted = (msg % args) if args else msg + except Exception: + formatted = f"{msg} {args}" LOG.warning(_truncate_log(formatted), **kwargs) def log_error(msg: str, *args: Any, **kwargs: Any) -> None: """Log at ERROR level with 2048 char limit.""" - formatted = (msg % args) if args else msg + try: + formatted = (msg % args) if args else msg + except Exception: + formatted = f"{msg} {args}" LOG.error(_truncate_log(formatted), **kwargs) def log_critical(msg: str, *args: Any, **kwargs: Any) -> None: """Log at CRITICAL level with 2048 char limit.""" - formatted = (msg % args) if args else msg + try: + formatted = (msg % args) if args else msg + except Exception: + formatted = f"{msg} {args}" LOG.critical(_truncate_log(formatted), **kwargs) @@ -111,6 +127,273 @@ def log_critical(msg: str, *args: Any, **kwargs: Any) -> None: # Use scheduler-managed cache directory (automatically synced from database on restart) CONFIGS_SSL_BASE = Path(os.sep, "var", "cache", "bunkerweb", "ssl") MIN_TTL = 4500 # 75 minutes: minimum safety threshold. Smart refresh uses max(MIN_TTL, 20% of response lifetime) +OPENSSL_BIN = "/usr/bin/openssl" + +_FINGERPRINT_RE = re.compile(r"^[0-9a-fA-F]{64}$") +_OCSP_RESPONDER_DNS_CACHE_MAX = 256 +DNS_CACHE_TTL = 300 # 5 minutes: limit poisoning / stale DNS impact during a job +_OCSP_RESPONDER_DNS_CACHE: "OrderedDict[str, Tuple[List[str], float]]" = OrderedDict() + + +def _is_safe_ip_str(ip_str: str) -> bool: + """ + Check whether an IP string is safe for outbound connections (SSRF defense). + Mirrors the allow/deny logic in `is_safe_url()` but for a raw IP. + """ + import ipaddress + + try: + ip_obj = ipaddress.ip_address(ip_str) + return not ( + ip_obj.is_unspecified + or ip_obj.is_private + or ip_obj.is_loopback + or ip_obj.is_link_local + or ip_obj.is_multicast + or ip_obj.is_reserved + ) + except Exception: + return False + + +def _normalize_fingerprint(fingerprint: Optional[str]) -> Optional[str]: + """ + Normalize an OCSP cache fingerprint into a lowercase 64-hex string. + Returns None if fingerprint does not match the expected format. + """ + if not fingerprint: + return None + fp = str(fingerprint).strip() + if not _FINGERPRINT_RE.match(fp): + return None + return fp.lower() + + +def _resolve_hostname_to_ips(hostname: str, port: int) -> List[str]: + """ + Resolve `hostname` to a unique list of IP strings (A/AAAA) for `port`. + Filters out non-IP / unsafe results (SSRF defense). + """ + import socket + + try: + addr_info = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + except Exception: + return [] + + ips: set = set() + for info in addr_info: + ip_str = info[4][0] + if isinstance(ip_str, str) and _is_safe_ip_str(ip_str): + ips.add(ip_str) + + return sorted(ips) + + +def _get_ocsp_responder_ips(ocsp_url: str, default_port: int) -> Tuple[str, List[str]]: + """ + Return (hostname, ips) for a given OCSP responder URL. + - hostname is the original DNS name (used for TLS SNI + cert validation). + - ips are resolved and safe IPs used for connecting. + """ + parsed = urlparse(ocsp_url) + hostname = parsed.hostname or "" + if not hostname: + return "", [] + + if hostname in _OCSP_RESPONDER_DNS_CACHE: + # Refresh LRU position + ips, cached_at = _OCSP_RESPONDER_DNS_CACHE[hostname] + if time.time() - cached_at < DNS_CACHE_TTL: + _OCSP_RESPONDER_DNS_CACHE.move_to_end(hostname) + return hostname, ips + # Expired โ€” re-resolve + try: + del _OCSP_RESPONDER_DNS_CACHE[hostname] + except KeyError: + pass + + port = parsed.port or default_port + ips = _resolve_hostname_to_ips(hostname, port) + _OCSP_RESPONDER_DNS_CACHE[hostname] = (ips, time.time()) + # Bounded cache to avoid unbounded memory growth. + while len(_OCSP_RESPONDER_DNS_CACHE) > _OCSP_RESPONDER_DNS_CACHE_MAX: + _OCSP_RESPONDER_DNS_CACHE.popitem(last=False) + return hostname, ips + + +def _read_http_response_body(conn: Any, max_body_bytes: int) -> Tuple[int, bytes]: + """ + Read an HTTP response from a (possibly SSL-wrapped) socket-like object. + Returns (status_code, body_bytes). Body is capped to `max_body_bytes`. + """ + status_code = 0 + body = b"" + buffer = b"" + + # Read headers + while b"\r\n\r\n" not in buffer: + chunk = conn.recv(4096) + if not chunk: + break + buffer += chunk + if len(buffer) > 1024 * 1024: + # Unreasonably large headers + raise RuntimeError("HTTP headers too large") + + if b"\r\n\r\n" not in buffer: + raise RuntimeError("Invalid HTTP response (missing header terminator)") + + header_bytes, remainder = buffer.split(b"\r\n\r\n", 1) + header_lines = header_bytes.split(b"\r\n") + if not header_lines: + raise RuntimeError("Invalid HTTP response (no status line)") + + # Example: b"HTTP/1.1 200 OK" + parts = header_lines[0].split(b" ", 2) + if len(parts) >= 2: + try: + status_code = int(parts[1].decode("ascii", errors="ignore")) + except Exception: + status_code = 0 + + # Parse Content-Length if present (best-effort) + content_length = None + for line in header_lines[1:]: + if b":" not in line: + continue + k, v = line.split(b":", 1) + if k.strip().lower() == b"content-length": + try: + content_length = int(v.strip().decode("ascii", errors="ignore")) + except Exception: + content_length = None + + body = remainder + if content_length is not None: + # Cap the amount we read to avoid memory issues + target = min(content_length, max_body_bytes) + while len(body) < target: + chunk = conn.recv(min(4096, target - len(body))) + if not chunk: + break + body += chunk + if len(body) >= target: + body = body[:target] + break + else: + # Read until EOF but cap the total body size + while len(body) < max_body_bytes: + chunk = conn.recv(min(4096, max_body_bytes - len(body))) + if not chunk: + break + body += chunk + body = body[:max_body_bytes] + + return status_code, body + + +def _post_ocsp_over_ip_with_sni( + ocsp_url: str, + ocsp_request_data: bytes, + ocsp_hostname: str, + ip_str: str, + timeout: int, +) -> Tuple[Optional[bytes], Optional[int], str]: + """ + Perform an OCSP HTTP POST by connecting to `ip_str` but using TLS SNI and + certificate validation for `ocsp_hostname`. + """ + parsed = urlparse(ocsp_url) + scheme = parsed.scheme.lower() + if scheme not in ("http", "https"): + return None, None, "" + + import http.client as http_client + import socket + + host_port = parsed.port or (443 if scheme == "https" else 80) + path = parsed.path or "/" + if parsed.query: + path = f"{path}?{parsed.query}" + + headers = { + "Content-Type": "application/ocsp-request", + "User-Agent": "BunkerWeb OCSP Fetcher (SSRF-Protected)", + "Connection": "close", + } + + conn: Any = None + sock: Optional[socket.socket] = None + wrapped_sock: Any = None + try: + if scheme == "https": + ssl_context = ssl.create_default_context() + # Use the original hostname for SNI + certificate validation, but connect to the resolved IP. + conn = http_client.HTTPSConnection( + ocsp_hostname, + port=host_port, + timeout=timeout, + context=ssl_context, + ) + sock = socket.create_connection((ip_str, host_port), timeout=timeout) + sock.settimeout(timeout) + wrapped_sock = ssl_context.wrap_socket(sock, server_hostname=ocsp_hostname) + conn.sock = wrapped_sock + else: + conn = http_client.HTTPConnection( + ocsp_hostname, + port=host_port, + timeout=timeout, + ) + sock = socket.create_connection((ip_str, host_port), timeout=timeout) + sock.settimeout(timeout) + conn.sock = sock + + conn.request("POST", path, body=ocsp_request_data, headers=headers) + resp = conn.getresponse() + status_code = resp.status + reason = getattr(resp, "reason", "") or "" + body = resp.read(102400) + try: + resp.close() + except Exception: + pass + + if status_code < 200 or status_code >= 300: + log_warning( + "โš ๏ธ OCSP HTTP status %d from %s (%s) for %s", + status_code, + ocsp_hostname, + ip_str, + ocsp_url, + ) + return None, status_code, reason + + return body, status_code, reason + finally: + try: + if conn is not None: + conn.close() + except Exception: + pass + + +def _log_ocsp_responder_dns_table() -> None: + """ + Log a consolidated uniq table (OCSP responder hostnames -> resolved IPs). + """ + if not _OCSP_RESPONDER_DNS_CACHE: + log_debug("โ„น๏ธ OCSP responder DNS table: no DNS lookups performed") + return + + # Print as a Markdown-like table for easy copy/paste + rows = [] + for hostname, (ips, _) in sorted(_OCSP_RESPONDER_DNS_CACHE.items()): + ip_list = ", ".join(ips) if ips else "" + rows.append(f"| {hostname} | {ip_list} |") + + log_info("๐Ÿ“‡ OCSP responder DNS table (uniq responders):\n| Responder Hostname | Resolved IPs |\n|---|---|\n%s", "\n".join(rows)) def _get_sharded_ocsp_path(fingerprint: str) -> Path: @@ -129,12 +412,13 @@ def _get_sharded_ocsp_path(fingerprint: str) -> Path: fingerprint: 089433bd22ca2b9536b597a9fc7ca86cdd1d1df0193caaa205043b40f1ea435b returns: /var/cache/bunkerweb/ssl/0/8/089433bd22ca2b9536b597a9fc7ca86cdd1d1df0193caaa205043b40f1ea435b/ """ - if not fingerprint or len(fingerprint) < 2: + normalized = _normalize_fingerprint(fingerprint) + if not normalized: return CONFIGS_SSL_BASE / "unknown" # Use first hex digit (0-f) and second hex digit (0-f) as tree levels - hex1 = fingerprint[0].lower() - hex2 = fingerprint[1].lower() - return CONFIGS_SSL_BASE / hex1 / hex2 / fingerprint + hex1 = normalized[0] + hex2 = normalized[1] + return CONFIGS_SSL_BASE / hex1 / hex2 / normalized def _init_sharded_ocsp_directories() -> bool: @@ -238,7 +522,9 @@ def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int Acquire a file-based lock for a specific certificate to prevent race conditions when multiple scheduler instances fetch OCSP responses concurrently. - Lock files are stored in /tmp/bunkerweb/ and use timestamp-based staleness detection. + Lock files are stored in a non-world-writable runtime directory (preferred: /run/bunkerweb, + then /var/run/bunkerweb, then a safe fallback under /tmp). Timestamp-based staleness detection + is used for identifying crashed/abandoned locks. If a lock file hasn't been updated in stale_threshold seconds, it's considered stale (indicating the previous process crashed or is hung). @@ -250,14 +536,74 @@ def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int Returns: File descriptor on success, None if lock unavailable or timed out """ - lock_dir = Path(os.sep, "tmp", "bunkerweb") - try: - lock_dir.mkdir(parents=True, exist_ok=True, mode=0o755) - except Exception: - return None + # Per-certificate lock is keyed by the certificate public key fingerprint. + # This avoids host/cert-name collisions and matches the fingerprint-based OCSP cache layout. + cert_fp = _normalize_fingerprint(cert_name) + lock_dir: Optional[Path] = None + lock_file: Optional[Path] = None + + # Fingerprint-based lock dir: root-folder/ssl/x/x (2-level sharding). + if cert_fp and cert_name != "main": + lock_dir = CONFIGS_SSL_BASE / cert_fp[0] / cert_fp[1] + lock_file = lock_dir / f"ocsp-{cert_fp}.lock" + try: + lock_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + if lock_dir.is_symlink(): + log_error("โŒ OCSP fingerprint lock directory %s is a symlink (refusing).", lock_dir) + return None + st = lock_dir.stat() + if st.st_mode & 0o022: + log_error( + "โŒ OCSP fingerprint lock directory %s has unsafe permissions (mode=%o).", + lock_dir, + st.st_mode & 0o777, + ) + return None + except Exception as e: + log_debug("โš ๏ธ OCSP could not prepare fingerprint lock directory %s: %s", lock_dir, e) + return None + + # Fallback (e.g. cert_name == "main" or unexpected non-fingerprint input): runtime lock dir. + if lock_file is None or lock_dir is None: + # Lock directories must not be attacker-controlled. + # `/tmp` is world-writable, so we prefer runtime dirs first and only fall back + # to `/tmp` if it's not a symlink and has safe permissions. + lock_dir_candidates = [ + Path(os.sep, "run", "bunkerweb"), + Path(os.sep, "var", "run", "bunkerweb"), + Path(os.sep, "tmp", "bunkerweb"), + ] + + for candidate in lock_dir_candidates: + try: + candidate.mkdir(parents=True, exist_ok=True, mode=0o700) + except Exception as e: + log_debug("โš ๏ธ OCSP lock candidate mkdir failed for %s: %s", candidate, e) + continue + + try: + if candidate.is_symlink(): + log_error("โŒ OCSP lock directory %s is a symlink (refusing).", candidate) + continue + + st = candidate.stat() + # Reject world/group-writable directories. + if st.st_mode & 0o022: + log_error("โŒ OCSP lock directory %s has unsafe permissions (mode=%o).", candidate, st.st_mode & 0o777) + continue + + lock_dir = candidate + break + except Exception as e: + log_debug("โš ๏ธ OCSP lock candidate stat failed for %s: %s", candidate, e) + continue + + if not lock_dir: + return None + + sanitized_name = _sanitize_filename(cert_name) + lock_file = lock_dir / f"ocsp-{sanitized_name}.lock" - sanitized_name = _sanitize_filename(cert_name) - lock_file = lock_dir / f"ocsp-{sanitized_name}.lock" start_time = time.time() while time.time() - start_time < timeout: @@ -269,15 +615,13 @@ def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int if age > stale_threshold: log_warning( "โš ๏ธ OCSP detected stale lock for %s (age %.0fs > threshold %ds). " - "Previous process may have crashed. Removing stale lock.", + "Previous process may have crashed. Proceeding without unlinking to avoid a race.", cert_name, age, stale_threshold ) - try: - lock_file.unlink() - except Exception: - pass # If we can't delete, we'll wait more - except Exception: - pass + # Do not unlink: another process may have just acquired the lock + # after updating mtime, and deleting here would be a race. + except Exception as e: + log_debug("โš ๏ธ OCSP stale lock detection failed for %s: %s", lock_file, e) try: fd = os.open(str(lock_file), os.O_CREAT | os.O_WRONLY, 0o600) @@ -287,8 +631,8 @@ def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int # Lock acquired successfullyโ€”write timestamp to detect stale locks try: os.write(fd, str(int(time.time())).encode()) - except Exception: - pass # Non-critical timestamp write + except Exception as e: + log_debug("โš ๏ธ OCSP could not write lock timestamp for %s: %s", cert_name, e) return fd except BlockingIOError: # Lock held by another process, close fd and exit or retry @@ -317,11 +661,13 @@ def _acquire_cert_lock(cert_name: str, timeout: int = 300, stale_threshold: int # Timeout reached: log clear message based on lock type if cert_name == "main": + chosen_lock_dir = str(lock_dir) if lock_dir else "/tmp/bunkerweb" log_error( "โŒ OCSP job timed out waiting for the main lock (%ds timeout). " "Another OCSP job is still running. This may indicate a hung or very slow job. " - "Check for stale lock files: rm -f /tmp/bunkerweb/ocsp*.lock", - timeout + "Check for stale lock files: rm -f %s/ocsp*.lock", + chosen_lock_dir, + timeout, ) else: log_warning( @@ -340,15 +686,45 @@ def _release_cert_lock(fd: Optional[int], cert_name: str = "undefined") -> None: if fd is None: return - sanitized_name = _sanitize_filename(cert_name) - lock_file = Path(os.sep, "tmp", "bunkerweb", f"ocsp-{sanitized_name}.lock") + cert_fp = _normalize_fingerprint(cert_name) + lock_file_name: str + lock_file: Optional[Path] = None + lock_dir_candidates = [ + Path(os.sep, "run", "bunkerweb"), + Path(os.sep, "var", "run", "bunkerweb"), + Path(os.sep, "tmp", "bunkerweb"), + ] + + # Fingerprint-based lock location: root-folder/ssl/x/x + if cert_fp and cert_name != "main": + lock_file = CONFIGS_SSL_BASE / cert_fp[0] / cert_fp[1] / f"ocsp-{cert_fp}.lock" + else: + sanitized_name = _sanitize_filename(cert_name) + lock_file_name = f"ocsp-{sanitized_name}.lock" try: + # Unlink while still holding the flock to avoid a release race where + # another process could acquire the same lock file name before we unlock. + if lock_file is not None: + try: + if lock_file.exists() and not lock_file.is_symlink(): + lock_file.unlink() + except Exception as e: + log_debug("โš ๏ธ OCSP fingerprint lock file unlink failed for %s: %s", lock_file, e) + else: + for lock_dir in lock_dir_candidates: + lock_file = lock_dir / lock_file_name + try: + if lock_file.exists() and not lock_file.is_symlink(): + lock_file.unlink() + break + except Exception as e: + log_debug("โš ๏ธ OCSP lock file unlink failed for %s: %s", lock_file, e) + continue + fcntl.flock(fd, fcntl.LOCK_UN) os.close(fd) - if lock_file.exists(): - lock_file.unlink() - except Exception: - pass + except Exception as e: + log_debug("โš ๏ธ OCSP could not release lock for %s: %s", cert_name, e) def _refresh_cert_lock(fd: Optional[int], cert_name: str) -> None: @@ -384,23 +760,124 @@ def _cleanup_stale_locks(stale_threshold: int = 300) -> None: Args: stale_threshold: Lock age (seconds) to consider stale (default 300 = 5 minutes) """ - lock_dir = Path(os.sep, "tmp", "bunkerweb") - if not lock_dir.exists(): - return - current_time = time.time() cleaned = 0 - for lock_file in lock_dir.glob("ocsp-*.lock"): + lock_dir_candidates = [ + Path(os.sep, "run", "bunkerweb"), + Path(os.sep, "var", "run", "bunkerweb"), + Path(os.sep, "tmp", "bunkerweb"), + ] + + for lock_dir in lock_dir_candidates: + if not lock_dir.exists(): + continue try: - mtime = lock_file.stat().st_mtime - age = current_time - mtime - if age > stale_threshold: - lock_file.unlink() - cleaned += 1 - log_debug("๐Ÿงน OCSP removed stale lock file %s (age %.0fs)", lock_file.name, age) + if lock_dir.is_symlink(): + continue + st = lock_dir.stat() + # Reject world/group-writable directories. + if st.st_mode & 0o022: + continue except Exception as e: - log_debug("โš ๏ธ OCSP could not clean lock file %s: %s", lock_file.name, e) + log_debug("โš ๏ธ OCSP lock directory stat failed for %s: %s", lock_dir, e) + continue + + for lock_file in lock_dir.glob("ocsp-*.lock"): + try: + mtime = lock_file.stat().st_mtime + age = current_time - mtime + if age > stale_threshold: + # Only unlink if we can also acquire the lock. + # This avoids the stale-cleanup unlink race where another process + # is legitimately holding the lock. + fd = None + try: + fd = os.open(str(lock_file), os.O_RDWR | os.O_CREAT, 0o600) + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + # Unlink while the lock is still held to avoid a tiny race + # between unlocking and deleting the file. + try: + lock_file.unlink() + except Exception as e: + log_debug("โš ๏ธ OCSP could not unlink stale lock file %s: %s", lock_file.name, e) + + cleaned += 1 + log_debug("๐Ÿงน OCSP removed stale lock file %s (age %.0fs)", lock_file.name, age) + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + fd = None + except BlockingIOError: + if fd is not None: + try: + os.close(fd) + except Exception as e: + log_debug("โš ๏ธ OCSP could not close stale lock fd: %s", e) + # Another process is holding the lock. + continue + except Exception as e: + if fd is not None: + try: + os.close(fd) + except Exception as close_e: + log_debug("โš ๏ธ OCSP could not close stale lock fd after exception: %s", close_e) + log_debug("โš ๏ธ OCSP failed to unlink stale lock file %s (age %.0fs): %s", lock_file.name, age, e) + continue + except Exception as e: + log_debug("โš ๏ธ OCSP could not clean lock file %s: %s", lock_file.name, e) + + # Fingerprint-based lock cleanup (root-folder/ssl/x/x) + if CONFIGS_SSL_BASE.is_dir(): + try: + for lock_dir in CONFIGS_SSL_BASE.glob("*/*"): + try: + if not lock_dir.is_dir() or lock_dir.is_symlink(): + continue + st = lock_dir.stat() + if st.st_mode & 0o022: + continue + except Exception as e: + log_debug("โš ๏ธ OCSP lock directory stat failed for fingerprint shard %s: %s", lock_dir, e) + continue + + for lock_file in lock_dir.glob("ocsp-*.lock"): + try: + mtime = lock_file.stat().st_mtime + age = current_time - mtime + if age <= stale_threshold: + continue + + fd = None + try: + fd = os.open(str(lock_file), os.O_RDWR | os.O_CREAT, 0o600) + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + lock_file.unlink() + except Exception as e: + log_debug("โš ๏ธ OCSP could not unlink stale fingerprint lock file %s: %s", lock_file.name, e) + + cleaned += 1 + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + fd = None + except BlockingIOError: + if fd is not None: + try: + os.close(fd) + except Exception as close_e: + log_debug("โš ๏ธ OCSP could not close stale fingerprint lock fd: %s", close_e) + continue + except Exception as e: + if fd is not None: + try: + os.close(fd) + except Exception as close_e: + log_debug("โš ๏ธ OCSP could not close stale fingerprint lock fd after exception: %s", close_e) + log_debug("โš ๏ธ OCSP failed to cleanup stale fingerprint lock file %s: %s", lock_file.name, e) + except Exception as e: + log_debug("โš ๏ธ OCSP could not stat stale fingerprint lock file %s: %s", lock_file, e) + except Exception as e: + log_debug("โš ๏ธ OCSP fingerprint lock cleanup failed: %s", e) if cleaned > 0: log_info("๐Ÿงน OCSP cleaned up %d stale lock file(s) from previous runs", cleaned) @@ -438,9 +915,8 @@ def is_safe_url(url: str) -> bool: for info in addr_info: ip_str = info[4][0] - ip_obj = ipaddress.ip_address(ip_str) - if ip_obj.is_private or ip_obj.is_loopback or ip_obj.is_link_local or ip_obj.is_multicast or ip_obj.is_reserved: - log_warning("โš ๏ธ SSRF protection: blocked request to %s (resolves to internal IP %s)", hostname, ip_str) + if not _is_safe_ip_str(ip_str): + log_warning("โš ๏ธ SSRF protection: blocked request to %s (resolves to unsafe IP %s)", hostname, ip_str) return False return True @@ -497,26 +973,125 @@ def _fetch_issuer_from_aia(leaf: x509.Certificate, cert_name: str = "") -> Optio continue log_debug("๐ŸŒ OCSP fetching issuer certificate from AIA: %s", issuer_url) - if not is_safe_url(issuer_url): - log_error("โŒ OCSP unsafe AIA issuer URL detected: %s", issuer_url) - return None + # Pin DNS resolution before connecting to mitigate DNS rebinding. + # We'll resolve the hostname to safe IPs and connect to those IPs directly + # while preserving SNI + certificate validation against the original hostname. + issuer_hostname = parsed.hostname + if not issuer_hostname: + continue - req = Request(issuer_url, headers={"User-Agent": "BunkerWeb OCSP Fetcher (SSRF-Protected)"}) - try: - # Use a short timeout so we don't hang - with urlopen(req, timeout=10) as response: - issuer_der = response.read(1_048_576) # Cap at 1MB to prevent memory exhaustion - # Try DER first (most common for caIssuers), then PEM + default_port = 443 if parsed.scheme == "https" else 80 + port = parsed.port or default_port + ips = _resolve_hostname_to_ips(issuer_hostname, port) + if not ips: + log_warning( + "โš ๏ธ OCSP could not resolve safe IPs for AIA issuer hostname %s (%s)", + issuer_hostname, + issuer_url, + ) + continue + + path = parsed.path or "/" + if parsed.query: + path = f"{path}?{parsed.query}" + + import http.client as http_client + import socket + + # Try each resolved IP (first successful response wins). + for ip_str in ips: + conn: Any = None + sock: Optional[socket.socket] = None + wrapped_sock: Any = None try: - return x509.load_der_x509_certificate(issuer_der) - except Exception: - return x509.load_pem_x509_certificate(issuer_der) - except HTTPError as e: - log_warning("โš ๏ธ OCSP HTTP %d error fetching issuer from AIA for %s from %s: %s", e.code, cert_name, issuer_url, e.reason) - except URLError as e: - log_warning("โš ๏ธ OCSP network error fetching issuer from AIA for %s from %s: %s", cert_name, issuer_url, e) - except Exception as e: - log_warning("โš ๏ธ OCSP failed to fetch issuer from AIA for %s from %s: %s", cert_name, issuer_url, e) + if parsed.scheme == "https": + ssl_context = ssl.create_default_context() + conn = http_client.HTTPSConnection( + issuer_hostname, + port=port, + timeout=10, + context=ssl_context, + ) + sock = socket.create_connection((ip_str, port), timeout=10) + sock.settimeout(10) + wrapped_sock = ssl_context.wrap_socket(sock, server_hostname=issuer_hostname) + conn.sock = wrapped_sock + else: + conn = http_client.HTTPConnection( + issuer_hostname, + port=port, + timeout=10, + ) + sock = socket.create_connection((ip_str, port), timeout=10) + sock.settimeout(10) + conn.sock = sock + + conn.request( + "GET", + path, + headers={"User-Agent": "BunkerWeb OCSP Fetcher (SSRF-Protected)", "Connection": "close"}, + ) + resp = conn.getresponse() + status_code = resp.status + if status_code < 200 or status_code >= 300: + log_warning( + "โš ๏ธ OCSP AIA issuer HTTP %d for %s from %s (%s -> %s)", + status_code, + cert_name, + issuer_url, + issuer_hostname, + ip_str, + ) + try: + resp.close() + except Exception as close_e: + log_debug( + "โš ๏ธ OCSP AIA issuer: failed to close HTTP response (%s -> %s): %s", + issuer_hostname, + ip_str, + close_e, + ) + continue + + issuer_der = resp.read(1_048_576) # Cap at 1MB + try: + resp.close() + except Exception as close_e: + log_debug( + "โš ๏ธ OCSP AIA issuer: failed to close HTTP response after read (%s -> %s): %s", + issuer_hostname, + ip_str, + close_e, + ) + + if not issuer_der: + continue + + # Try DER first (most common for caIssuers), then PEM + try: + return x509.load_der_x509_certificate(issuer_der) + except Exception: + return x509.load_pem_x509_certificate(issuer_der) + except Exception as e: + log_warning( + "โš ๏ธ OCSP failed to fetch issuer from AIA for %s from %s (%s -> %s): %s", + cert_name, + issuer_url, + issuer_hostname, + ip_str, + e, + ) + finally: + try: + if conn is not None: + conn.close() + except Exception as close_e: + log_debug( + "โš ๏ธ OCSP AIA issuer: failed to close HTTP connection (%s -> %s): %s", + issuer_hostname, + ip_str, + close_e, + ) except x509.ExtensionNotFound: log_debug("๐Ÿ”’ OCSP no AIA extension in leaf cert for %s", cert_name) except Exception as e: @@ -617,6 +1192,9 @@ def _parse_chain(pem_data: bytes, cert_name: str = "") -> Tuple[x509.Certificate # Default OCSP TTL fallback (1 day) if Next Update is missing DEFAULT_OCSP_TTL = 86400 +# Backoff duration after certain HTTP errors (e.g. responder temporarily bad) +HTTP_ERROR_BACKOFF_SECONDS = 300 # 5 minutes + def _ocsp_response_lifetimes(ocsp_response: x509_ocsp.OCSPResponse) -> Tuple[Optional[int], Optional[int]]: """ @@ -649,6 +1227,85 @@ def _ocsp_response_lifetimes(ocsp_response: x509_ocsp.OCSPResponse) -> Tuple[Opt return remaining, total_lifetime +def _write_ocsp_http_error_backoff( + cert_fp: Optional[str], + ocsp_url: str, + http_code: int, + reason: str, + backoff_seconds: int = HTTP_ERROR_BACKOFF_SECONDS, +) -> None: + """ + Persist an HTTP error backoff marker into ocsp.json so we can avoid + re-fetching for a short duration. + + This is used by _process_cert() to skip OCSP refresh attempts shortly + after certain transient/non-transient HTTP failures (e.g. 400/500). + """ + if not cert_fp: + return + + try: + ocsp_cert_dir = _get_sharded_ocsp_path(cert_fp) + ocsp_cert_dir.mkdir(parents=True, exist_ok=True) + meta_path = ocsp_cert_dir / "ocsp.json" + + now = datetime.now(timezone.utc) + retry_after = now + timedelta(seconds=backoff_seconds) + + meta = { + "fingerprint": cert_fp, + "ocsp_url": ocsp_url, + "http_error": {"code": http_code, "reason": reason or ""}, + "retry_after": retry_after.isoformat(), + # Keep "expires" for compatibility with the existing success metadata schema/logging. + "expires": retry_after.isoformat(), + "error_type": "http_backoff", + } + + meta_path.write_text(json.dumps(meta, separators=(",", ":")), encoding="utf-8") + meta_path.chmod(0o644) + except Exception as e: + log_debug("โš ๏ธ OCSP could not write http_error backoff metadata: %s", e) + + +def _get_http_error_backoff_remaining(cert_fp: Optional[str]) -> int: + """ + Return remaining seconds for an http_error backoff stored in ocsp.json. + Returns 0 if no backoff marker exists or if it is expired/invalid. + """ + if not cert_fp: + return 0 + + try: + meta_path = _get_sharded_ocsp_path(cert_fp) / "ocsp.json" + if not meta_path.is_file(): + return 0 + + raw = meta_path.read_text(encoding="utf-8") + meta = json.loads(raw) if raw else None + if not isinstance(meta, dict): + return 0 + + # Only trust backoff markers we wrote. + if meta.get("error_type") != "http_backoff": + return 0 + + retry_after = meta.get("retry_after") + if not retry_after or not isinstance(retry_after, str): + return 0 + + # Accept ISO-8601 timestamps produced by .isoformat() + retry_dt = datetime.fromisoformat(retry_after) + if retry_dt.tzinfo is None: + retry_dt = retry_dt.replace(tzinfo=timezone.utc) + + now = datetime.now(timezone.utc) + remaining = int((retry_dt - now).total_seconds()) + return max(0, remaining) + except Exception: + return 0 + + def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", timeout: int = 10) -> Tuple[Optional[bytes], int]: """ Fetch OCSP response using cryptography + urllib. @@ -660,6 +1317,9 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim log_error("โŒ OCSP failed to parse chain for %s: %s", cert_name, e) return None, 0 + # Used for writing backoff metadata on HTTP errors. + cert_fp = _get_cert_pubkey_fingerprint(pem_data) + try: # Try SHA256 first, fallback to SHA1 if responder returns non-successful status (RFC 6960 compatibility) ocsp_der = None @@ -677,24 +1337,52 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim ) # --- Build HTTP request --- - if not is_safe_url(ocsp_url): - log_error("โŒ OCSP unsafe responder URL detected: %s", ocsp_url) + parsed = urlparse(ocsp_url) + scheme = parsed.scheme.lower() if parsed.scheme else "" + if scheme not in ("http", "https"): + log_error("โŒ OCSP invalid responder scheme %s for %s: %s", scheme, cert_name, ocsp_url) return None, 0 - req = Request( - ocsp_url, - data=ocsp_request_data, - headers={ - "Content-Type": "application/ocsp-request", - "User-Agent": "BunkerWeb OCSP Fetcher (SSRF-Protected)", - }, - method="POST", - ) - # Explicitly use default SSL context with certificate verification enabled - ssl_context = ssl.create_default_context() - with urlopen(req, timeout=timeout, context=ssl_context) as response: - # cap response size at 100KB to prevent memory exhaustion from malicious responder - ocsp_der = response.read(102400) + default_port = 443 if scheme == "https" else 80 + ocsp_hostname, ips = _get_ocsp_responder_ips(ocsp_url, default_port=default_port) + if not ocsp_hostname or not ips: + log_error("โŒ OCSP could not resolve safe IPs for responder %s (host=%s)", cert_name, ocsp_hostname) + return None, 0 + + # Fetch by connecting to each resolved IP, while keeping TLS SNI for `ocsp_hostname`. + ocsp_der = None + for ip_str in ips: + http_code: Optional[int] = None + http_reason: str = "" + try: + ocsp_der, http_code, http_reason = _post_ocsp_over_ip_with_sni( + ocsp_url=ocsp_url, + ocsp_request_data=ocsp_request_data, + ocsp_hostname=ocsp_hostname, + ip_str=ip_str, + timeout=timeout, + ) + except Exception as e: + log_warning( + "โš ๏ธ OCSP fetch failed for %s (%s) -> %s using %s: %s", + cert_name, + ocsp_hostname, + ip_str, + alg_name, + e, + ) + ocsp_der = None + if ocsp_der: + break + + # For HTTP 400/500, persist a short retry backoff. + if http_code in (400, 500): + _write_ocsp_http_error_backoff( + cert_fp=cert_fp, + ocsp_url=ocsp_url, + http_code=http_code, + reason=http_reason, + ) if not ocsp_der: log_warning("โš ๏ธ OCSP empty response from %s for %s", ocsp_url, cert_name) @@ -715,6 +1403,13 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim except HTTPError as e: log_warning("โš ๏ธ OCSP HTTP %d error for %s using %s: %s", e.code, cert_name, alg_name, e.reason) + if e.code in (400, 500): + _write_ocsp_http_error_backoff( + cert_fp=cert_fp, + ocsp_url=ocsp_url, + http_code=e.code, + reason=e.reason or "", + ) ocsp_der = None continue except Exception as e: @@ -759,7 +1454,7 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim # -respin checks the response # -partial_chain allows the issuer to act as a trust anchor even if it's an intermediate cmd = [ - "openssl", "ocsp", + OPENSSL_BIN, "ocsp", "-respin", f_der.name, "-issuer", f_issuer.name, "-cert", f_leaf.name, @@ -767,6 +1462,10 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim "-partial_chain" ] + if not Path(OPENSSL_BIN).is_file(): + log_error("โŒ OCSP verification requires openssl at %s", OPENSSL_BIN) + return None, 0 + try: # Add a timeout so a stuck openssl process cannot hang the whole job p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=30) @@ -807,6 +1506,18 @@ def fetch_ocsp_response(pem_data: bytes, ocsp_url: str, cert_name: str = "", tim elif e.code >= 500: error_desc += f" (Server Error - {e.reason})" log_error("โŒ OCSP %s from responder for %s at %s", error_desc, cert_name, ocsp_url) + + # For HTTP 400/500, write a short backoff marker into ocsp.json so the job + # doesn't immediately retry the same failing responder for every certificate. + if e.code in (400, 500): + cert_fp = _get_cert_pubkey_fingerprint(pem_data) + _write_ocsp_http_error_backoff( + cert_fp=cert_fp, + ocsp_url=ocsp_url, + http_code=e.code, + reason=e.reason or "", + ) + return None, 0 except URLError as e: # Network error โ€” DNS, connection refused, timeout, SSL error, etc. @@ -1102,6 +1813,63 @@ def _load_le_certs_from_db(db: Any) -> Dict[str, bytes]: if not re.match(r"^[A-Za-z0-9_.*-]+$", cert_name): log_warning("โš ๏ธ OCSP sanitization: skipping unsafe cert_name from tarball: %s", cert_name) continue + + # Defensive hardening: validate symlink target stays within the expected + # archive// subtree. We only read from the in-memory tarball, + # but a malicious tar entry could otherwise point to unexpected content. + try: + linkname = getattr(member, "linkname", "") + if not isinstance(linkname, str) or not linkname: + log_warning("โš ๏ธ OCSP LE tarball: missing/invalid symlink target for %s", cert_name) + continue + # Disallow absolute targets. + if linkname.startswith("/"): + log_warning("โš ๏ธ OCSP LE tarball: symlink target is absolute for %s", cert_name) + continue + # Ensure the normalized resolved target remains within archive//. + import posixpath + from pathlib import PurePosixPath + + base_dir = PurePosixPath(member.name).parent + combined = posixpath.normpath(str(base_dir / PurePosixPath(linkname))) + if combined.startswith("./"): + combined = combined[2:] + if combined.startswith("../"): + log_warning("โš ๏ธ OCSP LE tarball: symlink target escapes archive for %s", cert_name) + continue + + # Be tolerant to tarball prefix differences by validating against the + # extracted cache base where "live//" comes from. + member_parts = PurePosixPath(member.name).parts + live_idx = None + for i, part in enumerate(member_parts): + if part == "live": + live_idx = i + break + if live_idx is None: + log_warning("โš ๏ธ OCSP LE tarball: could not locate 'live' base for %s", cert_name) + continue + + root_prefix_parts = [p for p in member_parts[:live_idx] if p not in (".", "")] + root_prefix = "/".join(root_prefix_parts) + + archive_prefix = ( + f"{root_prefix}/archive/{cert_name}/" if root_prefix else f"archive/{cert_name}/" + ) + live_prefix = f"{root_prefix}/live/{cert_name}/" if root_prefix else f"live/{cert_name}/" + + if not (combined.startswith(archive_prefix) or combined.startswith(live_prefix)): + log_warning( + "โš ๏ธ OCSP LE tarball: symlink target not in expected subtree for %s (got=%s, expected_prefixes=%s,%s)", + cert_name, + combined, + archive_prefix, + live_prefix, + ) + continue + except Exception as e: + log_warning("โš ๏ธ OCSP LE tarball: failed to validate symlink target for %s: %s", cert_name, e) + continue try: f = tar.extractfile(member) if f: @@ -1199,19 +1967,19 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: file_name = entry.get("file_name", "") if not file_name.startswith("ocsp/") or not entry.get("data"): continue - - cert_name_raw = file_name[len("ocsp/"):] - # Prevent path traversal from crafted database entries - if not re.match(r"^[A-Za-z0-9_.*-]+$", cert_name_raw): - log_warning("โš ๏ธ OCSP sanitization: skipping unsafe cert_name from database: %s", cert_name_raw) + cert_or_fp_raw = file_name[len("ocsp/"):] + # We only restore actual OCSP responses from database. + # Marker entries are stored as: ocsp/{cert_name} (data contains the fingerprint). + # Response entries are stored as: ocsp/{fingerprint}. + fingerprint = _normalize_fingerprint(cert_or_fp_raw) + if not fingerprint: continue - - sanitized_name = _sanitize_filename(cert_name_raw) + db_data = entry["data"] db_checksum = (entry.get("checksum") or hashlib.sha256(db_data).hexdigest()).lower() try: - ocsp_cert_dir = CONFIGS_SSL_BASE / sanitized_name + ocsp_cert_dir = _get_sharded_ocsp_path(fingerprint) ocsp_path = ocsp_cert_dir / "ocsp.der" if ocsp_path.is_file(): @@ -1219,10 +1987,10 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: disk_checksum = hashlib.sha256(ocsp_path.read_bytes()).hexdigest().lower() if disk_checksum == db_checksum: ok_count += 1 - log_debug("โœ“ OCSP disk file for %s matches database (checksum=%s)", cert_name_raw, db_checksum[:8]) + log_debug("โœ“ OCSP disk file for %s matches database (checksum=%s)", fingerprint, db_checksum[:8]) continue # Checksum mismatch โ€” replace with database version - log_info("๐Ÿ”„ OCSP disk file for %s has wrong checksum (disk=%s, db=%s), replacing", cert_name_raw, disk_checksum[:8], db_checksum[:8]) + log_info("๐Ÿ”„ OCSP disk file for %s has wrong checksum (disk=%s, db=%s), replacing", fingerprint, disk_checksum[:8], db_checksum[:8]) ocsp_path.write_bytes(db_data) ocsp_path.chmod(0o644) replaced_count += 1 @@ -1232,9 +2000,9 @@ def restore_ocsp_from_database(db: Optional[Any] = None) -> None: ocsp_path.write_bytes(db_data) ocsp_path.chmod(0o644) restored_count += 1 - log_debug("โœ“ OCSP restored cached response for %s from database", cert_name_raw) + log_debug("โœ“ OCSP restored cached response for %s from database", fingerprint) except Exception as e: - log_debug("โš ๏ธ OCSP could not sync cache for %s: %s", cert_name_raw, e) + log_debug("โš ๏ธ OCSP could not sync cache for %s: %s", fingerprint, e) if restored_count > 0 or replaced_count > 0: log_info("โœ“ OCSP sync complete: restored=%d, replaced=%d, unchanged=%d", restored_count, replaced_count, ok_count) @@ -1264,18 +2032,19 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta # uses the normalized/safe version. pem_data = _clean_pem(pem_data) cert_checksum = _calculate_cert_checksum(pem_data) + cert_fp = _get_cert_pubkey_fingerprint(pem_data) # Check per-service OCSP stapling setting if not _is_ocsp_enabled_for_service(service_name): log_debug("๐Ÿงน OCSP stapling disabled for service %s, cleaning up cache for %s", service_name, cert_name) - cleanup_ocsp_cache(db, cert_name) + cleanup_ocsp_cache(db, cert_name, fingerprint=cert_fp) stats["le_certs_skipped"] = stats.get("le_certs_skipped", 0) + 1 return (cert_name, None, 0, cert_checksum, pem_data, None, False) # Check if certificate checksum matches what we have in database # Checksums are stored by fingerprint (cert_checksum/{fingerprint}), so compute it first cached_checksum = None - fingerprint_for_checksum = _get_cert_pubkey_fingerprint(pem_data) + fingerprint_for_checksum = cert_fp if db and fingerprint_for_checksum: try: checksum_key = f"cert_checksum/{fingerprint_for_checksum}" @@ -1303,10 +2072,24 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta log_debug("๐ŸŒ OCSP responder for certificate %s: %s", cert_name, ocsp_url) # === Compute fingerprint for sharded path lookup === - fingerprint = _get_cert_pubkey_fingerprint(pem_data) + fingerprint = cert_fp if not fingerprint: log_warning("โš ๏ธ OCSP could not compute fingerprint for %s, treating as new fetch", cert_name) + # === HTTP error backoff (400/500) === + # If we previously got an HTTP 400/500 for this certificate's OCSP responder, + # avoid hammering the same endpoint until the backoff expires. + if not force_fetch: + backoff_remaining = _get_http_error_backoff_remaining(cert_fp) + if backoff_remaining > 0: + stats["ocsp_http_error_backoff_skipped"] = stats.get("ocsp_http_error_backoff_skipped", 0) + 1 + log_info( + "โธ๏ธ OCSP HTTP backoff active for %s (ocsp.json retry_after in %ds), skipping fetch", + cert_name, + backoff_remaining, + ) + return (cert_name, None, backoff_remaining, cert_checksum, pem_data, ocsp_url, False) + # === Check if cached OCSP response is still fresh (disk + database) === # This two-tier check handles aborted downloads, database inconsistencies, and ephemeral storage cached_ttl, total_lifetime = get_cached_ocsp_ttl(cert_name, pem_data, fingerprint) @@ -1356,7 +2139,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta "๐Ÿงน OCSP cached response for %s is already expired and refresh failed. Cleaning up invalid cache.", cert_name, ) - cleanup_ocsp_cache(db, cert_name) + cleanup_ocsp_cache(db, cert_name, fingerprint=cert_fp) elif current_ttl <= current_refresh_threshold: log_warning( "๐Ÿšจ OCSP CRITICAL: Cached response for %s is near expiration (TTL=%ds <= refresh_threshold=%ds [20%% of %ds]) and refresh failed. " @@ -1366,7 +2149,7 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta current_refresh_threshold, total_lifetime, ) - cleanup_ocsp_cache(db, cert_name) + cleanup_ocsp_cache(db, cert_name, fingerprint=cert_fp) else: log_warning( "โš ๏ธ OCSP could NOT refresh response for %s, keeping existing cache (TTL=%ds, above threshold %ds) for now", @@ -1533,7 +2316,12 @@ def _is_ocsp_enabled_for_service(service_name: str) -> bool: return value.lower() == "yes" -def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None, purge_db: bool = True) -> None: +def cleanup_ocsp_cache( + db: Optional[Any] = None, + cert_name: Optional[str] = None, + fingerprint: Optional[str] = None, + purge_db: bool = True, +) -> None: """ Remove OCSP stapling leftovers from disk and optionally database. If cert_name is provided, only clean up that specific certificate. @@ -1549,10 +2337,39 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None # Clean up a single certificate's OCSP cache sanitized_name = _sanitize_filename(cert_name) - ocsp_dir = CONFIGS_SSL_BASE / sanitized_name - if ocsp_dir.is_dir(): - shutil.rmtree(ocsp_dir, ignore_errors=True) - log_info("๐Ÿงน OCSP removed cache directory for %s", cert_name) + resolved_fp = _normalize_fingerprint(fingerprint) + + # If fingerprint wasn't provided, try to resolve it from the marker stored in DB. + # Marker entries are stored as: ocsp/{cert_name} with data = fingerprint (ASCII hex). + if not resolved_fp and db and purge_db: + try: + marker_data = db.get_job_cache_file( + file_name=f"ocsp/{sanitized_name}", + job_name="ocsp-refresh", + ) + if marker_data: + decoded = marker_data.decode("utf-8", errors="ignore").strip() + resolved_fp = _normalize_fingerprint(decoded) + except Exception as e: + log_debug("โš ๏ธ OCSP could not resolve fingerprint marker for %s: %s", cert_name, e) + resolved_fp = None + + # Disk cleanup: prefer fingerprint-based sharded storage. + if resolved_fp: + ocsp_fp_dir = _get_sharded_ocsp_path(resolved_fp) + if ocsp_fp_dir.is_dir(): + shutil.rmtree(ocsp_fp_dir, ignore_errors=True) + log_info("๐Ÿงน OCSP removed sharded cache for %s (fingerprint=%s)", cert_name, resolved_fp[:16] + "...") + else: + ocsp_dir = CONFIGS_SSL_BASE / sanitized_name + if ocsp_dir.is_dir(): + shutil.rmtree(ocsp_dir, ignore_errors=True) + log_info("๐Ÿงน OCSP removed cache directory for %s", cert_name) + + # Best-effort legacy flat cache cleanup (kept for backward compatibility). + legacy_ocsp_dir = CONFIGS_SSL_BASE / sanitized_name + if legacy_ocsp_dir.is_dir(): + shutil.rmtree(legacy_ocsp_dir, ignore_errors=True) # Also remove any SAN symlinks that point to this cert's OCSP cache if CONFIGS_SSL_BASE.is_dir(): @@ -1571,15 +2388,27 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None entry.rmdir() except OSError: pass - except Exception: - pass + except Exception as e: + log_debug("โš ๏ธ OCSP failed to remove SAN symlink %s: %s", san_ocsp, e) if db and purge_db: try: - # Delete both response and checksum from database - db.delete_job_cache(file_name=f"ocsp/{sanitized_name}", job_name="ocsp-refresh") - db.delete_job_cache(file_name=f"cert_checksum/{sanitized_name}", job_name="ocsp-refresh") - log_debug("๐Ÿงน OCSP database records removed for %s", cert_name) + # Always remove the cert-name marker (differential tracking). + db.delete_job_cache( + file_name=f"ocsp/{sanitized_name}", + job_name="ocsp-refresh", + ) + # If we resolved a fingerprint, also remove the corresponding response+checksum. + if resolved_fp: + db.delete_job_cache( + file_name=f"ocsp/{resolved_fp}", + job_name="ocsp-refresh", + ) + db.delete_job_cache( + file_name=f"cert_checksum/{resolved_fp}", + job_name="ocsp-refresh", + ) + log_debug("๐Ÿงน OCSP database records removed for %s (fingerprint resolved=%s)", cert_name, bool(resolved_fp)) except Exception as e: log_debug("๐Ÿงน OCSP could not remove database entry for %s: %s", cert_name, e) else: @@ -1593,16 +2422,16 @@ def cleanup_ocsp_cache(db: Optional[Any] = None, cert_name: Optional[str] = None try: ocsp_file.unlink() log_info("๐Ÿงน OCSP removed cached response %s", ocsp_file) - except Exception: - pass + except Exception as e: + log_debug("โš ๏ธ OCSP failed to unlink ocsp.der file %s: %s", ocsp_file, e) # Try to remove ocsp.json metadata if it exists meta_file = Path(root) / "ocsp.json" if meta_file.is_file(): try: meta_file.unlink() - except Exception: - pass + except Exception as e: + log_debug("โš ๏ธ OCSP failed to unlink ocsp.json metadata %s: %s", meta_file, e) # Remove empty directories (walk in reverse order ensures we clean up bottom-up) try: @@ -1641,14 +2470,30 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: # Build set of valid cert names from LE certs valid_cert_names: set = set(le_certs.keys()) + valid_fingerprints: set = set() + + # Build the set of valid OCSP cache fingerprints to safely prune sharded disk entries. + for _, pem_data in le_certs.items(): + try: + fp = _get_cert_pubkey_fingerprint(_clean_pem(pem_data)) + if fp: + valid_fingerprints.add(fp) + except Exception: + continue # Add custom cert names if db: try: custom_certs = _load_custom_certs_from_db(db) - for key in custom_certs: + for key, pem_data in custom_certs.items(): # key already starts with customcert- valid_cert_names.add(key) + try: + fp = _get_cert_pubkey_fingerprint(_clean_pem(pem_data)) + if fp: + valid_fingerprints.add(fp) + except Exception: + continue except Exception as e: log_warning("โš ๏ธ OCSP could not load custom certs for orphan check: %s", e) return # Don't clean up if we can't verify what's valid @@ -1668,10 +2513,38 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: if not file_name.startswith("ocsp/"): continue cert_name_raw = file_name[len("ocsp/"):] - if cert_name_raw not in valid_cert_names: - log_info("๐Ÿงน OCSP removing orphaned entry for deleted service: %s", cert_name_raw) + + # Database layout: + # - marker entries: ocsp/ (data = fingerprint) + # - response entries: ocsp/ (data = ocsp.der bytes) + # Only delete response entries if the fingerprint is not present anymore. + if cert_name_raw in valid_cert_names: + continue + + resolved_fp = _normalize_fingerprint(cert_name_raw) + if resolved_fp and resolved_fp in valid_fingerprints: + continue + + if resolved_fp: + # Orphaned OCSP response entry: remove DB record + checksum (disk cleanup is fingerprint-aware below). + try: + db.delete_job_cache( + file_name=f"ocsp/{resolved_fp}", + job_name="ocsp-refresh", + ) + db.delete_job_cache( + file_name=f"cert_checksum/{resolved_fp}", + job_name="ocsp-refresh", + ) + log_info("๐Ÿงน OCSP removed orphaned fingerprint DB entries: fp=%s", resolved_fp[:16] + "...") + orphaned_count += 1 + except Exception as e: + log_debug("โš ๏ธ OCSP failed to remove orphaned fingerprint DB entries fp=%s: %s", resolved_fp, e) + else: + # Orphaned marker entry (deleted service): reuse cleanup logic. + log_info("๐Ÿงน OCSP removing orphaned marker entry for deleted service: %s", cert_name_raw) cleanup_ocsp_cache(db, cert_name_raw) - orphaned_count = orphaned_count + 1 + orphaned_count += 1 except Exception as e: log_warning("โš ๏ธ OCSP could not check database for orphaned entries: %s", e) @@ -1692,6 +2565,23 @@ def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: cleanup_ocsp_cache(db, cert_name_raw) orphaned_count = orphaned_count + 1 + # Check sharded fingerprint cache directories for orphaned OCSP files. + # Sharded layout is: /var/cache/bunkerweb/ssl////ocsp.der + if CONFIGS_SSL_BASE.is_dir() and valid_fingerprints: + for root, dirs, files in os.walk(CONFIGS_SSL_BASE, topdown=False): + if "ocsp.der" not in files: + continue + fingerprint = _normalize_fingerprint(Path(root).name) + if not fingerprint: + continue + if fingerprint not in valid_fingerprints: + try: + log_info("๐Ÿงน OCSP removing orphaned sharded disk cache for fingerprint: %s", fingerprint) + shutil.rmtree(root, ignore_errors=True) + orphaned_count = orphaned_count + 1 + except Exception: + continue + if orphaned_count > 0: log_info("๐Ÿงน OCSP cleaned up %d orphaned cache entries for deleted services", orphaned_count) stats["orphaned_cleaned"] = orphaned_count @@ -1760,11 +2650,13 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic continue # At this point, data should be a DER-encoded OCSP response - sanitized_name = _sanitize_filename(cert_name_raw) + fingerprint = _normalize_fingerprint(cert_name_raw) + if not fingerprint: + continue verify_count += 1 # Check if file exists on disk - ocsp_cert_dir = CONFIGS_SSL_BASE / sanitized_name + ocsp_cert_dir = _get_sharded_ocsp_path(fingerprint) ocsp_path = ocsp_cert_dir / "ocsp.der" # Check if database response is already expired @@ -1778,8 +2670,8 @@ def _verify_and_restore_ocsp_files(db: Optional[Any] = None, stats: Optional[dic try: db.delete_job_cache(file_name=file_name, job_name="ocsp-refresh") log_debug("๐Ÿงน OCSP removed expired database entry %s", file_name) - except Exception: - pass + except Exception as e: + log_debug("โš ๏ธ OCSP could not remove expired database entry %s: %s", file_name, e) continue except Exception as e: log_warning("โš ๏ธ OCSP could not parse response from database for %s during verification: %s", cert_name_raw, e) @@ -1960,8 +2852,18 @@ def _persist_ocsp_results_to_disk( stats["errors"] = stats.get("errors", 0) + 1 continue - # Acquire lock to prevent race conditions with concurrent OCSP fetches - lock_fd = _acquire_cert_lock(cert_name) + # Acquire lock to prevent race conditions with concurrent OCSP fetches. + # Lock is keyed by certificate public key fingerprint (not hostname/cert_name). + lock_fd = _acquire_cert_lock(cert_fp) + if lock_fd is None: + # Avoid writing ocsp.der / ocsp.json without a lock. + stats["errors"] = stats.get("errors", 0) + 1 + log_warning( + "โญ๏ธ OCSP skipping disk write for %s (fingerprint: %s) due to lock acquisition failure", + cert_name, + cert_fp[:16] + "...", + ) + continue try: # Create sharded directory using fingerprint (0-f distribution) # Example: /var/cache/bunkerweb/ssl/0/089433bd22ca2b9536b597a9fc7ca86cdd1d1df0193caaa205043b40f1ea435b/ocsp.der @@ -2025,7 +2927,8 @@ def _persist_ocsp_results_to_disk( log_info("โ„น๏ธ OCSP kept existing OCSP response file for %s (new fetch failed)", cert_name) stats["errors"] = stats.get("errors", 0) + 1 finally: - _release_cert_lock(lock_fd) + if lock_fd is not None: + _release_cert_lock(lock_fd, cert_fp) except Exception as e: log_error("โŒ OCSP exception while writing response for %s to disk: %s", cert_name, e) stats["errors"] = stats.get("errors", 0) + 1 @@ -2099,6 +3002,11 @@ def refresh_job_lock(cert_name: str = "") -> None: # Acquire main lock for the entire OCSP refresh operation lock_fd_main = _acquire_cert_lock("main", timeout=300, stale_threshold=1800) + if lock_fd_main is None: + # If we can't acquire the main lock, avoid concurrent refresh runs. + # This prevents overlapping disk/database writes and reduces race conditions. + log_error("โŒ OCSP could not acquire main lock; another instance may be running") + return 1 # Initialize database connection โ€” this is our primary data source db = None @@ -2398,6 +3306,9 @@ def refresh_job_lock(cert_name: str = "") -> None: log_info("๐Ÿ” OCSP running end-of-job verification and restoration...") _verify_and_restore_ocsp_files(db, stats) + # Log a consolidated view of all OCSP responders resolved during this run. + _log_ocsp_responder_dns_table() + # Decide exit status based on results if stats["errors"] > 0: status = 2 From b8814b36c52e1488e868c1a116fcdd1b5d7714cf Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 19 Mar 2026 16:45:19 +0100 Subject: [PATCH 54/59] fix path issue --- src/common/core/ssl/jobs/ocsp-refresh.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index a5d5ca67d1..8d2c3f7388 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -3079,7 +3079,12 @@ def refresh_job_lock(cert_name: str = "") -> None: # Wait for scheduler's directory purge to finish after service restart, # then restore cached OCSP responses from database to disk. # This handles ephemeral storage and post-restart cache directory cleanup. - ocsp_files_exist = any((CONFIGS_SSL_BASE / d / "ocsp.der").is_file() for d in CONFIGS_SSL_BASE.iterdir()) if CONFIGS_SSL_BASE.is_dir() else False + # Note: OCSP files may live under legacy flat dirs (ssl//ocsp.der) or + # tree-sharded dirs (ssl////ocsp.der). Only checking + # direct children of ssl/ misses the sharded layout and falsely logs "no cached files". + ocsp_files_exist = ( + any(CONFIGS_SSL_BASE.rglob("ocsp.der")) if CONFIGS_SSL_BASE.is_dir() else False + ) if not ocsp_files_exist: log_info("๐Ÿ”„ OCSP no cached files on disk, waiting 2s for scheduler purge to finish before restoring from database...") time.sleep(2) From 42598172e2877493b96c615f88a7c992f5f560ba Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 19 Mar 2026 17:44:05 +0100 Subject: [PATCH 55/59] fix closing section error --- src/common/confs/server-http/ssl-certificate-lua.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 8dd513f494..15e53fe3cc 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -1214,6 +1214,7 @@ ssl_certificate_by_lua_block { end -- end cert_key_pairs check end -- end call_plugin result end -- end new_plugin result + end else safe_log(DEBUG, "skipped execution of " .. plugin_id .. " because method ssl_certificate() is not defined") end -- end ssl_certificate defined check From 1a6aab529719aefdc6ecea93e99b010fd0412919 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 19 Mar 2026 17:55:00 +0100 Subject: [PATCH 56/59] fix key mapping --- .../server-http/ssl-certificate-lua.conf | 101 ++++++++++-------- 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 15e53fe3cc..d790247997 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -1148,13 +1148,28 @@ ssl_certificate_by_lua_block { -- Clear old certificates before setting new ones (fail-safe) pcall(clear_certs) - -- Parse multiple certificates and keys (support RSA, ECDSA, PQC, etc.) - local cert_pem = ret.status[1] or "" - local key_pem = ret.status[2] or "" - - local certs = parse_pem_certificates(cert_pem) - local keys = parse_pem_keys(key_pem) - local cert_key_pairs = pair_certs_and_keys(certs, keys) + -- Some plugins return PEM strings; others return already-parsed ngx.ssl objects. + local cert_data = ret.status[1] + local key_data = ret.status[2] + + local cert_key_pairs = {} + local ocsp_possible = false + + if type(cert_data) == "string" and type(key_data) == "string" then + -- Parse multiple certificates and keys (support RSA, ECDSA, PQC, etc.) + local cert_pem = cert_data + local key_pem = key_data + + local certs = parse_pem_certificates(cert_pem) + local keys = parse_pem_keys(key_pem) + cert_key_pairs = pair_certs_and_keys(certs, keys) + ocsp_possible = true + else + -- Assume ngx.ssl.parse_pem_* was already called by the plugin + cert_key_pairs = { + { cert = cert_data, key = key_data, cert_id = "parsed", matched = true }, + } + end if #cert_key_pairs == 0 then safe_log(ERR, "No valid certificate/key pairs extracted from " .. plugin_id) @@ -1163,47 +1178,45 @@ ssl_certificate_by_lua_block { local all_certs_set = true - -- Loop through each certificate/key pair (supports RSA, ECDSA, PQC, etc.) - for idx, pair in ipairs(cert_key_pairs) do - safe_log(DEBUG, "Setting certificate #" .. idx .. " (" .. pair.cert_id .. ") from " .. plugin_id .. (pair.matched and " [matched key]" or " [unmatched - no key found]")) + -- Loop through each certificate/key pair + for idx, pair in ipairs(cert_key_pairs) do + safe_log(DEBUG, "Setting certificate #" .. idx .. " (" .. pair.cert_id .. ") from " .. plugin_id .. (pair.matched and " [matched key]" or " [unmatched - no key found]")) - -- Try to set the certificate - local ok_cert, err_cert = set_cert(pair.cert) - if not ok_cert then - safe_log(ERR, "error while setting certificate #" .. idx .. " (" .. pair.cert_id .. ") from " .. plugin_id .. ": " .. (err_cert or "unknown")) - all_certs_set = false - else - -- Certificate set successfully, now try to set private key (if available) - local ok_key = true - local err_key = nil - - if pair.key then - ok_key, err_key = set_priv_key(pair.key) - if not ok_key then - safe_log(ERR, "error while setting private key #" .. idx .. " (" .. pair.cert_id .. ") from " .. plugin_id .. ": " .. (err_key or "unknown")) - all_certs_set = false - end + -- Try to set the certificate + local ok_cert, err_cert = set_cert(pair.cert) + if not ok_cert then + safe_log(ERR, "error while setting certificate #" .. idx .. " (" .. pair.cert_id .. ") from " .. plugin_id .. ": " .. (err_cert or "unknown")) + all_certs_set = false else - safe_log(NOTICE, "Certificate #" .. idx .. " (" .. pair.cert_id .. ") has no matched private key - continuing without key (may fail during TLS handshake)") - end - - -- Only set OCSP if key was successfully set (or not needed) - if ok_key then - -- Certificate and key both set successfully (or cert-only is acceptable) - -- Try to set OCSP stapling from cache for this specific certificate (fail-safe) - safe_log(DEBUG, "Setting OCSP for certificate #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ")") - local ok_ocsp, ocsp_cert_acceptable = pcall(set_ocsp_from_cache, pair.cert) - if not ok_ocsp then - safe_log(ERR, "OCSP function error for cert #" .. idx .. " (" .. pair.cert_id .. "): " .. tostring(ocsp_cert_acceptable)) - elseif ocsp_cert_acceptable == false then - -- OCSP-Must-Staple requirement not met - log but continue (failsafe) - safe_log(ERR, "OCSP-Must-Staple requirement not met for cert #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ") - proceeding anyway") + -- Certificate set successfully, now try to set private key (if available) + local ok_key = true + local err_key = nil + + if pair.key then + ok_key, err_key = set_priv_key(pair.key) + if not ok_key then + safe_log(ERR, "error while setting private key #" .. idx .. " (" .. pair.cert_id .. ") from " .. plugin_id .. ": " .. (err_key or "unknown")) + all_certs_set = false + end else - safe_log(DEBUG, "OCSP loaded successfully for certificate #" .. idx .. " (" .. pair.cert_id .. ")") + safe_log(NOTICE, "Certificate #" .. idx .. " (" .. pair.cert_id .. ") has no matched private key - continuing without key (may fail during TLS handshake)") + end + + -- Only set OCSP if we can compute the fingerprint (PEM input) and key was set successfully + if ok_key and ocsp_possible and type(pair.cert) == "string" then + safe_log(DEBUG, "Setting OCSP for certificate #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ")") + local ok_ocsp, ocsp_cert_acceptable = pcall(set_ocsp_from_cache, pair.cert) + if not ok_ocsp then + safe_log(ERR, "OCSP function error for cert #" .. idx .. " (" .. pair.cert_id .. "): " .. tostring(ocsp_cert_acceptable)) + elseif ocsp_cert_acceptable == false then + -- OCSP-Must-Staple requirement not met - log but continue (failsafe) + safe_log(ERR, "OCSP-Must-Staple requirement not met for cert #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ") - proceeding anyway") + else + safe_log(DEBUG, "OCSP loaded successfully for certificate #" .. idx .. " (" .. pair.cert_id .. ")") + end end - end -- end if ok_key - end -- end ok_cert - end -- end for each cert/key pair + end -- end ok_cert + end -- end for each cert/key pair if all_certs_set then safe_log(INFO, "all " .. #cert_key_pairs .. " certificate(s) and key(s) set by " .. plugin_id) From 5309d9bbfa01c413c7e04a87c7fc186959b3839e Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 19 Mar 2026 20:07:21 +0100 Subject: [PATCH 57/59] fix calc of fingerprints --- .../server-http/ssl-certificate-lua.conf | 433 ++++++++++++++---- src/common/core/customcert/customcert.lua | 35 +- src/common/core/letsencrypt/letsencrypt.lua | 31 +- src/common/core/ssl/jobs/ocsp-refresh.py | 23 +- 4 files changed, 421 insertions(+), 101 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index d790247997..ebbd5c5e70 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -175,9 +175,26 @@ ssl_certificate_by_lua_block { local tostring = tostring local insert = table.insert local lower = string.lower + -- Convert binary data to lowercase hex (replacement for ngx.encode_base16) + local function to_hex(bin) + if not bin then + return nil + end + local t = {} + for i = 1, #bin do + t[i] = string.format("%02x", string.byte(bin, i)) + end + return table.concat(t) + end local match = string.match local concat = table.concat + -- Lua pattern note: `string.match()` uses Lua patterns, not regex. + -- So we validate fingerprints with length + allowed-characters checks. + local function is_fp64_lower_hex(fp) + return type(fp) == "string" and #fp == 64 and fp:match("^[0-9a-f]+$") ~= nil + end + -- Helper: parse multiple PEM certificates from a single string -- Returns array of individual PEM certificate strings local function parse_pem_certificates(pem_data) @@ -283,15 +300,24 @@ ssl_certificate_by_lua_block { return identifier end - -- Helper: extract public key fingerprint (SHA256) from certificate or private key (supports PEM and DER) + -- Helper: extract public key fingerprint (SHA256) from certificate or private key PEM string -- Uses native lua-resty-openssl for zero-shell-overhead fingerprinting + -- IMPORTANT: Only accepts PEM/DER strings. Do NOT pass cdata pointers from ngx.ssl.parse_pem_cert() + -- as they are STACK_OF(X509)* โ€” casting to X509* and calling resty.openssl causes a C-level segfault. local function get_pubkey_fingerprint(cert_data, is_key) - if not cert_data or #cert_data == 0 then + if not cert_data then + return nil + end + if type(cert_data) ~= "string" then + safe_log(DEBUG, "OCSP get_pubkey_fingerprint: rejecting non-string input (type=" .. type(cert_data) .. ") to prevent segfault") + return nil + end + if #cert_data == 0 then return nil end local fingerprint = nil - pcall(function() + local ok_rest, rest_err = pcall(function() local x509 = require("resty.openssl.x509") local pkey = require("resty.openssl.pkey") local digest_lib = require("resty.openssl.digest") @@ -313,7 +339,7 @@ ssl_certificate_by_lua_block { local digest_ctx = digest_lib.new('sha256') if digest_ctx then digest_ctx:update(pubkey_der) - fingerprint = ngx.encode_base16(digest_ctx:final()):lower() + fingerprint = to_hex(digest_ctx:final()):lower() end else -- Load as certificate (auto-detects PEM/DER) @@ -322,19 +348,163 @@ ssl_certificate_by_lua_block { safe_log(DEBUG, "OCSP failed to load certificate for fingerprinting") return end - -- Use built-in pubkey_digest method to compute SHA256 of public key - local digest_bytes, err = cert_obj:pubkey_digest('sha256') - if digest_bytes then - fingerprint = ngx.encode_base16(digest_bytes):lower() - else - safe_log(DEBUG, "OCSP pubkey_digest failed: " .. (err or "unknown")) + + -- Compute SHA256 of public key SubjectPublicKeyInfo DER (matches ocsp-refresh.py). + -- `cert_obj:pubkey_digest()` has shown mismatches vs the Python implementation. + -- So we export the pubkey in DER and hash it ourselves. + local digest_ctx = digest_lib.new('sha256') + if digest_ctx then + local pub = nil + local ok_pub, pub_err = pcall(function() + -- resty.openssl.x509 exposes get_pubkey(), which returns a pkey object. + pub = cert_obj:get_pubkey() + end) + if not ok_pub then + safe_log(DEBUG, "OCSP export pubkey: cert_obj:get_pubkey() failed: " .. tostring(pub_err)) + end + if pub then + local pubkey_der, err = pub:tostring("public", "DER") + if pubkey_der then + safe_log(DEBUG, "OCSP export pubkey SPKI DER ok (len=" .. tostring(#pubkey_der) .. ")") + digest_ctx:update(pubkey_der) + fingerprint = to_hex(digest_ctx:final()):lower() + else + safe_log(DEBUG, "OCSP failed to export cert pubkey SPKI DER: " .. (err or "unknown")) + end + else + safe_log(DEBUG, "OCSP export pubkey: cert_obj:get_pubkey() returned nil") + end + end + + -- Fallback (previous behavior) if export+hash failed + if not fingerprint then + safe_log(DEBUG, "OCSP export+hash SPKI DER failed; falling back to cert_obj:pubkey_digest('sha256')") + local digest_bytes, err = cert_obj:pubkey_digest('sha256') + if digest_bytes then + fingerprint = to_hex(digest_bytes):lower() + else + safe_log(DEBUG, "OCSP pubkey_digest failed: " .. (err or "unknown")) + end end end end) + if not ok_rest then + safe_log(DEBUG, "OCSP get_pubkey_fingerprint resty.openssl error: " .. tostring(rest_err)) + end + + -- Fallback to OpenSSL if lua-resty-openssl fingerprinting failed. + -- This is slower and should normally not be needed, but it prevents silent OCSP failures. + if not fingerprint then + pcall(function() + local worker_id = ngx.worker.id() or "0" + local tmp_cert = "/tmp/ocsp_pubkey_fp_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" + local f = io.open(tmp_cert, "w") + if not f then + return + end + f:write(cert_data) + f:close() + + -- Compute SHA256 over SubjectPublicKeyInfo DER (matches ocsp-refresh.py logic) + -- Output example: "SHA2-256(stdin)= " + local cmd = "openssl x509 -pubkey -noout -in " .. tmp_cert .. " 2>/dev/null" + .. " | openssl pkey -pubin -outform DER 2>/dev/null" + .. " | openssl dgst -sha256 2>/dev/null" + local handle = io.popen(cmd, "r") + if not handle then + pcall(function() os.remove(tmp_cert) end) + return + end + + local out = handle:read("*a") + handle:close() + pcall(function() os.remove(tmp_cert) end) + + if out then + local fp = out:match("=\\s*([0-9A-Fa-f]{64})") + if fp then + fingerprint = lower(fp) + end + end + end) + end return fingerprint end + -- OCSP uses the fingerprint to locate /var/cache/bunkerweb/ssl/{hex1}/{hex2}/{fingerprint}/ocsp.der. + -- The Python job (`ocsp-refresh.py`) computes this fingerprint by hashing the public key + -- in SubjectPublicKeyInfo DER. + -- + -- `resty.openssl` may produce a different hash for the "cert_obj:pubkey_digest('sha256')" path + -- depending on OpenSSL bindings/encoding. + -- + -- To guarantee cache hits, we remap the "resty fingerprint" to the "OpenSSL/Python-compatible" + -- fingerprint once, and then reuse that mapping from `internalstore`. + local function get_ocsp_pubkey_fingerprint(cert_pem) + local resty_fp = get_pubkey_fingerprint(cert_pem, false) + if not resty_fp then + return nil + end + if not is_fp64_lower_hex(resty_fp) then + return resty_fp + end + + local map_key = "TLS:SSL:ocsp_fp_map:" .. resty_fp + local ok_map, mapped = pcall(function() + return internalstore:get(map_key, true) + end) + if ok_map and is_fp64_lower_hex(mapped) then + return mapped + end + + -- Compute OpenSSL/Python-compatible fingerprint and cache mapping. + local fingerprint_openssl = nil + pcall(function() + local worker_id = ngx.worker.id() or "0" + local tmp_cert = "/tmp/ocsp_pubkey_fp_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" + local f = io.open(tmp_cert, "w") + if not f then + return + end + f:write(cert_pem) + f:close() + + -- Capture stderr in output to help debugging. + local cmd = "openssl x509 -pubkey -noout -in " .. tmp_cert .. " 2>&1" + .. " | openssl pkey -pubin -outform DER 2>/dev/null" + .. " | openssl dgst -sha256 2>/dev/null" + local handle = io.popen(cmd, "r") + if not handle then + return + end + local out = handle:read("*a") + handle:close() + pcall(function() os.remove(tmp_cert) end) + + if out then + local fp = out:match("=\\s*([0-9A-Fa-f]{64})") + if fp then + fingerprint_openssl = lower(fp) + end + if not fingerprint_openssl then + safe_log(DEBUG, "OCSP OpenSSL fingerprint parse failed; openssl_out=" .. tostring(out):sub(1, 120)) + end + end + end) + + if is_fp64_lower_hex(fingerprint_openssl) then + safe_log(DEBUG, "OCSP get_ocsp_pubkey_fingerprint resty=" .. resty_fp:sub(1, 16) .. "... openssl=" .. fingerprint_openssl:sub(1, 16) .. "...") + pcall(function() + internalstore:set(map_key, fingerprint_openssl, 86400, true) -- 24h mapping cache + end) + return fingerprint_openssl + end + + safe_log(DEBUG, "OCSP get_ocsp_pubkey_fingerprint openssl_fp=nil (resty=" .. resty_fp:sub(1, 16) .. "...)") + return resty_fp + end + -- Helper: pair certificates with their corresponding keys by matching public key fingerprints -- Returns array of {cert = "...", key = "...", cert_id = "...", matched = true/false} tables local function pair_certs_and_keys(certs, keys) @@ -975,7 +1145,7 @@ ssl_certificate_by_lua_block { -- Helper: set OCSP stapling from cache (fingerprint-based lookup) -- Input: cert_pem - certificate in PEM format string (or nil if from parsed plugin object) -- Returns: true if cert is acceptable, false if OCSP-Must-Staple requirement not met - local function set_ocsp_from_cache(cert_pem) + local function set_ocsp_from_cache(cert_pem, cert_fp_hint) safe_log(DEBUG, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) if not is_ocsp_stapling_enabled() then @@ -983,14 +1153,37 @@ ssl_certificate_by_lua_block { return true -- Certificate acceptable even without OCSP end - if not cert_pem or type(cert_pem) ~= "string" or #cert_pem == 0 then - safe_log(DEBUG, "OCSP no certificate PEM available (parsed object from ngx.ssl), skipping fingerprint-based lookup") - return true -- Cannot compute fingerprint without PEM string + -- Plugins may pass either: + -- - a PEM string (custom parsing) + -- - an opaque cdata cert pointer returned by ngx.ssl.parse_pem_cert() + -- We handle both so OCSP can work with letsencrypt/customcert plugins. + if not cert_pem then + safe_log(DEBUG, "OCSP no certificate provided, skipping") + return true + end + if type(cert_pem) == "string" and #cert_pem == 0 then + safe_log(DEBUG, "OCSP empty PEM certificate, skipping fingerprint lookup") + return true + end + + -- SAFETY: cdata pointers from ngx.ssl.parse_pem_cert() are STACK_OF(X509)*, not X509*. + -- Passing them to resty.openssl or ffi.cast("X509*", ...) causes a C-level segfault + -- that pcall cannot catch. Reject cdata early and log a clear message. + if type(cert_pem) ~= "string" then + safe_log(NOTICE, "OCSP cannot fingerprint cdata certificate pointer (type=" .. type(cert_pem) .. ") for " .. (server_name or "unknown") .. " - pass original PEM string for OCSP stapling") + return true end -- Extract certificate metadata (Must-Staple) + -- At this point cert_pem is guaranteed to be a non-empty string (cdata rejected above) + -- cert_pem may contain the full chain; extract leaf for metadata extraction + local leaf_for_meta = cert_pem + local meta_certs = parse_pem_certificates(cert_pem) + if meta_certs and #meta_certs > 0 then + leaf_for_meta = meta_certs[1] + end local has_must_staple = false - local ok_read, cert_meta = pcall(read_certificate_metadata, cert_pem) + local ok_read, cert_meta = pcall(read_certificate_metadata, leaf_for_meta) if ok_read and cert_meta then has_must_staple = cert_meta.must_staple if has_must_staple then @@ -998,95 +1191,124 @@ ssl_certificate_by_lua_block { end end - -- Compute certificate public key fingerprint (unique identifier) - local cert_fp = get_pubkey_fingerprint(cert_pem, false) - if not cert_fp then - safe_log(NOTICE, "OCSP could not compute certificate fingerprint for " .. (server_name or "unknown")) - if has_must_staple then - safe_log(ERR, "OCSP-Must-Staple required but cannot verify certificate - will attempt TLS anyway") + -- Compute certificate public key fingerprint(s) (unique identifiers). + -- `cert_pem` may contain the full chain; the real leaf is not always the first PEM block. + -- To handle that safely, we try multiple certificate-block fingerprints until we find an existing `ocsp.der`. + local resp = nil + local tried_fps = {} + local candidate_fps = {} + + -- Prefer pre-computed fingerprint hint from the plugin (index 5 in letsencrypt/customcert internalstore). + -- Still validate by file existence (handshake-time chain ordering can differ). + if cert_fp_hint and type(cert_fp_hint) == "string" then + local fp = tostring(cert_fp_hint):lower() + if is_fp64_lower_hex(fp) and not tried_fps[fp] then + candidate_fps[#candidate_fps + 1] = fp + tried_fps[fp] = true end - return true end - safe_log(DEBUG, "OCSP certificate fingerprint: " .. cert_fp:sub(1, 16) .. "...") - - -- Use tree-structured sharded path: /var/cache/bunkerweb/ssl/{hex1}/{hex2}/{full_fingerprint}/ocsp.der - -- hex1 and hex2 are individual hex digits creating 16ร—16 = 256 directory tree - local hex1 = cert_fp:sub(1, 1) - local hex2 = cert_fp:sub(2, 2) - local ocsp_path = "/var/cache/bunkerweb/ssl/" .. hex1 .. "/" .. hex2 .. "/" .. cert_fp .. "/ocsp.der" - local resp = nil + local chain_certs = parse_pem_certificates(cert_pem) + if not chain_certs or #chain_certs == 0 then + chain_certs = { cert_pem } + end + if #chain_certs > 1 then + safe_log(DEBUG, "OCSP will try multiple certificate blocks from chain of " .. #chain_certs .. " certificate(s)") + end - -- 1) Try local shared dict lookup first (per-worker cache) - local cache_key = "TLS:SSL:ocsp:" .. cert_fp - local ok_cache, cache_result = pcall(function() - return internalstore:get(cache_key, true) - end) - if ok_cache then - if cache_result then - resp = cache_result - safe_log(DEBUG, "OCSP found response in shared memory cache") + -- Add fingerprints for each certificate block (may include the real leaf and intermediates) + for idx, cert_block_pem in ipairs(chain_certs) do + local fp = get_ocsp_pubkey_fingerprint(cert_block_pem) + if fp and not tried_fps[fp] then + candidate_fps[#candidate_fps + 1] = fp + tried_fps[fp] = true end + if type(fp) == "string" then + safe_log(DEBUG, "OCSP cert block #" .. idx .. " pem_len=" .. tostring(#cert_block_pem) .. " fp=" .. fp:sub(1, 16) .. "...") + else + safe_log(DEBUG, "OCSP cert block #" .. idx .. " pem_len=" .. tostring(cert_block_pem and #cert_block_pem or 0) .. " fp=nil") + end + end + + if cert_fp_hint and type(cert_fp_hint) == "string" then + safe_log(DEBUG, "OCSP cert_fp_hint=" .. (cert_fp_hint:sub(1, 16) .. "...")) else - safe_log(DEBUG, "OCSP shared memory lookup skipped: " .. tostring(cache_result)) + safe_log(DEBUG, "OCSP cert_fp_hint=nil") end - -- 2) Fallback to disk file (fingerprint-based direct lookup) - if not resp then - safe_log(DEBUG, "OCSP looking for cached response at: " .. ocsp_path) - pcall(function() - local f, err = io.open(ocsp_path, "rb") - if f then - local data = f:read("*a") - f:close() - - if data and #data > 0 then - safe_log(DEBUG, "OCSP found response file (" .. #data .. " bytes)") - - -- Load optional metadata written by ocsp-refresh.py - local meta_path = base_path .. cert_fp .. "/ocsp.json" - local meta = nil - pcall(function() - local fmeta = io.open(meta_path, "r") - if fmeta then - local meta_raw = fmeta:read("*a") - fmeta:close() - if meta_raw and #meta_raw > 0 then - local ok_decode, decoded = pcall(cjson.decode, meta_raw) - if ok_decode and type(decoded) == "table" then - meta = decoded - safe_log(DEBUG, "OCSP metadata loaded (expires: " .. tostring(meta.expires or "unknown") .. ")") - end - end - end - end) + safe_log(DEBUG, "OCSP candidate_fps count=" .. tostring(#candidate_fps)) - -- Since we matched by fingerprint, the OCSP response is guaranteed to match - resp = data - safe_log(INFO, "OCSP loaded response from: " .. ocsp_path) + if #candidate_fps == 0 then + safe_log(NOTICE, "OCSP could not compute any certificate fingerprints for " .. (server_name or "unknown")) + if has_must_staple then + safe_log(ERR, "OCSP-Must-Staple required but cannot verify certificate - will attempt TLS anyway") + end + return true + end - -- Cache in shared memory (TTL 300s) - pcall(function() - local ok_cache_set = internalstore:set(cache_key, resp, 300, true) - if not ok_cache_set then - safe_log(DEBUG, "OCSP failed to cache response in shared memory") - end - end) - else - safe_log(DEBUG, "OCSP response file is empty: " .. ocsp_path) - end - else - if err and lower(err):find("permission denied", 1, true) then - safe_log(ERR, "OCSP permission denied reading " .. ocsp_path) + for _, cert_fp in ipairs(candidate_fps) do + safe_log(DEBUG, "OCSP trying candidate fingerprint raw=" .. tostring(cert_fp)) + if is_fp64_lower_hex(cert_fp) then + safe_log(DEBUG, "OCSP trying fingerprint: " .. cert_fp:sub(1, 16) .. "...") + + -- Use tree-structured sharded path: /var/cache/bunkerweb/ssl/{hex1}/{hex2}/{full_fingerprint}/ocsp.der + local hex1 = cert_fp:sub(1, 1) + local hex2 = cert_fp:sub(2, 2) + local ocsp_path = "/var/cache/bunkerweb/ssl/" .. hex1 .. "/" .. hex2 .. "/" .. cert_fp .. "/ocsp.der" + local cache_key = "TLS:SSL:ocsp:" .. cert_fp + safe_log(DEBUG, "OCSP lookup ocsp_path=" .. ocsp_path) + + -- 1) Try local shared dict lookup first (per-worker cache) + local ok_cache, cache_result = pcall(function() + return internalstore:get(cache_key, true) + end) + if ok_cache and cache_result then + resp = cache_result + safe_log(DEBUG, "OCSP found response in shared memory cache for fp=" .. cert_fp:sub(1, 16) .. "...") + break + end + + -- 2) Fallback to disk file + safe_log(DEBUG, "OCSP looking for cached response at: " .. ocsp_path) + pcall(function() + local f, err = io.open(ocsp_path, "rb") + if f then + local data = f:read("*a") + f:close() + + if data and #data > 0 then + resp = data + safe_log(INFO, "OCSP loaded response from: " .. ocsp_path) + + -- Cache in shared memory (TTL 300s) + pcall(function() + local ok_cache_set = internalstore:set(cache_key, resp, 300, true) + if not ok_cache_set then + safe_log(DEBUG, "OCSP failed to cache response in shared memory") + end + end) + else + safe_log(DEBUG, "OCSP response file is empty: " .. ocsp_path) + end else - safe_log(DEBUG, "OCSP response file not found: " .. ocsp_path) + if err and lower(err):find("permission denied", 1, true) then + safe_log(ERR, "OCSP permission denied reading " .. ocsp_path) + else + safe_log(DEBUG, "OCSP response file not found: " .. ocsp_path) + end end + end) + + if resp then + break end - end) + else + safe_log(DEBUG, "OCSP skipping invalid fingerprint format: " .. tostring(cert_fp)) + end end if not resp then - safe_log(DEBUG, "OCSP response not found for certificate with fingerprint: " .. cert_fp:sub(1, 16) .. "...") + safe_log(DEBUG, "OCSP response not found for any tried certificate public-key fingerprint(s) (server_name=" .. (server_name or "unknown") .. ")") if has_must_staple then safe_log(ERR, "OCSP-Must-Staple required but OCSP response not found - will attempt TLS anyway") end @@ -1166,8 +1388,30 @@ ssl_certificate_by_lua_block { ocsp_possible = true else -- Assume ngx.ssl.parse_pem_* was already called by the plugin + -- Check if original PEM strings are available at indices 3,4 (e.g., from letsencrypt plugin) + local orig_cert_pem = nil + if ret.status[3] and type(ret.status[3]) == "string" then + orig_cert_pem = ret.status[3] + safe_log(DEBUG, "Original PEM certificate available for OCSP fingerprinting (" .. #orig_cert_pem .. " bytes)") + end + -- Optional fingerprint hint at index 5 (computed in letsencrypt/customcert load_data) + local orig_cert_fp = nil + if ret.status[5] and type(ret.status[5]) == "string" then + local fp = ret.status[5]:lower():match("^[0-9a-f]{64}$") + if fp then + orig_cert_fp = fp + safe_log(DEBUG, "Original cert fingerprint available for OCSP (" .. fp:sub(1, 16) .. "... )") + end + end cert_key_pairs = { - { cert = cert_data, key = key_data, cert_id = "parsed", matched = true }, + { + cert = cert_data, + key = key_data, + cert_id = "parsed", + matched = true, + cert_pem_for_ocsp = orig_cert_pem, + cert_fp_for_ocsp = orig_cert_fp, + }, } end @@ -1202,10 +1446,13 @@ ssl_certificate_by_lua_block { safe_log(NOTICE, "Certificate #" .. idx .. " (" .. pair.cert_id .. ") has no matched private key - continuing without key (may fail during TLS handshake)") end - -- Only set OCSP if we can compute the fingerprint (PEM input) and key was set successfully - if ok_key and ocsp_possible and type(pair.cert) == "string" then - safe_log(DEBUG, "Setting OCSP for certificate #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ")") - local ok_ocsp, ocsp_cert_acceptable = pcall(set_ocsp_from_cache, pair.cert) + -- Attempt OCSP stapling whenever key + cert are available. + -- Use original PEM string for OCSP fingerprinting (safe); fall back to cdata only if no PEM available. + if ok_key then + local ocsp_cert = pair.cert_pem_for_ocsp or pair.cert + local ocsp_fp_hint = pair.cert_fp_for_ocsp + safe_log(DEBUG, "Setting OCSP for certificate #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ", cert_type=" .. type(ocsp_cert) .. ")") + local ok_ocsp, ocsp_cert_acceptable = pcall(set_ocsp_from_cache, ocsp_cert, ocsp_fp_hint) if not ok_ocsp then safe_log(ERR, "OCSP function error for cert #" .. idx .. " (" .. pair.cert_id .. "): " .. tostring(ocsp_cert_acceptable)) elseif ocsp_cert_acceptable == false then diff --git a/src/common/core/customcert/customcert.lua b/src/common/core/customcert/customcert.lua index 4c927b934e..19deecfa8e 100644 --- a/src/common/core/customcert/customcert.lua +++ b/src/common/core/customcert/customcert.lua @@ -15,6 +15,18 @@ local get_multiple_variables = utils.get_multiple_variables local has_variable = utils.has_variable local read_files = utils.read_files +-- Convert binary data to lowercase hex (replacement for ngx.encode_base16) +local function to_hex(bin) + if not bin then + return nil + end + local t = {} + for i = 1, #bin do + t[i] = string.format("%02x", string.byte(bin, i)) + end + return table.concat(t) +end + function customcert:initialize(ctx) -- Call parent initialize plugin.initialize(self, "customcert", ctx) @@ -122,11 +134,30 @@ function customcert:load_data(data, server_name) if not priv_key then return false, "error while parsing pem priv key : " .. err end - -- Cache data + + -- Pre-compute leaf certificate fingerprint for OCSP lookup (same strategy as letsencrypt) + -- This avoids relying on runtime PEM parsing inside `ssl-certificate-lua.conf`. + local cert_fingerprint = nil + pcall(function() + local x509 = require("resty.openssl.x509") + -- Keep in sync with letsencrypt.lua regex: take the first cert PEM block. + local leaf_pem = data[1]:match("(%-%-%-%-BEGIN CERTIFICATE%-%-%-%-.-%-%-%-%-END CERTIFICATE%-%-%-%-)") + if leaf_pem then + local cert_obj = x509.new(leaf_pem) + if cert_obj then + local digest_bytes = cert_obj:pubkey_digest("sha256") + if digest_bytes then + cert_fingerprint = to_hex(digest_bytes):lower() + end + end + end + end) + + -- Cache data (preserve original PEM strings at indices 3,4 for OCSP fingerprinting) for key in server_name:gmatch("%S+") do local cache_key = "plugin_customcert_" .. key local ok - ok, err = self.internalstore:set(cache_key, { cert_chain, priv_key }, nil, true) + ok, err = self.internalstore:set(cache_key, { cert_chain, priv_key, data[1], data[2], cert_fingerprint }, nil, true) if not ok then return false, "error while setting data into internalstore : " .. err end diff --git a/src/common/core/letsencrypt/letsencrypt.lua b/src/common/core/letsencrypt/letsencrypt.lua index f5e8eb9907..404ec4189a 100644 --- a/src/common/core/letsencrypt/letsencrypt.lua +++ b/src/common/core/letsencrypt/letsencrypt.lua @@ -34,6 +34,18 @@ local sort = table.sort local lower = string.lower local gsub = string.gsub +-- Convert binary data to lowercase hex (replacement for ngx.encode_base16) +local function to_hex(bin) + if not bin then + return nil + end + local t = {} + for i = 1, #bin do + t[i] = string.format("%02x", string.byte(bin, i)) + end + return table.concat(t) +end + -- Mirror certbot-new wildcard grouping so certificate identifiers stay in sync. local function sanitize_domain_labels(domain) if not domain or domain == "" then @@ -485,11 +497,26 @@ function letsencrypt:load_data(data, server_name) if not priv_key then return false, "error while parsing pem priv key : " .. err end - -- Cache data + -- Pre-compute leaf certificate fingerprint for OCSP lookup (avoids PEM parsing on every TLS handshake) + local cert_fingerprint = nil + pcall(function() + local x509 = require("resty.openssl.x509") + local leaf_pem = data[1]:match("(%-%-%-%-%-BEGIN CERTIFICATE%-%-%-%-%-.-%-%-%-%-%-END CERTIFICATE%-%-%-%-%-)") + if leaf_pem then + local cert_obj = x509.new(leaf_pem) + if cert_obj then + local digest_bytes = cert_obj:pubkey_digest("sha256") + if digest_bytes then + cert_fingerprint = to_hex(digest_bytes):lower() + end + end + end + end) + -- Cache data: {parsed_cert, parsed_key, cert_pem, key_pem, fingerprint} for key in server_name:gmatch("%S+") do local cache_key = "plugin_letsencrypt_" .. key local ok - ok, err = self.internalstore:set(cache_key, { cert_chain, priv_key }, nil, true) + ok, err = self.internalstore:set(cache_key, { cert_chain, priv_key, data[1], data[2], cert_fingerprint }, nil, true) if not ok then return false, "error while setting data into internalstore : " .. err end diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index 8d2c3f7388..be986c846c 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -1591,8 +1591,9 @@ def get_cached_ocsp_ttl(cert_name: str, cert_pem: Optional[bytes] = None, finger assert total_lifetime is not None log_info( - "โšก OCSP cached response for %s: remaining=%ds (%.1f days), total_lifetime=%ds (%.1f days)", + "โšก OCSP cached response for %s: fp=%s remaining=%ds (%.1f days), total_lifetime=%ds (%.1f days)", cert_name, + (fingerprint[:16] + "...") if fingerprint else "unknown", remaining, remaining / 86400.0, total_lifetime, @@ -2102,9 +2103,16 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta # Skip fetch ONLY if not forced AND TTL is above refresh_threshold AND above 50% lifetime if not force_fetch and cached_ttl > refresh_threshold and cached_ttl > half_lifetime: + fp_prefix = (fingerprint[:16] + "...") if fingerprint else "unknown" log_info( - "โœ“ OCSP cached response for %s still valid for %ds (%.1f days), above thresholds (refresh_threshold=%ds [20%% of %ds], 50%% threshold=%ds), skipping fetch", - cert_name, cached_ttl, cached_ttl / 86400.0, refresh_threshold, total_lifetime, half_lifetime, + "โœ“ OCSP cached response for %s still valid for %ds (%.1f days) [fp=%s], above thresholds (refresh_threshold=%ds [20%% of %ds], 50%% threshold=%ds), skipping fetch", + cert_name, + cached_ttl, + cached_ttl / 86400.0, + fp_prefix, + refresh_threshold, + total_lifetime, + half_lifetime, ) stats["ocsp_cached_responses"] = stats.get("ocsp_cached_responses", 0) + 1 return (cert_name, None, cached_ttl, cert_checksum, pem_data, ocsp_url, False) @@ -2164,7 +2172,14 @@ def _process_cert(cert_name: str, pem_data: bytes, db: Optional[Any] = None, sta # Calculate checksum for integrity verification (lowercase for consistency) ocsp_checksum = hashlib.sha256(ocsp_der).hexdigest().lower() ttl_readable = f"{ttl / 86400.0:.1f} days" if ttl >= 86400 else f"{ttl / 3600.0:.1f} hours" - log_info("โšก OCSP final TTL for %s is %ds (%s) (checksum=%s)", cert_name, ttl, ttl_readable, ocsp_checksum[:8]) + log_info( + "โšก OCSP final TTL for %s is %ds (%s) [fp=%s] (checksum=%s)", + cert_name, + ttl, + ttl_readable, + (fingerprint[:16] + "...") if fingerprint else "unknown", + ocsp_checksum[:8], + ) # === Return result for batched database writes === return (cert_name, ocsp_der, ttl, cert_checksum, pem_data, ocsp_url, True) From 697130a94286f07ead5d321c7169fe361b1939d0 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 20 Mar 2026 09:27:21 +0100 Subject: [PATCH 58/59] optimize ocsp validation/lookup+ verbose logging - verify ocsp response file vailidity using ngx.ocsp.validate_ocsp_response() - optimize ocsp lookup logic (leaf cert first) - add more debug information - add test scenarios --- .../server-http/ssl-certificate-lua.conf | 560 ++++++++++++++++-- 1 file changed, 506 insertions(+), 54 deletions(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index ebbd5c5e70..37e9744f8a 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -158,12 +158,61 @@ ssl_certificate_by_lua_block { end end - local ocsp = optional_require("ngx.ocsp", true) -- Report failure (OCSP stapling feature) + -- OCSP module. + -- Debug toggle: disable ngx.ocsp on purpose to simulate a failure. + -- Default: enabled (unless BW_DISABLE_NGX_OCSP is explicitly set). + local disable_ngx_ocsp = false + pcall(function() + local v = os.getenv("BW_DISABLE_NGX_OCSP") + if v then + v = tostring(v):lower() + if v == "1" or v == "yes" or v == "true" then + disable_ngx_ocsp = true + elseif v == "0" or v == "no" or v == "false" then + disable_ngx_ocsp = false + end + end + end) + + local ocsp = nil + if not disable_ngx_ocsp then + ocsp = optional_require("ngx.ocsp", true) -- Report failure (OCSP stapling feature) + else + safe_log(DEBUG, "OCSP DISABLED: ngx.ocsp disabled via BW_DISABLE_NGX_OCSP") + end local cclusterstore = optional_require("bunkerweb.clusterstore", false) -- Silent failure (cache optimization) - -- Try to load resty.openssl for native certificate parsing (report failure - performance optimization) - local resty_x509 = optional_require("resty.openssl.x509", true) - local has_resty_ssl = (resty_x509 ~= nil) + -- Try to load resty.openssl for native certificate parsing (report failure - performance optimization). + -- Debug toggles: + -- - disable resty.openssl to force OpenSSL CLI fallbacks via `BW_DISABLE_RESTY_OPENSSL=yes|1|true` + -- - enable resty.openssl via `BW_ENABLE_RESTY_OPENSSL=yes|1|true` + -- Default: enabled (resty.openssl will be loaded). + local disable_resty_openssl = false + pcall(function() + local v_disable = os.getenv("BW_DISABLE_RESTY_OPENSSL") + if v_disable then + v_disable = tostring(v_disable):lower() + if v_disable == "1" or v_disable == "yes" or v_disable == "true" then + disable_resty_openssl = true + end + end + + local v_enable = os.getenv("BW_ENABLE_RESTY_OPENSSL") + if v_enable then + v_enable = tostring(v_enable):lower() + if v_enable == "1" or v_enable == "yes" or v_enable == "true" then + disable_resty_openssl = false + end + end + end) + + local resty_x509 = nil + if not disable_resty_openssl then + resty_x509 = optional_require("resty.openssl.x509", true) + else + safe_log(DEBUG, "RESTY.OPENSSL DISABLED: resty.openssl.x509 is disabled; using OpenSSL CLI fallbacks") + end + local has_resty_ssl = (resty_x509 ~= nil) and not disable_resty_openssl local clear_certs = ssl and ssl.clear_certs local set_cert = ssl and ssl.set_cert @@ -260,18 +309,34 @@ ssl_certificate_by_lua_block { -- Try resty.openssl first (fast) if has_resty_ssl then - pcall(function() + local rest_err = nil + local ok_id, rest_identifier = pcall(function() local cert, err = resty_x509.new(cert_pem) + rest_err = err if cert then local serial = cert:get_serial_number() if serial then return tostring(serial):upper() end end + return nil end) + + if ok_id and rest_identifier then + return rest_identifier + end + safe_log( + DEBUG, + "OCSP get_cert_identifier resty.openssl could not extract serial; falling back to openssl server_name=" .. + (server_name or "nil") + ) + if rest_err then + safe_log(DEBUG, "OCSP get_cert_identifier resty.openssl error: " .. tostring(rest_err)) + end end -- Fallback: extract via openssl command + safe_log(DEBUG, "OCSP get_cert_identifier falling back to openssl server_name=" .. (server_name or "nil")) local identifier = nil pcall(function() local worker_id = ngx.worker.id() or "0" @@ -395,6 +460,7 @@ ssl_certificate_by_lua_block { -- Fallback to OpenSSL if lua-resty-openssl fingerprinting failed. -- This is slower and should normally not be needed, but it prevents silent OCSP failures. if not fingerprint then + safe_log(DEBUG, "OCSP get_pubkey_fingerprint falling back to openssl server_name=" .. (server_name or "nil")) pcall(function() local worker_id = ngx.worker.id() or "0" local tmp_cert = "/tmp/ocsp_pubkey_fp_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" @@ -436,7 +502,7 @@ ssl_certificate_by_lua_block { -- The Python job (`ocsp-refresh.py`) computes this fingerprint by hashing the public key -- in SubjectPublicKeyInfo DER. -- - -- `resty.openssl` may produce a different hash for the "cert_obj:pubkey_digest('sha256')" path + -- `resty.openssl` may produce a different hash for the "cert_obj:pubkey_digest(sha256)" path -- depending on OpenSSL bindings/encoding. -- -- To guarantee cache hits, we remap the "resty fingerprint" to the "OpenSSL/Python-compatible" @@ -461,6 +527,7 @@ ssl_certificate_by_lua_block { -- Compute OpenSSL/Python-compatible fingerprint and cache mapping. local fingerprint_openssl = nil pcall(function() + safe_log(DEBUG, "OCSP get_ocsp_pubkey_fingerprint computing openssl remap server_name=" .. (server_name or "nil")) local worker_id = ngx.worker.id() or "0" local tmp_cert = "/tmp/ocsp_pubkey_fp_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" local f = io.open(tmp_cert, "w") @@ -731,12 +798,15 @@ ssl_certificate_by_lua_block { if serial_num then return tostring(serial_num) end + safe_log(DEBUG, "OCSP get_cert_serial resty.openssl returned cert but no serial; falling back to openssl server_name=" .. (server_name or "nil")) else - safe_log(DEBUG, "OCSP resty.openssl parse error: " .. tostring(err)) + safe_log(DEBUG, "OCSP get_cert_serial resty.openssl parse error; falling back to openssl server_name=" .. (server_name or "nil")) + safe_log(DEBUG, "OCSP resty.openssl error: " .. tostring(err)) end end -- Fallback to openssl command (slower, but more compatible) - wrap in pcall to avoid block crash + safe_log(DEBUG, "OCSP get_cert_serial falling back to openssl server_name=" .. (server_name or "nil")) local serial = nil pcall(function() local worker_id = ngx.worker.id() or "0" @@ -808,7 +878,9 @@ ssl_certificate_by_lua_block { end end + safe_log(DEBUG, "OCSP get_ocsp_serial resty could not extract serial; falling back to openssl server_name=" .. (server_name or "nil")) -- Fallback: use openssl command (wrap in pcall for safety) + safe_log(DEBUG, "OCSP get_ocsp_serial falling back to openssl ocsp -respin server_name=" .. (server_name or "nil")) local serial = nil pcall(function() local worker_id = ngx.worker.id() or "0" @@ -896,6 +968,56 @@ ssl_certificate_by_lua_block { end end + -- Helper: fast validation using cached metadata fingerprint + -- This avoids parsing OCSP DER (openssl/resty) just to detect corrupted/incorrect cached responses. + -- Returns: + -- - true if fingerprint matches or metadata is missing + -- - false if fingerprint explicitly mismatches + local function verify_ocsp_fingerprint_match(expected_cert_fp, ocsp_dir) + if not expected_cert_fp or not ocsp_dir then + return true + end + + local meta_path = ocsp_dir .. "/ocsp.json" + local meta_raw = nil + pcall(function() + local fmeta = io.open(meta_path, "r") + if fmeta then + meta_raw = fmeta:read("*a") + fmeta:close() + end + end) + + if not meta_raw or #meta_raw == 0 then + safe_log(DEBUG, "OCSP fingerprint verify skipped (missing ocsp.json) expected_fp=" .. tostring(expected_cert_fp) .. " server_name=" .. (server_name or "nil")) + return true + end + + local ok_decode, decoded = pcall(cjson.decode, meta_raw) + if not ok_decode or type(decoded) ~= "table" then + safe_log(DEBUG, "OCSP fingerprint verify skipped (invalid ocsp.json) expected_fp=" .. tostring(expected_cert_fp) .. " server_name=" .. (server_name or "nil")) + return true + end + + local meta_fp = decoded.fingerprint + if type(meta_fp) ~= "string" then + safe_log(DEBUG, "OCSP fingerprint verify skipped (missing fingerprint in ocsp.json) expected_fp=" .. tostring(expected_cert_fp) .. " server_name=" .. (server_name or "nil")) + return true + end + + meta_fp = meta_fp:lower() + if meta_fp == expected_cert_fp then + return true + end + + safe_log( + ERR, + "OCSP fingerprint mismatch: meta_fp=" .. meta_fp .. " expected_fp=" .. tostring(expected_cert_fp) .. + " ocsp_dir=" .. ocsp_dir .. " server_name=" .. (server_name or "nil") + ) + return false + end + -- Helper: read and parse certificate once, extract Must-Staple + responder URL -- Input: cert_pem - certificate in PEM format (passed from plugin, NOT read via ssl.cert_pem()) -- Returns: { cert_parsed = resty cert object or nil, must_staple = bool, responder_url = string or nil } @@ -949,6 +1071,10 @@ ssl_certificate_by_lua_block { end -- Fallback: use openssl command to extract both Must-Staple and responder URL + if not has_resty_ssl then + safe_log(DEBUG, "RESTY.OPENSSL DISABLED: resty.openssl not available in read_certificate_metadata; using openssl fallback server_name=" .. (server_name or "nil")) + end + safe_log(DEBUG, "RESTY.OPENSSL DISABLED: OCSP read_certificate_metadata falling back to openssl server_name=" .. (server_name or "nil")) pcall(function() local worker_id = ngx.worker.id() or "0" local tmp_cert = "/tmp/ocsp_cert_meta_w" .. worker_id .. "_c" .. ngx.var.connection .. ".pem" @@ -1029,7 +1155,11 @@ ssl_certificate_by_lua_block { end end + if has_resty_ssl then + safe_log(DEBUG, "RESTY.OPENSSL DISABLED: resty.openssl did not yield an OCSP URL; using openssl fallback server_name=" .. (server_name or "nil")) + end -- Fallback to openssl command (wrap in pcall for safety) + safe_log(DEBUG, "RESTY.OPENSSL DISABLED: OCSP get_ocsp_responder_url falling back to openssl -ocsp_uri server_name=" .. (server_name or "nil")) local ocsp_url = nil pcall(function() local worker_id = ngx.worker.id() or "0" @@ -1105,6 +1235,10 @@ ssl_certificate_by_lua_block { end -- Fallback to openssl command (wrap in pcall for safety) + if has_resty_ssl then + safe_log(DEBUG, "RESTY.OPENSSL DISABLED: resty.openssl did not detect Must-Staple; using openssl fallback server_name=" .. (server_name or "nil")) + end + safe_log(DEBUG, "RESTY.OPENSSL DISABLED: OCSP has_ocsp_must_staple falling back to openssl server_name=" .. (server_name or "nil")) local must_staple = false pcall(function() local worker_id = ngx.worker.id() or "0" @@ -1147,12 +1281,32 @@ ssl_certificate_by_lua_block { -- Returns: true if cert is acceptable, false if OCSP-Must-Staple requirement not met local function set_ocsp_from_cache(cert_pem, cert_fp_hint) safe_log(DEBUG, "OCSP set_ocsp_from_cache() called for server_name=" .. (server_name or "nil")) + local now = ngx.now or ngx.time + local hrtime = ngx.hrtime + local floor = math.floor + + -- Prefer high-resolution timing (hrtime is nanoseconds) so we do not round small ops to 0ms. + local function ns_since(t0) + if hrtime then + -- hrtime() returns nanoseconds as an integer. + return hrtime() - t0 + end + -- Fallback: convert seconds to nanoseconds (best effort). + return floor((now() - t0) * 1000000000 + 0.5) + end if not is_ocsp_stapling_enabled() then safe_log(DEBUG, "OCSP stapling disabled via SSL_USE_OCSP_STAPLING setting") return true -- Certificate acceptable even without OCSP end + -- Fail fast if ngx.ocsp module is not available. + -- In that case, skip OCSP stapling rather than trying to validate and failing late. + if not ocsp then + safe_log(ERR, "OCSP DISABLED: ngx.ocsp module failed to load; skipping OCSP stapling for server_name=" .. (server_name or "nil")) + return true + end + -- Plugins may pass either: -- - a PEM string (custom parsing) -- - an opaque cdata cert pointer returned by ngx.ssl.parse_pem_cert() @@ -1177,13 +1331,18 @@ ssl_certificate_by_lua_block { -- Extract certificate metadata (Must-Staple) -- At this point cert_pem is guaranteed to be a non-empty string (cdata rejected above) -- cert_pem may contain the full chain; extract leaf for metadata extraction + local t_total_start = hrtime and hrtime() or now() local leaf_for_meta = cert_pem + local t_parse_meta_start = hrtime and hrtime() or now() local meta_certs = parse_pem_certificates(cert_pem) + safe_log(DEBUG, "OCSP parse_pem_certificates(meta) time_ns=" .. tostring(ns_since(t_parse_meta_start)) .. " server_name=" .. (server_name or "nil")) if meta_certs and #meta_certs > 0 then leaf_for_meta = meta_certs[1] end local has_must_staple = false + local t_meta_start = hrtime and hrtime() or now() local ok_read, cert_meta = pcall(read_certificate_metadata, leaf_for_meta) + safe_log(DEBUG, "OCSP read_certificate_metadata time_ns=" .. tostring(ns_since(t_meta_start)) .. " server_name=" .. (server_name or "nil")) if ok_read and cert_meta then has_must_staple = cert_meta.must_staple if has_must_staple then @@ -1194,9 +1353,11 @@ ssl_certificate_by_lua_block { -- Compute certificate public key fingerprint(s) (unique identifiers). -- `cert_pem` may contain the full chain; the real leaf is not always the first PEM block. -- To handle that safely, we try multiple certificate-block fingerprints until we find an existing `ocsp.der`. + local t_fp_phase_start = hrtime and hrtime() or now() local resp = nil local tried_fps = {} local candidate_fps = {} + local fp_to_cert_pem = {} -- Prefer pre-computed fingerprint hint from the plugin (index 5 in letsencrypt/customcert internalstore). -- Still validate by file existence (handshake-time chain ordering can differ). @@ -1208,35 +1369,52 @@ ssl_certificate_by_lua_block { end end + local t_parse_chain_start = hrtime and hrtime() or now() local chain_certs = parse_pem_certificates(cert_pem) + safe_log(DEBUG, "OCSP parse_pem_certificates(chain) time_ns=" .. tostring(ns_since(t_parse_chain_start)) .. " server_name=" .. (server_name or "nil")) if not chain_certs or #chain_certs == 0 then chain_certs = { cert_pem } end + + -- Optional optimization: mapping subject -> PEM. + -- If unavailable, issuer validation still works by trying all chain certs as possible issuers. + local chain_subject_to_pem = {} + + local der_chain_cache = {} if #chain_certs > 1 then - safe_log(DEBUG, "OCSP will try multiple certificate blocks from chain of " .. #chain_certs .. " certificate(s)") + safe_log(DEBUG, "OCSP will try multiple certificate blocks from chain of " .. #chain_certs .. " certificate(s) server_name=" .. (server_name or "nil")) end -- Add fingerprints for each certificate block (may include the real leaf and intermediates) + local t_fp_candidates_start = hrtime and hrtime() or now() for idx, cert_block_pem in ipairs(chain_certs) do local fp = get_ocsp_pubkey_fingerprint(cert_block_pem) - if fp and not tried_fps[fp] then - candidate_fps[#candidate_fps + 1] = fp - tried_fps[fp] = true + if fp then + -- Ensure we keep the PEM block for each computed fingerprint, even + -- if it was already present from a plugin fingerprint hint. + if not fp_to_cert_pem[fp] then + fp_to_cert_pem[fp] = cert_block_pem + end + if not tried_fps[fp] then + candidate_fps[#candidate_fps + 1] = fp + tried_fps[fp] = true + end end if type(fp) == "string" then - safe_log(DEBUG, "OCSP cert block #" .. idx .. " pem_len=" .. tostring(#cert_block_pem) .. " fp=" .. fp:sub(1, 16) .. "...") + safe_log(DEBUG, "OCSP cert block #" .. idx .. " pem_len=" .. tostring(#cert_block_pem) .. " fp=" .. fp:sub(1, 16) .. "... server_name=" .. (server_name or "nil")) else - safe_log(DEBUG, "OCSP cert block #" .. idx .. " pem_len=" .. tostring(cert_block_pem and #cert_block_pem or 0) .. " fp=nil") + safe_log(DEBUG, "OCSP cert block #" .. idx .. " pem_len=" .. tostring(cert_block_pem and #cert_block_pem or 0) .. " fp=nil server_name=" .. (server_name or "nil")) end end + safe_log(DEBUG, "OCSP candidate_fps compute time_ns=" .. tostring(ns_since(t_fp_candidates_start)) .. " server_name=" .. (server_name or "nil")) if cert_fp_hint and type(cert_fp_hint) == "string" then - safe_log(DEBUG, "OCSP cert_fp_hint=" .. (cert_fp_hint:sub(1, 16) .. "...")) + safe_log(DEBUG, "OCSP cert_fp_hint=" .. (cert_fp_hint:sub(1, 16) .. "...") .. " server_name=" .. (server_name or "nil")) else - safe_log(DEBUG, "OCSP cert_fp_hint=nil") + safe_log(DEBUG, "OCSP cert_fp_hint=nil server_name=" .. (server_name or "nil")) end - safe_log(DEBUG, "OCSP candidate_fps count=" .. tostring(#candidate_fps)) + safe_log(DEBUG, "OCSP candidate_fps count=" .. tostring(#candidate_fps) .. " server_name=" .. (server_name or "nil")) if #candidate_fps == 0 then safe_log(NOTICE, "OCSP could not compute any certificate fingerprints for " .. (server_name or "unknown")) @@ -1246,30 +1424,258 @@ ssl_certificate_by_lua_block { return true end - for _, cert_fp in ipairs(candidate_fps) do - safe_log(DEBUG, "OCSP trying candidate fingerprint raw=" .. tostring(cert_fp)) + -- Cryptographic OCSP validation (fast + secure) using ngx.ocsp. + -- Returns: + -- - true => validated (cryptographically) and should be cached as verified + -- - false => validated and rejected + -- - nil => validation skipped (insufficient data), caller may decide how to proceed + local function ocsp_validate_response_for_fp(cert_fp, cert_for_fp_pem, ocsp_der) + local t_fn_start_hr = hrtime and hrtime() or nil + local t_fn_start_sec = now() + safe_log(DEBUG, "OCSP ocsp_validate_response_for_fp start fp=" .. tostring(cert_fp and cert_fp:sub(1, 16) .. "...") .. " server_name=" .. (server_name or "nil")) + local function finish(retv) + local time_ns = nil + if t_fn_start_hr then + time_ns = hrtime() - t_fn_start_hr + end + -- `ngx.hrtime()` can be too fine/granular to show movement (sometimes 0); log us/ms too. + local time_us = floor(((now() - t_fn_start_sec) * 1000000) + 0.5) + local time_ms = floor(((now() - t_fn_start_sec) * 1000) + 0.5) + safe_log( + DEBUG, + "OCSP ocsp_validate_response_for_fp end fp=" .. tostring(cert_fp and cert_fp:sub(1, 16) .. "...") .. + " result=" .. tostring(retv) .. + " time_ns=" .. tostring(time_ns) .. + " time_us=" .. tostring(time_us) .. + " time_ms=" .. tostring(time_ms) .. + " server_name=" .. (server_name or "nil") + ) + return retv + end + + if not ocsp then + return finish(false) + end + -- Do not require resty.openssl.x509 for validation. + -- We can still build the DER chain via ngx.ssl and validate via ngx.ocsp. + if not ssl or not ssl.cert_pem_to_der then + return finish(false) + end + if not cert_for_fp_pem or type(cert_for_fp_pem) ~= "string" then + return finish(false) + end + + -- If validation previously failed for this cert fingerprint, avoid re-parsing/re-verifying + -- for a short period. This prevents getting "stuck" on repeated bad/poisoned cache entries. + local ocsp_validate_failed_key = "TLS:SSL:ocsp_validate_failed:" .. cert_fp + local ocsp_validate_failed = false + pcall(function() + ocsp_validate_failed = internalstore:get(ocsp_validate_failed_key, true) == true + end) + if ocsp_validate_failed then + return finish(false) + end + + local ocsp_validate_failure_ttl = 60 + local ocsp_validate_timeout_total_ns = 700000000 -- 700ms budget across issuer attempts + local ocsp_validate_max_issuer_candidates = 4 + local t_val_total_start = hrtime and hrtime() or now() + local issuer_name = nil + if has_resty_ssl and resty_x509 and resty_x509.new then + -- Optional fast-path: use resty.openssl to derive issuer DN. + local leaf_cert_obj = resty_x509.new(cert_for_fp_pem) + if leaf_cert_obj then + local issuer_name_obj = leaf_cert_obj:get_issuer_name() + if issuer_name_obj then + issuer_name = tostring(issuer_name_obj) + end + end + end + + if not chain_certs or #chain_certs < 2 then + return finish(false) + end + + -- Try to validate against multiple possible issuer certificates. + -- ngx.ocsp.validate_ocsp_response() verifies the OCSP signature and binds it to the + -- certificate via OCSP CertID, so we can safely try issuers in any order. + local issuer_pems_to_try = {} + if issuer_name then + local issuer_pem = chain_subject_to_pem and chain_subject_to_pem[issuer_name] or nil + if issuer_pem then + issuer_pems_to_try[#issuer_pems_to_try + 1] = issuer_pem + end + end + local seen_issuer = {} + for _, p in ipairs(issuer_pems_to_try) do + seen_issuer[p] = true + end + for _, cert_block_pem in ipairs(chain_certs) do + if cert_block_pem and cert_block_pem ~= cert_for_fp_pem then + -- Avoid duplicates when the DN-mapped issuer is already added. + if not seen_issuer[cert_block_pem] then + issuer_pems_to_try[#issuer_pems_to_try + 1] = cert_block_pem + seen_issuer[cert_block_pem] = true + end + end + end + + if #issuer_pems_to_try == 0 then + return finish(false) + end + if #issuer_pems_to_try > ocsp_validate_max_issuer_candidates then + -- Keep the most likely issuer candidates to avoid excessive parsing. + -- Order does not matter for security because CertID matching is inside validate(). + issuer_pems_to_try = { unpack(issuer_pems_to_try, 1, ocsp_validate_max_issuer_candidates) } + end + + local cached_by_issuer = der_chain_cache[cert_fp] + if not cached_by_issuer then + cached_by_issuer = {} + der_chain_cache[cert_fp] = cached_by_issuer + end + + for issuer_idx, issuer_candidate_pem in ipairs(issuer_pems_to_try) do + -- Soft total-time budget: stop trying more issuers if we already spent too long. + if (hrtime and hrtime() or now()) - t_val_total_start > ocsp_validate_timeout_total_ns then + safe_log( + ERR, + "OCSP validate budget exceeded for fp=" .. cert_fp:sub(1, 16) .. "... server_name=" .. (server_name or "nil") + ) + break + end + + local der_chain = cached_by_issuer[issuer_idx] + if not der_chain then + local ordered_chain_pem = cert_for_fp_pem .. "\n" .. issuer_candidate_pem + local der_cert_chain, err = ssl.cert_pem_to_der(ordered_chain_pem) + if der_cert_chain then + cached_by_issuer[issuer_idx] = der_cert_chain + der_chain = der_cert_chain + else + safe_log(DEBUG, "OCSP validate skipped (cert_pem_to_der failed): " .. tostring(err) .. " server_name=" .. (server_name or "nil")) + der_chain = nil + end + end + + if der_chain then + local ok_pcall, validate_ok, validate_err_or_next = pcall(function() + return ocsp.validate_ocsp_response(ocsp_der, der_chain) + end) + if ok_pcall and validate_ok == true then + return finish(true) + end + end + end + + -- Validation failed: remember it briefly to avoid repeated expensive checks. + pcall(function() + internalstore:set(ocsp_validate_failed_key, true, ocsp_validate_failure_ttl, true) + end) + return finish(false) + end + + -- Leaf-first sequential lookup: + -- ocsp-refresh.py stores OCSP responses for the leaf certificate public-key fingerprint. + -- The chain PEM order is not guaranteed, so we re-order candidate_fps by first fingerprint + -- that actually has an on-disk `ocsp.der` cache file. + local candidate_fps_ordered = candidate_fps + local leaf_fp = nil + local t_leaf_scan_start = hrtime and hrtime() or now() + pcall(function() + for _, cert_fp in ipairs(candidate_fps) do + if is_fp64_lower_hex(cert_fp) then + local hex1 = cert_fp:sub(1, 1) + local hex2 = cert_fp:sub(2, 2) + local ocsp_path_tmp = "/var/cache/bunkerweb/ssl/" .. hex1 .. "/" .. hex2 .. "/" .. cert_fp .. "/ocsp.der" + local fcheck = io.open(ocsp_path_tmp, "rb") + if fcheck then + fcheck:close() + leaf_fp = cert_fp + safe_log(DEBUG, "OCSP leaf fingerprint detected (ocsp.der exists) server_name=" .. (server_name or "nil") .. " fp=" .. leaf_fp:sub(1, 16) .. "...") + break + end + end + end + end) + + safe_log(DEBUG, "OCSP leaf scan time_ns=" .. tostring(ns_since(t_leaf_scan_start)) .. " server_name=" .. (server_name or "nil")) + + if leaf_fp and is_fp64_lower_hex(leaf_fp) then + candidate_fps_ordered = {} + candidate_fps_ordered[#candidate_fps_ordered + 1] = leaf_fp + for _, cert_fp in ipairs(candidate_fps) do + if cert_fp ~= leaf_fp then + candidate_fps_ordered[#candidate_fps_ordered + 1] = cert_fp + end + end + end + + safe_log(DEBUG, "OCSP fingerprint phase total_ns=" .. tostring(ns_since(t_fp_phase_start)) .. " server_name=" .. (server_name or "nil")) + + for _, cert_fp in ipairs(candidate_fps_ordered) do + local t_attempt_start = hrtime and hrtime() or now() + local attempt_hit = false + safe_log(DEBUG, "OCSP trying candidate fingerprint raw=" .. tostring(cert_fp) .. " server_name=" .. (server_name or "nil")) if is_fp64_lower_hex(cert_fp) then - safe_log(DEBUG, "OCSP trying fingerprint: " .. cert_fp:sub(1, 16) .. "...") + safe_log(DEBUG, "OCSP trying fingerprint: " .. cert_fp:sub(1, 16) .. "... server_name=" .. (server_name or "nil")) -- Use tree-structured sharded path: /var/cache/bunkerweb/ssl/{hex1}/{hex2}/{full_fingerprint}/ocsp.der local hex1 = cert_fp:sub(1, 1) local hex2 = cert_fp:sub(2, 2) local ocsp_path = "/var/cache/bunkerweb/ssl/" .. hex1 .. "/" .. hex2 .. "/" .. cert_fp .. "/ocsp.der" local cache_key = "TLS:SSL:ocsp:" .. cert_fp - safe_log(DEBUG, "OCSP lookup ocsp_path=" .. ocsp_path) + safe_log(DEBUG, "OCSP lookup ocsp_path=" .. ocsp_path .. " server_name=" .. (server_name or "nil")) -- 1) Try local shared dict lookup first (per-worker cache) local ok_cache, cache_result = pcall(function() return internalstore:get(cache_key, true) end) if ok_cache and cache_result then - resp = cache_result - safe_log(DEBUG, "OCSP found response in shared memory cache for fp=" .. cert_fp:sub(1, 16) .. "...") - break + local verified_key = "TLS:SSL:ocsp_verified:" .. cert_fp + local ocsp_verified = false + pcall(function() + ocsp_verified = internalstore:get(verified_key, true) + end) + + local ocsp_ok = ocsp_verified == true + local ocsp_validated_now = ocsp_verified == true + if not ocsp_ok then + local hex1 = cert_fp:sub(1, 1) + local hex2 = cert_fp:sub(2, 2) + local ocsp_dir = "/var/cache/bunkerweb/ssl/" .. hex1 .. "/" .. hex2 .. "/" .. cert_fp + local fp_ok = verify_ocsp_fingerprint_match(cert_fp, ocsp_dir) + if fp_ok then + local cert_for_fp_pem = fp_to_cert_pem[cert_fp] or leaf_for_meta + local validate_res = ocsp_validate_response_for_fp(cert_fp, cert_for_fp_pem, cache_result) + if validate_res == true then + ocsp_ok = true + ocsp_validated_now = true + pcall(function() + internalstore:set(verified_key, true, 300, true) + end) + else + ocsp_ok = false + end + end + + if ocsp_ok then + resp = cache_result + attempt_hit = true + safe_log(DEBUG, "OCSP found response in shared memory cache for fp=" .. cert_fp:sub(1, 16) .. "... verified=" .. tostring(ocsp_validated_now) .. " server_name=" .. (server_name or "nil")) + break + else + safe_log(ERR, "OCSP cached response rejected (fingerprint mismatch); discarding cache for fp=" .. cert_fp:sub(1, 16) .. "... server_name=" .. (server_name or "nil")) + pcall(function() + internalstore:delete(cache_key) + internalstore:delete(verified_key) + end) + end + end end -- 2) Fallback to disk file - safe_log(DEBUG, "OCSP looking for cached response at: " .. ocsp_path) + safe_log(DEBUG, "OCSP looking for cached response at: " .. ocsp_path .. " server_name=" .. (server_name or "nil")) pcall(function() local f, err = io.open(ocsp_path, "rb") if f then @@ -1277,33 +1683,71 @@ ssl_certificate_by_lua_block { f:close() if data and #data > 0 then - resp = data - safe_log(INFO, "OCSP loaded response from: " .. ocsp_path) - - -- Cache in shared memory (TTL 300s) + local verified_key = "TLS:SSL:ocsp_verified:" .. cert_fp + local ocsp_verified = false pcall(function() - local ok_cache_set = internalstore:set(cache_key, resp, 300, true) - if not ok_cache_set then - safe_log(DEBUG, "OCSP failed to cache response in shared memory") - end + ocsp_verified = internalstore:get(verified_key, true) end) + + local ocsp_ok = ocsp_verified == true + if not ocsp_ok then + local hex1 = cert_fp:sub(1, 1) + local hex2 = cert_fp:sub(2, 2) + local ocsp_dir = "/var/cache/bunkerweb/ssl/" .. hex1 .. "/" .. hex2 .. "/" .. cert_fp + local fp_ok = verify_ocsp_fingerprint_match(cert_fp, ocsp_dir) + if fp_ok then + local cert_for_fp_pem = fp_to_cert_pem[cert_fp] or leaf_for_meta + local validate_res = ocsp_validate_response_for_fp(cert_fp, cert_for_fp_pem, data) + if validate_res == true then + ocsp_ok = true + pcall(function() + internalstore:set(verified_key, true, 300, true) + end) + else + ocsp_ok = false + end + end + end + + if ocsp_ok then + resp = data + attempt_hit = true + safe_log(INFO, "OCSP loaded response from: " .. ocsp_path .. " server_name=" .. (server_name or "nil")) + + -- Cache in shared memory (TTL 300s) + pcall(function() + local ok_cache_set = internalstore:set(cache_key, resp, 300, true) + if not ok_cache_set then + safe_log(DEBUG, "OCSP failed to cache response in shared memory server_name=" .. (server_name or "nil")) + end + end) + else + safe_log(ERR, "OCSP rejected (fingerprint mismatch); skipping invalid ocsp.der at: " .. ocsp_path .. " server_name=" .. (server_name or "nil")) + end else - safe_log(DEBUG, "OCSP response file is empty: " .. ocsp_path) + safe_log(DEBUG, "OCSP response file is empty: " .. ocsp_path .. " server_name=" .. (server_name or "nil")) end else if err and lower(err):find("permission denied", 1, true) then - safe_log(ERR, "OCSP permission denied reading " .. ocsp_path) + safe_log(ERR, "OCSP permission denied reading " .. ocsp_path .. " server_name=" .. (server_name or "nil")) else - safe_log(DEBUG, "OCSP response file not found: " .. ocsp_path) + safe_log(DEBUG, "OCSP response file not found: " .. ocsp_path .. " server_name=" .. (server_name or "nil")) end end end) + safe_log( + DEBUG, + "OCSP attempt fp=" .. cert_fp:sub(1, 16) .. "..." .. + " hit=" .. tostring(attempt_hit) .. + " time_ns=" .. tostring(ns_since(t_attempt_start)) .. + " server_name=" .. (server_name or "nil") + ) if resp then break end else - safe_log(DEBUG, "OCSP skipping invalid fingerprint format: " .. tostring(cert_fp)) + safe_log(DEBUG, "OCSP skipping invalid fingerprint format: " .. tostring(cert_fp) .. " server_name=" .. (server_name or "nil")) end end @@ -1335,6 +1779,7 @@ ssl_certificate_by_lua_block { end -- If we reach here without returning earlier, OCSP failed but is not required (or succeeded) + safe_log(DEBUG, "OCSP set_ocsp_from_cache total time_ns=" .. tostring(ns_since(t_total_start)) .. " server_name=" .. (server_name or "nil")) return true end @@ -1390,17 +1835,20 @@ ssl_certificate_by_lua_block { -- Assume ngx.ssl.parse_pem_* was already called by the plugin -- Check if original PEM strings are available at indices 3,4 (e.g., from letsencrypt plugin) local orig_cert_pem = nil - if ret.status[3] and type(ret.status[3]) == "string" then - orig_cert_pem = ret.status[3] - safe_log(DEBUG, "Original PEM certificate available for OCSP fingerprinting (" .. #orig_cert_pem .. " bytes)") - end - -- Optional fingerprint hint at index 5 (computed in letsencrypt/customcert load_data) local orig_cert_fp = nil - if ret.status[5] and type(ret.status[5]) == "string" then - local fp = ret.status[5]:lower():match("^[0-9a-f]{64}$") - if fp then - orig_cert_fp = fp - safe_log(DEBUG, "Original cert fingerprint available for OCSP (" .. fp:sub(1, 16) .. "... )") + -- Skip OCSP fingerprint preparation entirely when ngx.ocsp is disabled. + if ocsp then + if ret.status[3] and type(ret.status[3]) == "string" then + orig_cert_pem = ret.status[3] + safe_log(DEBUG, "Original PEM certificate available for OCSP fingerprinting (" .. #orig_cert_pem .. " bytes)") + end + -- Optional fingerprint hint at index 5 (computed in letsencrypt/customcert load_data) + if ret.status[5] and type(ret.status[5]) == "string" then + local fp = ret.status[5]:lower():match("^[0-9a-f]{64}$") + if fp then + orig_cert_fp = fp + safe_log(DEBUG, "Original cert fingerprint available for OCSP (" .. fp:sub(1, 16) .. "... )") + end end end cert_key_pairs = { @@ -1451,15 +1899,19 @@ ssl_certificate_by_lua_block { if ok_key then local ocsp_cert = pair.cert_pem_for_ocsp or pair.cert local ocsp_fp_hint = pair.cert_fp_for_ocsp - safe_log(DEBUG, "Setting OCSP for certificate #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ", cert_type=" .. type(ocsp_cert) .. ")") - local ok_ocsp, ocsp_cert_acceptable = pcall(set_ocsp_from_cache, ocsp_cert, ocsp_fp_hint) - if not ok_ocsp then - safe_log(ERR, "OCSP function error for cert #" .. idx .. " (" .. pair.cert_id .. "): " .. tostring(ocsp_cert_acceptable)) - elseif ocsp_cert_acceptable == false then - -- OCSP-Must-Staple requirement not met - log but continue (failsafe) - safe_log(ERR, "OCSP-Must-Staple requirement not met for cert #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ") - proceeding anyway") + if ocsp then + safe_log(DEBUG, "Setting OCSP for certificate #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ", cert_type=" .. type(ocsp_cert) .. ")") + local ok_ocsp, ocsp_cert_acceptable = pcall(set_ocsp_from_cache, ocsp_cert, ocsp_fp_hint) + if not ok_ocsp then + safe_log(ERR, "OCSP function error for cert #" .. idx .. " (" .. pair.cert_id .. "): " .. tostring(ocsp_cert_acceptable)) + elseif ocsp_cert_acceptable == false then + -- OCSP-Must-Staple requirement not met - log but continue (failsafe) + safe_log(ERR, "OCSP-Must-Staple requirement not met for cert #" .. idx .. " (" .. pair.cert_id .. ") (server_name=" .. server_name .. ") - proceeding anyway") + else + safe_log(DEBUG, "OCSP loaded successfully for certificate #" .. idx .. " (" .. pair.cert_id .. ")") + end else - safe_log(DEBUG, "OCSP loaded successfully for certificate #" .. idx .. " (" .. pair.cert_id .. ")") + safe_log(DEBUG, "Skipping OCSP setup because ngx.ocsp is disabled (server_name=" .. (server_name or "nil") .. ")") end end end -- end ok_cert From 0e2d186333b638723b8f151cbe848921c86633b9 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Fri, 20 Mar 2026 10:31:39 +0100 Subject: [PATCH 59/59] optimize orphan cleanup + hash colision check - optimize orphan cleanup - add hash collision check (a collision "should" never happen but we need to catch that situation) --- .../server-http/ssl-certificate-lua.conf | 52 +++++ src/common/core/ssl/jobs/ocsp-refresh.py | 205 +++++++++++++++++- 2 files changed, 256 insertions(+), 1 deletion(-) diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 37e9744f8a..14c57627a8 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -1656,6 +1656,34 @@ ssl_certificate_by_lua_block { end) else ocsp_ok = false + -- If metadata fingerprint matches but OCSP cryptographic validation fails, + -- check whether the OCSP response serial actually belongs to a different + -- certificate while sharing the same fingerprint. + -- + -- This is treated as a cache collision / hash collision symptom. + local report_key = "TLS:SSL:ocsp_hash_collision_reported:" .. cert_fp + local already_reported = false + pcall(function() + already_reported = internalstore:get(report_key, true) == true + end) + if not already_reported then + local serial_mismatch = false + pcall(function() + if cert_for_fp_pem and type(cert_for_fp_pem) == "string" then + serial_mismatch = (verify_ocsp_cert_match(cert_for_fp_pem, cache_result) == false) + end + end) + if serial_mismatch then + pcall(function() + internalstore:set(report_key, true, 300, true) -- throttle per fingerprint + end) + safe_log( + ERR, + "HASH COLLISION DETECTED: OCSP response serial mismatch for fp=" .. + cert_fp:sub(1, 16) .. "... server_name=" .. (server_name or "nil") + ) + end + end end end @@ -1705,6 +1733,30 @@ ssl_certificate_by_lua_block { end) else ocsp_ok = false + -- Detect cache collision symptom (serial mismatch while fingerprint matches). + local report_key = "TLS:SSL:ocsp_hash_collision_reported:" .. cert_fp + local already_reported = false + pcall(function() + already_reported = internalstore:get(report_key, true) == true + end) + if not already_reported then + local serial_mismatch = false + pcall(function() + if cert_for_fp_pem and type(cert_for_fp_pem) == "string" then + serial_mismatch = (verify_ocsp_cert_match(cert_for_fp_pem, data) == false) + end + end) + if serial_mismatch then + pcall(function() + internalstore:set(report_key, true, 300, true) -- throttle per fingerprint + end) + safe_log( + ERR, + "HASH COLLISION DETECTED: OCSP response serial mismatch for fp=" .. + cert_fp:sub(1, 16) .. "... server_name=" .. (server_name or "nil") + ) + end + end end end end diff --git a/src/common/core/ssl/jobs/ocsp-refresh.py b/src/common/core/ssl/jobs/ocsp-refresh.py index be986c846c..348c175e4c 100644 --- a/src/common/core/ssl/jobs/ocsp-refresh.py +++ b/src/common/core/ssl/jobs/ocsp-refresh.py @@ -2475,6 +2475,200 @@ def cleanup_ocsp_cache( log_info("๐Ÿงน OCSP all stapling caches cleaned up") +def _cleanup_expired_ocsp_entries( + db: Optional[Any] = None, + stats: Optional[dict] = None, +) -> int: + """ + Cleanup expired/stale OCSP response cache entries. + + This job maintains a fingerprint-sharded cache: + - /var/cache/bunkerweb/ssl////ocsp.der + - /var/cache/bunkerweb/ssl////ocsp.json + and mirrors OCSP response bytes in database cache entries: + - file_name="ocsp/" + """ + if stats is None: + stats = {} + + now = datetime.now(timezone.utc) + + # Fingerprints whose ocsp.{json,der} are believed expired. + expired_fingerprints: set = set() + + # Fingerprints whose ocsp.json exists but "expires" couldn't be parsed, + # so we might need to fall back to parsing ocsp.der. + meta_unparseable_fingerprints: set = set() + + expired_cleaned_count: int = 0 + + max_disk_meta_checks = 2000 + max_disk_der_checks = 2000 + max_db_checks = 500 + + def _parse_expires(expires_value: Any) -> Optional[datetime]: + """ + Parse the `expires` value written by this job into a timezone-aware datetime. + + Success metadata format: + - " + s" + + Error/backoff metadata format: + - "" (retry_after.isoformat()) + """ + if not expires_value or not isinstance(expires_value, str): + return None + raw = expires_value.strip() + if not raw or raw.lower() == "unknown": + return None + + try: + if " + " in raw: + base_str, tail_str = raw.rsplit(" + ", 1) + # Expected: "s" + m = re.match(r"^(\\d+)\\s*s$", tail_str.strip()) + if not m: + return None + ttl_seconds = int(m.group(1)) + base_dt = datetime.fromisoformat(base_str.strip()) + if base_dt.tzinfo is None: + base_dt = base_dt.replace(tzinfo=timezone.utc) + return base_dt + timedelta(seconds=ttl_seconds) + + dt = datetime.fromisoformat(raw) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + except Exception: + return None + + def _delete_fingerprint_cache(fingerprint: str) -> None: + nonlocal expired_cleaned_count + if not fingerprint: + return + if fingerprint in expired_fingerprints: + return + + ocsp_dir = _get_sharded_ocsp_path(fingerprint) + try: + shutil.rmtree(ocsp_dir, ignore_errors=True) + except Exception: + pass + + if db: + try: + db.delete_job_cache(file_name=f"ocsp/{fingerprint}", job_name="ocsp-refresh") + except Exception: + pass + + expired_fingerprints.add(fingerprint) + expired_cleaned_count += 1 + + # 1. Disk cleanup using ocsp.json "expires". + if CONFIGS_SSL_BASE.is_dir(): + try: + meta_files = list(CONFIGS_SSL_BASE.rglob("ocsp.json")) + except Exception: + meta_files = [] + + for meta_file in meta_files[:max_disk_meta_checks]: + try: + raw = meta_file.read_text(encoding="utf-8") + meta = json.loads(raw) if raw else None + if not isinstance(meta, dict): + continue + + fingerprint = _normalize_fingerprint(meta.get("fingerprint")) or _normalize_fingerprint( + meta_file.parent.name + ) + if not fingerprint: + continue + + expires_dt = _parse_expires(meta.get("expires")) + if expires_dt is None: + meta_unparseable_fingerprints.add(fingerprint) + continue + + if expires_dt > now: + continue + + _delete_fingerprint_cache(fingerprint) + except Exception: + # Best-effort: don't fail the job due to a corrupted metadata file. + continue + + # 2. Disk cleanup fallback using ocsp.der parsing. + # Parse ocsp.der when ocsp.json is missing OR could not be interpreted. + if CONFIGS_SSL_BASE.is_dir(): + try: + der_files = list(CONFIGS_SSL_BASE.rglob("ocsp.der")) + except Exception: + der_files = [] + + checked = 0 + for ocsp_der in der_files: + if checked >= max_disk_der_checks: + break + checked += 1 + + try: + fingerprint = _normalize_fingerprint(ocsp_der.parent.name) + if not fingerprint or fingerprint in expired_fingerprints: + continue + + # Parse only when ocsp.json couldn't be trusted or is missing. + ocsp_json = ocsp_der.parent / "ocsp.json" + if ocsp_json.is_file() and fingerprint not in meta_unparseable_fingerprints: + continue + + ocsp_data = ocsp_der.read_bytes() + ocsp_response = x509_ocsp.load_der_ocsp_response(ocsp_data) + remaining, _ = _ocsp_response_lifetimes(ocsp_response) + if remaining is not None and remaining <= 0: + _delete_fingerprint_cache(fingerprint) + except Exception: + continue + + # 3. Database cleanup: delete expired OCSP response entries. + if db: + try: + cache_files = db.get_jobs_cache_files(job_name="ocsp-refresh", with_data=True) + except Exception: + cache_files = [] + + checked_db = 0 + for entry in cache_files: + try: + file_name = entry.get("file_name", "") + if not file_name.startswith("ocsp/"): + continue + fp_raw = file_name[len("ocsp/"):] + fingerprint = _normalize_fingerprint(fp_raw) + if not fingerprint or fingerprint in expired_fingerprints: + continue + + data = entry.get("data") + if not data: + continue + + if checked_db >= max_db_checks: + break + checked_db += 1 + + # Marker entries should not reach here (we require a valid fingerprint key). + ocsp_response = x509_ocsp.load_der_ocsp_response(data) + remaining, _ = _ocsp_response_lifetimes(ocsp_response) + if remaining is not None and remaining <= 0: + _delete_fingerprint_cache(fingerprint) + except Exception: + continue + + stats["expired_cleaned"] = stats.get("expired_cleaned", 0) + expired_cleaned_count + if expired_cleaned_count > 0: + log_info("๐Ÿงน OCSP removed %d expired cache item(s)", expired_cleaned_count) + return expired_cleaned_count + + def _cleanup_orphaned_ocsp(db: Optional[Any], le_certs: Dict[str, bytes], stats: Optional[dict] = None) -> None: """ Remove OCSP cache entries (disk + database) for services that no longer have @@ -2993,6 +3187,7 @@ def refresh_job_lock(cert_name: str = "") -> None: "ocsp_restored": 0, "ocsp_corrected": 0, "orphaned_cleaned": 0, + "expired_cleaned": 0, } # Parse command line arguments @@ -3105,6 +3300,14 @@ def refresh_job_lock(cert_name: str = "") -> None: time.sleep(2) restore_ocsp_from_database(db) + # Cleanup expired/stale OCSP cache entries early (before TTL-skip optimization). + # This prevents expired OCSP responses from staying on disk/db when the job runs in a + # "recently refreshed" optimization mode. + try: + _cleanup_expired_ocsp_entries(db, stats) + except Exception as e: + log_debug("โš ๏ธ OCSP expired cleanup failed: %s", e) + # === Efficiency optimization: skip full refresh if run recently and no changes detected === last_refresh_key = "last_full_refresh" now_ts = int(time.time()) @@ -3112,7 +3315,7 @@ def refresh_job_lock(cert_name: str = "") -> None: if not force_all: last_refresh_entry = db.get_job_cache_file(file_name=last_refresh_key, job_name="ocsp-refresh", with_info=True) - if last_refresh_entry and last_refresh_entry.get("data"): + if last_refresh_entry and last_refresh_entry.get("data") and stats.get("expired_cleaned", 0) == 0: try: last_refresh_time = int(last_refresh_entry["data"].decode("utf-8")) if now_ts - last_refresh_time < 1800: # 30 minutes window