From 6b43ff037d7d404f79151cb6e1c8fd7849606483 Mon Sep 17 00:00:00 2001 From: Egor Baranov Date: Mon, 13 Jul 2026 11:44:29 +0300 Subject: [PATCH 1/3] IGNITE-28885 Add JMX Prometheus Exporter to Ducktests for JVM Memory/GC Monitoring --- modules/ducktests/tests/docker/Dockerfile | 6 ++ .../ignitetest/services/utils/ignite_aware.py | 31 +++++++ .../ignitetest/services/utils/ignite_spec.py | 9 ++ .../services/utils/jmx_exporter/__init__.py | 92 +++++++++++++++++++ .../utils/jmx_exporter/jmx_exporter.yml | 66 +++++++++++++ 5 files changed, 204 insertions(+) create mode 100644 modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/jmx_exporter.yml diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index 59ed67463d5de..605350467d199 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -158,6 +158,12 @@ ARG JMXTERM_ARTIFACT="$JMXTERM_NAME-$JMXTERM_VERSION-uber.jar" RUN cd /opt && curl -OL https://github.com/jiaqi/jmxterm/releases/download/v$JMXTERM_VERSION/$JMXTERM_ARTIFACT \ && mv $JMXTERM_ARTIFACT $JMXTERM_NAME.jar +# Install JMX Prometheus Exporter (javaagent for exposing JMX metrics to Prometheus) +ARG JMX_EXPORTER_VERSION="1.1.0" +ARG JMX_EXPORTER_ARTIFACT="jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" +RUN cd /opt && curl -OL "https://github.com/prometheus/jmx_exporter/releases/download/v${JMX_EXPORTER_VERSION}/${JMX_EXPORTER_ARTIFACT}" \ + && mv $JMX_EXPORTER_ARTIFACT jmx_exporter.jar + # --------------------------------------------------------------------- # 7. User Isolation Permissions (Ducker Execution Account Setup) # --------------------------------------------------------------------- diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 7d475844a2ed1..cfd02cf128c50 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -37,6 +37,8 @@ from ignitetest.services.utils.background_thread import BackgroundThreadService from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue from ignitetest.services.utils.ignite_spec import resolve_spec, SHARED_PREPARED_FILE +from ignitetest.services.utils.jmx_exporter import is_jmx_exporter_enabled, get_jmx_exporter_yml_content, \ + JMX_EXPORTER_JAR_PATH, JMX_EXPORTER_YML_NAME from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin, JmxClient from ignitetest.services.utils.jvm_utils import JvmProcessMixin, JvmVersionMixin from ignitetest.services.utils.log_utils import monitor_log @@ -44,6 +46,12 @@ from ignitetest.utils.enum import constructible +JMX_EXPORTER_DOWNLOAD_URL = ( + "https://github.com/prometheus/jmx_exporter/releases/download/v1.1.0/" + "jmx_prometheus_javaagent-1.1.0.jar" +) + + class IgniteAwareService(BackgroundThreadService, IgnitePathAware, JvmProcessMixin, JvmVersionMixin, metaclass=ABCMeta): """ The base class to build services aware of Ignite. @@ -184,8 +192,24 @@ def init_persistent(self, node): """ super().init_persistent(node) + # Ensure jmx_exporter.jar is available on the node + if is_jmx_exporter_enabled(self.globals): + self._ensure_jmx_exporter_jar(node) + self._prepare_configs(node) + @staticmethod + def _ensure_jmx_exporter_jar(node): + """ + Download jmx_exporter.jar to the node if it's not already present. + Uses curl and conditional execution — idempotent. + """ + cmd = ( + f"test -f {JMX_EXPORTER_JAR_PATH} || " + f"curl -sL {JMX_EXPORTER_DOWNLOAD_URL} -o {JMX_EXPORTER_JAR_PATH}" + ) + node.account.ssh(cmd, allow_fail=False) + def init_shared(self, node): """ Init shared directory. Content of shared directory must be equal on all test nodes. @@ -231,6 +255,13 @@ def _prepare_configs(self, node): self.logger.debug("Config %s for node %s: %s" % (name, node.account.hostname, config_txt)) + # Create jmx_exporter.yml config if enabled + if is_jmx_exporter_enabled(self.globals): + yml_on_node = os.path.join(self.config_dir, JMX_EXPORTER_YML_NAME) + yml_content = get_jmx_exporter_yml_content() + node.account.create_file(yml_on_node, yml_content) + self.logger.debug("JMX Exporter config %s created on node %s" % (yml_on_node, node.account.hostname)) + setattr(node, "consistent_id", node.account.externally_routable_ip) def worker(self, idx, node, **kwargs): diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index a99aa163026f4..15137d512d4ac 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -36,6 +36,8 @@ from ignitetest.services.utils.metrics.metrics import is_opencensus_metrics_enabled, configure_opencensus_metrics, \ is_jmx_metrics_enabled, configure_jmx_metrics from ignitetest.services.utils.jmx_remote.jmx_remote_params import get_jmx_remote_params +from ignitetest.services.utils.jmx_exporter import get_jmx_exporter_params, \ + jmx_agent_jvm_opt, JMX_EXPORTER_YML_NAME from ignitetest.utils.ignite_test import JFR_ENABLED, SAFEPOINT_LOGS_ENABLED from ignitetest.utils.version import DEV_BRANCH @@ -130,6 +132,13 @@ def __get_default_jvm_opts(self): "-Dcom.sun.management.jmxremote.authenticate=false", "-Dcom.sun.management.jmxremote.ssl=false"]) + jmx_exp_cfg = get_jmx_exporter_params(self.service.context.globals) + if jmx_exp_cfg.enabled: + # Конфиг jmx_exporter.yml будет создан в config_dir на ноде + yml_on_node = os.path.join(self.service.config_dir, JMX_EXPORTER_YML_NAME) + default_jvm_opts = merge_jvm_settings(default_jvm_opts, + [jmx_agent_jvm_opt(jmx_exp_cfg.port, yml_on_node)]) + return default_jvm_opts def config_templates(self): diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py new file mode 100644 index 0000000000000..bf4947d969e52 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py @@ -0,0 +1,92 @@ +""" +JMX Prometheus Exporter helper for ducktests. + +Starts jmx_prometheus_javaagent as a JVM agent to expose JVM/Ignite metrics +on :{port}/metrics in Prometheus text format for external scraping. + +Works for both Docker and remote nodes: + - jmx_exporter.jar is expected at JMX_EXPORTER_JAR_PATH on each node + - jmx_exporter.yml is created in the node's config directory at startup +""" + +import os +from dataclasses import dataclass, field + +JMX_EXPORTER_JAR_PATH = "/opt/jmx_exporter.jar" + +JMX_EXPORTER_YML_NAME = "jmx_exporter.yml" + + +@dataclass(frozen=True) +class JmxExporterParams: + """ + JMX Prometheus Exporter settings from globals config. + + Example in globals: + { + "jmx_exporter": { + "enabled": true, + "port": 8083 + } + } + """ + enabled: bool = True + port: int = 8083 + + +def get_jmx_exporter_params(globals_cfg: dict = None) -> JmxExporterParams: + """ + Read JMX Exporter parameters from globals configuration. + """ + if globals_cfg is None: + globals_cfg = {} + + cfg = globals_cfg.get("jmx_exporter", {}) + if not isinstance(cfg, dict): + return JmxExporterParams() + + return JmxExporterParams( + enabled=bool(cfg.get("enabled", False)), + port=int(cfg.get("port", 8083)), + ) + + +def is_jmx_exporter_enabled(globals_cfg: dict = None) -> bool: + """ + Check if JMX Exporter is enabled in globals. + """ + return get_jmx_exporter_params(globals_cfg).enabled + + +def get_jmx_exporter_yml_content() -> str: + """ + Return the bundled jmx_exporter.yml content as a string. + + Uses importlib.resources (Python 3.11+) with a direct file read fallback. + """ + try: + from importlib.resources import files + res = files(__package__).joinpath("jmx_exporter.yml") + return res.read_text(encoding="utf-8") + except (ModuleNotFoundError, AttributeError, FileNotFoundError): + # Fallback: read from the package directory directly + pkg_dir = os.path.dirname(os.path.abspath(__file__)) + yml_path = os.path.join(pkg_dir, "jmx_exporter.yml") + with open(yml_path, "r", encoding="utf-8") as f: + return f.read() + + +def jmx_agent_jvm_opt(port: int = 8083, config_path: str = None) -> str: + """ + Build -javaagent JVM option for jmx_prometheus_javaagent. + + :param port: HTTP port for /metrics endpoint + :param config_path: Absolute path to jmx_exporter.yml ON THE NODE + (e.g., /mnt/service/config/jmx_exporter.yml) + :return: JVM argument string, e.g. + -javaagent:/opt/jmx_exporter.jar=8083:/mnt/service/config/jmx_exporter.yml + """ + if config_path is None: + config_path = JMX_EXPORTER_YML_NAME + + return f"-javaagent:{JMX_EXPORTER_JAR_PATH}={port}:{config_path}" diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/jmx_exporter.yml b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/jmx_exporter.yml new file mode 100644 index 0000000000000..6fb66c9136992 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/jmx_exporter.yml @@ -0,0 +1,66 @@ +# JVM Memory & GC metrics for Prometheus scraping +# Sufficient to reproduce jconsole "Memory" tab in Grafana. +# +# Exposed on :8083/metrics + +lowercaseOutputName: true +whitelistObjectNames: + - "java.lang:type=Memory" + - "java.lang:type=MemoryPool,*" + - "java.lang:type=GarbageCollector,*" + +rules: + # ------------------------------------------------------------------ + # Heap / Non-Heap overview (java.lang:type=Memory) + # Attributes: heapMemoryUsage (CompositeData: used/init/max/committed) + # nonHeapMemoryUsage (CompositeData) + # ------------------------------------------------------------------ + - pattern: "java.lang(\w+)" + name: "jvm_memory_heap_bytes_$1" + type: GAUGE + help: "JVM heap memory — used / init / max / committed" + + - pattern: "java.lang(\w+)" + name: "jvm_memory_nonheap_bytes_$1" + type: GAUGE + help: "JVM non-heap memory — used / init / max / committed" + + # ------------------------------------------------------------------ + # Per-pool usage (java.lang:type=MemoryPool,name=) + # Attributes: usage (CompositeData: used/init/max/committed) + # collectionUsage (CompositeData, post-GC usage) + # ------------------------------------------------------------------ + - pattern: "java.lang(\w+)" + name: "jvm_memory_pool_usage_bytes" + labels: + pool: "$1" + attribute: "$2" + type: GAUGE + help: "JVM memory pool usage — used / init / max / committed" + + - pattern: "java.lang(\w+)" + name: "jvm_memory_pool_collection_bytes" + labels: + pool: "$1" + attribute: "$2" + type: GAUGE + help: "JVM memory pool post-GC usage — used / init / max / committed" + + # ------------------------------------------------------------------ + # Garbage Collectors (java.lang:type=GarbageCollector,name=) + # Attributes: collectionCount, collectionTime + # ------------------------------------------------------------------ + - pattern: "java.lang" + name: "jvm_gc_collection_count_total" + labels: + gc: "$1" + type: COUNTER + help: "Total number of GC collections" + + - pattern: "java.lang" + name: "jvm_gc_collection_time_seconds_total" + labels: + gc: "$1" + type: COUNTER + help: "Total GC collection time (ms → seconds via divisor)" + valueFactor: 0.001 From 78b2f2356de30d1f7d4bd947c39dfe9c98ddd181 Mon Sep 17 00:00:00 2001 From: Egor Baranov Date: Wed, 15 Jul 2026 10:31:57 +0300 Subject: [PATCH 2/3] IGNITE-28885 Add JMX Prometheus Exporter to Ducktests for JVM Memory/GC Monitoring --- .../ignitetest/services/utils/jmx_exporter/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py index bf4947d969e52..afbbaaa7907c4 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py @@ -30,8 +30,8 @@ class JmxExporterParams: } } """ - enabled: bool = True - port: int = 8083 + enabled: bool + port: int def get_jmx_exporter_params(globals_cfg: dict = None) -> JmxExporterParams: @@ -43,10 +43,10 @@ def get_jmx_exporter_params(globals_cfg: dict = None) -> JmxExporterParams: cfg = globals_cfg.get("jmx_exporter", {}) if not isinstance(cfg, dict): - return JmxExporterParams() + return JmxExporterParams(enabled=True, port=8083) return JmxExporterParams( - enabled=bool(cfg.get("enabled", False)), + enabled=bool(cfg.get("enabled", True)), port=int(cfg.get("port", 8083)), ) From 06ccddf1485e7930ef871c63fcdd751283f4d806 Mon Sep 17 00:00:00 2001 From: Egor Baranov Date: Wed, 15 Jul 2026 10:39:22 +0300 Subject: [PATCH 3/3] IGNITE-28885 Add JMX Prometheus Exporter to Ducktests for JVM Memory/GC Monitoring --- .../services/utils/jmx_exporter/__init__.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py index afbbaaa7907c4..f7d64a4903311 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py @@ -10,7 +10,8 @@ """ import os -from dataclasses import dataclass, field +from dataclasses import dataclass +from importlib.resources import files JMX_EXPORTER_JAR_PATH = "/opt/jmx_exporter.jar" @@ -61,19 +62,10 @@ def is_jmx_exporter_enabled(globals_cfg: dict = None) -> bool: def get_jmx_exporter_yml_content() -> str: """ Return the bundled jmx_exporter.yml content as a string. - - Uses importlib.resources (Python 3.11+) with a direct file read fallback. """ - try: - from importlib.resources import files - res = files(__package__).joinpath("jmx_exporter.yml") - return res.read_text(encoding="utf-8") - except (ModuleNotFoundError, AttributeError, FileNotFoundError): - # Fallback: read from the package directory directly - pkg_dir = os.path.dirname(os.path.abspath(__file__)) - yml_path = os.path.join(pkg_dir, "jmx_exporter.yml") - with open(yml_path, "r", encoding="utf-8") as f: - return f.read() + file = (files(__package__) + .joinpath("jmx_exporter.yml")) + return file.read_text(encoding="utf-8") def jmx_agent_jvm_opt(port: int = 8083, config_path: str = None) -> str: