IGNITE-28885 Add JMX Prometheus Exporter to Ducktests for Monitoring

modules/ducktests/tests/docker/Dockerfile

@@ -158,6 +158,12 @@ ARG JMXTERM_ARTIFACT="$JMXTERM_NAME-$JMXTERM_VERSION-uber.jar"
 RUN cd /opt && curl -OL https://github.com/jiaqi/jmxterm/releases/download/v$JMXTERM_VERSION/$JMXTERM_ARTIFACT \
     && mv $JMXTERM_ARTIFACT $JMXTERM_NAME.jar
 
+# Install JMX Prometheus Exporter (javaagent for exposing JMX metrics to Prometheus)
+ARG JMX_EXPORTER_VERSION="1.1.0"
+ARG JMX_EXPORTER_ARTIFACT="jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar"
+RUN cd /opt && curl -OL "https://github.com/prometheus/jmx_exporter/releases/download/v${JMX_EXPORTER_VERSION}/${JMX_EXPORTER_ARTIFACT}" \
+    && mv $JMX_EXPORTER_ARTIFACT jmx_exporter.jar
+
 # ---------------------------------------------------------------------
 # 7. User Isolation Permissions (Ducker Execution Account Setup)
 # ---------------------------------------------------------------------

modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py

@@ -37,13 +37,21 @@
 from ignitetest.services.utils.background_thread import BackgroundThreadService
 from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue
 from ignitetest.services.utils.ignite_spec import resolve_spec, SHARED_PREPARED_FILE
+from ignitetest.services.utils.jmx_exporter import is_jmx_exporter_enabled, get_jmx_exporter_yml_content, \
+    JMX_EXPORTER_JAR_PATH, JMX_EXPORTER_YML_NAME
 from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin, JmxClient
 from ignitetest.services.utils.jvm_utils import JvmProcessMixin, JvmVersionMixin
 from ignitetest.services.utils.log_utils import monitor_log
 from ignitetest.services.utils.path import IgnitePathAware
 from ignitetest.utils.enum import constructible
 
 
+JMX_EXPORTER_DOWNLOAD_URL = (
+    "https://github.com/prometheus/jmx_exporter/releases/download/v1.1.0/"
+    "jmx_prometheus_javaagent-1.1.0.jar"
+)
+
+
 class IgniteAwareService(BackgroundThreadService, IgnitePathAware, JvmProcessMixin, JvmVersionMixin, metaclass=ABCMeta):
     """
     The base class to build services aware of Ignite.
@@ -184,8 +192,24 @@ def init_persistent(self, node):
         """
         super().init_persistent(node)
 
+        # Ensure jmx_exporter.jar is available on the node
+        if is_jmx_exporter_enabled(self.globals):
+            self._ensure_jmx_exporter_jar(node)
+
         self._prepare_configs(node)
 
+    @staticmethod
+    def _ensure_jmx_exporter_jar(node):
+        """
+        Download jmx_exporter.jar to the node if it's not already present.
+        Uses curl and conditional execution — idempotent.
+        """
+        cmd = (
+            f"test -f {JMX_EXPORTER_JAR_PATH} || "
+            f"curl -sL {JMX_EXPORTER_DOWNLOAD_URL} -o {JMX_EXPORTER_JAR_PATH}"
+        )
+        node.account.ssh(cmd, allow_fail=False)
+
     def init_shared(self, node):
         """
         Init shared directory. Content of shared directory must be equal on all test nodes.
@@ -231,6 +255,13 @@ def _prepare_configs(self, node):
 
             self.logger.debug("Config %s for node %s: %s" % (name, node.account.hostname, config_txt))
 
+        # Create jmx_exporter.yml config if enabled
+        if is_jmx_exporter_enabled(self.globals):
+            yml_on_node = os.path.join(self.config_dir, JMX_EXPORTER_YML_NAME)
+            yml_content = get_jmx_exporter_yml_content()
+            node.account.create_file(yml_on_node, yml_content)
+            self.logger.debug("JMX Exporter config %s created on node %s" % (yml_on_node, node.account.hostname))
+
         setattr(node, "consistent_id", node.account.externally_routable_ip)
 
     def worker(self, idx, node, **kwargs):

modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py

@@ -36,6 +36,8 @@
 from ignitetest.services.utils.metrics.metrics import is_opencensus_metrics_enabled, configure_opencensus_metrics, \
     is_jmx_metrics_enabled, configure_jmx_metrics
 from ignitetest.services.utils.jmx_remote.jmx_remote_params import get_jmx_remote_params
+from ignitetest.services.utils.jmx_exporter import get_jmx_exporter_params, \
+    jmx_agent_jvm_opt, JMX_EXPORTER_YML_NAME
 from ignitetest.utils.ignite_test import JFR_ENABLED, SAFEPOINT_LOGS_ENABLED
 from ignitetest.utils.version import DEV_BRANCH
 
@@ -130,6 +132,13 @@ def __get_default_jvm_opts(self):
                                                    "-Dcom.sun.management.jmxremote.authenticate=false",
                                                    "-Dcom.sun.management.jmxremote.ssl=false"])
 
+        jmx_exp_cfg = get_jmx_exporter_params(self.service.context.globals)
+        if jmx_exp_cfg.enabled:
+            # Конфиг jmx_exporter.yml будет создан в config_dir на ноде
+            yml_on_node = os.path.join(self.service.config_dir, JMX_EXPORTER_YML_NAME)
+            default_jvm_opts = merge_jvm_settings(default_jvm_opts,
+                                                  [jmx_agent_jvm_opt(jmx_exp_cfg.port, yml_on_node)])
+
         return default_jvm_opts
 
     def config_templates(self):

modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py

@@ -0,0 +1,84 @@
+"""
+JMX Prometheus Exporter helper for ducktests.
+
+Starts jmx_prometheus_javaagent as a JVM agent to expose JVM/Ignite metrics
+on :{port}/metrics in Prometheus text format for external scraping.
+
+Works for both Docker and remote nodes:
+ - jmx_exporter.jar is expected at JMX_EXPORTER_JAR_PATH on each node
+ - jmx_exporter.yml is created in the node's config directory at startup
+"""
+
+import os
+from dataclasses import dataclass
+from importlib.resources import files
+
+JMX_EXPORTER_JAR_PATH = "/opt/jmx_exporter.jar"
+
+JMX_EXPORTER_YML_NAME = "jmx_exporter.yml"
+
+
+@dataclass(frozen=True)
+class JmxExporterParams:
+    """
+    JMX Prometheus Exporter settings from globals config.
+
+    Example in globals:
+        {
+            "jmx_exporter": {
+                "enabled": true,
+                "port": 8083
+            }
+        }
+    """
+    enabled: bool
+    port: int
+
+
+def get_jmx_exporter_params(globals_cfg: dict = None) -> JmxExporterParams:
+    """
+    Read JMX Exporter parameters from globals configuration.
+    """
+    if globals_cfg is None:
+        globals_cfg = {}
+
+    cfg = globals_cfg.get("jmx_exporter", {})
+    if not isinstance(cfg, dict):
+        return JmxExporterParams(enabled=True, port=8083)
+
+    return JmxExporterParams(
+        enabled=bool(cfg.get("enabled", True)),
+        port=int(cfg.get("port", 8083)),
+    )
+
+
+def is_jmx_exporter_enabled(globals_cfg: dict = None) -> bool:
+    """
+    Check if JMX Exporter is enabled in globals.
+    """
+    return get_jmx_exporter_params(globals_cfg).enabled
+
+
+def get_jmx_exporter_yml_content() -> str:
+    """
+    Return the bundled jmx_exporter.yml content as a string.
+    """
+    file = (files(__package__)
+           .joinpath("jmx_exporter.yml"))
+    return file.read_text(encoding="utf-8")
+
+
+def jmx_agent_jvm_opt(port: int = 8083, config_path: str = None) -> str:
+    """
+    Build -javaagent JVM option for jmx_prometheus_javaagent.
+
+    :param port: HTTP port for /metrics endpoint
+    :param config_path: Absolute path to jmx_exporter.yml ON THE NODE
+                        (e.g., /mnt/service/config/jmx_exporter.yml)
+    :return: JVM argument string, e.g.
+             -javaagent:/opt/jmx_exporter.jar=8083:/mnt/service/config/jmx_exporter.yml
+    """
+    if config_path is None:
+        config_path = JMX_EXPORTER_YML_NAME
+
+    return f"-javaagent:{JMX_EXPORTER_JAR_PATH}={port}:{config_path}"

modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/jmx_exporter.yml

@@ -0,0 +1,66 @@
+# JVM Memory & GC metrics for Prometheus scraping
+# Sufficient to reproduce jconsole "Memory" tab in Grafana.
+#
+# Exposed on :8083/metrics
+
+lowercaseOutputName: true
+whitelistObjectNames:
+  - "java.lang:type=Memory"
+  - "java.lang:type=MemoryPool,*"
+  - "java.lang:type=GarbageCollector,*"
+
+rules:
+  # ------------------------------------------------------------------
+  # Heap / Non-Heap overview  (java.lang:type=Memory)
+  #   Attributes: heapMemoryUsage (CompositeData: used/init/max/committed)
+  #               nonHeapMemoryUsage (CompositeData)
+  # ------------------------------------------------------------------
+  - pattern: "java.lang<type=Memory><heapMemoryUsage>(\w+)"
+    name: "jvm_memory_heap_bytes_$1"
+    type: GAUGE
+    help: "JVM heap memory — used / init / max / committed"
+
+  - pattern: "java.lang<type=Memory><nonHeapMemoryUsage>(\w+)"
+    name: "jvm_memory_nonheap_bytes_$1"
+    type: GAUGE
+    help: "JVM non-heap memory — used / init / max / committed"
+
+  # ------------------------------------------------------------------
+  # Per-pool usage  (java.lang:type=MemoryPool,name=<pool>)
+  #   Attributes: usage (CompositeData: used/init/max/committed)
+  #               collectionUsage (CompositeData, post-GC usage)
+  # ------------------------------------------------------------------
+  - pattern: "java.lang<type=MemoryPool,name=(.*?)><usage>(\w+)"
+    name: "jvm_memory_pool_usage_bytes"
+    labels:
+      pool: "$1"
+      attribute: "$2"
+    type: GAUGE
+    help: "JVM memory pool usage — used / init / max / committed"
+
+  - pattern: "java.lang<type=MemoryPool,name=(.*?)><collectionUsage>(\w+)"
+    name: "jvm_memory_pool_collection_bytes"
+    labels:
+      pool: "$1"
+      attribute: "$2"
+    type: GAUGE
+    help: "JVM memory pool post-GC usage — used / init / max / committed"
+
+  # ------------------------------------------------------------------
+  # Garbage Collectors  (java.lang:type=GarbageCollector,name=<gc>)
+  #   Attributes: collectionCount, collectionTime
+  # ------------------------------------------------------------------
+  - pattern: "java.lang<type=GarbageCollector,name=(.*?)><collectionCount>"
+    name: "jvm_gc_collection_count_total"
+    labels:
+      gc: "$1"
+    type: COUNTER
+    help: "Total number of GC collections"
+
+  - pattern: "java.lang<type=GarbageCollector,name=(.*?)><collectionTime>"
+    name: "jvm_gc_collection_time_seconds_total"
+    labels:
+      gc: "$1"
+    type: COUNTER
+    help: "Total GC collection time (ms → seconds via divisor)"
+    valueFactor: 0.001