From cf8f1592781ec90ed89ed40c4b7756f003b1ad31 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 7 Jul 2026 12:43:59 +0300 Subject: [PATCH 01/44] IGNITE-27833 WIP --- .../ducktests/tests/docker/requirements.txt | 1 + .../services/network_group/__init__.py | 21 ++ .../services/network_group/configuration.py | 65 +++++ .../services/network_group/manager.py | 260 ++++++++++++++++++ .../services/network_group/tc_rule_args.py | 69 +++++ .../ignitetest/services/utils/jmx_utils.py | 10 +- .../tests/network_group/__init__.py | 59 ++++ .../partition_resilience_test.py | 113 ++++++++ 8 files changed, 595 insertions(+), 3 deletions(-) create mode 100644 modules/ducktests/tests/ignitetest/services/network_group/__init__.py create mode 100644 modules/ducktests/tests/ignitetest/services/network_group/configuration.py create mode 100644 modules/ducktests/tests/ignitetest/services/network_group/manager.py create mode 100644 modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py create mode 100644 modules/ducktests/tests/ignitetest/tests/network_group/__init__.py create mode 100644 modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py diff --git a/modules/ducktests/tests/docker/requirements.txt b/modules/ducktests/tests/docker/requirements.txt index aba4f25a86d1d..c9f9a003dcf2a 100644 --- a/modules/ducktests/tests/docker/requirements.txt +++ b/modules/ducktests/tests/docker/requirements.txt @@ -16,3 +16,4 @@ filelock==3.8.2 ducktape==0.13.0 looseversion==1.3.0 +tcconfig==0.30.1 \ No newline at end of file diff --git a/modules/ducktests/tests/ignitetest/services/network_group/__init__.py b/modules/ducktests/tests/ignitetest/services/network_group/__init__.py new file mode 100644 index 0000000000000..321d706c84f58 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/network_group/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Network Group Management Service + +This module provides tools for simulating complex network topologies by +defining traffic impairments between logical groups. +""" \ No newline at end of file diff --git a/modules/ducktests/tests/ignitetest/services/network_group/configuration.py b/modules/ducktests/tests/ignitetest/services/network_group/configuration.py new file mode 100644 index 0000000000000..7f3fde990f5f9 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/network_group/configuration.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional, Dict + + +@dataclass(frozen=True) +class CrossNetworkGroupConfiguration: + """ + Defines the network impairment profile between two network groups. + """ + delay: Optional[str] = None # tcset time expression, e.g. "100ms" + loss: Optional[float] = None # fraction in [0.0, 1.0], e.g. 0.1 (10%) + rate: str = "1gbit" # Default to high-speed interface + + def __post_init__(self): + if self.loss is not None and not 0.0 <= self.loss <= 1.0: + raise ValueError(f"loss must be within [0.0, 1.0], got {self.loss}") + + if self.delay is not None and not isinstance(self.delay, str): + raise TypeError(f"delay must be a tcset time expression string (e.g. '100ms'), got {self.delay!r}") + + @property + def is_empty(self) -> bool: + """True if the configuration defines no impairments.""" + return not self.delay and self.loss is None + + +class NetworkGroupStore: + """ + A registry for managing traffic impairments between different network groups. + """ + def __init__(self): + self.matrix: Dict[str, Dict[str, CrossNetworkGroupConfiguration]] = {} + + def set_config(self, group_a: str, group_b: str, impairment: CrossNetworkGroupConfiguration): + """ + Sets bidirectional rules between two network groups. + + Args: + group_a: The first network group identifier. + group_b: The second network group identifier. + impairment: The :class:`CrossNetworkGroupConfiguration` applied to all cross-group traffic directions. + """ + for src, dst in [(group_a, group_b), (group_b, group_a)]: + self.matrix.setdefault(src, {})[dst] = impairment + + def get_config(self, src_group: str, dst_group: str) -> Optional[CrossNetworkGroupConfiguration]: + """ + :return: :class:`CrossNetworkGroupConfiguration` for traffic from src to dst or None if not defined. + """ + return self.matrix.get(src_group, {}).get(dst_group) \ No newline at end of file diff --git a/modules/ducktests/tests/ignitetest/services/network_group/manager.py b/modules/ducktests/tests/ignitetest/services/network_group/manager.py new file mode 100644 index 0000000000000..452cc346a7a1f --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/network_group/manager.py @@ -0,0 +1,260 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re +import socket +import struct +import sys +from itertools import permutations +from typing import Dict, Iterator, List, Optional, Tuple + +from ducktape.services.service import Service + +from ignitetest.services.network_group.configuration import NetworkGroupStore, CrossNetworkGroupConfiguration +from ignitetest.services.network_group.tc_rule_args import ( + ACTION_ADD, ACTION_CHANGE, ACTION_OVERWRITE, + to_tcset_cmd, to_tcdel_all_cmd, to_tcdel_ip_cmd +) +from ignitetest.services.utils.decorators import memoize + + +# Get the default network interface (e.g., eth0, ens3) +CMD_GET_NETWORK_INTERFACE = "ip route | grep default | awk -- '{printf $5}'" + +# Pseudo-action used only on initial deployment: the first rule per node +# overwrites any stale state, subsequent rules are appended. +ACTION_DEPLOY = "deploy" + +# Complete bidirectional traffic drop, simulating a split-brain. +PARTITION_CFG = CrossNetworkGroupConfiguration(loss=1.0) + + +class NetworkGroupManager: + """ + Deploys and tears down traffic-control rules between logical node groups, + and toggles full network partitions between them at test time. + """ + def __init__(self, logger, network_group_store: NetworkGroupStore, network_group_registry: Dict[str, List[Service]]): + self.logger = logger + + self.network_group_store = network_group_store + self.network_group_registry = network_group_registry + + def __enter__(self): + self.deploy() + + self._log_network("ON_DEPLOY") + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.destroy() + + self._log_network("ON_EXIT") + + def deploy(self): + """ + Compiles routing maps and deploys cross-group network constraints. + """ + self.destroy() + + all_pairs = list(permutations(self.network_group_registry.keys(), 2)) + + self._configure_cross_group_traffic(group_pairs=all_pairs, action=ACTION_DEPLOY) + + def destroy(self): + """Restores network interfaces back to their un-throttled state.""" + for node in self._iter_all_nodes(): + interface = self._get_default_network_interface(node) + + node.account.ssh(to_tcdel_all_cmd(interface)) + + def enable_network_partition(self, group_a: str, group_b: str): + """ + Creates a complete, bidirectional network partition between two groups. + All cross-group traffic is dropped (100% loss), simulating a split-brain. + """ + self.logger.info(f"Enabling network partition between [{group_a}] <---> [{group_b}]") + + for src_group, dst_group in self._bidirectional(group_a, group_b): + # tcset rejects '--add' for a destination network that already has a rule, + # so modify the existing rule in-place when a default impairment is deployed. + has_existing_rule = self.network_group_store.get_config(src_group, dst_group) is not None + + action = ACTION_CHANGE if has_existing_rule else ACTION_ADD + + self._configure_cross_group_traffic(group_pairs=[(src_group, dst_group)], + action=action, override_cfg=PARTITION_CFG) + + self._log_network(f"PARTITION {group_a} <-> {group_b}") + + def disable_network_partition(self, group_a: str, group_b: str): + """ + Removes an active network partition between two groups by restoring + the originally stored rules in-place, or clearing the path entirely + if no default impairment was configured. + """ + self.logger.info(f"Disabling network partition between [{group_a}] <---> [{group_b}]") + + self._configure_cross_group_traffic(group_pairs=self._bidirectional(group_a, group_b), + action=ACTION_CHANGE) + + self._log_network(f"NET RESTORED {group_a} <-> {group_b}") + + def _configure_cross_group_traffic(self, group_pairs: List[Tuple[str, str]], action: str, + override_cfg: Optional[CrossNetworkGroupConfiguration] = None): + """ + Applies tcset/tcdel rules from every node of each source group towards + every node of the corresponding destination group. + """ + for src_group, dst_group in group_pairs: + cfg = override_cfg or self.network_group_store.get_config(src_group, dst_group) + + if cfg is None: + if action == ACTION_DEPLOY: + # No impairment configured between these groups: traffic flows unconstrained. + self.logger.debug(f"No configuration for {src_group} -> {dst_group}, skipping.") + continue + + if action != ACTION_CHANGE: + raise Exception(f"Group configuration not found for {src_group}-{dst_group}.") + + dst_ips = [dst_node.account.externally_routable_ip for dst_node in self._iter_group_nodes(dst_group)] + + for src_node in self._iter_group_nodes(src_group): + self._apply_node_rules(src_node, dst_ips, cfg, action) + + def _apply_node_rules(self, src_node, dst_ips: List[str], + cfg: Optional[CrossNetworkGroupConfiguration], action: str): + """ + Applies the given configuration from a single source node to all destination IPs. + """ + interface = self._get_default_network_interface(src_node) + + for idx, dst_ip in enumerate(dst_ips): + if cfg is None: + # Healing cycle with no originally stored impairment: clear the path. + src_node.account.ssh(to_tcdel_ip_cmd(interface, dst_ip)) + continue + + if action == ACTION_DEPLOY: + current_action = ACTION_OVERWRITE if idx == 0 else ACTION_ADD + else: + current_action = action + + cmd = to_tcset_cmd(interface=interface, dst_host_or_ip=dst_ip, config=cfg, action=current_action) + + if cmd: + src_node.account.ssh(cmd) + elif action == ACTION_DEPLOY: + raise ValueError(f"No network constraints defined from {src_node.account.hostname} to {dst_ip}.") + + def _bidirectional(self, group_a: str, group_b: str) -> List[Tuple[str, str]]: + return [(group_a, group_b), (group_b, group_a)] + + def _iter_group_nodes(self, group: str) -> Iterator: + for svc in self.network_group_registry[group]: + yield from svc.nodes + + def _iter_all_nodes(self) -> Iterator: + for group in self.network_group_registry: + yield from self._iter_group_nodes(group) + + def _log_network(self, log_tag: str): + """ + Logs a concise, structured overview of the active traffic control + queuing disciplines (qdiscs) and routing filters across all cluster nodes. + """ + self.logger.debug(f"Network State Overview: [START][{log_tag}]") + + for group, services in self.network_group_registry.items(): + for svc in services: + for node in svc.nodes: + qdisc_output = self._exec_tc_show_command(node, "qdisc") + filter_output = self._exec_tc_show_command(node, "filter") + + dst_ips = self._parse_filter_destinations(filter_output) + constraints = self._parse_qdisc_constraints(qdisc_output) + + targets_str = f" -> to [{', '.join(dst_ips)}]" if dst_ips and constraints != "noqueue" else "" + node_ip = socket.gethostbyname(node.account.externally_routable_ip) + + self.logger.debug(f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : {constraints}{targets_str}") + + self.logger.debug(f"Network State Overview: [END][{log_tag}]") + + def _exec_tc_show_command(self, node, sub_system: str) -> List[str]: + """ + Executes a 'tc show' shell query command on a remote node over SSH + and decodes the terminal response cleanly into strings. + """ + interface = self._get_default_network_interface(node) + + cmd = f"sudo tc {sub_system} show dev {interface}" + + raw_bytes = node.account.ssh_output(cmd, allow_fail=False) + + return raw_bytes.decode(sys.getdefaultencoding()).splitlines() + + @staticmethod + def _parse_filter_destinations(filter_lines: List[str]) -> List[str]: + """ + Parses raw 'tc filter' output lines to extract destination IPs. + Converts the internal u32 hexadecimal match filters back to human-readable strings. + """ + dst_ips = [] + + for line in filter_lines: + match = re.search(r"match\s+([0-9a-fA-F]{8})/ffffffff\s+at\s+16", line) + if match: + hex_ip = match.group(1) + try: + ip_bytes = struct.pack("!I", int(hex_ip, 16)) + dst_ips.append(socket.inet_ntoa(ip_bytes)) + except (ValueError, struct.error, OSError): + continue + + return dst_ips + + @staticmethod + def _parse_qdisc_constraints(qdisc_lines: List[str]) -> str: + """ + Parses raw 'tc qdisc' output lines to identify active traffic impairments. + Extracts active netem delay and loss parameters, ignoring verbose system handles. + """ + for line in qdisc_lines: + if "qdisc netem" in line: + delay_match = re.search(r"delay\s+(\d+\w+)", line) + loss_match = re.search(r"loss\s+(\d+%)", line) + + params = [] + if delay_match: + params.append(f"delay: {delay_match.group(1)}") + if loss_match: + params.append(f"loss: {loss_match.group(1)}") + + if params: + return f"netem({', '.join(params)})" + + return "noqueue" + + @memoize + def _get_default_network_interface(self, node): + return self._get_ssh_output(node, CMD_GET_NETWORK_INTERFACE) + + @staticmethod + def _get_ssh_output(node, cmd): + return node.account.ssh_output(cmd) \ + .decode(sys.getdefaultencoding()) \ + .strip() diff --git a/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py b/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py new file mode 100644 index 0000000000000..465d8eb506a1b --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import socket +from typing import Optional + +from ignitetest.services.network_group.configuration import CrossNetworkGroupConfiguration + +# In non-interactive SSH sessions, 'sudo' enforces a strict 'secure_path' +# and ignores the user's environment. We explicitly pass common installation +# paths (global, virtualenv, and local user directories) via 'env PATH' so +# the system can locate the 'tcset' binary regardless of how tcconfig was installed. +SUDO_PREFIX = 'sudo env "PATH=$PATH:/home/ducker/.local/bin:/opt/venv/bin"' + +# tcset rule actions. +ACTION_OVERWRITE = "--overwrite" +ACTION_ADD = "--add" +ACTION_CHANGE = "--change" + + +def to_tcset_cmd(interface: str, dst_host_or_ip: str, config: CrossNetworkGroupConfiguration, + action: str = ACTION_OVERWRITE) -> Optional[str]: + """ + Compiles the full 'tcset' command string. + Returns None if there are no network limitations configured. + """ + if config.is_empty: + return None + + dst_ip = socket.gethostbyname(dst_host_or_ip) + + args = [ + f"{SUDO_PREFIX} tcset {interface}", + f"--dst-network {dst_ip}/32", + action + ] + + if config.delay: + args.append(f"--delay {config.delay}") + + if config.loss is not None: + args.append(f"--loss {config.loss * 100:g}%") + + return " ".join(args) + + +def to_tcdel_all_cmd(interface: str) -> str: + """ + Compiles the absolute clear command for an interface. + """ + return f"{SUDO_PREFIX} tcdel {interface} --all" + + +def to_tcdel_ip_cmd(interface: str, dst_host_or_ip: str) -> str: + """ + Compiles the absolute clear command for destination host. + """ + return f"{SUDO_PREFIX} tcdel {interface} --peer {dst_host_or_ip}" diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py index 48cfe00003f53..65e850acec39e 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py @@ -81,7 +81,6 @@ def jmx_util_cmd(self): return os.path.join(f"java {extra_flag} -jar {self.install_root}/jmxterm.jar -v silent -n") - @memoize def find_mbean(self, pattern, negative_pattern=None, domain='org.apache'): """ Find mbean by specified pattern and domain on node. @@ -180,6 +179,13 @@ def int_order(self): val = self.__find__("intOrder=(\\d+),") return int(val) if val else -1 + @property + def is_coordinator(self): + """ + :return: True if node is coordinator. False otherwise. + """ + return self.coordinator == self.node_id + def __find__(self, pattern): res = re.search(pattern, self._local_raw) return res.group(1) if res else None @@ -189,7 +195,6 @@ class IgniteJmxMixin: """ Mixin to IgniteService node, exposing useful properties, obtained from JMX. """ - @memoize def jmx_client(self): """ :return: JmxClient instance. @@ -220,7 +225,6 @@ def kernal_mbean(self): """ return self.jmx_client().find_mbean('.*group=Kernal.*name=IgniteKernal') - @memoize def disco_mbean(self): """ :return: DiscoverySpi MBean. diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/__init__.py b/modules/ducktests/tests/ignitetest/tests/network_group/__init__.py new file mode 100644 index 0000000000000..9f6b859e3980c --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/network_group/__init__.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import Dict, List + +from ducktape.services.service import Service + +from ignitetest.services.network_group.configuration import NetworkGroupStore +from ignitetest.services.network_group.manager import NetworkGroupManager +from ignitetest.utils.ignite_test import IgniteTest + + +class NetworkGroupAbstractTest(IgniteTest, ABC): + """ + Abstract test for network-controlled tests. + Ensures network is deployed before the test and cleaned up after. + """ + def _configure_services(self, **kwargs): + pass + + def _configure_network_group_store(self, **kwargs) -> NetworkGroupStore: + return NetworkGroupStore() + + def _configure_network_group_registry(self, **kwargs) -> Dict[str, List[Service]]: + return {} + + @abstractmethod + def _run(self, network_mgr: NetworkGroupManager, **kwargs): + pass + + def configure_network_and_run(self, **kwargs): + self.logger.debug("Deploying network topology...") + + self._configure_services(**kwargs) + + grp_store = self._configure_network_group_store(**kwargs) + grp_registry = self._configure_network_group_registry(**kwargs) + + try: + with NetworkGroupManager(self.logger, grp_store, grp_registry) as network_mgr: + self.logger.debug("Network configuration complete. Starting test logic ...") + + self._run(network_mgr, **kwargs) + finally: + # NetworkGroupManager.__exit__ has restored the network by this point. + self.logger.debug("Network is restored.") \ No newline at end of file diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py new file mode 100644 index 0000000000000..fd2cfeaec4aba --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from time import sleep + +from ducktape.mark import matrix + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.network_group.configuration import NetworkGroupStore, CrossNetworkGroupConfiguration +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration +from ignitetest.tests.network_group import NetworkGroupAbstractTest +from ignitetest.utils import cluster, ignite_versions +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + +NUM_NODES = 12 + +DC_1_NAME = "DC1" +DC_2_NAME = "DC2" + + +class MultiDCPartitionResilienceTest(NetworkGroupAbstractTest): + """ + Tests for cluster network partition resilience in MultiDC + """ + @cluster(num_nodes=NUM_NODES) + @ignite_versions(str(DEV_BRANCH)) + @matrix(cross_dc_latency_ms=[1, 5, 10], partition_time_sec=[0.5, 1, 5], is_node_count_odd=[True, False]) + def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms, partition_time_sec, + is_node_count_odd): + self.configure_network_and_run(ignite_version=ignite_version, cross_dc_latency_ms=cross_dc_latency_ms, + partition_time_sec=partition_time_sec, is_node_count_odd=is_node_count_odd) + + def _configure_network_group_store(self, **kwargs) -> NetworkGroupStore: + store = super()._configure_network_group_store(**kwargs) + + # tcset expects a time expression with units, not a bare integer. + dc1_dc2_cfg = CrossNetworkGroupConfiguration(delay=f"{kwargs['cross_dc_latency_ms']}ms") + store.set_config(DC_1_NAME, DC_2_NAME, dc1_dc2_cfg) + + return store + + def _configure_services(self, **kwargs): + self.ign_cfg = IgniteConfiguration( + version=IgniteVersion(kwargs['ignite_version']), + peer_class_loading_enabled=False + ) + + dc_1_nodes_num, dc_2_nodes_num = self._dc_node_counts(kwargs['is_node_count_odd']) + + self.svc_dc_1 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_1_nodes_num, + jvm_opts=[f"-DIGNITE_DATA_CENTER_ID={DC_1_NAME}"]) + self.svc_dc_2 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_2_nodes_num, + jvm_opts=[f"-DIGNITE_DATA_CENTER_ID={DC_2_NAME}"]) + + def _configure_network_group_registry(self, **kwargs): + return { + DC_1_NAME: [self.svc_dc_1], + DC_2_NAME: [self.svc_dc_2] + } + + def _run(self, network_mgr, **kwargs): + for svc in [self.svc_dc_1, self.svc_dc_2]: + svc.start() + + network_mgr.enable_network_partition(DC_1_NAME, DC_2_NAME) + + sleep(kwargs['partition_time_sec']) + + network_mgr.disable_network_partition(DC_1_NAME, DC_2_NAME) + + self._verify_cluster_survived(kwargs['is_node_count_odd']) + + self.svc_dc_1.stop() + self.svc_dc_2.stop() + + def _verify_cluster_survived(self, is_node_count_odd: bool): + total_alive = sum(len(svc.alive_nodes) for svc in [self.svc_dc_1, self.svc_dc_2]) + + # The majority side of the partition must survive. + min_alive_nodes = min(self._dc_node_counts(is_node_count_odd)) + + assert total_alive >= min_alive_nodes, (f"At least {min_alive_nodes} nodes should be alive! " + f"[actual={total_alive}]") + + coordinator_node = self.svc_dc_1.nodes[0] + + assert self.svc_dc_1.alive(coordinator_node), "Coordinator should remain alive" + + disco_info_dc_1 = coordinator_node.discovery_info() + + assert disco_info_dc_1.is_coordinator, f"{self.svc_dc_1.who_am_i(coordinator_node)} is not a coordinator" + + @staticmethod + def _dc_node_counts(is_node_count_odd: bool): + """ + :return: (dc_1_nodes_num, dc_2_nodes_num). DC2 gets one node fewer when + an odd total cluster size is requested. + """ + dc_1 = NUM_NODES // 2 + dc_2 = NUM_NODES // 2 - 1 if is_node_count_odd else NUM_NODES // 2 + + return dc_1, dc_2 From ba1fe701ca69640aaaffab7818baea62ab1d6dd5 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 7 Jul 2026 14:08:58 +0300 Subject: [PATCH 02/44] IGNITE-27833 WIP --- .../network_group/partition_resilience_test.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py index fd2cfeaec4aba..2a7ee61c69ff8 100644 --- a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py @@ -35,7 +35,7 @@ class MultiDCPartitionResilienceTest(NetworkGroupAbstractTest): """ @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[1, 5, 10], partition_time_sec=[0.5, 1, 5], is_node_count_odd=[True, False]) + @matrix(cross_dc_latency_ms=[5, 10, 20, 50], partition_time_sec=[0.5, 1, 5], is_node_count_odd=[True, False]) def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms, partition_time_sec, is_node_count_odd): self.configure_network_and_run(ignite_version=ignite_version, cross_dc_latency_ms=cross_dc_latency_ms, @@ -51,10 +51,7 @@ def _configure_network_group_store(self, **kwargs) -> NetworkGroupStore: return store def _configure_services(self, **kwargs): - self.ign_cfg = IgniteConfiguration( - version=IgniteVersion(kwargs['ignite_version']), - peer_class_loading_enabled=False - ) + self.ign_cfg = IgniteConfiguration(version=IgniteVersion(kwargs['ignite_version'])) dc_1_nodes_num, dc_2_nodes_num = self._dc_node_counts(kwargs['is_node_count_odd']) @@ -87,11 +84,9 @@ def _run(self, network_mgr, **kwargs): def _verify_cluster_survived(self, is_node_count_odd: bool): total_alive = sum(len(svc.alive_nodes) for svc in [self.svc_dc_1, self.svc_dc_2]) - # The majority side of the partition must survive. - min_alive_nodes = min(self._dc_node_counts(is_node_count_odd)) + alive_nodes = sum(self._dc_node_counts(is_node_count_odd)) - assert total_alive >= min_alive_nodes, (f"At least {min_alive_nodes} nodes should be alive! " - f"[actual={total_alive}]") + assert total_alive == alive_nodes, f"{alive_nodes} nodes should be alive! [actual={total_alive}]" coordinator_node = self.svc_dc_1.nodes[0] From bbfd01a6165204facbf14cac1878819f4b132810 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 8 Jul 2026 11:07:48 +0300 Subject: [PATCH 03/44] IGNITE-27833 WIP --- .../services/network_group/manager.py | 167 +++++++++++++----- .../partition_resilience_test.py | 54 ++++-- 2 files changed, 162 insertions(+), 59 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/network_group/manager.py b/modules/ducktests/tests/ignitetest/services/network_group/manager.py index 452cc346a7a1f..9caffb55e5ac4 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/manager.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/manager.py @@ -16,7 +16,9 @@ import socket import struct import sys +from concurrent.futures import ThreadPoolExecutor from itertools import permutations +from time import monotonic from typing import Dict, Iterator, List, Optional, Tuple from ducktape.services.service import Service @@ -39,13 +41,26 @@ # Complete bidirectional traffic drop, simulating a split-brain. PARTITION_CFG = CrossNetworkGroupConfiguration(loss=1.0) +# Upper bound on concurrent SSH sessions used to apply tc rules cluster-wide. +MAX_PARALLEL_SSH_SESSIONS = 16 + +# A rule spec: (src_group, dst_group, action, config). config=None means +# "clear the path" (only valid together with ACTION_CHANGE). +RuleSpec = Tuple[str, str, str, Optional[CrossNetworkGroupConfiguration]] + class NetworkGroupManager: """ Deploys and tears down traffic-control rules between logical node groups, and toggles full network partitions between them at test time. + + All tc commands targeting a single node are batched into one SSH + invocation, and nodes are configured in parallel, so that a partition + (or its removal) takes effect near-atomically across the cluster instead + of rolling out node by node. """ - def __init__(self, logger, network_group_store: NetworkGroupStore, network_group_registry: Dict[str, List[Service]]): + def __init__(self, logger, network_group_store: NetworkGroupStore, + network_group_registry: Dict[str, List[Service]]): self.logger = logger self.network_group_store = network_group_store @@ -67,26 +82,45 @@ def deploy(self): """ Compiles routing maps and deploys cross-group network constraints. """ + self._prefetch_network_interfaces() + self.destroy() - all_pairs = list(permutations(self.network_group_registry.keys(), 2)) + specs = [] + + for src_group, dst_group in permutations(self.network_group_registry.keys(), 2): + cfg = self.network_group_store.get_config(src_group, dst_group) - self._configure_cross_group_traffic(group_pairs=all_pairs, action=ACTION_DEPLOY) + if cfg is None: + # No impairment configured between these groups: traffic flows unconstrained. + self.logger.debug(f"No configuration for {src_group} -> {dst_group}, skipping.") + continue + + specs.append((src_group, dst_group, ACTION_DEPLOY, cfg)) + + self._apply_specs(specs, tag="DEPLOY") def destroy(self): """Restores network interfaces back to their un-throttled state.""" + tasks = [] + for node in self._iter_all_nodes(): interface = self._get_default_network_interface(node) - node.account.ssh(to_tcdel_all_cmd(interface)) + tasks.append((node, to_tcdel_all_cmd(interface))) + + self._ssh_parallel(tasks, tag="DESTROY") def enable_network_partition(self, group_a: str, group_b: str): """ Creates a complete, bidirectional network partition between two groups. All cross-group traffic is dropped (100% loss), simulating a split-brain. + Both directions and all nodes are configured in a single parallel wave. """ self.logger.info(f"Enabling network partition between [{group_a}] <---> [{group_b}]") + specs = [] + for src_group, dst_group in self._bidirectional(group_a, group_b): # tcset rejects '--add' for a destination network that already has a rule, # so modify the existing rule in-place when a default impairment is deployed. @@ -94,8 +128,9 @@ def enable_network_partition(self, group_a: str, group_b: str): action = ACTION_CHANGE if has_existing_rule else ACTION_ADD - self._configure_cross_group_traffic(group_pairs=[(src_group, dst_group)], - action=action, override_cfg=PARTITION_CFG) + specs.append((src_group, dst_group, action, PARTITION_CFG)) + + self._apply_specs(specs, tag="PARTITION_ON") self._log_network(f"PARTITION {group_a} <-> {group_b}") @@ -104,63 +139,111 @@ def disable_network_partition(self, group_a: str, group_b: str): Removes an active network partition between two groups by restoring the originally stored rules in-place, or clearing the path entirely if no default impairment was configured. + Both directions and all nodes are restored in a single parallel wave. """ self.logger.info(f"Disabling network partition between [{group_a}] <---> [{group_b}]") - self._configure_cross_group_traffic(group_pairs=self._bidirectional(group_a, group_b), - action=ACTION_CHANGE) + specs = [] + + for src_group, dst_group in self._bidirectional(group_a, group_b): + # A None config makes _collect_node_cmds emit 'tcdel --peer' for each path. + cfg = self.network_group_store.get_config(src_group, dst_group) + + specs.append((src_group, dst_group, ACTION_CHANGE, cfg)) + + self._apply_specs(specs, tag="PARTITION_OFF") self._log_network(f"NET RESTORED {group_a} <-> {group_b}") - def _configure_cross_group_traffic(self, group_pairs: List[Tuple[str, str]], action: str, - override_cfg: Optional[CrossNetworkGroupConfiguration] = None): + def _apply_specs(self, specs: List[RuleSpec], tag: str): """ - Applies tcset/tcdel rules from every node of each source group towards - every node of the corresponding destination group. + Compiles all rule specs into one batched command per source node and + executes them across nodes in parallel. """ - for src_group, dst_group in group_pairs: - cfg = override_cfg or self.network_group_store.get_config(src_group, dst_group) + node_cmds = self._collect_node_cmds(specs) - if cfg is None: - if action == ACTION_DEPLOY: - # No impairment configured between these groups: traffic flows unconstrained. - self.logger.debug(f"No configuration for {src_group} -> {dst_group}, skipping.") - continue + # One SSH round-trip per node: all rules for a node take effect together. + tasks = [(node, " && ".join(cmds)) for node, cmds in node_cmds] - if action != ACTION_CHANGE: - raise Exception(f"Group configuration not found for {src_group}-{dst_group}.") + self._ssh_parallel(tasks, tag=tag) - dst_ips = [dst_node.account.externally_routable_ip for dst_node in self._iter_group_nodes(dst_group)] + def _collect_node_cmds(self, specs: List[RuleSpec]): + """ + Groups the tc commands produced by all specs by source node. + + :return: List of (node, [command, ...]) with insertion order preserved, + so that on deployment the first rule per node is an '--overwrite' and + every subsequent rule (across all destination groups) is an '--add'. + """ + per_node = {} + + for src_group, dst_group, action, cfg in specs: + dst_ips = [node.account.externally_routable_ip for node in self._iter_group_nodes(dst_group)] for src_node in self._iter_group_nodes(src_group): - self._apply_node_rules(src_node, dst_ips, cfg, action) + interface = self._get_default_network_interface(src_node) + + _, cmds = per_node.setdefault(id(src_node), (src_node, [])) + + for dst_ip in dst_ips: + if cfg is None: + # Healing cycle with no originally stored impairment: clear the path. + cmds.append(to_tcdel_ip_cmd(interface, dst_ip)) + continue + + if action == ACTION_DEPLOY: + current_action = ACTION_OVERWRITE if not cmds else ACTION_ADD + else: + current_action = action + + cmd = to_tcset_cmd(interface=interface, dst_host_or_ip=dst_ip, config=cfg, + action=current_action) + + if cmd: + cmds.append(cmd) + elif action == ACTION_DEPLOY: + raise ValueError( + f"No network constraints defined from {src_node.account.hostname} to {dst_ip}." + ) + + return [entry for entry in per_node.values() if entry[1]] - def _apply_node_rules(self, src_node, dst_ips: List[str], - cfg: Optional[CrossNetworkGroupConfiguration], action: str): + def _ssh_parallel(self, tasks: List[Tuple[object, str]], tag: str): """ - Applies the given configuration from a single source node to all destination IPs. + Executes one command per node concurrently and propagates the first failure. """ - interface = self._get_default_network_interface(src_node) + if not tasks: + return - for idx, dst_ip in enumerate(dst_ips): - if cfg is None: - # Healing cycle with no originally stored impairment: clear the path. - src_node.account.ssh(to_tcdel_ip_cmd(interface, dst_ip)) - continue + started = monotonic() + + def run(task): + node, cmd = task + node.account.ssh(cmd) - if action == ACTION_DEPLOY: - current_action = ACTION_OVERWRITE if idx == 0 else ACTION_ADD - else: - current_action = action + with ThreadPoolExecutor(max_workers=min(MAX_PARALLEL_SSH_SESSIONS, len(tasks))) as pool: + futures = [pool.submit(run, task) for task in tasks] - cmd = to_tcset_cmd(interface=interface, dst_host_or_ip=dst_ip, config=cfg, action=current_action) + for future in futures: + future.result() - if cmd: - src_node.account.ssh(cmd) - elif action == ACTION_DEPLOY: - raise ValueError(f"No network constraints defined from {src_node.account.hostname} to {dst_ip}.") + self.logger.debug(f"[{tag}] tc rules applied on {len(tasks)} node(s) in {monotonic() - started:.2f}s") - def _bidirectional(self, group_a: str, group_b: str) -> List[Tuple[str, str]]: + def _prefetch_network_interfaces(self): + """ + Warms up the per-node network interface cache in parallel, so that + command compilation later on requires no sequential SSH round-trips. + """ + nodes = list(self._iter_all_nodes()) + + if not nodes: + return + + with ThreadPoolExecutor(max_workers=min(MAX_PARALLEL_SSH_SESSIONS, len(nodes))) as pool: + list(pool.map(self._get_default_network_interface, nodes)) + + @staticmethod + def _bidirectional(group_a: str, group_b: str) -> List[Tuple[str, str]]: return [(group_a, group_b), (group_b, group_a)] def _iter_group_nodes(self, group: str) -> Iterator: diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py index 2a7ee61c69ff8..d6fab97fb056a 100644 --- a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py @@ -15,9 +15,11 @@ from time import sleep from ducktape.mark import matrix +from ducktape.utils.util import wait_until from ignitetest.services.ignite import IgniteService from ignitetest.services.network_group.configuration import NetworkGroupStore, CrossNetworkGroupConfiguration +from ignitetest.services.utils.control_utility import ControlUtility from ignitetest.services.utils.ignite_configuration import IgniteConfiguration from ignitetest.tests.network_group import NetworkGroupAbstractTest from ignitetest.utils import cluster, ignite_versions @@ -28,6 +30,11 @@ DC_1_NAME = "DC1" DC_2_NAME = "DC2" +# How long to wait for the half-ring to reach the expected cluster state +# after the partition heals. Discovery failure detection, ring closure and +# the state transition are asynchronous, so an immediate check is racy. +CLUSTER_STATE_TIMEOUT_SEC = 60 + class MultiDCPartitionResilienceTest(NetworkGroupAbstractTest): """ @@ -35,11 +42,14 @@ class MultiDCPartitionResilienceTest(NetworkGroupAbstractTest): """ @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[5, 10, 20, 50], partition_time_sec=[0.5, 1, 5], is_node_count_odd=[True, False]) - def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms, partition_time_sec, - is_node_count_odd): + # NOTE: for the partition to be reliably *detected*, partition_time_sec must + # exceed discovery failureDetectionTimeout + connectionRecoveryTimeout + # (~10s + 10s by default). Sub-detection values (e.g. 0.5s) exercise the + # "partition heals before the cluster reacts" case instead. + @matrix(cross_dc_latency_ms=[20], partition_time_sec=[30]) + def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms, partition_time_sec): self.configure_network_and_run(ignite_version=ignite_version, cross_dc_latency_ms=cross_dc_latency_ms, - partition_time_sec=partition_time_sec, is_node_count_odd=is_node_count_odd) + partition_time_sec=partition_time_sec) def _configure_network_group_store(self, **kwargs) -> NetworkGroupStore: store = super()._configure_network_group_store(**kwargs) @@ -53,7 +63,7 @@ def _configure_network_group_store(self, **kwargs) -> NetworkGroupStore: def _configure_services(self, **kwargs): self.ign_cfg = IgniteConfiguration(version=IgniteVersion(kwargs['ignite_version'])) - dc_1_nodes_num, dc_2_nodes_num = self._dc_node_counts(kwargs['is_node_count_odd']) + dc_1_nodes_num, dc_2_nodes_num = NUM_NODES // 2, NUM_NODES // 2 self.svc_dc_1 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_1_nodes_num, jvm_opts=[f"-DIGNITE_DATA_CENTER_ID={DC_1_NAME}"]) @@ -76,17 +86,18 @@ def _run(self, network_mgr, **kwargs): network_mgr.disable_network_partition(DC_1_NAME, DC_2_NAME) - self._verify_cluster_survived(kwargs['is_node_count_odd']) + self._verify_half_ring_status(self.svc_dc_1, "ACTIVE") + self._verify_half_ring_status(self.svc_dc_2, "READ_ONLY") + + self._verify_cluster_survived() self.svc_dc_1.stop() self.svc_dc_2.stop() - def _verify_cluster_survived(self, is_node_count_odd: bool): + def _verify_cluster_survived(self): total_alive = sum(len(svc.alive_nodes) for svc in [self.svc_dc_1, self.svc_dc_2]) - alive_nodes = sum(self._dc_node_counts(is_node_count_odd)) - - assert total_alive == alive_nodes, f"{alive_nodes} nodes should be alive! [actual={total_alive}]" + assert total_alive == NUM_NODES, f"{NUM_NODES} nodes should be alive! [actual={total_alive}]" coordinator_node = self.svc_dc_1.nodes[0] @@ -97,12 +108,21 @@ def _verify_cluster_survived(self, is_node_count_odd: bool): assert disco_info_dc_1.is_coordinator, f"{self.svc_dc_1.who_am_i(coordinator_node)} is not a coordinator" @staticmethod - def _dc_node_counts(is_node_count_odd: bool): + def _verify_half_ring_status(svc: IgniteService, status: str, timeout_sec: int = CLUSTER_STATE_TIMEOUT_SEC): """ - :return: (dc_1_nodes_num, dc_2_nodes_num). DC2 gets one node fewer when - an odd total cluster size is requested. + Polls the half-ring cluster state until it reaches the expected status. + The state transition after a partition is asynchronous (failure detection, + ring closure, state change exchange), so a one-shot assertion is racy. """ - dc_1 = NUM_NODES // 2 - dc_2 = NUM_NODES // 2 - 1 if is_node_count_odd else NUM_NODES // 2 - - return dc_1, dc_2 + control_utility = ControlUtility(svc) + + def has_expected_status(): + try: + return status in control_utility.cluster_state() + except Exception: + # control.sh may transiently fail while the ring is re-forming. + return False + + wait_until(has_expected_status, timeout_sec=timeout_sec, backoff_sec=1, + err_msg=f"Half-ring did not reach {status} status within {timeout_sec}s " + f"[last known state check failed or mismatched]") From b59c2a034821fdb0a03b04757f575c6d12e634c5 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 8 Jul 2026 11:09:07 +0300 Subject: [PATCH 04/44] IGNITE-27833 WIP --- modules/ducktests/tests/docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index 59ed67463d5de..db6f4222f0ec4 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -81,6 +81,7 @@ RUN apt-get update && apt-get install -y \ mc \ git \ build-essential \ + iproute2 \ && apt-get -y clean \ && rm -rf /var/lib/apt/lists/* From 48971e1cc2255a24b3399e882e62c644994e0838 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 8 Jul 2026 13:40:36 +0300 Subject: [PATCH 05/44] IGNITE-27833 WIP --- .../tests/ignitetest/services/network_group/manager.py | 9 ++++++++- .../tests/network_group/partition_resilience_test.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/network_group/manager.py b/modules/ducktests/tests/ignitetest/services/network_group/manager.py index 9caffb55e5ac4..297887f153860 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/manager.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/manager.py @@ -261,6 +261,8 @@ def _log_network(self, log_tag: str): """ self.logger.debug(f"Network State Overview: [START][{log_tag}]") + node_to_status_map = {} + for group, services in self.network_group_registry.items(): for svc in services: for node in svc.nodes: @@ -273,7 +275,12 @@ def _log_network(self, log_tag: str): targets_str = f" -> to [{', '.join(dst_ips)}]" if dst_ips and constraints != "noqueue" else "" node_ip = socket.gethostbyname(node.account.externally_routable_ip) - self.logger.debug(f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : {constraints}{targets_str}") + node_status = f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : {constraints}{targets_str}" + + node_to_status_map.update({id(node): node_status}) + + for node_id, node_status in node_to_status_map.items(): + self.logger.debug(node_status) self.logger.debug(f"Network State Overview: [END][{log_tag}]") diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py index d6fab97fb056a..09d8e893512b0 100644 --- a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py @@ -87,7 +87,7 @@ def _run(self, network_mgr, **kwargs): network_mgr.disable_network_partition(DC_1_NAME, DC_2_NAME) self._verify_half_ring_status(self.svc_dc_1, "ACTIVE") - self._verify_half_ring_status(self.svc_dc_2, "READ_ONLY") + self._verify_half_ring_status(self.svc_dc_2, "ACTIVE") self._verify_cluster_survived() From 9a47f78c1644d3a463373cd88557942165f79455 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 8 Jul 2026 14:18:11 +0300 Subject: [PATCH 06/44] IGNITE-27833 iptables used for network partitioning --- .../services/network_group/manager.py | 93 ++++++++++--------- .../services/network_group/tc_rule_args.py | 76 ++++++++++++++- 2 files changed, 126 insertions(+), 43 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/network_group/manager.py b/modules/ducktests/tests/ignitetest/services/network_group/manager.py index 297887f153860..5fc453eadcecc 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/manager.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/manager.py @@ -19,14 +19,15 @@ from concurrent.futures import ThreadPoolExecutor from itertools import permutations from time import monotonic -from typing import Dict, Iterator, List, Optional, Tuple +from typing import Dict, Iterator, List, Tuple from ducktape.services.service import Service from ignitetest.services.network_group.configuration import NetworkGroupStore, CrossNetworkGroupConfiguration from ignitetest.services.network_group.tc_rule_args import ( - ACTION_ADD, ACTION_CHANGE, ACTION_OVERWRITE, - to_tcset_cmd, to_tcdel_all_cmd, to_tcdel_ip_cmd + ACTION_ADD, ACTION_OVERWRITE, + to_tcset_cmd, to_tcdel_all_cmd, + partition_chain_name, to_partition_enable_cmd, to_partition_disable_cmd, to_partition_teardown_cmd ) from ignitetest.services.utils.decorators import memoize @@ -38,15 +39,11 @@ # overwrites any stale state, subsequent rules are appended. ACTION_DEPLOY = "deploy" -# Complete bidirectional traffic drop, simulating a split-brain. -PARTITION_CFG = CrossNetworkGroupConfiguration(loss=1.0) - # Upper bound on concurrent SSH sessions used to apply tc rules cluster-wide. MAX_PARALLEL_SSH_SESSIONS = 16 -# A rule spec: (src_group, dst_group, action, config). config=None means -# "clear the path" (only valid together with ACTION_CHANGE). -RuleSpec = Tuple[str, str, str, Optional[CrossNetworkGroupConfiguration]] +# A rule spec: (src_group, dst_group, action, config). +RuleSpec = Tuple[str, str, str, CrossNetworkGroupConfiguration] class NetworkGroupManager: @@ -54,10 +51,15 @@ class NetworkGroupManager: Deploys and tears down traffic-control rules between logical node groups, and toggles full network partitions between them at test time. - All tc commands targeting a single node are batched into one SSH - invocation, and nodes are configured in parallel, so that a partition - (or its removal) takes effect near-atomically across the cluster instead - of rolling out node by node. + Baseline impairments (delay/loss/rate) are deployed once via tcset. + Partitions are layered on top as per-pair iptables DROP chains. The netem + rules are never touched by a partition, so healing is a pure chain flush + that automatically restores the originally deployed impairments. + + All commands targeting a single node are batched into one SSH invocation, + and nodes are configured in parallel, so that a partition (or its removal) + takes effect near-atomically across the cluster instead of rolling out + node by node. """ def __init__(self, logger, network_group_store: NetworkGroupStore, network_group_registry: Dict[str, List[Service]]): @@ -101,60 +103,67 @@ def deploy(self): self._apply_specs(specs, tag="DEPLOY") def destroy(self): - """Restores network interfaces back to their un-throttled state.""" + """ + Restores network interfaces back to their un-throttled state and + removes any leftover partition chains from iptables. + """ tasks = [] for node in self._iter_all_nodes(): interface = self._get_default_network_interface(node) - tasks.append((node, to_tcdel_all_cmd(interface))) + tasks.append((node, f"{to_tcdel_all_cmd(interface)} && {to_partition_teardown_cmd()}")) self._ssh_parallel(tasks, tag="DESTROY") def enable_network_partition(self, group_a: str, group_b: str): """ - Creates a complete, bidirectional network partition between two groups. - All cross-group traffic is dropped (100% loss), simulating a split-brain. - Both directions and all nodes are configured in a single parallel wave. + Creates a complete, bidirectional network partition between two groups + by installing iptables DROP rules for all cross-group traffic, + simulating a split-brain. The netem impairments deployed via tcset are + left untouched underneath. """ self.logger.info(f"Enabling network partition between [{group_a}] <---> [{group_b}]") - specs = [] + chain = partition_chain_name(group_a, group_b) + + tasks = [] for src_group, dst_group in self._bidirectional(group_a, group_b): - # tcset rejects '--add' for a destination network that already has a rule, - # so modify the existing rule in-place when a default impairment is deployed. - has_existing_rule = self.network_group_store.get_config(src_group, dst_group) is not None + remote_ips = self._resolve_group_ips(dst_group) - action = ACTION_CHANGE if has_existing_rule else ACTION_ADD + cmd = to_partition_enable_cmd(chain, remote_ips) - specs.append((src_group, dst_group, action, PARTITION_CFG)) + for node in self._iter_group_nodes(src_group): + tasks.append((node, cmd)) - self._apply_specs(specs, tag="PARTITION_ON") + self._ssh_parallel(tasks, tag="PARTITION_ON") self._log_network(f"PARTITION {group_a} <-> {group_b}") def disable_network_partition(self, group_a: str, group_b: str): """ - Removes an active network partition between two groups by restoring - the originally stored rules in-place, or clearing the path entirely - if no default impairment was configured. - Both directions and all nodes are restored in a single parallel wave. + Heals an active network partition between two groups by flushing the + pair's iptables DROP chain on every node - a single native call per + node. The originally deployed tcset impairments were never modified, + so they are back in effect immediately without any re-application. """ self.logger.info(f"Disabling network partition between [{group_a}] <---> [{group_b}]") - specs = [] - - for src_group, dst_group in self._bidirectional(group_a, group_b): - # A None config makes _collect_node_cmds emit 'tcdel --peer' for each path. - cfg = self.network_group_store.get_config(src_group, dst_group) + cmd = to_partition_disable_cmd(partition_chain_name(group_a, group_b)) - specs.append((src_group, dst_group, ACTION_CHANGE, cfg)) + tasks = [(node, cmd) + for group in (group_a, group_b) + for node in self._iter_group_nodes(group)] - self._apply_specs(specs, tag="PARTITION_OFF") + self._ssh_parallel(tasks, tag="PARTITION_OFF") self._log_network(f"NET RESTORED {group_a} <-> {group_b}") + def _resolve_group_ips(self, group: str) -> List[str]: + return [socket.gethostbyname(node.account.externally_routable_ip) + for node in self._iter_group_nodes(group)] + def _apply_specs(self, specs: List[RuleSpec], tag: str): """ Compiles all rule specs into one batched command per source node and @@ -186,11 +195,6 @@ def _collect_node_cmds(self, specs: List[RuleSpec]): _, cmds = per_node.setdefault(id(src_node), (src_node, [])) for dst_ip in dst_ips: - if cfg is None: - # Healing cycle with no originally stored impairment: clear the path. - cmds.append(to_tcdel_ip_cmd(interface, dst_ip)) - continue - if action == ACTION_DEPLOY: current_action = ACTION_OVERWRITE if not cmds else ACTION_ADD else: @@ -275,7 +279,12 @@ def _log_network(self, log_tag: str): targets_str = f" -> to [{', '.join(dst_ips)}]" if dst_ips and constraints != "noqueue" else "" node_ip = socket.gethostbyname(node.account.externally_routable_ip) - node_status = f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : {constraints}{targets_str}" + drop_count = self._get_ssh_output( + node, "sudo iptables -S 2>/dev/null | grep -c -- '-j DROP' || true") + partition_str = f" | partition DROP rules: {drop_count}" if drop_count not in ("", "0") else "" + + node_status = f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : " \ + f"{constraints}{targets_str}{partition_str}" node_to_status_map.update({id(node): node_status}) diff --git a/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py b/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py index 465d8eb506a1b..c7b028c5def76 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py @@ -12,8 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import re import socket -from typing import Optional +from typing import Iterable, Optional from ignitetest.services.network_group.configuration import CrossNetworkGroupConfiguration @@ -28,6 +29,19 @@ ACTION_ADD = "--add" ACTION_CHANGE = "--change" +# -w: wait on the xtables lock instead of failing +IPTABLES = "sudo iptables -w" + +PARTITION_CHAIN_PREFIX = "NP_" + +# Extraction filter to find user-defined chains starting with our prefix +# - iptables -S outputs "-N CHAIN_NAME" for new chains +# - awk filters rows where column 1 is "-N" and column 2 matches our prefix +FIND_CUSTOM_CHAINS_AWK = f"awk '$1==\"-N\" && $2 ~ /^{PARTITION_CHAIN_PREFIX}/ {{print $2}}'" + +# iptables chain names are limited to 28 characters. +MAX_CHAIN_NAME_LEN = 28 + def to_tcset_cmd(interface: str, dst_host_or_ip: str, config: CrossNetworkGroupConfiguration, action: str = ACTION_OVERWRITE) -> Optional[str]: @@ -67,3 +81,63 @@ def to_tcdel_ip_cmd(interface: str, dst_host_or_ip: str) -> str: Compiles the absolute clear command for destination host. """ return f"{SUDO_PREFIX} tcdel {interface} --peer {dst_host_or_ip}" + + +def partition_chain_name(group_a: str, group_b: str) -> str: + """ + Deterministic, order-independent iptables chain name for a group pair. + """ + a, b = sorted((group_a, group_b)) + + raw = f"{PARTITION_CHAIN_PREFIX}{a}_{b}" + + return re.sub(r"[^A-Za-z0-9_]", "_", raw)[:MAX_CHAIN_NAME_LEN] + + +def to_partition_enable_cmd(chain: str, remote_ips: Iterable[str]) -> str: + """ + Compiles a single shell command that (idempotently) creates the partition + chain, hooks it into INPUT/OUTPUT, and drops all traffic to and from the + given remote IPs. + + Dropping both '-s' and '-d' on each side means the partition between a + node pair is effective as soon as *either* endpoint has applied its rules, + minimizing the window in which the partition is only half-visible. + """ + cmds = [ + f"{{ {IPTABLES} -N {chain} 2>/dev/null || true; }}", + f"{{ {IPTABLES} -C INPUT -j {chain} 2>/dev/null || {IPTABLES} -I INPUT 1 -j {chain}; }}", + f"{{ {IPTABLES} -C OUTPUT -j {chain} 2>/dev/null || {IPTABLES} -I OUTPUT 1 -j {chain}; }}", + ] + + for ip in remote_ips: + cmds.append(f"{IPTABLES} -A {chain} -s {ip}/32 -j DROP") + cmds.append(f"{IPTABLES} -A {chain} -d {ip}/32 -j DROP") + + return " && ".join(cmds) + + +def to_partition_disable_cmd(chain: str) -> str: + """ + Compiles the healing command: flushes the pair's DROP rules in one call. + The (now empty) chain and its INPUT/OUTPUT hooks are intentionally left in + place, so re-enabling the same partition later stays a pure append and the + final teardown remains the single owner of chain removal. + """ + return f"{IPTABLES} -F {chain} 2>/dev/null || true" + + +def to_partition_teardown_cmd() -> str: + """ + Compiles the full cleanup command: unhooks, flushes, and deletes + every partition chain on the node, restoring a pristine iptables state. + """ + find_chains_subshell = f"sudo iptables -S 2>/dev/null | {FIND_CUSTOM_CHAINS_AWK}" + + return ( + f"for c in $({find_chains_subshell}); do " + f"{IPTABLES} -D INPUT -j $c 2>/dev/null; " + f"{IPTABLES} -D OUTPUT -j $c 2>/dev/null; " + f"{IPTABLES} -F $c && {IPTABLES} -X $c; " + f"done; true" + ) From 9932859a264b03223a1f1e05020e2d046d62e981 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 8 Jul 2026 14:38:08 +0300 Subject: [PATCH 07/44] IGNITE-27833 readable iptables logs for network partitoning --- .../services/network_group/manager.py | 69 +++++++++++++++++-- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/network_group/manager.py b/modules/ducktests/tests/ignitetest/services/network_group/manager.py index 5fc453eadcecc..ac985ec89ca6d 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/manager.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/manager.py @@ -27,7 +27,8 @@ from ignitetest.services.network_group.tc_rule_args import ( ACTION_ADD, ACTION_OVERWRITE, to_tcset_cmd, to_tcdel_all_cmd, - partition_chain_name, to_partition_enable_cmd, to_partition_disable_cmd, to_partition_teardown_cmd + partition_chain_name, to_partition_enable_cmd, to_partition_disable_cmd, to_partition_teardown_cmd, + PARTITION_CHAIN_PREFIX ) from ignitetest.services.utils.decorators import memoize @@ -279,9 +280,10 @@ def _log_network(self, log_tag: str): targets_str = f" -> to [{', '.join(dst_ips)}]" if dst_ips and constraints != "noqueue" else "" node_ip = socket.gethostbyname(node.account.externally_routable_ip) - drop_count = self._get_ssh_output( - node, "sudo iptables -S 2>/dev/null | grep -c -- '-j DROP' || true") - partition_str = f" | partition DROP rules: {drop_count}" if drop_count not in ("", "0") else "" + iptables_lines = self._get_ssh_output( + node, "sudo iptables -S 2>/dev/null || true").splitlines() + partition_str = self._format_partition_drops( + self._parse_partition_drops(iptables_lines)) node_status = f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : " \ f"{constraints}{targets_str}{partition_str}" @@ -326,6 +328,65 @@ def _parse_filter_destinations(filter_lines: List[str]) -> List[str]: return dst_ips + @staticmethod + def _parse_partition_drops(iptables_lines: List[str]) -> Dict[str, Dict[str, set]]: + """ + Parses 'iptables -S' output lines into per-partition-chain drop sets. + + :return: {chain_name: {'s': {inbound-dropped ips}, 'd': {outbound-dropped ips}}} + """ + pattern = re.compile( + rf"^-A\s+({re.escape(PARTITION_CHAIN_PREFIX)}\S+)\s+" + rf"-(s|d)\s+(\d{{1,3}}(?:\.\d{{1,3}}){{3}})(?:/32)?\s+-j\s+DROP$" + ) + + drops: Dict[str, Dict[str, set]] = {} + + for line in iptables_lines: + match = pattern.match(line.strip()) + + if match: + chain, direction, ip = match.groups() + + drops.setdefault(chain, {"s": set(), "d": set()})[direction].add(ip) + + return drops + + @staticmethod + def _format_partition_drops(drops: Dict[str, Dict[str, set]]) -> str: + """ + Renders parsed partition drops into a compact, human-readable suffix. + + Fully cut peers (both inbound and outbound DROP present) are shown as + '<-X->'. Peers with only a one-way drop are flagged explicitly as + 'in-X'/'out-X' — on a healthy partition these lists are empty, so any + occurrence pinpoints a node with partially applied rules. + """ + if not drops: + return "" + + def fmt(ips: set) -> str: + return ", ".join(sorted(ips, key=lambda ip: tuple(map(int, ip.split("."))))) + + chain_summaries = [] + + for chain in sorted(drops): + inbound, outbound = drops[chain]["s"], drops[chain]["d"] + + both, in_only, out_only = inbound & outbound, inbound - outbound, outbound - inbound + + details = [] + if both: + details.append(f"<-X-> [{fmt(both)}]") + if in_only: + details.append(f"in-X only [{fmt(in_only)}]") + if out_only: + details.append(f"out-X only [{fmt(out_only)}]") + + chain_summaries.append(f"{chain} {' '.join(details)}") + + return " | partition: " + "; ".join(chain_summaries) + @staticmethod def _parse_qdisc_constraints(qdisc_lines: List[str]) -> str: """ From ef2a0ef1fb6d64610f3d7ba6ba96719062b2a2b4 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 8 Jul 2026 15:13:48 +0300 Subject: [PATCH 08/44] IGNITE-27833 half-ring alive status verified --- .../partition_resilience_test.py | 53 ++++++------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py index 09d8e893512b0..653c73617a8ce 100644 --- a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py @@ -15,7 +15,6 @@ from time import sleep from ducktape.mark import matrix -from ducktape.utils.util import wait_until from ignitetest.services.ignite import IgniteService from ignitetest.services.network_group.configuration import NetworkGroupStore, CrossNetworkGroupConfiguration @@ -30,11 +29,6 @@ DC_1_NAME = "DC1" DC_2_NAME = "DC2" -# How long to wait for the half-ring to reach the expected cluster state -# after the partition heals. Discovery failure detection, ring closure and -# the state transition are asynchronous, so an immediate check is racy. -CLUSTER_STATE_TIMEOUT_SEC = 60 - class MultiDCPartitionResilienceTest(NetworkGroupAbstractTest): """ @@ -42,10 +36,6 @@ class MultiDCPartitionResilienceTest(NetworkGroupAbstractTest): """ @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH)) - # NOTE: for the partition to be reliably *detected*, partition_time_sec must - # exceed discovery failureDetectionTimeout + connectionRecoveryTimeout - # (~10s + 10s by default). Sub-detection values (e.g. 0.5s) exercise the - # "partition heals before the cluster reacts" case instead. @matrix(cross_dc_latency_ms=[20], partition_time_sec=[30]) def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms, partition_time_sec): self.configure_network_and_run(ignite_version=ignite_version, cross_dc_latency_ms=cross_dc_latency_ms, @@ -86,43 +76,32 @@ def _run(self, network_mgr, **kwargs): network_mgr.disable_network_partition(DC_1_NAME, DC_2_NAME) - self._verify_half_ring_status(self.svc_dc_1, "ACTIVE") - self._verify_half_ring_status(self.svc_dc_2, "ACTIVE") - - self._verify_cluster_survived() + self._verify_half_ring_healthy(self.svc_dc_1) + self._verify_half_ring_healthy(self.svc_dc_2) self.svc_dc_1.stop() self.svc_dc_2.stop() - def _verify_cluster_survived(self): - total_alive = sum(len(svc.alive_nodes) for svc in [self.svc_dc_1, self.svc_dc_2]) + @staticmethod + def _verify_half_ring_healthy(svc: IgniteService): + exp_alive_nodes = NUM_NODES // 2 + act_alive_nodes = len(svc.alive_nodes) - assert total_alive == NUM_NODES, f"{NUM_NODES} nodes should be alive! [actual={total_alive}]" + assert act_alive_nodes == exp_alive_nodes, f"{exp_alive_nodes} nodes should be alive! [actual={act_alive_nodes}]" - coordinator_node = self.svc_dc_1.nodes[0] + coordinator_node = svc.nodes[0] - assert self.svc_dc_1.alive(coordinator_node), "Coordinator should remain alive" + assert svc.alive(coordinator_node), "Coordinator node should remain alive" - disco_info_dc_1 = coordinator_node.discovery_info() + disco_info = coordinator_node.discovery_info() - assert disco_info_dc_1.is_coordinator, f"{self.svc_dc_1.who_am_i(coordinator_node)} is not a coordinator" + assert disco_info.is_coordinator, f"{svc.who_am_i(coordinator_node)} is not a coordinator" - @staticmethod - def _verify_half_ring_status(svc: IgniteService, status: str, timeout_sec: int = CLUSTER_STATE_TIMEOUT_SEC): - """ - Polls the half-ring cluster state until it reaches the expected status. - The state transition after a partition is asynchronous (failure detection, - ring closure, state change exchange), so a one-shot assertion is racy. - """ control_utility = ControlUtility(svc) - def has_expected_status(): - try: - return status in control_utility.cluster_state() - except Exception: - # control.sh may transiently fail while the ring is re-forming. - return False + cluster_state = control_utility.cluster_state() + + assert "ACTIVE" == cluster_state.state, f"Half-ring state should remain ACTIVE [actual={cluster_state.state}]" - wait_until(has_expected_status, timeout_sec=timeout_sec, backoff_sec=1, - err_msg=f"Half-ring did not reach {status} status within {timeout_sec}s " - f"[last known state check failed or mismatched]") + assert len(cluster_state.baseline) == exp_alive_nodes, \ + f"Half-ring baseline is not expected [exp={exp_alive_nodes}, actual={cluster_state.baseline}]" From 64e255300dea50f533f81ebced74bc840ae1cfd2 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 9 Jul 2026 09:32:39 +0300 Subject: [PATCH 09/44] IGNITE-27833 cache distribution check --- .../ducktest/tests/dto/IndexedDataRecord.java | 79 ++++++++ .../tests/mdc/MdcCacheAwareApplication.java | 61 ++++++ .../tests/mdc/MdcDataCheckerApplication.java | 55 ++++++ .../mdc/MdcDataGeneratorApplication.java | 33 ++++ ...MdcPutAdmissibilityCheckerApplication.java | 76 ++++++++ .../DataLoaderAndCheckerApplication.java | 68 +------ .../tests/ignitetest/services/ignite_app.py | 4 +- .../services/utils/cdc/kafka_to_ignite.py | 2 +- .../services/utils/control_utility.py | 106 +++++++++++ .../ignitetest/services/utils/ignite_spec.py | 2 + .../partition_resilience_test.py | 174 ++++++++++++++++-- 11 files changed, 576 insertions(+), 84 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dto/IndexedDataRecord.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dto/IndexedDataRecord.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dto/IndexedDataRecord.java new file mode 100644 index 0000000000000..00250c3b3444e --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dto/IndexedDataRecord.java @@ -0,0 +1,79 @@ +package org.apache.ignite.internal.ducktest.tests.dto; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** */ +public class IndexedDataRecord { + /** The base index converted to a string. */ + private final String strRepresentation; + + /** The primitive index boxed as an Integer. */ + private final Integer boxedIdx; + + /** True if the base index is an even number, false otherwise. */ + private final Boolean isEven; + + /** The first character of the string representation. */ + private final char leadingCharacter; + + /** An array containing the index, its double, and its square. */ + private final Integer[] seqArr; + + /** A list view of the sequence array. */ + private final List seqList; + + /** + * Constructs an immutable record by generating derived properties from the provided index. + * + * @param idx the base integer used to compute all internal fields + */ + public IndexedDataRecord(int idx) { + this.strRepresentation = String.valueOf(idx); + this.boxedIdx = idx; + this.isEven = (idx % 2 == 0); + this.leadingCharacter = this.strRepresentation.charAt(0); + this.seqArr = new Integer[] {idx, idx * 2, idx * idx}; + this.seqList = Arrays.asList(this.seqArr); + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + IndexedDataRecord that = (IndexedDataRecord)o; + + return leadingCharacter == that.leadingCharacter + && Objects.equals(strRepresentation, that.strRepresentation) + && Objects.equals(boxedIdx, that.boxedIdx) + && Objects.equals(isEven, that.isEven) + && Arrays.equals(seqArr, that.seqArr) + && Objects.equals(seqList, that.seqList); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int result = Objects.hash(strRepresentation, boxedIdx, isEven, leadingCharacter, seqList); + + result = 31 * result + Arrays.hashCode(seqArr); + + return result; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "IndexedDataRecord{" + + "stringRepresentation='" + strRepresentation + '\'' + + ", boxedIndex=" + boxedIdx + + ", isEven=" + isEven + + ", leadingCharacter=" + leadingCharacter + + ", sequenceArray=" + Arrays.toString(seqArr) + + ", sequenceList=" + seqList + + '}'; + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java new file mode 100644 index 0000000000000..12d2fb6e23fba --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java @@ -0,0 +1,61 @@ +package org.apache.ignite.internal.ducktest.tests.mdc; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.tests.dto.IndexedDataRecord; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; +import org.apache.ignite.topology.MdcTopologyValidator; + +import static org.apache.ignite.cache.CacheMode.PARTITIONED; + +/** + * Base class for MDC test applications. + * Encapsulates the cache configuration (with {@link MdcTopologyValidator}) so that + * generator and checkers always operate on an identically configured cache. + */ +public abstract class MdcCacheAwareApplication extends IgniteAwareApplication { + /** JVM option name holding the data center identifier. */ + public static final String DC_ID_PROP = "IGNITE_DATA_CENTER_ID"; + + /** */ + protected static String DFLT_CACHE_NAME = "default"; + + /** */ + protected static int DFLT_BACKUPS = 0; + + /** + * @param jNode Parameters. + * @return Cache configured with the MDC topology validator. + */ + protected IgniteCache mdcCache(JsonNode jNode) { + String cacheName = jNode.path("cacheName").asText(DFLT_CACHE_NAME); + int backups = jNode.path("backups").asInt(DFLT_BACKUPS); + + String mainDc = jNode.path("mainDc").asText(); + + MdcTopologyValidator topValidator = new MdcTopologyValidator(); + + topValidator.setMainDatacenter(mainDc); + + CacheConfiguration cacheCfg = new CacheConfiguration() + .setName(cacheName) + .setTopologyValidator(topValidator) + .setCacheMode(PARTITIONED) + .setBackups(backups); + + return ignite.getOrCreateCache(cacheCfg); + } + + /** */ + protected IgniteCache mdcCache(String cacheName) { + return ignite.cache(cacheName); + } + + /** + * @return Data center id this client belongs to (passed via {@code -DIGNITE_DATA_CENTER_ID=...}). + */ + protected static String dcId() { + return System.getProperty(DC_ID_PROP, ""); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java new file mode 100644 index 0000000000000..6de7cb544d2ee --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java @@ -0,0 +1,55 @@ +package org.apache.ignite.internal.ducktest.tests.mdc; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.ducktest.tests.dto.IndexedDataRecord; + +/** + * Verifies that all entries written by {@link MdcDataGeneratorApplication} are readable + * and hold expected values. Intended to be run from a client in each DC after the + * network partition: reads must succeed everywhere, even in the read-only DC. + */ +public class MdcDataCheckerApplication extends MdcCacheAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws IgniteInterruptedCheckedException { + String cacheName = jNode.get("cacheName").asText(); + int from = jNode.path("from").asInt(0); + int to = jNode.path("to").asInt(10_000); + + markInitialized(); + waitForActivation(); + + IgniteCache cache = mdcCache(cacheName); + + log.info("Data check started [dc=" + dcId() + ", cache=" + cache.getName() + + ", from=" + from + ", to=" + to + "]"); + + int missed = 0; + int corrupted = 0; + + for (int i = from; i < to && !terminated(); i++) { + IndexedDataRecord obj = cache.get(i); + + if (obj == null) { + missed++; + + log.error("Entry is missed [dc=" + dcId() + ", key=" + i + "]"); + } + else if (!obj.equals(new IndexedDataRecord(i))) { + corrupted++; + + log.error("Entry is corrupted [dc=" + dcId() + ", key=" + i + ", val=" + obj + "]"); + } + } + + if (missed > 0 || corrupted > 0) { + throw new IllegalStateException("Data check failed [dc=" + dcId() + + ", missed=" + missed + ", corrupted=" + corrupted + "]"); + } + + log.info("Data check passed [dc=" + dcId() + ", entries=" + (to - from) + "]"); + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java new file mode 100644 index 0000000000000..8cfcd5f0990f6 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java @@ -0,0 +1,33 @@ +package org.apache.ignite.internal.ducktest.tests.mdc; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.ducktest.tests.dto.IndexedDataRecord; + +/** + * Populates the MDC cache with a deterministic data set: keys in {@code [from, to)}, + * values {@code IndexedDataRecord(key)}. Run it before the network partition. + */ +public class MdcDataGeneratorApplication extends MdcCacheAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws IgniteInterruptedCheckedException { + int from = jNode.path("from").asInt(0); + int to = jNode.path("to").asInt(10_000); + + markInitialized(); + waitForActivation(); + + IgniteCache cache = mdcCache(jNode); + + log.info("Data generation started [dc=" + dcId() + ", cache=" + cache.getName() + + ", from=" + from + ", to=" + to + "]"); + + for (int i = from; i < to && !terminated(); i++) + cache.put(i, new IndexedDataRecord(i)); + + log.info("Data generation finished [dc=" + dcId() + ", entries=" + (to - from) + "]"); + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java new file mode 100644 index 0000000000000..6ec4be1cda6a0 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java @@ -0,0 +1,76 @@ +package org.apache.ignite.internal.ducktest.tests.mdc; + +import javax.cache.CacheException; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.ducktest.tests.dto.IndexedDataRecord; + +/** + * Checks whether the put load is admissible or inadmissible for the DC this client + * belongs to. + *

+ * With {@code expectAdmissible=true} every probe put must succeed (primary DC). + * With {@code expectAdmissible=false} every probe put must fail with a + * {@link CacheException} thrown by the topology validator (read-only DC). + *

+ * Probe keys start at {@code keyOffset} (defaults to 1_000_000) so they never + * intersect with the data set verified by {@link MdcDataCheckerApplication}. + */ +public class MdcPutAdmissibilityCheckerApplication extends MdcCacheAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws IgniteInterruptedCheckedException { + String cacheName = jNode.path("cacheName").asText(DFLT_CACHE_NAME); + boolean expectAdmissible = jNode.get("expectAdmissible").asBoolean(); + int probes = jNode.path("probes").asInt(100); + int keyOffset = jNode.path("keyOffset").asInt(1_000_000); + + markInitialized(); + waitForActivation(); + + IgniteCache cache = mdcCache(cacheName); + + log.info("Put admissibility check started [dc=" + dcId() + ", cache=" + cache.getName() + + ", expectAdmissible=" + expectAdmissible + ", probes=" + probes + "]"); + + int succeeded = 0; + int rejected = 0; + + for (int i = 0; i < probes && !terminated(); i++) { + int key = keyOffset + i; + + try { + cache.put(key, new IndexedDataRecord(key)); + + succeeded++; + + if (!expectAdmissible) + log.error("Put unexpectedly succeeded in read-only DC [dc=" + dcId() + ", key=" + key + "]"); + } + catch (CacheException e) { + rejected++; + + if (expectAdmissible) + log.error("Put unexpectedly rejected [dc=" + dcId() + ", key=" + key + "]", e); + else + log.info("Put rejected as expected [dc=" + dcId() + ", key=" + key + + ", msg=" + e.getMessage() + "]"); + } + } + + if (expectAdmissible && rejected > 0) { + throw new IllegalStateException("Put load is inadmissible while expected to be admissible [dc=" + + dcId() + ", succeeded=" + succeeded + ", rejected=" + rejected + "]"); + } + + if (!expectAdmissible && succeeded > 0) { + throw new IllegalStateException("Put load is admissible while expected to be inadmissible [dc=" + + dcId() + ", succeeded=" + succeeded + ", rejected=" + rejected + "]"); + } + + log.info("Put admissibility check passed [dc=" + dcId() + + ", succeeded=" + succeeded + ", rejected=" + rejected + "]"); + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/persistence_upgrade_test/DataLoaderAndCheckerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/persistence_upgrade_test/DataLoaderAndCheckerApplication.java index 82108b3e5ef03..6a857b4505cb3 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/persistence_upgrade_test/DataLoaderAndCheckerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/persistence_upgrade_test/DataLoaderAndCheckerApplication.java @@ -17,13 +17,11 @@ package org.apache.ignite.internal.ducktest.tests.persistence_upgrade_test; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.IgniteCache; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.ducktest.tests.dto.IndexedDataRecord; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; import org.apache.ignite.internal.util.typedef.internal.U; @@ -40,16 +38,16 @@ public class DataLoaderAndCheckerApplication extends IgniteAwareApplication { markInitialized(); waitForActivation(); - CacheConfiguration cacheCfg = new CacheConfiguration<>("cache"); + CacheConfiguration cacheCfg = new CacheConfiguration<>("cache"); cacheCfg.setBackups(backups); - IgniteCache cache = ignite.getOrCreateCache(cacheCfg); + IgniteCache cache = ignite.getOrCreateCache(cacheCfg); log.info(check ? "Checking..." : " Preparing..."); for (int i = 0; i < entryCnt; i++) { - CustomObject obj = new CustomObject(i); + IndexedDataRecord obj = new IndexedDataRecord(i); if (!check) cache.put(i, obj); @@ -64,62 +62,4 @@ public class DataLoaderAndCheckerApplication extends IgniteAwareApplication { markFinished(); } - - /** - * - */ - private static class CustomObject { - /** String value. */ - private final String sVal; - - /** Integer value. */ - private final Integer iVal; - - /** Boolean value. */ - private final Boolean bVal; - - /** char value. */ - private final char cVal; - - /** Integer array value. */ - private final Integer[] iArVal; - - /** Integer List. */ - private final List iLiVal; - - /** - * @param idx Index. - */ - public CustomObject(int idx) { - sVal = String.valueOf(idx); - iVal = idx; - bVal = idx % 2 == 0; - cVal = sVal.charAt(0); - iArVal = new Integer[] {idx, idx * 2, idx * idx}; - iLiVal = Arrays.asList(iArVal); - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - CustomObject obj = (CustomObject)o; - return cVal == obj.cVal - && Objects.equals(sVal, obj.sVal) - && Objects.equals(iVal, obj.iVal) - && Objects.equals(bVal, obj.bVal) - && Arrays.equals(iArVal, obj.iArVal) - && Objects.equals(iLiVal, obj.iLiVal); - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - int result = Objects.hash(sVal, iVal, bVal, cVal, iLiVal); - result = 31 * result + Arrays.hashCode(iArVal); - return result; - } - } - } diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 8ccd51e6f4e72..268a63354dc0f 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -34,7 +34,7 @@ class IgniteApplicationService(IgniteAwareService): APP_FINISH_EVT_MSG = "IGNITE_APPLICATION_FINISHED" APP_BROKEN_EVT_MSG = "IGNITE_APPLICATION_BROKEN" - def __init__(self, context, config, java_class_name, num_nodes=1, params="", startup_timeout_sec=60, + def __init__(self, context, config, java_class_name=None, num_nodes=1, params="", startup_timeout_sec=60, shutdown_timeout_sec=60, modules=None, main_java_class=SERVICE_JAVA_CLASS_NAME, jvm_opts=None, merge_with_default=True): super().__init__(context, config, num_nodes, startup_timeout_sec, shutdown_timeout_sec, main_java_class, @@ -43,7 +43,7 @@ def __init__(self, context, config, java_class_name, num_nodes=1, params="", sta self.java_class_name = java_class_name self.params = params - def await_started(self): + def await_started(self, nodes=None): super().await_started() self.__check_status(self.APP_INIT_EVT_MSG, timeout=self.startup_timeout_sec) diff --git a/modules/ducktests/tests/ignitetest/services/utils/cdc/kafka_to_ignite.py b/modules/ducktests/tests/ignitetest/services/utils/cdc/kafka_to_ignite.py index 1ffb501ecd1f5..e5cba2a0b834b 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/cdc/kafka_to_ignite.py +++ b/modules/ducktests/tests/ignitetest/services/utils/cdc/kafka_to_ignite.py @@ -121,7 +121,7 @@ def get_config(dst_cluster, client_type): version=dst_cluster.config.version ) - def await_started(self): + def await_started(self, nodes=None): """ Awaits kafka-to-ignite.sh started. """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index d95f97b0a1362..7f29af98440a8 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -181,6 +181,83 @@ def idle_verify_dump(self, node=None): return re.search(r'/.*.txt', data).group(0) + def cache_distribution(self, node_id=None, cache_names=None, user_attributes=None): + """ + Prints partition distribution. + + :param node_id: Node id to get distribution for, all nodes if None. + :param cache_names: Cache name, or list of cache names, all caches if None. + :param user_attributes: Node attribute name or list of names to add to the output + (e.g. "IGNITE_DATA_CENTER_ID"). + :return: CacheDistribution. + """ + if isinstance(cache_names, str): + cache_names = [cache_names] + + if isinstance(user_attributes, str): + user_attributes = [user_attributes] + + cmd = f"--cache distribution {node_id if node_id else 'null'}" + + if cache_names: + cmd += f" {','.join(cache_names)}" + + if user_attributes: + cmd += f" --user-attributes {','.join(user_attributes)}" + + result = self.__run(cmd) + + return self.__parse_cache_distribution(result, user_attributes) + + @staticmethod + def __parse_cache_distribution(output, user_attributes=None): + group_pattern = re.compile(r"\[next group: id=(?P-?\d+), name=(?P[^\]]+)\]") + + # Trailing attribute values appear after nodeAddresses, comma-separated, + # in the same order they were passed to --user-attributes. + row_pattern = re.compile(r"(?P-?\d+)," + r"(?P\d+)," + r"(?P[0-9a-fA-F]+)," + r"(?P[PB])," + r"(?P[A-Z_]+)," + r"(?P\d+)," + r"(?P\d+)," + r"\[(?P[^\]]*)\]" + r"(?:,(?P.*))?$") + + groups = {} + cur_group = None + + for line in output.splitlines(): + line = line.strip() + + match = group_pattern.search(line) + if match: + cur_group = CacheGroupDistribution(group_id=int(match.group("group_id")), + name=match.group("name"), + partitions={}) + groups[cur_group.name] = cur_group + continue + + match = row_pattern.match(line) + if match and cur_group is not None: + attrs = {} + if user_attributes and match.group("attr_values") is not None: + values = [v.strip() for v in match.group("attr_values").split(",")] + attrs = dict(zip(user_attributes, values)) + + copy = PartitionCopy(node_id=match.group("node_id"), + primary=match.group("primary") == "P", + state=match.group("state"), + update_counter=int(match.group("update_counter")), + partition_size=int(match.group("partition_size")), + node_addresses=[a.strip() for a in match.group("node_addresses").split(",") if a], + user_attributes=attrs) + + cur_group.partitions.setdefault(int(match.group("partition")), []).append(copy) + + return CacheDistribution(groups=groups) + def check_consistency(self, args): """ Consistency check. @@ -520,6 +597,35 @@ class TxVerboseInfo(NamedTuple): states: list +class PartitionCopy(NamedTuple): + """ + Single copy (primary or backup) of a partition on a node. + """ + node_id: str + primary: bool + state: str + update_counter: int + partition_size: int + node_addresses: list + user_attributes: dict + + +class CacheGroupDistribution(NamedTuple): + """ + Distribution of a single cache group: partition id -> list of PartitionCopy. + """ + group_id: int + name: str + partitions: dict + + +class CacheDistribution(NamedTuple): + """ + Distribution info for all printed cache groups: group name -> CacheGroupDistribution. + """ + groups: dict + + class ControlUtilityError(RemoteCommandError): """ Error is raised when control utility failed. diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index a99aa163026f4..1edad489ee363 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -358,6 +358,8 @@ def __get_default_jvm_opts(self): ] def command(self, node): + assert self.service.java_class_name is not None, "Missing required 'java_class_name' to execute command." + args = [ str(self.service.config.service_type.name), self.service.java_class_name, diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py index 653c73617a8ce..a86dc7e6e1e19 100644 --- a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py @@ -17,29 +17,39 @@ from ducktape.mark import matrix from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.network_group.configuration import NetworkGroupStore, CrossNetworkGroupConfiguration from ignitetest.services.utils.control_utility import ControlUtility from ignitetest.services.utils.ignite_configuration import IgniteConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_services from ignitetest.tests.network_group import NetworkGroupAbstractTest from ignitetest.utils import cluster, ignite_versions from ignitetest.utils.version import DEV_BRANCH, IgniteVersion -NUM_NODES = 12 +SRV_NODES = 10 +CLI_NODES = 2 +TOTAL_NODES = SRV_NODES + CLI_NODES DC_1_NAME = "DC1" DC_2_NAME = "DC2" +GENERATOR_JAVA_CLASS_NAME = "org.apache.ignite.internal.ducktest.tests.mdc.MdcDataGeneratorApplication" +GET_CHECKER_JAVA_CLASS_NAME = "org.apache.ignite.internal.ducktest.tests.mdc.MdcDataCheckerApplication" +PUT_CHECKER_JAVA_CLASS_NAME = "org.apache.ignite.internal.ducktest.tests.mdc.MdcPutAdmissibilityCheckerApplication" + +CACHE_NAME = "replicated" + +DATA_CENTER_ATTR = "IGNITE_DATA_CENTER_ID" class MultiDCPartitionResilienceTest(NetworkGroupAbstractTest): """ Tests for cluster network partition resilience in MultiDC """ - @cluster(num_nodes=NUM_NODES) + @cluster(num_nodes=TOTAL_NODES) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[20], partition_time_sec=[30]) - def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms, partition_time_sec): - self.configure_network_and_run(ignite_version=ignite_version, cross_dc_latency_ms=cross_dc_latency_ms, - partition_time_sec=partition_time_sec) + @matrix(cross_dc_latency_ms=[20]) + def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms): + self.configure_network_and_run(ignite_version=ignite_version, cross_dc_latency_ms=cross_dc_latency_ms) def _configure_network_group_store(self, **kwargs) -> NetworkGroupStore: store = super()._configure_network_group_store(**kwargs) @@ -53,38 +63,63 @@ def _configure_network_group_store(self, **kwargs) -> NetworkGroupStore: def _configure_services(self, **kwargs): self.ign_cfg = IgniteConfiguration(version=IgniteVersion(kwargs['ignite_version'])) - dc_1_nodes_num, dc_2_nodes_num = NUM_NODES // 2, NUM_NODES // 2 + dc_1_nodes_num, dc_2_nodes_num = SRV_NODES // 2, SRV_NODES // 2 + + jvm_opts_dc_1 = [f"-D{DATA_CENTER_ATTR}={DC_1_NAME}"] + jvm_opts_dc_2 = [f"-D{DATA_CENTER_ATTR}={DC_2_NAME}"] - self.svc_dc_1 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_1_nodes_num, - jvm_opts=[f"-DIGNITE_DATA_CENTER_ID={DC_1_NAME}"]) - self.svc_dc_2 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_2_nodes_num, - jvm_opts=[f"-DIGNITE_DATA_CENTER_ID={DC_2_NAME}"]) + self.svc_dc_1 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_1_nodes_num, jvm_opts=jvm_opts_dc_1) + self.svc_dc_2 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_2_nodes_num, jvm_opts=jvm_opts_dc_2) + + cli_cfg = self.ign_cfg._replace( + client_mode=True, + discovery_spi=from_ignite_services([self.svc_dc_1, self.svc_dc_2]) + ) + + self.app_dc_1 = IgniteApplicationService(self.test_context, cli_cfg, jvm_opts=jvm_opts_dc_1) + self.app_dc_2 = IgniteApplicationService(self.test_context, cli_cfg, jvm_opts=jvm_opts_dc_2) def _configure_network_group_registry(self, **kwargs): return { - DC_1_NAME: [self.svc_dc_1], - DC_2_NAME: [self.svc_dc_2] + DC_1_NAME: [self.svc_dc_1, self.app_dc_1], + DC_2_NAME: [self.svc_dc_2, self.app_dc_2] } def _run(self, network_mgr, **kwargs): for svc in [self.svc_dc_1, self.svc_dc_2]: svc.start() - network_mgr.enable_network_partition(DC_1_NAME, DC_2_NAME) + self._populate_cluster_with_data() - sleep(kwargs['partition_time_sec']) + control_utility = ControlUtility(self.svc_dc_1) - network_mgr.disable_network_partition(DC_1_NAME, DC_2_NAME) + distribution = control_utility.cache_distribution( + cache_names=CACHE_NAME, + user_attributes=DATA_CENTER_ATTR) + + self.assert_cross_dc_distribution_by_attribute( + distribution, + dc_attr=DATA_CENTER_ATTR, + expected_dcs=[DC_1_NAME, DC_2_NAME]) + + network_mgr.enable_network_partition(DC_1_NAME, DC_2_NAME) + + sleep(10) self._verify_half_ring_healthy(self.svc_dc_1) self._verify_half_ring_healthy(self.svc_dc_2) + self._check_data_accessible() + self._check_data_change_access() + + network_mgr.disable_network_partition(DC_1_NAME, DC_2_NAME) + self.svc_dc_1.stop() self.svc_dc_2.stop() @staticmethod def _verify_half_ring_healthy(svc: IgniteService): - exp_alive_nodes = NUM_NODES // 2 + exp_alive_nodes = SRV_NODES // 2 act_alive_nodes = len(svc.alive_nodes) assert act_alive_nodes == exp_alive_nodes, f"{exp_alive_nodes} nodes should be alive! [actual={act_alive_nodes}]" @@ -105,3 +140,108 @@ def _verify_half_ring_healthy(svc: IgniteService): assert len(cluster_state.baseline) == exp_alive_nodes, \ f"Half-ring baseline is not expected [exp={exp_alive_nodes}, actual={cluster_state.baseline}]" + + def _populate_cluster_with_data(self): + self.app_dc_1.java_class_name = GENERATOR_JAVA_CLASS_NAME + self.app_dc_2.java_class_name = GENERATOR_JAVA_CLASS_NAME + + self.app_dc_1.params = {"mainDc": DC_1_NAME, "cacheName": CACHE_NAME, "backups": 2, "from": 0, "to": 100} + self.app_dc_2.params = {"mainDc": DC_1_NAME, "cacheName": CACHE_NAME, "backups": 2, "from": 100, "to": 200} + + self.app_dc_1.start() + self.app_dc_2.start() + + self.app_dc_1.wait() + self.app_dc_2.wait() + + self.app_dc_1.stop() + self.app_dc_2.stop() + + def _check_data_accessible(self): + self.app_dc_1.java_class_name = GET_CHECKER_JAVA_CLASS_NAME + self.app_dc_2.java_class_name = GET_CHECKER_JAVA_CLASS_NAME + + self.app_dc_1.params = {"cacheName": CACHE_NAME, "from": 0, "to": 200} + self.app_dc_2.params = {"cacheName": CACHE_NAME, "from": 0, "to": 200} + + self.app_dc_1.start() + self.app_dc_2.start() + + self.app_dc_1.wait() + self.app_dc_2.wait() + + self.app_dc_1.stop() + self.app_dc_2.stop() + + def _check_data_change_access(self): + self.app_dc_1.java_class_name = PUT_CHECKER_JAVA_CLASS_NAME + self.app_dc_2.java_class_name = PUT_CHECKER_JAVA_CLASS_NAME + + self.app_dc_1.params = {"cacheName": CACHE_NAME, "expectAdmissible": True} + self.app_dc_2.params = {"cacheName": CACHE_NAME, "expectAdmissible": False} + + self.app_dc_1.start() + self.app_dc_2.start() + + self.app_dc_1.wait() + self.app_dc_2.wait() + + self.app_dc_1.stop() + self.app_dc_2.stop() + + def assert_cross_dc_distribution_by_attribute(self, distribution, dc_attr, expected_dcs, owning_only=True): + """ + Asserts that every partition of every cache group has at least one copy in every DC, + using a node attribute (requested via --user-attributes) as the DC marker. + + :param distribution: CacheDistribution returned by ControlUtility.cache_distribution(), + requested with user_attributes=[dc_attr]. + :param dc_attr: Attribute name holding the DC id, e.g. "IGNITE_DATA_CENTER_ID". + :param expected_dcs: Collection of DC ids that must own a copy of every partition. + :param owning_only: Count only copies in OWNING state as present. + """ + + def dc_of(copy): + return copy.user_attributes.get(dc_attr) + + self._assert_cross_dc(distribution, set(expected_dcs), dc_of, owning_only, + layout_hint=f"DC attribute: {dc_attr}, expected DCs: {sorted(expected_dcs)}") + + def assert_cross_dc_distribution(self, distribution, dc_addresses, owning_only=True): + """ + Same assertion, but DC membership is derived from node IP addresses. + + :param dc_addresses: Dict: DC name -> collection of node IP addresses of that DC, + e.g. {"DC1": ["192.168.64.9", ...], "DC2": [...]}. + """ + dc_addresses = {dc: set(addrs) for dc, addrs in dc_addresses.items()} + + def dc_of(copy): + for dc, addrs in dc_addresses.items(): + if any(addr in addrs for addr in copy.node_addresses): + return dc + return None + + self._assert_cross_dc(distribution, set(dc_addresses), dc_of, owning_only, + layout_hint=f"DC layout: { {dc: sorted(addrs) for dc, addrs in dc_addresses.items()} }") + + @staticmethod + def _assert_cross_dc(distribution, expected_dcs, dc_of, owning_only, layout_hint): + violations = [] + + for group in distribution.groups.values(): + for part, copies in sorted(group.partitions.items()): + counted = [c for c in copies if not owning_only or c.state == "OWNING"] + + missing = expected_dcs - {dc_of(c) for c in counted} + + if missing: + copies_dump = ", ".join( + f"{c.node_id}({'P' if c.primary else 'B'},{c.state},dc={dc_of(c)},{c.node_addresses})" + for c in copies) + + violations.append(f"group={group.name}(id={group.group_id}), partition={part}, " + f"missing DCs={sorted(missing)}, copies=[{copies_dump}]") + + assert not violations, \ + "Partition distribution is not cross-DC:\n " + "\n ".join(violations) + "\n" + layout_hint From 4a043a0496ceed1aeae0f696d70094d7a87cbf66 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 9 Jul 2026 12:06:38 +0300 Subject: [PATCH 10/44] IGNITE-27833 cache distribution fixed --- .../tests/mdc/MdcCacheAwareApplication.java | 11 ++++- .../partition_resilience_test.py | 45 ++++++++++--------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java index 12d2fb6e23fba..9262e67f6c9cb 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.rendezvous.MdcAffinityBackupFilter; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.ducktest.tests.dto.IndexedDataRecord; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; @@ -24,6 +26,9 @@ public abstract class MdcCacheAwareApplication extends IgniteAwareApplication { /** */ protected static int DFLT_BACKUPS = 0; + /** */ + protected static int DFLT_DCS_NUM = 2; + /** * @param jNode Parameters. * @return Cache configured with the MDC topology validator. @@ -33,6 +38,7 @@ protected IgniteCache mdcCache(JsonNode jNode) { int backups = jNode.path("backups").asInt(DFLT_BACKUPS); String mainDc = jNode.path("mainDc").asText(); + int dcsNum = jNode.path("dcsNum").asInt(DFLT_DCS_NUM); MdcTopologyValidator topValidator = new MdcTopologyValidator(); @@ -42,7 +48,10 @@ protected IgniteCache mdcCache(JsonNode jNode) { .setName(cacheName) .setTopologyValidator(topValidator) .setCacheMode(PARTITIONED) - .setBackups(backups); + .setBackups(backups) + .setAffinity(new RendezvousAffinityFunction() + .setPartitions(32) + .setAffinityBackupFilter(new MdcAffinityBackupFilter(dcsNum, backups))); return ignite.getOrCreateCache(cacheCfg); } diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py index a86dc7e6e1e19..50a6f2b05e495 100644 --- a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py @@ -21,7 +21,7 @@ from ignitetest.services.network_group.configuration import NetworkGroupStore, CrossNetworkGroupConfiguration from ignitetest.services.utils.control_utility import ControlUtility from ignitetest.services.utils.ignite_configuration import IgniteConfiguration -from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_services +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.tests.network_group import NetworkGroupAbstractTest from ignitetest.utils import cluster, ignite_versions from ignitetest.utils.version import DEV_BRANCH, IgniteVersion @@ -71,13 +71,11 @@ def _configure_services(self, **kwargs): self.svc_dc_1 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_1_nodes_num, jvm_opts=jvm_opts_dc_1) self.svc_dc_2 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_2_nodes_num, jvm_opts=jvm_opts_dc_2) - cli_cfg = self.ign_cfg._replace( - client_mode=True, - discovery_spi=from_ignite_services([self.svc_dc_1, self.svc_dc_2]) - ) + cli_cfg_dc_1 = self.ign_cfg._replace(client_mode=True, discovery_spi=from_ignite_cluster(self.svc_dc_1)) + cli_cfg_dc_2 = self.ign_cfg._replace(client_mode=True, discovery_spi=from_ignite_cluster(self.svc_dc_2)) - self.app_dc_1 = IgniteApplicationService(self.test_context, cli_cfg, jvm_opts=jvm_opts_dc_1) - self.app_dc_2 = IgniteApplicationService(self.test_context, cli_cfg, jvm_opts=jvm_opts_dc_2) + self.app_dc_1 = IgniteApplicationService(self.test_context, cli_cfg_dc_1, jvm_opts=jvm_opts_dc_1) + self.app_dc_2 = IgniteApplicationService(self.test_context, cli_cfg_dc_2, jvm_opts=jvm_opts_dc_2) def _configure_network_group_registry(self, **kwargs): return { @@ -109,6 +107,8 @@ def _run(self, network_mgr, **kwargs): self._verify_half_ring_healthy(self.svc_dc_1) self._verify_half_ring_healthy(self.svc_dc_2) + self._verify_split_brain(self.svc_dc_1, self.svc_dc_2) + self._check_data_accessible() self._check_data_change_access() @@ -124,14 +124,6 @@ def _verify_half_ring_healthy(svc: IgniteService): assert act_alive_nodes == exp_alive_nodes, f"{exp_alive_nodes} nodes should be alive! [actual={act_alive_nodes}]" - coordinator_node = svc.nodes[0] - - assert svc.alive(coordinator_node), "Coordinator node should remain alive" - - disco_info = coordinator_node.discovery_info() - - assert disco_info.is_coordinator, f"{svc.who_am_i(coordinator_node)} is not a coordinator" - control_utility = ControlUtility(svc) cluster_state = control_utility.cluster_state() @@ -139,14 +131,23 @@ def _verify_half_ring_healthy(svc: IgniteService): assert "ACTIVE" == cluster_state.state, f"Half-ring state should remain ACTIVE [actual={cluster_state.state}]" assert len(cluster_state.baseline) == exp_alive_nodes, \ - f"Half-ring baseline is not expected [exp={exp_alive_nodes}, actual={cluster_state.baseline}]" + f"Half-ring baseline is not expected [exp={exp_alive_nodes}, actual_baseline={cluster_state.baseline}]" + + def _verify_split_brain(self, svc_dc_1: IgniteService, svc_dc_2: IgniteService): + control_utility_dc_1 = ControlUtility(svc_dc_1) + cluster_state_dc_1 = control_utility_dc_1.cluster_state() + + control_utility_dc_2 = ControlUtility(svc_dc_2) + cluster_state_dc_2 = control_utility_dc_2.cluster_state() + + # Check baseline doesn't intersect, each has it's own coordinator def _populate_cluster_with_data(self): self.app_dc_1.java_class_name = GENERATOR_JAVA_CLASS_NAME self.app_dc_2.java_class_name = GENERATOR_JAVA_CLASS_NAME - self.app_dc_1.params = {"mainDc": DC_1_NAME, "cacheName": CACHE_NAME, "backups": 2, "from": 0, "to": 100} - self.app_dc_2.params = {"mainDc": DC_1_NAME, "cacheName": CACHE_NAME, "backups": 2, "from": 100, "to": 200} + self.app_dc_1.params = {"mainDc": DC_1_NAME, "cacheName": CACHE_NAME, "backups": 1, "from": 0, "to": 100} + self.app_dc_2.params = {"mainDc": DC_1_NAME, "cacheName": CACHE_NAME, "backups": 1, "from": 100, "to": 200} self.app_dc_1.start() self.app_dc_2.start() @@ -164,8 +165,8 @@ def _check_data_accessible(self): self.app_dc_1.params = {"cacheName": CACHE_NAME, "from": 0, "to": 200} self.app_dc_2.params = {"cacheName": CACHE_NAME, "from": 0, "to": 200} - self.app_dc_1.start() - self.app_dc_2.start() + self.app_dc_1.start(clean=False) + self.app_dc_2.start(clean=False) self.app_dc_1.wait() self.app_dc_2.wait() @@ -180,8 +181,8 @@ def _check_data_change_access(self): self.app_dc_1.params = {"cacheName": CACHE_NAME, "expectAdmissible": True} self.app_dc_2.params = {"cacheName": CACHE_NAME, "expectAdmissible": False} - self.app_dc_1.start() - self.app_dc_2.start() + self.app_dc_1.start(clean=False) + self.app_dc_2.start(clean=False) self.app_dc_1.wait() self.app_dc_2.wait() From f8a5bdf84f86c2e61c0688962deae08207829c18 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 9 Jul 2026 12:15:18 +0300 Subject: [PATCH 11/44] IGNITE-27833 verify baseline --- .../services/utils/control_utility.py | 17 ++++++++- .../partition_resilience_test.py | 38 ++++++++++++++++--- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index 7f29af98440a8..c8f6cbe19bad4 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -467,6 +467,8 @@ def __parse_cluster_state(output): ",\\sS(tate|TATE)=(?P[^\\s,]+)" "(,\\sOrder=(?P\\d+))?") + coordinator_pattern = re.compile("\\(Coordinator: [^)]*Order=(?P\\d+)\\)") + match = state_pattern.search(output) state = match.group("cluster_state") if match else None @@ -481,7 +483,19 @@ def __parse_cluster_state(output): order=int(match.group("order")) if match.group("order") else None) baseline.append(node) - return ClusterState(state=state, topology_version=topology, baseline=baseline) + coordinator = None + + match = coordinator_pattern.search(output) + + if match: + order = int(match.group("order")) + + coordinator = next((node for node in baseline if node.order == order), None) + + if coordinator is None: + raise AssertionError(f"Coordinator with order={order} is not found in baseline [baseline={baseline}]") + + return ClusterState(state=state, topology_version=topology, baseline=baseline, coordinator=coordinator) def __run(self, cmd, node=None): if node is None: @@ -558,6 +572,7 @@ class ClusterState(NamedTuple): state: str topology_version: int baseline: list + coordinator: BaselineNode = None class TxInfo(NamedTuple): diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py index 50a6f2b05e495..b1d6da4a01e4c 100644 --- a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py @@ -133,14 +133,40 @@ def _verify_half_ring_healthy(svc: IgniteService): assert len(cluster_state.baseline) == exp_alive_nodes, \ f"Half-ring baseline is not expected [exp={exp_alive_nodes}, actual_baseline={cluster_state.baseline}]" - def _verify_split_brain(self, svc_dc_1: IgniteService, svc_dc_2: IgniteService): - control_utility_dc_1 = ControlUtility(svc_dc_1) - cluster_state_dc_1 = control_utility_dc_1.cluster_state() + @staticmethod + def _verify_split_brain(svc_dc_1: IgniteService, svc_dc_2: IgniteService): + """ + Verifies that after the network partition the cluster has split into two independent + half-rings: their baselines don't intersect and each half elected its own coordinator. + """ + cluster_state_dc_1 = ControlUtility(svc_dc_1).cluster_state() + cluster_state_dc_2 = ControlUtility(svc_dc_2).cluster_state() + + baseline_dc_1 = {node.consistent_id for node in cluster_state_dc_1.baseline} + baseline_dc_2 = {node.consistent_id for node in cluster_state_dc_2.baseline} + + common_nodes = baseline_dc_1 & baseline_dc_2 + + assert not common_nodes, \ + f"Half-ring baselines should not intersect " \ + f"[common={sorted(common_nodes)}, dc1={sorted(baseline_dc_1)}, dc2={sorted(baseline_dc_2)}]" + + coordinator_dc_1 = cluster_state_dc_1.coordinator + coordinator_dc_2 = cluster_state_dc_2.coordinator + + assert coordinator_dc_1, f"Coordinator is not found in {DC_1_NAME} half-ring baseline output!" + assert coordinator_dc_2, f"Coordinator is not found in {DC_2_NAME} half-ring baseline output!" + + assert coordinator_dc_1.consistent_id != coordinator_dc_2.consistent_id, \ + f"Half-rings should have different coordinators [coordinator={coordinator_dc_1.consistent_id}]" - control_utility_dc_2 = ControlUtility(svc_dc_2) - cluster_state_dc_2 = control_utility_dc_2.cluster_state() + assert coordinator_dc_1.consistent_id in baseline_dc_1, \ + f"{DC_1_NAME} coordinator should belong to its own half-ring baseline " \ + f"[coordinator={coordinator_dc_1.consistent_id}, baseline={sorted(baseline_dc_1)}]" - # Check baseline doesn't intersect, each has it's own coordinator + assert coordinator_dc_2.consistent_id in baseline_dc_2, \ + f"{DC_2_NAME} coordinator should belong to its own half-ring baseline " \ + f"[coordinator={coordinator_dc_2.consistent_id}, baseline={sorted(baseline_dc_2)}]" def _populate_cluster_with_data(self): self.app_dc_1.java_class_name = GENERATOR_JAVA_CLASS_NAME From 09d484c451c46c565c6c3888278478ff14ffd59d Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 9 Jul 2026 13:20:00 +0300 Subject: [PATCH 12/44] IGNITE-27833 cluster recovery --- .../partition_resilience_test.py | 123 +++++++++--------- 1 file changed, 61 insertions(+), 62 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py index b1d6da4a01e4c..70906c0391f77 100644 --- a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py @@ -38,6 +38,7 @@ PUT_CHECKER_JAVA_CLASS_NAME = "org.apache.ignite.internal.ducktest.tests.mdc.MdcPutAdmissibilityCheckerApplication" CACHE_NAME = "replicated" +BACKUPS = 1 DATA_CENTER_ATTR = "IGNITE_DATA_CENTER_ID" @@ -87,58 +88,58 @@ def _run(self, network_mgr, **kwargs): for svc in [self.svc_dc_1, self.svc_dc_2]: svc.start() - self._populate_cluster_with_data() + self._populate_cluster_with_data(self.app_dc_1, 0, 100) + self._populate_cluster_with_data(self.app_dc_2, 100, 200) - control_utility = ControlUtility(self.svc_dc_1) - - distribution = control_utility.cache_distribution( - cache_names=CACHE_NAME, - user_attributes=DATA_CENTER_ATTR) - - self.assert_cross_dc_distribution_by_attribute( - distribution, - dc_attr=DATA_CENTER_ATTR, - expected_dcs=[DC_1_NAME, DC_2_NAME]) + self._verify_cache_distribution() network_mgr.enable_network_partition(DC_1_NAME, DC_2_NAME) sleep(10) - self._verify_half_ring_healthy(self.svc_dc_1) - self._verify_half_ring_healthy(self.svc_dc_2) - self._verify_split_brain(self.svc_dc_1, self.svc_dc_2) - self._check_data_accessible() - self._check_data_change_access() + self._check_data_accessible(self.app_dc_1, 0, 200) + self._check_data_accessible(self.app_dc_2, 0, 200) + + self._check_client_data_change_access(self.app_dc_1, True) + self._check_client_data_change_access(self.app_dc_2, False) network_mgr.disable_network_partition(DC_1_NAME, DC_2_NAME) - self.svc_dc_1.stop() self.svc_dc_2.stop() + self.svc_dc_2.start(clean=False) - @staticmethod - def _verify_half_ring_healthy(svc: IgniteService): - exp_alive_nodes = SRV_NODES // 2 - act_alive_nodes = len(svc.alive_nodes) + self.svc_dc_2.await_rebalance() - assert act_alive_nodes == exp_alive_nodes, f"{exp_alive_nodes} nodes should be alive! [actual={act_alive_nodes}]" + self._check_client_data_change_access(self.app_dc_1, True) + self._check_client_data_change_access(self.app_dc_2, True) - control_utility = ControlUtility(svc) + self._verify_cache_distribution() - cluster_state = control_utility.cluster_state() + self.svc_dc_1.stop() + self.svc_dc_2.stop() - assert "ACTIVE" == cluster_state.state, f"Half-ring state should remain ACTIVE [actual={cluster_state.state}]" + def _verify_cache_distribution(self): + control_utility = ControlUtility(self.svc_dc_1) - assert len(cluster_state.baseline) == exp_alive_nodes, \ - f"Half-ring baseline is not expected [exp={exp_alive_nodes}, actual_baseline={cluster_state.baseline}]" + distribution = control_utility.cache_distribution( + cache_names=CACHE_NAME, + user_attributes=DATA_CENTER_ATTR) - @staticmethod - def _verify_split_brain(svc_dc_1: IgniteService, svc_dc_2: IgniteService): + self.assert_cross_dc_distribution_by_attribute( + distribution, + dc_attr=DATA_CENTER_ATTR, + expected_dcs=[DC_1_NAME, DC_2_NAME]) + + def _verify_split_brain(self, svc_dc_1: IgniteService, svc_dc_2: IgniteService): """ Verifies that after the network partition the cluster has split into two independent half-rings: their baselines don't intersect and each half elected its own coordinator. """ + self._verify_half_ring_healthy(svc_dc_1) + self._verify_half_ring_healthy(svc_dc_2) + cluster_state_dc_1 = ControlUtility(svc_dc_1).cluster_state() cluster_state_dc_2 = ControlUtility(svc_dc_2).cluster_state() @@ -168,53 +169,51 @@ def _verify_split_brain(svc_dc_1: IgniteService, svc_dc_2: IgniteService): f"{DC_2_NAME} coordinator should belong to its own half-ring baseline " \ f"[coordinator={coordinator_dc_2.consistent_id}, baseline={sorted(baseline_dc_2)}]" - def _populate_cluster_with_data(self): - self.app_dc_1.java_class_name = GENERATOR_JAVA_CLASS_NAME - self.app_dc_2.java_class_name = GENERATOR_JAVA_CLASS_NAME + @staticmethod + def _verify_half_ring_healthy(svc: IgniteService): + exp_alive_nodes = SRV_NODES // 2 + act_alive_nodes = len(svc.alive_nodes) - self.app_dc_1.params = {"mainDc": DC_1_NAME, "cacheName": CACHE_NAME, "backups": 1, "from": 0, "to": 100} - self.app_dc_2.params = {"mainDc": DC_1_NAME, "cacheName": CACHE_NAME, "backups": 1, "from": 100, "to": 200} + assert act_alive_nodes == exp_alive_nodes, f"{exp_alive_nodes} nodes should be alive! [actual={act_alive_nodes}]" - self.app_dc_1.start() - self.app_dc_2.start() + control_utility = ControlUtility(svc) - self.app_dc_1.wait() - self.app_dc_2.wait() + cluster_state = control_utility.cluster_state() - self.app_dc_1.stop() - self.app_dc_2.stop() + assert "ACTIVE" == cluster_state.state, f"Half-ring state should remain ACTIVE [actual={cluster_state.state}]" - def _check_data_accessible(self): - self.app_dc_1.java_class_name = GET_CHECKER_JAVA_CLASS_NAME - self.app_dc_2.java_class_name = GET_CHECKER_JAVA_CLASS_NAME + assert len(cluster_state.baseline) == exp_alive_nodes, \ + f"Half-ring baseline is not expected [exp={exp_alive_nodes}, actual_baseline={cluster_state.baseline}]" - self.app_dc_1.params = {"cacheName": CACHE_NAME, "from": 0, "to": 200} - self.app_dc_2.params = {"cacheName": CACHE_NAME, "from": 0, "to": 200} + @staticmethod + def _populate_cluster_with_data(cli: IgniteApplicationService, from_idx: int, to_idx: int): + cli.java_class_name = GENERATOR_JAVA_CLASS_NAME - self.app_dc_1.start(clean=False) - self.app_dc_2.start(clean=False) + cli.params = {"mainDc": DC_1_NAME, "cacheName": CACHE_NAME, "backups": BACKUPS, "from": from_idx, "to": to_idx} - self.app_dc_1.wait() - self.app_dc_2.wait() + cli.start() + cli.wait() + cli.stop() - self.app_dc_1.stop() - self.app_dc_2.stop() + @staticmethod + def _check_data_accessible(cli: IgniteApplicationService, from_idx: int, to_idx: int): + cli.java_class_name = GET_CHECKER_JAVA_CLASS_NAME - def _check_data_change_access(self): - self.app_dc_1.java_class_name = PUT_CHECKER_JAVA_CLASS_NAME - self.app_dc_2.java_class_name = PUT_CHECKER_JAVA_CLASS_NAME + cli.params = {"cacheName": CACHE_NAME, "from": from_idx, "to": to_idx} - self.app_dc_1.params = {"cacheName": CACHE_NAME, "expectAdmissible": True} - self.app_dc_2.params = {"cacheName": CACHE_NAME, "expectAdmissible": False} + cli.start(clean=False) + cli.wait() + cli.stop() - self.app_dc_1.start(clean=False) - self.app_dc_2.start(clean=False) + @staticmethod + def _check_client_data_change_access(cli: IgniteApplicationService, expect_admissible: bool): + cli.java_class_name = PUT_CHECKER_JAVA_CLASS_NAME - self.app_dc_1.wait() - self.app_dc_2.wait() + cli.params = {"cacheName": CACHE_NAME, "expectAdmissible": expect_admissible} - self.app_dc_1.stop() - self.app_dc_2.stop() + cli.start(clean=False) + cli.wait() + cli.stop() def assert_cross_dc_distribution_by_attribute(self, distribution, dc_attr, expected_dcs, owning_only=True): """ From c0733ab7ff22e707a9d688ff8339ebb7168a8af4 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 9 Jul 2026 14:04:53 +0300 Subject: [PATCH 13/44] IGNITE-27833 read from backup --- .../internal/ducktest/tests/mdc/MdcCacheAwareApplication.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java index 9262e67f6c9cb..abd1e06325b0e 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java @@ -49,6 +49,7 @@ protected IgniteCache mdcCache(JsonNode jNode) { .setTopologyValidator(topValidator) .setCacheMode(PARTITIONED) .setBackups(backups) + .setReadFromBackup(true) .setAffinity(new RendezvousAffinityFunction() .setPartitions(32) .setAffinityBackupFilter(new MdcAffinityBackupFilter(dcsNum, backups))); From 5fbfb475ff565133287cf5eb4ea197fad851beb8 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 9 Jul 2026 14:07:07 +0300 Subject: [PATCH 14/44] IGNITE-27833 jmx_utils rollback --- .../tests/ignitetest/services/utils/jmx_utils.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py index 65e850acec39e..48cfe00003f53 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py @@ -81,6 +81,7 @@ def jmx_util_cmd(self): return os.path.join(f"java {extra_flag} -jar {self.install_root}/jmxterm.jar -v silent -n") + @memoize def find_mbean(self, pattern, negative_pattern=None, domain='org.apache'): """ Find mbean by specified pattern and domain on node. @@ -179,13 +180,6 @@ def int_order(self): val = self.__find__("intOrder=(\\d+),") return int(val) if val else -1 - @property - def is_coordinator(self): - """ - :return: True if node is coordinator. False otherwise. - """ - return self.coordinator == self.node_id - def __find__(self, pattern): res = re.search(pattern, self._local_raw) return res.group(1) if res else None @@ -195,6 +189,7 @@ class IgniteJmxMixin: """ Mixin to IgniteService node, exposing useful properties, obtained from JMX. """ + @memoize def jmx_client(self): """ :return: JmxClient instance. @@ -225,6 +220,7 @@ def kernal_mbean(self): """ return self.jmx_client().find_mbean('.*group=Kernal.*name=IgniteKernal') + @memoize def disco_mbean(self): """ :return: DiscoverySpi MBean. From 72a2d3cba41211f22cd06a206c0ca0bc422b76b3 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Fri, 10 Jul 2026 14:20:28 +0300 Subject: [PATCH 15/44] IGNITE-27833 WIP --- .../tests/mdc/MdcCacheAwareApplication.java | 131 +++++++-- .../mdc/MdcContinuousLoadApplication.java | 271 ++++++++++++++++++ .../mdc/MdcThinClientLoadApplication.java | 123 ++++++++ .../internal/ducktest/utils/OpStats.java | 49 ++++ .../ignite/internal/ducktest/utils/Utils.java | 38 +++ 5 files changed, 593 insertions(+), 19 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java index abd1e06325b0e..92a7b030a6ab5 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java @@ -1,7 +1,15 @@ package org.apache.ignite.internal.ducktest.tests.mdc; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.affinity.rendezvous.MdcAffinityBackupFilter; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; @@ -9,63 +17,148 @@ import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; import org.apache.ignite.topology.MdcTopologyValidator; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_DATA_CENTER_ID; +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; /** * Base class for MDC test applications. - * Encapsulates the cache configuration (with {@link MdcTopologyValidator}) so that - * generator and checkers always operate on an identically configured cache. + * Encapsulates the cache configuration (with {@link MdcTopologyValidator} and + * {@link MdcAffinityBackupFilter}) so that the generator, the checkers and the load + * applications always operate on an identically configured cache. + *

+ * Supported cache parameters (all optional unless stated otherwise): + *

    + *
  • {@code cacheName} - cache name;
    • + *
  • {@code backups} - number of backups; {@code (backups + 1)} must be divisible by {@code dcsNum};
    • + *
  • {@code mainDc} - main data center for the topology validator (2 DC mode);
    • + *
  • {@code datacenters} - full DC set for majority-based validation (odd DC count mode), + * takes precedence over {@code mainDc};
    • + *
  • {@code dcsNum} - number of data centers, default 2;
    • + *
  • {@code atomicity} - {@link CacheAtomicityMode}, default {@code ATOMIC};
    • + *
  • {@code syncMode} - {@link CacheWriteSynchronizationMode}, default {@code FULL_SYNC}. + * Note: MDC-aware local reads require a mode other than {@code PRIMARY_SYNC};
    • + *
  • {@code readFromBackup} - default {@code true}, required for DC-local reads;
    • + *
  • {@code partitions} - affinity partitions number, default 32.
    • + *
*/ public abstract class MdcCacheAwareApplication extends IgniteAwareApplication { - /** JVM option name holding the data center identifier. */ - public static final String DC_ID_PROP = "IGNITE_DATA_CENTER_ID"; + /** Table name of the SQL-enabled MDC cache. The SQL schema is the quoted cache name. */ + protected static final String SQL_TABLE = "LOAD"; /** */ - protected static String DFLT_CACHE_NAME = "default"; + protected static final String DFLT_CACHE_NAME = "default"; /** */ - protected static int DFLT_BACKUPS = 0; + protected static final CacheMode DFLT_CACHE_MODE = PARTITIONED; /** */ - protected static int DFLT_DCS_NUM = 2; + protected static final int DFLT_BACKUPS = 0; + + /** */ + protected static final int DFLT_DCS_NUM = 2; + + /** */ + protected static final int DFLT_PARTITIONS = 32; + + /** */ + protected static final CacheAtomicityMode DFLT_ATOMICITY_MODE = ATOMIC; + + /** */ + protected static final CacheWriteSynchronizationMode DFLT_WRITE_SYNC = FULL_SYNC; /** * @param jNode Parameters. - * @return Cache configured with the MDC topology validator. + * @return Cache configured with the MDC topology validator and backup filter. */ protected IgniteCache mdcCache(JsonNode jNode) { + return ignite.getOrCreateCache(this.mdcCacheConfiguration(jNode)); + } + + /** + * Same MDC cache configuration with a {@link QueryEntity} on top, so the cache is + * queryable via SQL: {@code SELECT _VAL FROM "".LOAD WHERE _KEY = ?}. + * + * @param jNode Parameters. + * @return SQL-enabled cache configured with the MDC topology validator and backup filter. + */ + protected IgniteCache mdcSqlCache(JsonNode jNode) { + CacheConfiguration cacheCfg = mdcCacheConfiguration(jNode); + + cacheCfg.setQueryEntities(Collections.singletonList( + new QueryEntity(Integer.class, Integer.class).setTableName(SQL_TABLE))); + + return ignite.getOrCreateCache(cacheCfg); // todo: check query engine + } + + /** + * @param jNode Parameters. + * @return MDC cache configuration compiled from the application parameters. + */ + protected CacheConfiguration mdcCacheConfiguration(JsonNode jNode) { String cacheName = jNode.path("cacheName").asText(DFLT_CACHE_NAME); int backups = jNode.path("backups").asInt(DFLT_BACKUPS); + int partitions = jNode.path("partitions").asInt(DFLT_PARTITIONS); + + CacheAtomicityMode atomicity = getEnum(jNode, "atomicity", DFLT_ATOMICITY_MODE); + CacheWriteSynchronizationMode writeSync = getEnum(jNode, "writeSync", DFLT_WRITE_SYNC); + CacheMode cacheMode = getEnum(jNode, "cacheMode", DFLT_CACHE_MODE); + + boolean readFromBackup = jNode.path("readFromBackup").asBoolean(true); - String mainDc = jNode.path("mainDc").asText(); int dcsNum = jNode.path("dcsNum").asInt(DFLT_DCS_NUM); MdcTopologyValidator topValidator = new MdcTopologyValidator(); - topValidator.setMainDatacenter(mainDc); + if (jNode.hasNonNull("datacenters")) { + Set dcs = new HashSet<>(); + + jNode.get("datacenters").forEach(dc -> dcs.add(dc.asText())); + + topValidator.setDatacenters(dcs); + } + else + topValidator.setMainDatacenter(jNode.path("mainDc").asText()); - CacheConfiguration cacheCfg = new CacheConfiguration() + return new CacheConfiguration() .setName(cacheName) .setTopologyValidator(topValidator) - .setCacheMode(PARTITIONED) + .setCacheMode(cacheMode) + .setAtomicityMode(atomicity) + .setWriteSynchronizationMode(writeSync) .setBackups(backups) - .setReadFromBackup(true) + .setReadFromBackup(readFromBackup) .setAffinity(new RendezvousAffinityFunction() - .setPartitions(32) + .setPartitions(partitions) .setAffinityBackupFilter(new MdcAffinityBackupFilter(dcsNum, backups))); - - return ignite.getOrCreateCache(cacheCfg); } - /** */ + /** + * @param cacheName Cache name. + * @return Existing cache. The cache must have been created by the generator beforehand. + */ protected IgniteCache mdcCache(String cacheName) { return ignite.cache(cacheName); } /** - * @return Data center id this client belongs to (passed via {@code -DIGNITE_DATA_CENTER_ID=...}). + * @return Data center id this client belongs to (passed via {@link IgniteSystemProperties#IGNITE_DATA_CENTER_ID}). */ protected static String dcId() { - return System.getProperty(DC_ID_PROP, ""); + return IgniteSystemProperties.getString(IGNITE_DATA_CENTER_ID); + } + + /** */ + protected static > E getEnum(JsonNode jNode, String fieldName, E dfltVal) { + JsonNode field = jNode.path(fieldName); + + if (field.isMissingNode() || field.isNull()) + return dfltVal; + try { + return Enum.valueOf(dfltVal.getDeclaringClass(), field.asText().toUpperCase()); + } catch (IllegalArgumentException e) { + return dfltVal; // Fallback if string doesn't match any enum constant + } } } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java new file mode 100644 index 0000000000000..d572148f9fc7d --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java @@ -0,0 +1,271 @@ +package org.apache.ignite.internal.ducktest.tests.mdc; + +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import javax.cache.CacheException; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.internal.ducktest.tests.dto.IndexedDataRecord; +import org.apache.ignite.internal.ducktest.utils.OpStats; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; + +import static org.apache.ignite.internal.ducktest.utils.Utils.fmtMs; +import static org.apache.ignite.internal.ducktest.utils.Utils.timed; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * Universal MDC load application. Runs a single-threaded synchronous load of the + * requested {@link Mode} either for a fixed number of iterations (a "burst") or until + * externally terminated (a "background" load spanning several test phases, e.g. a + * network partition and its healing). + *

+ * Parameters: + *

    + *
  • {@code mode} - one of {@code GET, PUT, TX_PUT, SQL_SELECT, SQL_PUT};
    • + *
  • {@code cacheName} - cache to operate on;
    • + *
  • {@code createCache} - if {@code true}, (re)creates the cache from the MDC + * configuration parameters (see {@link MdcCacheAwareApplication}); otherwise the + * cache must already exist;
    • + *
  • {@code keyFrom} / {@code keyTo} - key range. Read modes cycle within the range; + * write modes advance sequentially from {@code keyFrom} on every success, so on + * completion the keys {@code [keyFrom, keyFrom + opsCnt)} are guaranteed written;
    • + *
  • {@code iterations} - number of operations; {@code 0} means "run until terminated";
    • + *
  • {@code expectAdmissible} - write modes only. {@code true}: writes must succeed + * (default); {@code false}: every write must be rejected by the topology validator + * (read-only DC), any success fails the application;
    • + *
  • {@code tolerateErrors} - if {@code true}, operation failures are counted instead of + * failing fast. Intended for background loads crossing a partition boundary, where a + * short transient error window is possible;
    • + *
  • {@code opPauseMs} - pause between operations, default 0;
    • + *
  • {@code resultPrefix} - prefix for recorded results, so that several runs reusing one + * service produce uniquely named results;
    • + *
  • {@code txConcurrency} / {@code txIsolation} - transaction parameters for {@code TX_PUT}.
    • + *
+ */ +public class MdcContinuousLoadApplication extends MdcCacheAwareApplication { + /** Load modes. */ + private enum Mode { + /** Cache API reads with value verification. */ + GET, + + /** Cache API writes. */ + PUT, + + /** Transactional cache API writes (requires a TRANSACTIONAL cache). */ + TX_PUT, + + /** SQL reads with value verification (requires an SQL-enabled cache). */ + SQL_SELECT, + + /** SQL DML writes (requires an SQL-enabled cache). */ + SQL_PUT + } + + /** */ + public static final TransactionConcurrency DFLT_TX_CONCURRENCY = PESSIMISTIC; + + /** */ + public static final TransactionIsolation DFLT_TX_ISOLATION = REPEATABLE_READ; + + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws Exception { + Mode mode = Mode.valueOf(jNode.get("mode").asText().toUpperCase()); + + String cacheName = jNode.path("cacheName").asText(DFLT_CACHE_NAME); + boolean createCache = jNode.path("createCache").asBoolean(false); + + int keyFrom = jNode.path("keyFrom").asInt(0); + int keyTo = jNode.path("keyTo").asInt(Integer.MAX_VALUE); + long iterations = jNode.path("iterations").asLong(0); + + boolean expectAdmissible = jNode.path("expectAdmissible").asBoolean(true); + boolean tolerateErrors = jNode.path("tolerateErrors").asBoolean(false); + + long opPauseMs = jNode.path("opPauseMs").asLong(0); + + String pfx = jNode.path("resultPrefix").asText(""); + + TransactionConcurrency txConcurrency = getEnum(jNode, "txConcurrency", DFLT_TX_CONCURRENCY); + TransactionIsolation txIsolation = getEnum(jNode, "txIsolation", DFLT_TX_ISOLATION); + + markInitialized(); + waitForActivation(); + + boolean sqlMode = mode == Mode.SQL_SELECT || mode == Mode.SQL_PUT; + boolean writeMode = mode == Mode.PUT || mode == Mode.TX_PUT || mode == Mode.SQL_PUT; + + IgniteCache cache0 = null; + IgniteCache sqlCache0 = null; + + if (sqlMode) + sqlCache0 = createCache ? mdcSqlCache(jNode) : ignite.cache(cacheName); + else + cache0 = createCache ? mdcCache(jNode) : ignite.cache(cacheName); + + // Effectively-final copies for use inside timed lambdas. + IgniteCache cache = cache0; + IgniteCache sqlCache = sqlCache0; + + String mergeSql = String.format("MERGE INTO \"%s\".%s(_KEY, _VAL) VALUES(?, ?)", cacheName, SQL_TABLE); + String selectSql = String.format("SELECT _VAL FROM \"%s\".%s WHERE _KEY = ?", cacheName, SQL_TABLE); + + log.info("MDC load started [dc=" + dcId() + ", mode=" + mode + ", cache=" + cacheName + + ", keyFrom=" + keyFrom + ", keyTo=" + keyTo + ", iterations=" + iterations + + ", expectAdmissible=" + expectAdmissible + ", tolerateErrors=" + tolerateErrors + "]"); + + long opsCnt = 0; + long errCnt = 0; + + OpStats stats = new OpStats(); + + long maxStallMs = 0; + + long startTs = System.currentTimeMillis(); + long lastOkTs = startTs; + + int key0 = keyFrom; + + while (!terminated() && (iterations == 0 || opsCnt + errCnt < iterations)) { + int key = key0; + + boolean ok; + + try { + switch (mode) { + case GET: { + IndexedDataRecord val = timed(stats, () -> cache.get(key)); + + ok = val != null && val.equals(new IndexedDataRecord(key)); + + if (!ok) + log.error("Read entry is missed or corrupted [dc=" + dcId() + ", key=" + key + + ", val=" + val + "]"); + + break; + } + + case PUT: { + IndexedDataRecord val = new IndexedDataRecord(key); + + timed(stats, () -> cache.put(key, val)); + + ok = true; + + break; + } + + case TX_PUT: { + IndexedDataRecord val = new IndexedDataRecord(key); + + timed(stats, () -> { + try (Transaction tx = ignite.transactions().txStart(txConcurrency, txIsolation)) { + cache.put(key, val); + + tx.commit(); + } + }); + + ok = true; + + break; + } + + case SQL_PUT: { + SqlFieldsQuery qry = new SqlFieldsQuery(mergeSql).setArgs(key, key); + + timed(stats, () -> sqlCache.query(qry).getAll()); + + ok = true; + + break; + } + + case SQL_SELECT: { + SqlFieldsQuery qry = new SqlFieldsQuery(selectSql).setArgs(key); + + List> rows = timed(stats, () -> sqlCache.query(qry).getAll()); + + ok = !rows.isEmpty() && Objects.equals(rows.get(0).get(0), key); + + if (!ok) + log.error("SQL row is missed or corrupted [dc=" + dcId() + ", key=" + key + + ", rows=" + rows + "]"); + + break; + } + + default: + throw new IllegalArgumentException("Unknown mode: " + mode); + } + } + catch (CacheException | IgniteException e) { + ok = false; + + if (writeMode && !expectAdmissible) + log.info("Write rejected as expected [dc=" + dcId() + ", key=" + key + + ", msg=" + e.getMessage() + "]"); + else if (!tolerateErrors) + throw new IllegalStateException("Operation failed [dc=" + dcId() + ", mode=" + mode + + ", key=" + key + "]", e); + else + log.warn("Operation failed, tolerated [dc=" + dcId() + ", mode=" + mode + + ", key=" + key + ", msg=" + e.getMessage() + "]"); + } + + if (ok) { + opsCnt++; + + long now = System.currentTimeMillis(); + + maxStallMs = Math.max(maxStallMs, now - lastOkTs); + lastOkTs = now; + } + else + errCnt++; + + // Writes advance on success only, so [keyFrom, keyFrom + opsCnt) is guaranteed written. + // The inadmissible-probe mode advances always to probe distinct keys. Reads cycle the range. + if (ok || (writeMode && !expectAdmissible)) { + key0++; + + if (key0 >= keyTo) + key0 = keyFrom; + } + + if (opPauseMs > 0) + Thread.sleep(opPauseMs); + } + + long durationMs = System.currentTimeMillis() - startTs; + + if (writeMode && !expectAdmissible && opsCnt > 0) { + throw new IllegalStateException("Write load is admissible while expected to be inadmissible [dc=" + + dcId() + ", mode=" + mode + ", succeeded=" + opsCnt + ", rejected=" + errCnt + "]"); + } + + recordResult(pfx + "OpsCnt", String.valueOf(opsCnt)); + recordResult(pfx + "ErrCnt", String.valueOf(errCnt)); + recordResult(pfx + "DurationMs", String.valueOf(durationMs)); + + recordResult(pfx + "AvgOpMs", fmtMs(stats.avgNs())); + recordResult(pfx + "MinOpMs", fmtMs(stats.minNs())); + recordResult(pfx + "MaxOpMs", fmtMs(stats.maxNs())); + + recordResult(pfx + "DerivedTps", String.format(Locale.US, "%.1f", stats.tps())); + + recordResult(pfx + "MaxStallMs", String.valueOf(maxStallMs)); + + log.info("MDC load finished [dc=" + dcId() + ", mode=" + mode + ", ops=" + opsCnt + + ", errs=" + errCnt + ", durationMs=" + durationMs + + ", avgOpMs=" + fmtMs(stats.avgNs()) + ", maxOpMs=" + fmtMs(stats.maxNs()) + + ", maxStallMs=" + maxStallMs + "]"); + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java new file mode 100644 index 0000000000000..7f1e67ee4bf2f --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java @@ -0,0 +1,123 @@ +package org.apache.ignite.internal.ducktest.tests.mdc; + +import javax.cache.CacheException; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientException; +import org.apache.ignite.internal.ducktest.tests.dto.IndexedDataRecord; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; +import org.apache.ignite.internal.ducktest.utils.OpStats; + +import static org.apache.ignite.internal.ducktest.utils.Utils.fmtMs; +import static org.apache.ignite.internal.ducktest.utils.Utils.timed; + +/** + * Thin client MDC load application. + *

+ * The application performs synchronous {@code GET} or {@code PUT} operations against an + * existing cache and records the average operation latency, which the python side uses as + * a crude proxy for "which data center served the request": with a high cross-DC netem + * delay, DC-local routing yields a small average, cross-DC routing a large one. + *

+ * Parameters: {@code mode} ({@code GET}/{@code PUT}), {@code cacheName}, {@code keyFrom}, + * {@code keyTo}, {@code iterations}, {@code expectAdmissible} (PUT only), + * {@code resultPrefix}. + *

+ */ +public class MdcThinClientLoadApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws Exception { + String mode = jNode.get("mode").asText().toUpperCase(); + String cacheName = jNode.get("cacheName").asText(); + + int keyFrom = jNode.path("keyFrom").asInt(0); + int keyTo = jNode.path("keyTo").asInt(Integer.MAX_VALUE); + long iterations = jNode.path("iterations").asLong(100); + + boolean put = "PUT".equals(mode); + boolean expectAdmissible = jNode.path("expectAdmissible").asBoolean(true); + + String pfx = jNode.path("resultPrefix").asText(""); + + markInitialized(); + + ClientCache cache = client.cache(cacheName); + + log.info("MDC thin client load started [mode=" + mode + ", cache=" + cacheName + + ", keyFrom=" + keyFrom + ", keyTo=" + keyTo + ", iterations=" + iterations + + ", expectAdmissible=" + expectAdmissible + "]"); + + long opsCnt = 0; + long errCnt = 0; + + OpStats stats = new OpStats(); + + long startTs = System.currentTimeMillis(); + + int key0 = keyFrom; + + for (long i = 0; i < iterations && !terminated(); i++) { + int key = key0; + + boolean ok; + + try { + if (put) { + IndexedDataRecord val = new IndexedDataRecord(key); + + timed(stats, () -> cache.put(key, val)); + + ok = true; + } + else { + IndexedDataRecord val = timed(stats, () -> cache.get(key)); + + ok = val != null && val.equals(new IndexedDataRecord(key)); + + if (!ok) + log.error("Read entry is missed or corrupted [key=" + key + ", val=" + val + "]"); + } + } + catch (ClientException | CacheException e) { + ok = false; + + if (put && !expectAdmissible) + log.info("Put rejected as expected [key=" + key + ", msg=" + e.getMessage() + "]"); + else + throw new IllegalStateException("Operation failed [mode=" + mode + ", key=" + key + "]", e); + } + + if (ok) + opsCnt++; + else + errCnt++; + + // Writes advance always here: the inadmissible probe covers distinct keys, and for + // an admissible run any failure fails fast above, so success and attempt counts match. + key0++; + + if (key0 >= keyTo) + key0 = keyFrom; + } + + long durationMs = System.currentTimeMillis() - startTs; + + if (put && !expectAdmissible && opsCnt > 0) { + throw new IllegalStateException("Put load is admissible while expected to be inadmissible " + + "[succeeded=" + opsCnt + ", rejected=" + errCnt + "]"); + } + + recordResult(pfx + "OpsCnt", String.valueOf(opsCnt)); + recordResult(pfx + "ErrCnt", String.valueOf(errCnt)); + + recordResult(pfx + "AvgOpMs", fmtMs(stats.avgNs())); + recordResult(pfx + "MinOpMs", fmtMs(stats.minNs())); + recordResult(pfx + "MaxOpMs", fmtMs(stats.maxNs())); + + log.info("MDC thin client load finished [mode=" + mode + ", ops=" + opsCnt + + ", errs=" + errCnt + ", durationMs=" + durationMs + + ", avgOpMs=" + fmtMs(stats.avgNs()) + "]"); + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java new file mode 100644 index 0000000000000..206bcf28362e4 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java @@ -0,0 +1,49 @@ +package org.apache.ignite.internal.ducktest.utils; + +/** Accumulates latency of completed operations. */ +public class OpStats { + /** Sum of recorded latencies, nanoseconds. */ + private long totalNs; + + /** Minimum recorded latency, nanoseconds. */ + private long minNs = Long.MAX_VALUE; + + /** Maximum recorded latency, nanoseconds. */ + private long maxNs; + + /** Number of recorded operations. */ + private long cnt; + + /** Records a single operation latency. */ + public void record(long ns) { + totalNs += ns; + + minNs = Math.min(minNs, ns); + maxNs = Math.max(maxNs, ns); + + cnt++; + } + + /** @return Minimum latency in nanoseconds, or {@code -1} if nothing was recorded. */ + public long minNs() { + return cnt > 0 ? minNs : -1; + } + + /** @return Maximum latency in nanoseconds, or {@code -1} if nothing was recorded. */ + public long maxNs() { + return cnt > 0 ? maxNs : -1; + } + + /** @return Average latency in nanoseconds, or {@code -1} if nothing was recorded. */ + public double avgNs() { + return cnt > 0 ? totalNs / (double)cnt : -1; + } + + /** + * @return Derived throughput: recorded operations per second of pure operation time, + * or {@code -1} if nothing was recorded. + */ + public double tps() { + return totalNs > 0 ? cnt * 1e9 / totalNs : -1; + } +} \ No newline at end of file diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java new file mode 100644 index 0000000000000..d9f61e5bfc675 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java @@ -0,0 +1,38 @@ +package org.apache.ignite.internal.ducktest.utils; + +import java.util.Locale; +import java.util.function.Supplier; + +/** */ +public final class Utils { + /** + * Runs the operation, records its latency into {@code stats} and returns its result. + * If the operation throws, nothing is recorded and the exception propagates. + */ + public static T timed(OpStats stats, Supplier op) { + long startNs = System.nanoTime(); + + T val = op.get(); + + stats.record(System.nanoTime() - startNs); + + return val; + } + + /** + * Runs the void operation and records its latency into {@code stats}. + * If the operation throws, nothing is recorded and the exception propagates. + */ + public static void timed(OpStats stats, Runnable op) { + timed(stats, () -> { + op.run(); + + return null; + }); + } + + /** Formats a nanosecond latency as milliseconds with 3 decimal places. */ + public static String fmtMs(double ns) { + return String.format(Locale.US, "%.3f", ns < 0 ? -1.0 : ns / 1e6); + } +} From 4d6bae988c3c9a906c960c5d0ff62cf3a26356d9 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Fri, 10 Jul 2026 20:31:56 +0300 Subject: [PATCH 16/44] IGNITE-27833 WIP --- .../tests/mdc/MdcCacheAwareApplication.java | 2 +- .../mdc/MdcDataGeneratorApplication.java | 18 +- .../tests/ignitetest/services/mdc/__init__.py | 18 + .../ignitetest/services/mdc/mdc_cluster.py | 509 ++++++++++++++++++ .../ignitetest/services/utils/ignite_aware.py | 22 + .../ignitetest/services/utils/log_utils.py | 14 +- .../tests/ignitetest/tests/mdc/__init__.py | 18 + .../ignitetest/tests/mdc/distribution_test.py | 119 ++++ .../ignitetest/tests/mdc/latency_test.py | 113 ++++ .../tests/mdc/partition_load_test.py | 270 ++++++++++ .../tests/mdc/partition_resilience_test.py | 183 +++++++ .../ignitetest/tests/mdc/thin_client_test.py | 188 +++++++ .../tests/network_group/__init__.py | 59 -- .../partition_resilience_test.py | 273 ---------- 14 files changed, 1470 insertions(+), 336 deletions(-) create mode 100644 modules/ducktests/tests/ignitetest/services/mdc/__init__.py create mode 100644 modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py create mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/__init__.py create mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/distribution_test.py create mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/latency_test.py create mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py create mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py create mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py delete mode 100644 modules/ducktests/tests/ignitetest/tests/network_group/__init__.py delete mode 100644 modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java index 92a7b030a6ab5..e181289a1ba54 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java @@ -60,7 +60,7 @@ public abstract class MdcCacheAwareApplication extends IgniteAwareApplication { protected static final int DFLT_DCS_NUM = 2; /** */ - protected static final int DFLT_PARTITIONS = 32; + protected static final int DFLT_PARTITIONS = 512; /** */ protected static final CacheAtomicityMode DFLT_ATOMICITY_MODE = ATOMIC; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java index 8cfcd5f0990f6..e8727dfddc6f1 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java @@ -1,5 +1,7 @@ package org.apache.ignite.internal.ducktest.tests.mdc; +import java.util.Map; +import java.util.TreeMap; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.IgniteInterruptedCheckedException; @@ -14,6 +16,7 @@ public class MdcDataGeneratorApplication extends MdcCacheAwareApplication { @Override public void run(JsonNode jNode) throws IgniteInterruptedCheckedException { int from = jNode.path("from").asInt(0); int to = jNode.path("to").asInt(10_000); + int batchSize = jNode.path("batchSize").asInt(1_024); markInitialized(); waitForActivation(); @@ -23,8 +26,19 @@ public class MdcDataGeneratorApplication extends MdcCacheAwareApplication { log.info("Data generation started [dc=" + dcId() + ", cache=" + cache.getName() + ", from=" + from + ", to=" + to + "]"); - for (int i = from; i < to && !terminated(); i++) - cache.put(i, new IndexedDataRecord(i)); + Map batch = new TreeMap<>(); + + for (int i = from; i < to && !terminated(); i++) { + batch.put(i, new IndexedDataRecord(i)); + + if (batch.size() >= batchSize) { + cache.putAll(batch); + batch.clear(); + } + } + + if (!batch.isEmpty() && !terminated()) + cache.putAll(batch); log.info("Data generation finished [dc=" + dcId() + ", entries=" + (to - from) + "]"); diff --git a/modules/ducktests/tests/ignitetest/services/mdc/__init__.py b/modules/ducktests/tests/ignitetest/services/mdc/__init__.py new file mode 100644 index 0000000000000..ba4f3680ef3e1 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/mdc/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Multi DC Cluster Service +""" \ No newline at end of file diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py new file mode 100644 index 0000000000000..07e43079a662d --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -0,0 +1,509 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +MDC test fixture. + + mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc=1) + + with cross_dc_network(self.logger, mdc, delay_ms=20) as net: + mdc.start_servers() + mdc.generate_data(DC_1, CACHE, 0, 1000, backups=1) + mdc.verify_cache_distribution(CACHE, copies_per_dc=1) + + net.enable_network_partition(DC_1, DC_2) + ... +""" +from typing import Dict, List, Optional, Union + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.network_group.configuration import NetworkGroupStore, CrossNetworkGroupConfiguration +from ignitetest.services.network_group.manager import NetworkGroupManager +from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration +from ignitetest.utils.version import IgniteVersion + +DC_1 = "DC1" +DC_2 = "DC2" +DCS = (DC_1, DC_2) + +DATA_CENTER_ATTR = "IGNITE_DATA_CENTER_ID" + +_APP_PKG = "org.apache.ignite.internal.ducktest.tests.mdc." + +GENERATOR_APP = _APP_PKG + "MdcDataGeneratorApplication" +DATA_CHECKER_APP = _APP_PKG + "MdcDataCheckerApplication" +PUT_CHECKER_APP = _APP_PKG + "MdcPutAdmissibilityCheckerApplication" +LOAD_APP = _APP_PKG + "MdcContinuousLoadApplication" +THIN_LOAD_APP = _APP_PKG + "MdcThinClientLoadApplication" + +# Suspicious server log patterns: none of them is expected in any MDC scenario, todo: Replace with log checks or control.sh commands +# partitioned or not. Matched against the node console capture. +LRT_PATTERN = "Found long running transaction" +PME_FREEZE_PATTERN = "Failed to wait for partition map exchange" +LOST_PARTITIONS_PATTERN = "Detected lost partitions" + + +def dc_jvm_opts(dc: str) -> List[str]: + """ + :return: JVM options assigning a node to the given data center. + """ + return [f"-D{DATA_CENTER_ATTR}={dc}"] + + +def _per_dc(value: Union[int, Dict[str, int]]) -> Dict[str, int]: + """ + Normalizes an int-or-dict per-DC count into a dict, e.g. 3 -> {DC1: 3, DC2: 3}. + """ + return dict(value) if isinstance(value, dict) else {dc: value for dc in DCS} + + +class MdcCluster: + """ + Owns the per-DC Ignite services and reusable application services of an MDC test, + plus the MDC-specific verification helpers. + + :param test: The ducktape test instance. + :param ignite_version: Ignite version string. + :param srv_per_dc: Servers per DC, an int or a per-DC dict (asymmetric DCs). + :param runners_per_dc: Reusable run-to-completion app services per DC (generator, + checkers, load bursts). An int or a per-DC dict. + :param loaders_per_dc: Dedicated background load app services per DC. They run + concurrently with runner apps, hence separate containers. + :param client_connector: Whether to expose the thin client connector on servers. + """ + def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, int]] = 3, + runners_per_dc: Union[int, Dict[str, int]] = 1, + loaders_per_dc: Union[int, Dict[str, int]] = 0, # todo: не понял зачем это нужно + client_connector: bool = False): + self.test_context = test.test_context + self.logger = test.logger + + cfg_kwargs = {"version": IgniteVersion(ignite_version)} + + if client_connector: + cfg_kwargs["client_connector_configuration"] = ClientConnectorConfiguration() + + self.ignite_config = IgniteConfiguration(**cfg_kwargs) + + self.srv_per_dc = _per_dc(srv_per_dc) + + self.servers: Dict[str, IgniteService] = { + dc: IgniteService(self.test_context, self.ignite_config, num_nodes=num, jvm_opts=dc_jvm_opts(dc)) + for dc, num in self.srv_per_dc.items() if num > 0} + + self.runners: Dict[str, List[IgniteApplicationService]] = { + dc: [self._app_service(dc) for _ in range(num)] + for dc, num in _per_dc(runners_per_dc).items()} + + self.loaders: Dict[str, List[IgniteApplicationService]] = { + dc: [self._app_service(dc) for _ in range(num)] + for dc, num in _per_dc(loaders_per_dc).items()} + + # Extra services (e.g. thin clients) registered into a DC's network group. + self.extras: Dict[str, List] = {dc: [] for dc in DCS} + + # App services that have been started at least once: the first start is clean, + # subsequent ones preserve work dirs (and logs - hence unique result prefixes). + self._started_apps = set() + + def _app_service(self, dc: str) -> IgniteApplicationService: + client_cfg = self.ignite_config._replace(client_mode=True, discovery_spi=from_ignite_cluster(self.servers[dc])) + + return IgniteApplicationService(self.test_context, client_cfg, jvm_opts=dc_jvm_opts(dc)) + + def register(self, dc: str, service): + """ + Registers an extra service (e.g. a thin client app) into a DC's network group, + so netem impairments and partitions apply to it. Must be called before the + :class:`NetworkGroupManager` is entered. + """ + self.extras[dc].append(service) + + def network_registry(self) -> Dict[str, List]: + """ + :return: Network group registry: DC name -> all services belonging to that DC. + """ + registry = {} + + for dc in DCS: + services = [] + + if dc in self.servers: + services.append(self.servers[dc]) + + services += self.runners.get(dc, []) + services += self.loaders.get(dc, []) + services += self.extras.get(dc, []) + + if services: + registry[dc] = services + + return registry + + def thin_client_addresses(self) -> List[str]: + """ + :return: Thin client addresses of all server nodes across all DCs. + """ + port = self.ignite_config.client_connector_configuration.port + + return [f"{node.account.hostname}:{port}" + for dc in sorted(self.servers) for node in self.servers[dc].nodes] + + def start_servers(self): + """ + Starts all server services. + """ + for dc in sorted(self.servers): + self.servers[dc].start() + + def stop_servers(self): + """ + Stops all server services. + """ + for dc in sorted(self.servers): + self.servers[dc].stop() + + def restart(self, dc: str, clean: bool = False, await_rebalance: bool = True): + """ + Restarts a whole DC preserving its persistence (the pattern used to rejoin a + read-only half-ring back into the main cluster after a partition heals). + """ + self.servers[dc].stop() + self.servers[dc].start(clean=clean) + + if await_rebalance: + self.servers[dc].await_rebalance() + + def run_app(self, dc: str, java_class: str, params: dict, runner: int = 0) -> IgniteApplicationService: + """ + Runs a run-to-completion application on one of the DC's reusable runner services + and returns the service (for ``extract_result``). + """ + svc = self.runners[dc][runner] + + svc.java_class_name = java_class + svc.params = params + + svc.start(clean=self._first_start(svc)) + svc.wait() + svc.stop() + + return svc + + def start_loader(self, dc: str, params: dict, loader: int = 0, + java_class: str = LOAD_APP) -> IgniteApplicationService: + """ + Starts a background load application (runs until stopped). Any exception raised + by the application surfaces in :meth:`stop_loader`. + """ + svc = self.loaders[dc][loader] + + svc.java_class_name = java_class + svc.params = params + + svc.start(clean=self._first_start(svc)) + + return svc + + def stop_loader(self, dc: str, loader: int = 0) -> IgniteApplicationService: + """ + Stops a background load application. The application finishes its loop, records + results and exits; a failed application fails the test here. + """ + svc = self.loaders[dc][loader] + + svc.stop() + + return svc + + def _first_start(self, svc) -> bool: + first = id(svc) not in self._started_apps + + self._started_apps.add(id(svc)) + + return first + + # ------------------------------------------------------- domain applications + + def generate_data(self, dc: str, cache_name: str, from_idx: int, to_idx: int, backups: int, + main_dc: str = DC_1, **cache_params) -> IgniteApplicationService: + """ + Creates the MDC cache (if absent) and populates keys ``[from_idx, to_idx)``. + Extra cache parameters (``atomicity``, ``syncMode``, ``readFromBackup``, + ``partitions``, ...) are passed through to the cache configuration builder. + """ + params = {"cacheName": cache_name, "backups": backups, "mainDc": main_dc, + "from": from_idx, "to": to_idx, **cache_params} + + return self.run_app(dc, GENERATOR_APP, params) + + def check_data(self, dc: str, cache_name: str, from_idx: int, to_idx: int) -> IgniteApplicationService: + """ + Verifies that every key in ``[from_idx, to_idx)`` is readable and holds the + expected value, from a client in the given DC. + """ + if to_idx <= from_idx: + self.logger.debug(f"Nothing to check [cache={cache_name}, from={from_idx}, to={to_idx}]") + return self.runners[dc][0] + + params = {"cacheName": cache_name, "from": from_idx, "to": to_idx} + + return self.run_app(dc, DATA_CHECKER_APP, params) + + def check_put_admissibility(self, dc: str, cache_name: str, admissible: bool, + key_offset: int = 1_000_000) -> IgniteApplicationService: + """ + Verifies that put load from the given DC is admissible (primary DC visible) or + rejected by the topology validator (read-only DC). + """ + params = {"cacheName": cache_name, "expectAdmissible": admissible, "keyOffset": key_offset} + + return self.run_app(dc, PUT_CHECKER_APP, params) + + def run_load(self, dc: str, mode: str, cache_name: str, result_prefix: str, + runner: int = 0, **params) -> IgniteApplicationService: + """ + Runs a load burst (see ``MdcContinuousLoadApplication``) and returns the service. + ``result_prefix`` must be unique per burst because runner services are reused. + """ + load_params = {"mode": mode, "cacheName": cache_name, "resultPrefix": result_prefix, **params} + + return self.run_app(dc, LOAD_APP, load_params, runner=runner) + + def control(self, dc: str = DC_1) -> ControlUtility: + """ + :return: Control utility bound to the given DC's servers. + """ + return ControlUtility(self.servers[dc]) + + def verify_cache_distribution(self, cache_name: str, copies_per_dc: Optional[int] = None, dc: str = DC_1): + """ + Verifies that every partition of the cache has an OWNING copy in every DC, and + optionally that each DC holds exactly ``copies_per_dc`` copies. + + :return: The CacheDistribution for further custom assertions. + """ + distribution = self.control(dc).cache_distribution(cache_names=cache_name, user_attributes=DATA_CENTER_ATTR) + + assert_cross_dc_distribution_by_attribute(distribution, dc_attr=DATA_CENTER_ATTR, + expected_dcs=DCS, copies_per_dc=copies_per_dc) + + return distribution + + def verify_split_brain(self): + """ + Verifies that after the network partition the cluster has split into two independent + half-rings: their baselines don't intersect and each half elected its own coordinator. + """ + for dc in DCS: + self.verify_half_ring_healthy(dc) + + state = {dc: self.control(dc).cluster_state() for dc in DCS} + + baselines = {dc: {node.consistent_id for node in state[dc].baseline} for dc in DCS} + + common_nodes = baselines[DC_1] & baselines[DC_2] + + assert not common_nodes, \ + f"Half-ring baselines should not intersect " \ + f"[common={sorted(common_nodes)}, dc1={sorted(baselines[DC_1])}, dc2={sorted(baselines[DC_2])}]" + + for dc in DCS: + coordinator = state[dc].coordinator + + assert coordinator, f"Coordinator is not found in {dc} half-ring baseline output!" + + assert coordinator.consistent_id in baselines[dc], \ + f"{dc} coordinator should belong to its own half-ring baseline " \ + f"[coordinator={coordinator.consistent_id}, baseline={sorted(baselines[dc])}]" + + assert state[DC_1].coordinator.consistent_id != state[DC_2].coordinator.consistent_id, \ + f"Half-rings should have different coordinators " \ + f"[coordinator={state[DC_1].coordinator.consistent_id}]" + + def verify_half_ring_healthy(self, dc: str): + """ + Verifies that a half-ring is fully alive, ACTIVE, and its baseline matches its size. + """ + exp_alive_nodes = self.srv_per_dc[dc] + act_alive_nodes = len(self.servers[dc].alive_nodes) + + assert act_alive_nodes == exp_alive_nodes, \ + f"{exp_alive_nodes} nodes should be alive in {dc}! [actual={act_alive_nodes}]" + + cluster_state = self.control(dc).cluster_state() + + assert "ACTIVE" == cluster_state.state, \ + f"{dc} half-ring state should remain ACTIVE [actual={cluster_state.state}]" + + assert len(cluster_state.baseline) == exp_alive_nodes, \ + f"{dc} half-ring baseline is not expected " \ + f"[exp={exp_alive_nodes}, actual_baseline={cluster_state.baseline}]" + + def verify_whole_cluster_healthy(self): + """ + Verifies that both DCs form a single ACTIVE cluster: every server node is alive + and the baseline seen from DC1 covers all servers of both DCs. + """ + exp_total = sum(self.srv_per_dc.values()) + + act_alive = sum(len(self.servers[dc].alive_nodes) for dc in self.servers) + + assert act_alive == exp_total, f"All {exp_total} server nodes should be alive [actual={act_alive}]" + + cluster_state = self.control(DC_1).cluster_state() + + assert "ACTIVE" == cluster_state.state, f"Cluster should be ACTIVE [actual={cluster_state.state}]" + + assert len(cluster_state.baseline) == exp_total, \ + f"Cluster baseline should cover both DCs [exp={exp_total}, actual={cluster_state.baseline}]" + + def verify_servers_log_clean(self): + """ + Verifies the negative invariants on all server nodes: no long running transactions + were detected, no PME hang and no lost partitions were reported. + """ + for pattern in (LRT_PATTERN, PME_FREEZE_PATTERN, LOST_PARTITIONS_PATTERN): + for dc, svc in self.servers.items(): + hits = svc.check_event_absent(pattern) + + assert hits == 0, f"Suspicious events found in {dc} server logs [pattern='{pattern}', hits={hits}]" + + def verify_no_hanging_txs(self, dc: str = DC_1): + """ + Verifies that no active transactions are left on the cluster. + """ + txs = self.control(dc).tx() + + assert not isinstance(txs, list) or not txs, f"No active transactions expected [txs={txs}]" + + @staticmethod + def result_int(svc: IgniteApplicationService, name: str) -> int: + """ + :return: Application-recorded integer result. + """ + return int(svc.extract_result(name)) + + @staticmethod + def result_float(svc: IgniteApplicationService, name: str) -> float: + """ + :return: Application-recorded float result. + """ + return float(svc.extract_result(name)) + + +def cross_dc_network(logger, mdc: MdcCluster, delay_ms: Optional[int] = None, + loss: Optional[float] = None) -> NetworkGroupManager: + """ + Builds a :class:`NetworkGroupManager` (context manager) for the cluster with symmetric + DC1 <-> DC2 impairments. With no impairments the manager still owns partition + enable/disable and the final network cleanup. + + :param delay_ms: One-way cross-DC latency in milliseconds (the effective RTT is twice + that, since netem delay is applied on egress in both directions). + :param loss: Cross-DC packet loss fraction in [0.0, 1.0]. + """ + store = NetworkGroupStore() + + cfg = CrossNetworkGroupConfiguration(delay=f"{delay_ms}ms" if delay_ms is not None else None, loss=loss) + + if not cfg.is_empty: + store.set_config(DC_1, DC_2, cfg) + + return NetworkGroupManager(logger, store, mdc.network_registry()) + + +def assert_cross_dc_distribution_by_attribute(distribution, dc_attr, expected_dcs, owning_only=True, copies_per_dc=None): + """ + Asserts that every partition of every cache group has at least one copy in every DC, + using a node attribute (requested via --user-attributes) as the DC marker. + + :param distribution: CacheDistribution returned by ControlUtility.cache_distribution(), + requested with user_attributes=[dc_attr]. + :param dc_attr: Attribute name holding the DC id, e.g. "IGNITE_DATA_CENTER_ID". + :param expected_dcs: Collection of DC ids that must own a copy of every partition. + :param owning_only: Count only copies in OWNING state as present. + :param copies_per_dc: If set, each DC must hold exactly this many copies of every + partition - the MdcAffinityBackupFilter guarantee + ``(backups + 1) / dcsNum``. + """ + def dc_of(copy): + return copy.user_attributes.get(dc_attr) + + _assert_cross_dc(distribution, set(expected_dcs), dc_of, owning_only, copies_per_dc, + layout_hint=f"DC attribute: {dc_attr}, expected DCs: {sorted(expected_dcs)}") + + +def assert_cross_dc_distribution(distribution, dc_addresses, owning_only=True, copies_per_dc=None): + """ + Same assertion, but DC membership is derived from node IP addresses. + + :param dc_addresses: Dict: DC name -> collection of node IP addresses of that DC, + e.g. {"DC1": ["192.168.64.9", ...], "DC2": [...]}. + """ + dc_addresses = {dc: set(addrs) for dc, addrs in dc_addresses.items()} + + def dc_of(copy): + for dc, addrs in dc_addresses.items(): + if any(addr in addrs for addr in copy.node_addresses): + return dc + return None + + _assert_cross_dc(distribution, set(dc_addresses), dc_of, owning_only, copies_per_dc, + layout_hint=f"DC layout: { {dc: sorted(addrs) for dc, addrs in dc_addresses.items()} }") + + +def _assert_cross_dc(distribution, expected_dcs, dc_of, owning_only, copies_per_dc, layout_hint): + violations = [] + + for group in distribution.groups.values(): + for part, copies in sorted(group.partitions.items()): + counted = [c for c in copies if not owning_only or c.state == "OWNING"] + + per_dc = {dc: 0 for dc in expected_dcs} + + for copy in counted: + dc = dc_of(copy) + + if dc in per_dc: + per_dc[dc] += 1 + + missing = {dc for dc, cnt in per_dc.items() if cnt == 0} + + unbalanced = {} if copies_per_dc is None else \ + {dc: cnt for dc, cnt in per_dc.items() if cnt != copies_per_dc} + + if missing or unbalanced: + copies_dump = ", ".join( + f"{c.node_id}({'P' if c.primary else 'B'},{c.state},dc={dc_of(c)},{c.node_addresses})" + for c in copies) + + problems = [] + + if missing: + problems.append(f"missing DCs={sorted(missing)}") + + if unbalanced: + problems.append(f"copies per DC != {copies_per_dc}: {unbalanced}") + + violations.append(f"group={group.name}(id={group.group_id}), partition={part}, " + f"{', '.join(problems)}, copies=[{copies_dump}]") + + assert not violations, \ + "Partition distribution is not cross-DC:\n " + "\n ".join(violations) + "\n" + layout_hint diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 7d475844a2ed1..8bd758f339d04 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -284,6 +284,28 @@ def await_event(self, evt_message, timeout_sec, nodes=None, from_the_beginning=F self.await_event_on_node(evt_message, node, timeout_sec, from_the_beginning=from_the_beginning, backoff_sec=backoff_sec, log_file=log_file) + def check_event_absent_on_node(self, evt_message, node, from_the_beginning=True, log_file=None): + """ + Verify that a specific event message is NOT present in a node's log file. + :param evt_message: Event message. + :param node: Ignite service node. + :param from_the_beginning: If True, search from the beginning of the log file + (this is usually what you want for an absence check). + :param log_file: Explicit log file. + """ + log = os.path.join(self.log_dir, log_file) if log_file else node.log_file + + with monitor_log(node, log, from_the_beginning) as monitor: + assert not monitor.found(evt_message), \ + "Event [%s] was unexpectedly found on '%s'" % (evt_message, node.name) + + def check_event_absent(self, evt_message, from_the_beginning=True, log_file=None): + """ + Verify that a specific event message is NOT present on any node of the service. + """ + for node in self.nodes: + self.check_event_absent_on_node(evt_message, node, from_the_beginning=from_the_beginning, log_file=log_file) + @staticmethod def event_time(evt_message, node): """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/log_utils.py b/modules/ducktests/tests/ignitetest/services/utils/log_utils.py index 09b1f9871e4d8..ba97e72c7ff7e 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/log_utils.py +++ b/modules/ducktests/tests/ignitetest/services/utils/log_utils.py @@ -22,6 +22,18 @@ from ducktape.cluster.remoteaccount import LogMonitor +class IgniteLogMonitor(LogMonitor): + """ + Extends ducktape's LogMonitor with a one-shot presence check. + """ + def found(self, pattern): + """ + Check once whether the pattern is present in the log after the initial + offset recorded when the monitor was created. + """ + return self.acct.ssh("tail -c +%d %s | grep '%s'" % (self.offset + 1, self.log, pattern), allow_fail=True) == 0 + + @contextmanager def monitor_log(node, log, from_the_beginning=False): """ @@ -38,4 +50,4 @@ def monitor_log(node, log, from_the_beginning=False): offset = 0 if from_the_beginning else int(node.account.ssh_output("wc -c %s" % log).split()[0]) except Exception: offset = 0 - yield LogMonitor(node.account, log, offset) + yield IgniteLogMonitor(node.account, log, offset) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/__init__.py b/modules/ducktests/tests/ignitetest/tests/mdc/__init__.py new file mode 100644 index 0000000000000..c3e67412c9c9f --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/mdc/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Multi data center (MDC) tests. +""" diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/distribution_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/distribution_test.py new file mode 100644 index 0000000000000..9e6079e4941f5 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/mdc/distribution_test.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ducktape.mark import matrix + +from ignitetest.services.mdc.mdc_cluster import MdcCluster, DC_1, DC_2, cross_dc_network +from ignitetest.utils import cluster, ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH + +CACHE_NAME = "mdc-dist" + +DATA_SIZE = 1_000 + + +class MdcDistributionTest(IgniteTest): + """ + Tests for cross-DC partition copy placement by MdcAffinityBackupFilter. + """ + @cluster(num_nodes=7) + @ignite_versions(str(DEV_BRANCH)) + @matrix(backups=[1, 3], cross_dc_latency_ms=50) + def test_copies_balanced_across_dcs(self, ignite_version, backups, cross_dc_latency_ms): + """ + Every partition has exactly (backups + 1) / 2 OWNING copies in each DC, + and the guarantee survives a full restart of one DC with rebalance. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc={DC_1: 1, DC_2: 0}) + + copies_per_dc = (backups + 1) // 2 + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms): + mdc.start_servers() + + mdc.generate_data(DC_1, CACHE_NAME, 0, DATA_SIZE, backups=backups) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=copies_per_dc) + + mdc.restart(DC_2) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=copies_per_dc) + + mdc.check_data(DC_1, CACHE_NAME, 0, DATA_SIZE) + + mdc.stop_servers() + + @cluster(num_nodes=7) + @ignite_versions(str(DEV_BRANCH)) + @matrix(cross_dc_latency_ms=50) + def test_copies_balanced_in_asymmetric_dcs(self, ignite_version, cross_dc_latency_ms): + """ + DC sizes differ (2 vs 4 nodes), yet each DC still holds exactly one copy of + every partition: balance is per-DC, not per-node. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc={DC_1: 2, DC_2: 4}, + runners_per_dc={DC_1: 1, DC_2: 0}) + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms): + mdc.start_servers() + + mdc.generate_data(DC_1, CACHE_NAME, 0, DATA_SIZE, backups=1) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) + + mdc.check_data(DC_1, CACHE_NAME, 0, DATA_SIZE) + + mdc.stop_servers() + + @cluster(num_nodes=7) + @ignite_versions(str(DEV_BRANCH)) + @matrix(cross_dc_latency_ms=50) + def test_distribution_survives_node_loss(self, ignite_version, cross_dc_latency_ms): + """ + With backups=3 (two copies per DC) losing one node of a DC leaves at least one + copy of every partition in that DC, and after the baseline change and rebalance + the filter restores the exact two-copies-per-DC placement on the surviving nodes. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc={DC_1: 1, DC_2: 0}) + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms): + mdc.start_servers() + + mdc.generate_data(DC_1, CACHE_NAME, 0, DATA_SIZE, backups=3) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=2) + + mdc.servers[DC_2].stop_node(mdc.servers[DC_2].nodes[-1]) + + # Immediately after the loss the surviving copies must still cover both DCs + # (at least one OWNING copy of every partition per DC). + mdc.verify_cache_distribution(CACHE_NAME) + + mdc.check_data(DC_1, CACHE_NAME, 0, DATA_SIZE) + + # Adopt the new topology as baseline and let rebalance restore the full + # two-copies-per-DC placement on the remaining 3 + 2 nodes. + control = mdc.control(DC_1) + + control.set_baseline(control.cluster_state().topology_version) + + mdc.servers[DC_1].await_rebalance() + mdc.servers[DC_2].await_rebalance() + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=2) + + mdc.check_data(DC_1, CACHE_NAME, 0, DATA_SIZE) + + mdc.stop_servers() diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/latency_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/latency_test.py new file mode 100644 index 0000000000000..c7dbe569239e1 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/mdc/latency_test.py @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Crude cross-DC latency effects. + +With a large netem delay between the DCs the average synchronous operation latency +(measured inside the load application as wall time / operations) becomes a reliable +proxy for "did the operation cross the DC boundary": + +* GET with ``readFromBackup=true`` (MDC-aware local reads): every partition has a copy + in the client's DC, so reads are served locally and the average stays far below the + configured delay - i.e. read latency is NOT affected by the cross-DC link. +* GET with ``readFromBackup=false``: roughly half the keys have a remote primary, each + such read pays the full cross-DC round trip, so the average grows well above the + local case. +* PUT with ``FULL_SYNC``: unavoidably pays the cross-DC price - the write is acknowledged + only after the backup in the other DC responds, so the average exceeds the one-way delay. + +All thresholds are deliberately loose (2x+ margins): this is a demonstration of the +effect, not a benchmark. +""" +from ducktape.mark import matrix + +from ignitetest.services.mdc.mdc_cluster import MdcCluster, cross_dc_network, DC_1 +from ignitetest.utils import cluster, ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH + +CACHE_LOCAL_READS = "mdc-lat-rfb" +CACHE_REMOTE_READS = "mdc-lat-norfb" + +KEYS = 2048 + +LOCAL_GET_ITERS = 300 +REMOTE_GET_ITERS = 60 +PUT_ITERS = 30 + +PUT_OFFSET = 1_000_000 + + +class MdcLatencyTest(IgniteTest): + """ + Tests for cross-DC latency effects on cache reads and writes. + """ + @cluster(num_nodes=6) + @ignite_versions(str(DEV_BRANCH)) + @matrix(cross_dc_latency_ms=[50]) + def test_cross_dc_latency_effects(self, ignite_version, cross_dc_latency_ms): + """ + Local reads are unaffected by the cross-DC delay, non-local reads and FULL_SYNC + writes pay it. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=2, runners_per_dc={DC_1: 1}) + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms): + mdc.start_servers() + + # Both caches: backups=1 (a copy of every partition in each DC), FULL_SYNC. + # They differ only in readFromBackup - the switch enabling MDC-local reads. + mdc.generate_data(DC_1, CACHE_LOCAL_READS, 0, KEYS, backups=1) + mdc.generate_data(DC_1, CACHE_REMOTE_READS, 0, KEYS, backups=1, readFromBackup=False) + + svc = mdc.run_load(DC_1, "GET", CACHE_LOCAL_READS, "latLocalGet", + keyFrom=0, keyTo=KEYS, iterations=LOCAL_GET_ITERS) + + avg_local_get = mdc.result_float(svc, "latLocalGetAvgOpMs") + max_local_get = mdc.result_float(svc, "latLocalGetMaxOpMs") + + svc = mdc.run_load(DC_1, "GET", CACHE_REMOTE_READS, "latRemoteGet", + keyFrom=0, keyTo=KEYS, iterations=REMOTE_GET_ITERS) + + avg_remote_get = mdc.result_float(svc, "latRemoteGetAvgOpMs") + max_remote_get = mdc.result_float(svc, "latRemoteGetMaxOpMs") + + svc = mdc.run_load(DC_1, "PUT", CACHE_LOCAL_READS, "latPut", + keyFrom=PUT_OFFSET, iterations=PUT_ITERS) + + avg_put = mdc.result_float(svc, "latPutAvgOpMs") + min_put = mdc.result_float(svc, "latPutMinOpMs") + + self.logger.info(f"Cross-DC latency effects [delayMs={cross_dc_latency_ms}, " + f"avgLocalGetMs={avg_local_get}, avgRemoteGetMs={avg_remote_get}, " + f"avgPutMs={avg_put}]") + + # Local reads: served by the same-DC copy, so the cross-DC delay never applies. + assert avg_local_get < cross_dc_latency_ms * 0.1 and max_local_get < cross_dc_latency_ms, \ + f"MDC-aware local reads should not pay the cross-DC latency " \ + f"[avgMs={avg_local_get}, maxMs={max_local_get}, delayMs={cross_dc_latency_ms}]" + + assert avg_remote_get > cross_dc_latency_ms * 0.9 and max_remote_get > cross_dc_latency_ms, \ + f"Reads without readFromBackup should pay the cross-DC latency " \ + f"[avgMs={avg_remote_get}, maxMs={max_remote_get}, delayMs={cross_dc_latency_ms}]" + + # FULL_SYNC writes: the backup in the other DC must acknowledge each update, + # so a synchronous put cannot possibly complete under the one-way delay. + assert avg_put > 2 * cross_dc_latency_ms and min_put > 2 * cross_dc_latency_ms, \ + f"FULL_SYNC writes should pay the cross-DC latency " \ + f"[avgMs={avg_put}, minMs={min_put}, delayMs={cross_dc_latency_ms}]" + + mdc.stop_servers() diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py new file mode 100644 index 0000000000000..951b44c297f4d --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py @@ -0,0 +1,270 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Continuous cache/tx/SQL load through a network partition. + +The availability claims verified here: + +* GET load (cache API) runs in BOTH data centers from before the partition, through the + partition and its healing, in a single application run - so any exception crossing a + phase boundary fails the test on loader stop. Per-window throughput sampling and the + max-stall metric additionally show that availability never dropped for long. +* PUT / transactional / SQL DML load succeeds in the main-DC half in every phase, is + rejected (and only rejected) in the read-only half during the partition, and resumes + everywhere after healing. +* Everything that was acknowledged as written is readable afterwards from the OPPOSITE + data center ("as much as we succeeded to put, we get back"). +* Negative invariants: no long running transactions, no PME freeze, no lost partitions, + no hanging transactions, and idle_verify finds no conflicts after the recovery. +""" +from time import sleep + +from ducktape.mark import matrix + +from ignitetest.services.mdc.mdc_cluster import MdcCluster, cross_dc_network, DC_1, DC_2 +from ignitetest.utils import cluster, ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH + +ATOMIC_CACHE = "mdc-atomic" +TX_CACHE = "mdc-tx" +SQL_CACHE = "mdc-sql" + +# Pre-loaded data set, written half from each DC. +GEN_SIZE = 1_000 + +# Non-intersecting key offsets for the write bursts of each phase. +OFFSET_BEFORE_DC1 = 1_000_000 +OFFSET_BEFORE_DC2 = 1_500_000 +OFFSET_DURING_DC1 = 2_000_000 +OFFSET_AFTER_DC1 = 3_000_000 +OFFSET_AFTER_DC2 = 3_500_000 + +# Probe keys of the expected-to-be-rejected bursts (never intersect readable data). +OFFSET_REJECTED_PROBES = 9_000_000 + +PUT_BURST = 200 +TX_BURST = 60 +SQL_BURST = 100 +GET_BURST = 500 +REJECTED_BURST = 30 + +SPLIT_SETTLE_SECS = 10 + +# Boundary tolerance for the background load: a transient error window is possible at +# the instant the partition is enabled/healed, but must stay marginal. +BG_MAX_ERR_RATIO = 0.01 +BG_MAX_STALL_MS = 60_000 + + +class MdcPartitionLoadTest(IgniteTest): + """ + Tests for continuous load through an MDC network partition. + """ + @cluster(num_nodes=10) + @ignite_versions(str(DEV_BRANCH)) + @matrix(cross_dc_latency_ms=[20]) + def test_load_through_partition(self, ignite_version, cross_dc_latency_ms): + """ + Background gets in both DCs across the whole scenario; put/tx/sql bursts before, + during and after the partition; cross-DC readback of every acknowledged write. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc=1, loaders_per_dc=1) + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: + mdc.start_servers() + + mdc.generate_data(DC_1, ATOMIC_CACHE, 0, GEN_SIZE // 2, backups=1) + mdc.generate_data(DC_2, ATOMIC_CACHE, GEN_SIZE // 2, GEN_SIZE, backups=1) + + # Background gets: a single run per DC spanning every phase. Any exception + # surfaces on stop_loader(); transient boundary errors are counted, not thrown. + for dc in (DC_1, DC_2): + mdc.start_loader(dc, {"mode": "GET", "cacheName": ATOMIC_CACHE, + "keyFrom": 0, "keyTo": GEN_SIZE, + "tolerateErrors": True, "opPauseMs": 5, + "resultPrefix": f"bg{dc}"}) + + # written: (cache, kind, written_from_dc, offset, count) + written = [] + + # ---- Phase 1: before the partition, both DCs accept writes. + written += self._admissible_write_bursts(mdc, DC_1, "before", OFFSET_BEFORE_DC1) + written.append(self._admissible_put_burst(mdc, DC_2, "before", OFFSET_BEFORE_DC2)) + + before_sql = next(entry for entry in written if entry[0] == SQL_CACHE) + + net.enable_network_partition(DC_1, DC_2) + + sleep(SPLIT_SETTLE_SECS) + + mdc.verify_split_brain() + + # ---- Phase 2: during the partition. + # The main-DC half keeps accepting the full write load. + written += self._admissible_write_bursts(mdc, DC_1, "during", OFFSET_DURING_DC1) + + # The read-only half serves reads cleanly - cache API and SQL... + svc = mdc.run_load(DC_2, "GET", ATOMIC_CACHE, "duringDC2Get", + keyFrom=0, keyTo=GEN_SIZE, iterations=GET_BURST) + + assert mdc.result_int(svc, "duringDC2GetErrCnt") == 0, "Reads in the read-only DC must be clean" + + svc = mdc.run_load(DC_2, "SQL_SELECT", SQL_CACHE, "duringDC2Sql", + keyFrom=before_sql[3], keyTo=before_sql[3] + before_sql[4], + iterations=before_sql[4]) + + assert mdc.result_int(svc, "duringDC2SqlErrCnt") == 0, "SQL reads in the read-only DC must be clean" + + # ...and rejects every write - plain, transactional and SQL DML. + mdc.run_load(DC_2, "PUT", ATOMIC_CACHE, "duringDC2Put", keyFrom=OFFSET_REJECTED_PROBES, + iterations=REJECTED_BURST, expectAdmissible=False) + + mdc.run_load(DC_2, "TX_PUT", TX_CACHE, "duringDC2Tx", keyFrom=OFFSET_REJECTED_PROBES, + iterations=REJECTED_BURST, expectAdmissible=False) + + mdc.run_load(DC_2, "SQL_PUT", SQL_CACHE, "duringDC2SqlPut", keyFrom=OFFSET_REJECTED_PROBES, + iterations=REJECTED_BURST, expectAdmissible=False) + + # ---- Phase 3: heal, rejoin the read-only half, writes resume everywhere. + net.disable_network_partition(DC_1, DC_2) + + mdc.restart(DC_2) + + written += self._admissible_write_bursts(mdc, DC_1, "after", OFFSET_AFTER_DC1) + written.append(self._admissible_put_burst(mdc, DC_2, "after", OFFSET_AFTER_DC2)) + + # ---- Background loads: no exception on stop, availability never dropped. + for dc in (DC_1, DC_2): + svc = mdc.stop_loader(dc) + + ops = mdc.result_int(svc, f"bg{dc}OpsCnt") + errs = mdc.result_int(svc, f"bg{dc}ErrCnt") + max_stall = mdc.result_int(svc, f"bg{dc}MaxStallMs") + min_window = mdc.result_int(svc, f"bg{dc}MinWindowOps") + + self.logger.info(f"Background GET load [dc={dc}, ops={ops}, errs={errs}, " + f"maxStallMs={max_stall}, minWindowOps={min_window}]") + + assert ops > 0, f"Background get load performed no operations [dc={dc}]" + + assert errs <= max(1, int(ops * BG_MAX_ERR_RATIO)), \ + f"Background get load errors exceed the boundary tolerance [dc={dc}, ops={ops}, errs={errs}]" + + assert max_stall < BG_MAX_STALL_MS, \ + f"Background get load stalled for too long [dc={dc}, maxStallMs={max_stall}]" + + # ---- Readback: every acknowledged write is readable from the OPPOSITE DC. + for idx, (cache, kind, src_dc, offset, cnt) in enumerate(written): + assert cnt > 0, f"Write burst acknowledged nothing [cache={cache}, offset={offset}]" + + dst_dc = DC_2 if src_dc == DC_1 else DC_1 + + if kind == "sql": + svc = mdc.run_load(dst_dc, "SQL_SELECT", cache, f"rb{idx}{dst_dc}", + keyFrom=offset, keyTo=offset + cnt, iterations=cnt) + + assert mdc.result_int(svc, f"rb{idx}{dst_dc}ErrCnt") == 0, \ + f"SQL readback failed [cache={cache}, offset={offset}, cnt={cnt}, dc={dst_dc}]" + else: + mdc.check_data(dst_dc, cache, offset, offset + cnt) + + # ---- Negative invariants. + mdc.verify_no_hanging_txs() + + mdc.control(DC_1).idle_verify(",".join([ATOMIC_CACHE, TX_CACHE, SQL_CACHE])) + + mdc.verify_servers_log_clean() + + mdc.stop_servers() + + @cluster(num_nodes=8) + @ignite_versions(str(DEV_BRANCH)) + @matrix(cross_dc_latency_ms=[50], cross_dc_loss=[0.05]) + def test_degraded_link_without_partition(self, ignite_version, cross_dc_latency_ms, cross_dc_loss): + """ + A slow, lossy WAN that stays connected ("slow WAN" as opposed to "broken WAN"): + the cluster must NOT split, and cache, transactional and SQL load must complete + cleanly in both DCs (slower, but without a single error) - TCP absorbs the loss. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc=1) + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms, loss=cross_dc_loss): + mdc.start_servers() + + mdc.generate_data(DC_1, ATOMIC_CACHE, 0, GEN_SIZE // 2, backups=1) + mdc.generate_data(DC_2, ATOMIC_CACHE, GEN_SIZE // 2, GEN_SIZE, backups=1) + + for dc, offset in ((DC_1, OFFSET_BEFORE_DC1), (DC_2, OFFSET_BEFORE_DC2)): + svc = mdc.run_load(dc, "GET", ATOMIC_CACHE, f"deg{dc}Get", + keyFrom=0, keyTo=GEN_SIZE, iterations=GET_BURST) + + assert mdc.result_int(svc, f"deg{dc}GetErrCnt") == 0, f"Degraded-link reads must be clean [dc={dc}]" + + self._admissible_put_burst(mdc, dc, "deg", offset) + + self._admissible_burst(mdc, DC_1, "TX_PUT", TX_CACHE, "degDC1Tx", OFFSET_DURING_DC1, TX_BURST, + atomicity="TRANSACTIONAL") + + mdc.verify_whole_cluster_healthy() + + mdc.verify_cache_distribution(ATOMIC_CACHE, copies_per_dc=1) + + mdc.control(DC_1).idle_verify(",".join([ATOMIC_CACHE, TX_CACHE])) + + mdc.verify_servers_log_clean() + + mdc.stop_servers() + + # ------------------------------------------------------------------ helpers + + def _admissible_write_bursts(self, mdc: MdcCluster, dc: str, phase: str, offset: int): + """ + Runs the full trio of write bursts (cache API put, transactional put, SQL DML) + expected to fully succeed, and returns the written ranges for later readback. + """ + return [ + self._admissible_burst(mdc, dc, "PUT", ATOMIC_CACHE, f"{phase}{dc}Put", offset, PUT_BURST), + self._admissible_burst(mdc, dc, "TX_PUT", TX_CACHE, f"{phase}{dc}Tx", offset, TX_BURST, + atomicity="TRANSACTIONAL"), + self._admissible_burst(mdc, dc, "SQL_PUT", SQL_CACHE, f"{phase}{dc}Sql", offset, SQL_BURST), + ] + + def _admissible_put_burst(self, mdc: MdcCluster, dc: str, phase: str, offset: int): + """ + Runs a single cache API put burst expected to fully succeed. + """ + return self._admissible_burst(mdc, dc, "PUT", ATOMIC_CACHE, f"{phase}{dc}Put", offset, PUT_BURST) + + def _admissible_burst(self, mdc: MdcCluster, dc: str, mode: str, cache: str, prefix: str, + offset: int, iterations: int, **cache_params): + """ + Runs one write burst that must succeed completely: the load application fails fast + on any exception, and the acknowledged count must match the requested iterations. + + :return: (cache, kind, dc, offset, count) tuple for the readback stage. + """ + svc = mdc.run_load(dc, mode, cache, prefix, keyFrom=offset, iterations=iterations, + createCache=True, backups=1, mainDc=DC_1, **cache_params) + + ops = mdc.result_int(svc, f"{prefix}OpsCnt") + + assert ops == iterations, \ + f"Write burst is incomplete [dc={dc}, mode={mode}, cache={cache}, exp={iterations}, ops={ops}]" + + kind = "sql" if mode.startswith("SQL") else "data" + + return cache, kind, dc, offset, ops diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py new file mode 100644 index 0000000000000..1fd21e2c329f7 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -0,0 +1,183 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +MDC cluster resilience to network and data center failures. + +Covers the full split-brain lifecycle (partition -> active half + read-only half -> +heal -> read-only half rejoins via restart), the loss and return of the main DC, +and short network blips that must NOT split the cluster. +""" +from time import sleep + +from ducktape.mark import matrix + +from ignitetest.services.mdc.mdc_cluster import MdcCluster, cross_dc_network, DC_1, DC_2 +from ignitetest.utils import cluster, ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH + +CACHE_NAME = "mdc-resilience" + +BACKUPS = 1 + +# Time for discovery to detect the partition and for both half-rings to complete PME. +SPLIT_SETTLE_SECS = 10 + +# A blip must stay well below the failure detection timeout (10s by default). +BLIP_SECS = 1 +BLIP_SETTLE_SECS = 5 +BLIPS = 3 + + +class MdcPartitionResilienceTest(IgniteTest): + """ + Tests for cluster network partition resilience in MultiDC. + """ + @cluster(num_nodes=12) + @ignite_versions(str(DEV_BRANCH)) + @matrix(cross_dc_latency_ms=[20]) + def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms): + """ + The canonical split-brain lifecycle: partition -> split into two healthy half-rings + (DC1 active, DC2 read-only) -> all data readable everywhere -> heal -> DC2 rejoins + via restart -> writes restored everywhere, distribution and consistency verified. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=5, runners_per_dc=1) + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: + mdc.start_servers() + + mdc.generate_data(DC_1, CACHE_NAME, 0, 100, backups=BACKUPS) + mdc.generate_data(DC_2, CACHE_NAME, 100, 200, backups=BACKUPS) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) + + net.enable_network_partition(DC_1, DC_2) + + sleep(SPLIT_SETTLE_SECS) + + mdc.verify_split_brain() + + # All data written before the split is readable in both halves. + mdc.check_data(DC_1, CACHE_NAME, 0, 200) + mdc.check_data(DC_2, CACHE_NAME, 0, 200) + + # The half-ring holding the main DC accepts writes, the other one is read-only. + mdc.check_put_admissibility(DC_1, CACHE_NAME, True) + mdc.check_put_admissibility(DC_2, CACHE_NAME, False) + + net.disable_network_partition(DC_1, DC_2) + + # Split-brain does not self-heal: the read-only half rejoins via restart. + mdc.restart(DC_2) + + mdc.check_put_admissibility(DC_1, CACHE_NAME, True, key_offset=2_000_000) + mdc.check_put_admissibility(DC_2, CACHE_NAME, True, key_offset=3_000_000) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) + + mdc.control(DC_1).idle_verify(CACHE_NAME) + + mdc.verify_servers_log_clean() + + mdc.stop_servers() + + @cluster(num_nodes=6) + @ignite_versions(str(DEV_BRANCH)) + @matrix(cross_dc_latency_ms=20) + def test_main_dc_loss_and_return(self, ignite_version, cross_dc_latency_ms): + """ + The inverse of the canonical scenario: the MAIN data center goes down entirely. + The surviving DC must stay readable but reject writes (the topology validator does + not see the main DC), and writes must resume in both DCs once the main DC returns. + No network impairments are involved - the main DC is stopped, not partitioned. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=2, runners_per_dc=1) + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms): + mdc.start_servers() + + mdc.generate_data(DC_1, CACHE_NAME, 0, 100, backups=BACKUPS) + mdc.generate_data(DC_2, CACHE_NAME, 100, 200, backups=BACKUPS) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) + + mdc.servers[DC_1].stop() + + state = mdc.control(DC_2).cluster_state() + + assert "ACTIVE" == state.state, f"Surviving DC should remain ACTIVE [actual={state.state}]" + + # One copy of every partition lives in DC2, so all data stays readable... + mdc.check_data(DC_2, CACHE_NAME, 0, 200) + + # ...but the surviving DC is read-only while the main DC is invisible. + mdc.check_put_admissibility(DC_2, CACHE_NAME, False) + + mdc.servers[DC_1].start(clean=False) + + mdc.servers[DC_1].await_rebalance() + + mdc.check_put_admissibility(DC_1, CACHE_NAME, True, key_offset=2_000_000) + mdc.check_put_admissibility(DC_2, CACHE_NAME, True, key_offset=3_000_000) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) + + mdc.control(DC_1).idle_verify(CACHE_NAME) + + mdc.stop_servers() + + @cluster(num_nodes=8) + @ignite_versions(str(DEV_BRANCH)) + @matrix(cross_dc_latency_ms=[20]) + def test_short_partition_blips_do_not_split(self, ignite_version, cross_dc_latency_ms): + """ + A flapping WAN link: several short (below the failure detection timeout) full + cross-DC connectivity drops. The cluster must NOT split: after the blips it is + still one ACTIVE cluster, all data is intact and both DCs accept writes. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc=1) + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: + mdc.start_servers() + + mdc.generate_data(DC_1, CACHE_NAME, 0, 100, backups=BACKUPS) + mdc.generate_data(DC_2, CACHE_NAME, 100, 200, backups=BACKUPS) + + for i in range(BLIPS): + self.logger.info(f"Network blip {i + 1}/{BLIPS}") + + net.enable_network_partition(DC_1, DC_2) + + sleep(BLIP_SECS) + + net.disable_network_partition(DC_1, DC_2) + + sleep(BLIP_SETTLE_SECS) + + mdc.verify_whole_cluster_healthy() + + mdc.check_data(DC_1, CACHE_NAME, 0, 200) + mdc.check_data(DC_2, CACHE_NAME, 0, 200) + + mdc.check_put_admissibility(DC_1, CACHE_NAME, True, key_offset=2_000_000) + mdc.check_put_admissibility(DC_2, CACHE_NAME, True, key_offset=3_000_000) + + mdc.control(DC_1).idle_verify(CACHE_NAME) + + mdc.verify_servers_log_clean() + + mdc.stop_servers() diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py new file mode 100644 index 0000000000000..9e3991fef9c83 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py @@ -0,0 +1,188 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Thin client data-center-aware routing. + +Every thin client is configured with the addresses of ALL server nodes of BOTH DCs. +A client pinned to a data center (via the ``-DIGNITE_DATA_CENTER_ID=...`` JVM system +property, per the MDC docs) must route partition-aware reads to nodes of its own DC: +with a large cross-DC netem delay its average read latency stays small, while an +unpinned client's reads hit remote primaries for about half the keys and pay the +cross-DC round trip. Latency is the routing proxy - no metrics infrastructure needed. + +The partition part verifies the thin client experience of a split-brain: a client +pinned to the main-DC half keeps writing; a client pinned to the read-only half still +reads cleanly (falling back to the reachable nodes from its address list) but gets +every write rejected; and writes resume after the heal and rejoin. + +The thin client services are registered into their DC's network group, so both the +netem delay and the iptables partition apply to their traffic as well. +""" +from time import sleep + +from ducktape.mark import matrix + +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.ignite_configuration import IgniteThinClientConfiguration +from ignitetest.tests.mdc.fixture import DC_1, DC_2, THIN_LOAD_APP, MdcCluster, cross_dc_network, dc_jvm_opts +from ignitetest.utils import cluster, ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + +CACHE_NAME = "mdc-thin" + +KEYS = 100 + +PINNED_GET_ITERS = 200 +UNPINNED_GET_ITERS = 60 +PUT_ITERS = 30 + +OFFSET_DURING = 1_000_000 +OFFSET_REJECTED_PROBES = 9_000_000 +OFFSET_AFTER = 2_000_000 + +SPLIT_SETTLE_SECS = 10 + + +class MdcThinClientTest(IgniteTest): + """ + Tests for thin client DC-aware routing and behavior through a partition. + """ + @cluster(num_nodes=8) + @ignite_versions(str(DEV_BRANCH)) + @matrix(cross_dc_latency_ms=[200]) + def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_dc_latency_ms): + """ + DC-pinned thin client reads locally (latency far below the cross-DC delay), + an unpinned client pays the cross-DC price; through a partition the pinned + clients behave like their half-ring: main half writes, read-only half reads + but rejects writes, and writes resume after the heal. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=2, runners_per_dc={DC_1: 1}, + client_connector=True) + + # All thin clients get the full address list of both DCs: DC preference must come + # from routing, not from the address list. + pinned_dc1 = self._thin_client(mdc, dc_jvm_opts(DC_1)) + pinned_dc2 = self._thin_client(mdc, dc_jvm_opts(DC_2)) + unpinned = self._thin_client(mdc, []) + + # Register the thin client hosts into the network groups, so netem and the + # partition apply to them. The unpinned client observes from DC1's side. + mdc.register(DC_1, pinned_dc1) + mdc.register(DC_2, pinned_dc2) + mdc.register(DC_1, unpinned) + + first_start = set() + + def run_thin(svc, params): + svc.params = params + + svc.start(clean=id(svc) not in first_start) + + first_start.add(id(svc)) + + svc.wait() + svc.stop() + + return svc + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: + mdc.start_servers() + + # readFromBackup=true + FULL_SYNC (the fixture defaults) are exactly the + # documented preconditions of thin client partition-aware local reads. + mdc.generate_data(DC_1, CACHE_NAME, 0, KEYS, backups=1) + + # ---- DC-aware routing: latency comparison. + svc = run_thin(pinned_dc1, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, + "keyTo": KEYS, "iterations": PINNED_GET_ITERS, + "resultPrefix": "pinnedGet"}) + + avg_pinned = mdc.result_float(svc, "pinnedGetAvgOpMs") + + svc = run_thin(unpinned, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, + "keyTo": KEYS, "iterations": UNPINNED_GET_ITERS, + "resultPrefix": "freeGet"}) + + avg_unpinned = mdc.result_float(svc, "freeGetAvgOpMs") + + self.logger.info(f"Thin client routing latency [delayMs={cross_dc_latency_ms}, " + f"pinnedAvgMs={avg_pinned}, unpinnedAvgMs={avg_unpinned}]") + + assert avg_pinned < cross_dc_latency_ms * 0.5, \ + f"DC-pinned thin client reads should be served locally " \ + f"[avgMs={avg_pinned}, delayMs={cross_dc_latency_ms}]" + + assert avg_unpinned > avg_pinned * 2, \ + f"Unpinned thin client reads should pay the cross-DC latency " \ + f"[unpinnedAvgMs={avg_unpinned}, pinnedAvgMs={avg_pinned}]" + + # ---- Thin clients through a partition. + net.enable_network_partition(DC_1, DC_2) + + sleep(SPLIT_SETTLE_SECS) + + mdc.verify_split_brain() + + # The client pinned to the main half keeps writing... + run_thin(pinned_dc1, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_DURING, + "keyTo": OFFSET_DURING + PUT_ITERS, "iterations": PUT_ITERS, + "resultPrefix": "duringPut"}) + + # ...while the client pinned to the read-only half still reads cleanly + # (its DC1 addresses are unreachable, so it falls back to DC2 nodes)... + svc = run_thin(pinned_dc2, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, + "keyTo": KEYS, "iterations": PINNED_GET_ITERS, + "resultPrefix": "roGet"}) + + assert mdc.result_int(svc, "roGetErrCnt") == 0, \ + "Thin client reads in the read-only DC must be clean" + + # ...and has every write rejected by the topology validator. + run_thin(pinned_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, + "keyFrom": OFFSET_REJECTED_PROBES, + "keyTo": OFFSET_REJECTED_PROBES + PUT_ITERS, + "iterations": PUT_ITERS, "expectAdmissible": False, + "resultPrefix": "roPut"}) + + # ---- Heal, rejoin, thin client writes resume in the former read-only DC. + net.disable_network_partition(DC_1, DC_2) + + mdc.restart(DC_2) + + run_thin(pinned_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_AFTER, + "keyTo": OFFSET_AFTER + PUT_ITERS, "iterations": PUT_ITERS, + "resultPrefix": "afterPut"}) + + # Everything the thin clients acknowledged is readable via the thick client. + mdc.check_data(DC_1, CACHE_NAME, OFFSET_DURING, OFFSET_DURING + PUT_ITERS) + mdc.check_data(DC_1, CACHE_NAME, OFFSET_AFTER, OFFSET_AFTER + PUT_ITERS) + + mdc.stop_servers() + + def _thin_client(self, mdc: MdcCluster, jvm_opts) -> IgniteApplicationService: + """ + Builds a single-node thin client application service with the addresses of all + server nodes of both DCs. + """ + return IgniteApplicationService( + self.test_context, + IgniteThinClientConfiguration(addresses=mdc.thin_client_addresses(), + version=IgniteVersion(str(DEV_BRANCH))), + java_class_name=THIN_LOAD_APP, + num_nodes=1, + jvm_opts=jvm_opts) diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/__init__.py b/modules/ducktests/tests/ignitetest/tests/network_group/__init__.py deleted file mode 100644 index 9f6b859e3980c..0000000000000 --- a/modules/ducktests/tests/ignitetest/tests/network_group/__init__.py +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import ABC, abstractmethod -from typing import Dict, List - -from ducktape.services.service import Service - -from ignitetest.services.network_group.configuration import NetworkGroupStore -from ignitetest.services.network_group.manager import NetworkGroupManager -from ignitetest.utils.ignite_test import IgniteTest - - -class NetworkGroupAbstractTest(IgniteTest, ABC): - """ - Abstract test for network-controlled tests. - Ensures network is deployed before the test and cleaned up after. - """ - def _configure_services(self, **kwargs): - pass - - def _configure_network_group_store(self, **kwargs) -> NetworkGroupStore: - return NetworkGroupStore() - - def _configure_network_group_registry(self, **kwargs) -> Dict[str, List[Service]]: - return {} - - @abstractmethod - def _run(self, network_mgr: NetworkGroupManager, **kwargs): - pass - - def configure_network_and_run(self, **kwargs): - self.logger.debug("Deploying network topology...") - - self._configure_services(**kwargs) - - grp_store = self._configure_network_group_store(**kwargs) - grp_registry = self._configure_network_group_registry(**kwargs) - - try: - with NetworkGroupManager(self.logger, grp_store, grp_registry) as network_mgr: - self.logger.debug("Network configuration complete. Starting test logic ...") - - self._run(network_mgr, **kwargs) - finally: - # NetworkGroupManager.__exit__ has restored the network by this point. - self.logger.debug("Network is restored.") \ No newline at end of file diff --git a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py deleted file mode 100644 index 70906c0391f77..0000000000000 --- a/modules/ducktests/tests/ignitetest/tests/network_group/partition_resilience_test.py +++ /dev/null @@ -1,273 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from time import sleep - -from ducktape.mark import matrix - -from ignitetest.services.ignite import IgniteService -from ignitetest.services.ignite_app import IgniteApplicationService -from ignitetest.services.network_group.configuration import NetworkGroupStore, CrossNetworkGroupConfiguration -from ignitetest.services.utils.control_utility import ControlUtility -from ignitetest.services.utils.ignite_configuration import IgniteConfiguration -from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster -from ignitetest.tests.network_group import NetworkGroupAbstractTest -from ignitetest.utils import cluster, ignite_versions -from ignitetest.utils.version import DEV_BRANCH, IgniteVersion - -SRV_NODES = 10 -CLI_NODES = 2 -TOTAL_NODES = SRV_NODES + CLI_NODES - -DC_1_NAME = "DC1" -DC_2_NAME = "DC2" - -GENERATOR_JAVA_CLASS_NAME = "org.apache.ignite.internal.ducktest.tests.mdc.MdcDataGeneratorApplication" -GET_CHECKER_JAVA_CLASS_NAME = "org.apache.ignite.internal.ducktest.tests.mdc.MdcDataCheckerApplication" -PUT_CHECKER_JAVA_CLASS_NAME = "org.apache.ignite.internal.ducktest.tests.mdc.MdcPutAdmissibilityCheckerApplication" - -CACHE_NAME = "replicated" -BACKUPS = 1 - -DATA_CENTER_ATTR = "IGNITE_DATA_CENTER_ID" - -class MultiDCPartitionResilienceTest(NetworkGroupAbstractTest): - """ - Tests for cluster network partition resilience in MultiDC - """ - @cluster(num_nodes=TOTAL_NODES) - @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[20]) - def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms): - self.configure_network_and_run(ignite_version=ignite_version, cross_dc_latency_ms=cross_dc_latency_ms) - - def _configure_network_group_store(self, **kwargs) -> NetworkGroupStore: - store = super()._configure_network_group_store(**kwargs) - - # tcset expects a time expression with units, not a bare integer. - dc1_dc2_cfg = CrossNetworkGroupConfiguration(delay=f"{kwargs['cross_dc_latency_ms']}ms") - store.set_config(DC_1_NAME, DC_2_NAME, dc1_dc2_cfg) - - return store - - def _configure_services(self, **kwargs): - self.ign_cfg = IgniteConfiguration(version=IgniteVersion(kwargs['ignite_version'])) - - dc_1_nodes_num, dc_2_nodes_num = SRV_NODES // 2, SRV_NODES // 2 - - jvm_opts_dc_1 = [f"-D{DATA_CENTER_ATTR}={DC_1_NAME}"] - jvm_opts_dc_2 = [f"-D{DATA_CENTER_ATTR}={DC_2_NAME}"] - - self.svc_dc_1 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_1_nodes_num, jvm_opts=jvm_opts_dc_1) - self.svc_dc_2 = IgniteService(self.test_context, self.ign_cfg, num_nodes=dc_2_nodes_num, jvm_opts=jvm_opts_dc_2) - - cli_cfg_dc_1 = self.ign_cfg._replace(client_mode=True, discovery_spi=from_ignite_cluster(self.svc_dc_1)) - cli_cfg_dc_2 = self.ign_cfg._replace(client_mode=True, discovery_spi=from_ignite_cluster(self.svc_dc_2)) - - self.app_dc_1 = IgniteApplicationService(self.test_context, cli_cfg_dc_1, jvm_opts=jvm_opts_dc_1) - self.app_dc_2 = IgniteApplicationService(self.test_context, cli_cfg_dc_2, jvm_opts=jvm_opts_dc_2) - - def _configure_network_group_registry(self, **kwargs): - return { - DC_1_NAME: [self.svc_dc_1, self.app_dc_1], - DC_2_NAME: [self.svc_dc_2, self.app_dc_2] - } - - def _run(self, network_mgr, **kwargs): - for svc in [self.svc_dc_1, self.svc_dc_2]: - svc.start() - - self._populate_cluster_with_data(self.app_dc_1, 0, 100) - self._populate_cluster_with_data(self.app_dc_2, 100, 200) - - self._verify_cache_distribution() - - network_mgr.enable_network_partition(DC_1_NAME, DC_2_NAME) - - sleep(10) - - self._verify_split_brain(self.svc_dc_1, self.svc_dc_2) - - self._check_data_accessible(self.app_dc_1, 0, 200) - self._check_data_accessible(self.app_dc_2, 0, 200) - - self._check_client_data_change_access(self.app_dc_1, True) - self._check_client_data_change_access(self.app_dc_2, False) - - network_mgr.disable_network_partition(DC_1_NAME, DC_2_NAME) - - self.svc_dc_2.stop() - self.svc_dc_2.start(clean=False) - - self.svc_dc_2.await_rebalance() - - self._check_client_data_change_access(self.app_dc_1, True) - self._check_client_data_change_access(self.app_dc_2, True) - - self._verify_cache_distribution() - - self.svc_dc_1.stop() - self.svc_dc_2.stop() - - def _verify_cache_distribution(self): - control_utility = ControlUtility(self.svc_dc_1) - - distribution = control_utility.cache_distribution( - cache_names=CACHE_NAME, - user_attributes=DATA_CENTER_ATTR) - - self.assert_cross_dc_distribution_by_attribute( - distribution, - dc_attr=DATA_CENTER_ATTR, - expected_dcs=[DC_1_NAME, DC_2_NAME]) - - def _verify_split_brain(self, svc_dc_1: IgniteService, svc_dc_2: IgniteService): - """ - Verifies that after the network partition the cluster has split into two independent - half-rings: their baselines don't intersect and each half elected its own coordinator. - """ - self._verify_half_ring_healthy(svc_dc_1) - self._verify_half_ring_healthy(svc_dc_2) - - cluster_state_dc_1 = ControlUtility(svc_dc_1).cluster_state() - cluster_state_dc_2 = ControlUtility(svc_dc_2).cluster_state() - - baseline_dc_1 = {node.consistent_id for node in cluster_state_dc_1.baseline} - baseline_dc_2 = {node.consistent_id for node in cluster_state_dc_2.baseline} - - common_nodes = baseline_dc_1 & baseline_dc_2 - - assert not common_nodes, \ - f"Half-ring baselines should not intersect " \ - f"[common={sorted(common_nodes)}, dc1={sorted(baseline_dc_1)}, dc2={sorted(baseline_dc_2)}]" - - coordinator_dc_1 = cluster_state_dc_1.coordinator - coordinator_dc_2 = cluster_state_dc_2.coordinator - - assert coordinator_dc_1, f"Coordinator is not found in {DC_1_NAME} half-ring baseline output!" - assert coordinator_dc_2, f"Coordinator is not found in {DC_2_NAME} half-ring baseline output!" - - assert coordinator_dc_1.consistent_id != coordinator_dc_2.consistent_id, \ - f"Half-rings should have different coordinators [coordinator={coordinator_dc_1.consistent_id}]" - - assert coordinator_dc_1.consistent_id in baseline_dc_1, \ - f"{DC_1_NAME} coordinator should belong to its own half-ring baseline " \ - f"[coordinator={coordinator_dc_1.consistent_id}, baseline={sorted(baseline_dc_1)}]" - - assert coordinator_dc_2.consistent_id in baseline_dc_2, \ - f"{DC_2_NAME} coordinator should belong to its own half-ring baseline " \ - f"[coordinator={coordinator_dc_2.consistent_id}, baseline={sorted(baseline_dc_2)}]" - - @staticmethod - def _verify_half_ring_healthy(svc: IgniteService): - exp_alive_nodes = SRV_NODES // 2 - act_alive_nodes = len(svc.alive_nodes) - - assert act_alive_nodes == exp_alive_nodes, f"{exp_alive_nodes} nodes should be alive! [actual={act_alive_nodes}]" - - control_utility = ControlUtility(svc) - - cluster_state = control_utility.cluster_state() - - assert "ACTIVE" == cluster_state.state, f"Half-ring state should remain ACTIVE [actual={cluster_state.state}]" - - assert len(cluster_state.baseline) == exp_alive_nodes, \ - f"Half-ring baseline is not expected [exp={exp_alive_nodes}, actual_baseline={cluster_state.baseline}]" - - @staticmethod - def _populate_cluster_with_data(cli: IgniteApplicationService, from_idx: int, to_idx: int): - cli.java_class_name = GENERATOR_JAVA_CLASS_NAME - - cli.params = {"mainDc": DC_1_NAME, "cacheName": CACHE_NAME, "backups": BACKUPS, "from": from_idx, "to": to_idx} - - cli.start() - cli.wait() - cli.stop() - - @staticmethod - def _check_data_accessible(cli: IgniteApplicationService, from_idx: int, to_idx: int): - cli.java_class_name = GET_CHECKER_JAVA_CLASS_NAME - - cli.params = {"cacheName": CACHE_NAME, "from": from_idx, "to": to_idx} - - cli.start(clean=False) - cli.wait() - cli.stop() - - @staticmethod - def _check_client_data_change_access(cli: IgniteApplicationService, expect_admissible: bool): - cli.java_class_name = PUT_CHECKER_JAVA_CLASS_NAME - - cli.params = {"cacheName": CACHE_NAME, "expectAdmissible": expect_admissible} - - cli.start(clean=False) - cli.wait() - cli.stop() - - def assert_cross_dc_distribution_by_attribute(self, distribution, dc_attr, expected_dcs, owning_only=True): - """ - Asserts that every partition of every cache group has at least one copy in every DC, - using a node attribute (requested via --user-attributes) as the DC marker. - - :param distribution: CacheDistribution returned by ControlUtility.cache_distribution(), - requested with user_attributes=[dc_attr]. - :param dc_attr: Attribute name holding the DC id, e.g. "IGNITE_DATA_CENTER_ID". - :param expected_dcs: Collection of DC ids that must own a copy of every partition. - :param owning_only: Count only copies in OWNING state as present. - """ - - def dc_of(copy): - return copy.user_attributes.get(dc_attr) - - self._assert_cross_dc(distribution, set(expected_dcs), dc_of, owning_only, - layout_hint=f"DC attribute: {dc_attr}, expected DCs: {sorted(expected_dcs)}") - - def assert_cross_dc_distribution(self, distribution, dc_addresses, owning_only=True): - """ - Same assertion, but DC membership is derived from node IP addresses. - - :param dc_addresses: Dict: DC name -> collection of node IP addresses of that DC, - e.g. {"DC1": ["192.168.64.9", ...], "DC2": [...]}. - """ - dc_addresses = {dc: set(addrs) for dc, addrs in dc_addresses.items()} - - def dc_of(copy): - for dc, addrs in dc_addresses.items(): - if any(addr in addrs for addr in copy.node_addresses): - return dc - return None - - self._assert_cross_dc(distribution, set(dc_addresses), dc_of, owning_only, - layout_hint=f"DC layout: { {dc: sorted(addrs) for dc, addrs in dc_addresses.items()} }") - - @staticmethod - def _assert_cross_dc(distribution, expected_dcs, dc_of, owning_only, layout_hint): - violations = [] - - for group in distribution.groups.values(): - for part, copies in sorted(group.partitions.items()): - counted = [c for c in copies if not owning_only or c.state == "OWNING"] - - missing = expected_dcs - {dc_of(c) for c in counted} - - if missing: - copies_dump = ", ".join( - f"{c.node_id}({'P' if c.primary else 'B'},{c.state},dc={dc_of(c)},{c.node_addresses})" - for c in copies) - - violations.append(f"group={group.name}(id={group.group_id}), partition={part}, " - f"missing DCs={sorted(missing)}, copies=[{copies_dump}]") - - assert not violations, \ - "Partition distribution is not cross-DC:\n " + "\n ".join(violations) + "\n" + layout_hint From 02eaa708f96075785b73d75e858de50ebd35bb63 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 14 Jul 2026 14:11:17 +0300 Subject: [PATCH 17/44] IGNITE-27833 WIP --- .../ignitetest/tests/mdc/partition_load_test.py | 13 +++++-------- .../tests/mdc/partition_resilience_test.py | 4 +++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py index 951b44c297f4d..f22e77ce1c085 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py @@ -161,11 +161,9 @@ def test_load_through_partition(self, ignite_version, cross_dc_latency_ms): assert ops > 0, f"Background get load performed no operations [dc={dc}]" - assert errs <= max(1, int(ops * BG_MAX_ERR_RATIO)), \ - f"Background get load errors exceed the boundary tolerance [dc={dc}, ops={ops}, errs={errs}]" + assert errs == 0, f"Background get load errors exceed the boundary tolerance [dc={dc}, ops={ops}, errs={errs}]" - assert max_stall < BG_MAX_STALL_MS, \ - f"Background get load stalled for too long [dc={dc}, maxStallMs={max_stall}]" + assert max_stall < BG_MAX_STALL_MS, f"Background get load stalled for too long [dc={dc}, maxStallMs={max_stall}]" # ---- Readback: every acknowledged write is readable from the OPPOSITE DC. for idx, (cache, kind, src_dc, offset, cnt) in enumerate(written): @@ -193,7 +191,7 @@ def test_load_through_partition(self, ignite_version, cross_dc_latency_ms): @cluster(num_nodes=8) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[50], cross_dc_loss=[0.05]) + @matrix(cross_dc_latency_ms=[20], cross_dc_loss=[0.1]) def test_degraded_link_without_partition(self, ignite_version, cross_dc_latency_ms, cross_dc_loss): """ A slow, lossy WAN that stays connected ("slow WAN" as opposed to "broken WAN"): @@ -229,8 +227,6 @@ def test_degraded_link_without_partition(self, ignite_version, cross_dc_latency_ mdc.stop_servers() - # ------------------------------------------------------------------ helpers - def _admissible_write_bursts(self, mdc: MdcCluster, dc: str, phase: str, offset: int): """ Runs the full trio of write bursts (cache API put, transactional put, SQL DML) @@ -249,7 +245,8 @@ def _admissible_put_burst(self, mdc: MdcCluster, dc: str, phase: str, offset: in """ return self._admissible_burst(mdc, dc, "PUT", ATOMIC_CACHE, f"{phase}{dc}Put", offset, PUT_BURST) - def _admissible_burst(self, mdc: MdcCluster, dc: str, mode: str, cache: str, prefix: str, + @staticmethod + def _admissible_burst(mdc: MdcCluster, dc: str, mode: str, cache: str, prefix: str, offset: int, iterations: int, **cache_params): """ Runs one write burst that must succeed completely: the load application fails fast diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index 1fd21e2c329f7..b1dc3c4ecce60 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -39,7 +39,7 @@ # A blip must stay well below the failure detection timeout (10s by default). BLIP_SECS = 1 BLIP_SETTLE_SECS = 5 -BLIPS = 3 +BLIPS = 10 class MdcPartitionResilienceTest(IgniteTest): @@ -138,6 +138,8 @@ def test_main_dc_loss_and_return(self, ignite_version, cross_dc_latency_ms): mdc.control(DC_1).idle_verify(CACHE_NAME) + mdc.verify_servers_log_clean() + mdc.stop_servers() @cluster(num_nodes=8) From be8338472cc5c1cbb1e67fbab4cc54f4d6112382 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 14 Jul 2026 14:12:15 +0300 Subject: [PATCH 18/44] IGNITE-27833 WIP --- .../ignitetest/tests/mdc/distribution_test.py | 119 ------------------ .../ignitetest/tests/mdc/latency_test.py | 113 ----------------- 2 files changed, 232 deletions(-) delete mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/distribution_test.py delete mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/latency_test.py diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/distribution_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/distribution_test.py deleted file mode 100644 index 9e6079e4941f5..0000000000000 --- a/modules/ducktests/tests/ignitetest/tests/mdc/distribution_test.py +++ /dev/null @@ -1,119 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ducktape.mark import matrix - -from ignitetest.services.mdc.mdc_cluster import MdcCluster, DC_1, DC_2, cross_dc_network -from ignitetest.utils import cluster, ignite_versions -from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH - -CACHE_NAME = "mdc-dist" - -DATA_SIZE = 1_000 - - -class MdcDistributionTest(IgniteTest): - """ - Tests for cross-DC partition copy placement by MdcAffinityBackupFilter. - """ - @cluster(num_nodes=7) - @ignite_versions(str(DEV_BRANCH)) - @matrix(backups=[1, 3], cross_dc_latency_ms=50) - def test_copies_balanced_across_dcs(self, ignite_version, backups, cross_dc_latency_ms): - """ - Every partition has exactly (backups + 1) / 2 OWNING copies in each DC, - and the guarantee survives a full restart of one DC with rebalance. - """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc={DC_1: 1, DC_2: 0}) - - copies_per_dc = (backups + 1) // 2 - - with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms): - mdc.start_servers() - - mdc.generate_data(DC_1, CACHE_NAME, 0, DATA_SIZE, backups=backups) - - mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=copies_per_dc) - - mdc.restart(DC_2) - - mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=copies_per_dc) - - mdc.check_data(DC_1, CACHE_NAME, 0, DATA_SIZE) - - mdc.stop_servers() - - @cluster(num_nodes=7) - @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=50) - def test_copies_balanced_in_asymmetric_dcs(self, ignite_version, cross_dc_latency_ms): - """ - DC sizes differ (2 vs 4 nodes), yet each DC still holds exactly one copy of - every partition: balance is per-DC, not per-node. - """ - mdc = MdcCluster(self, ignite_version, srv_per_dc={DC_1: 2, DC_2: 4}, - runners_per_dc={DC_1: 1, DC_2: 0}) - - with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms): - mdc.start_servers() - - mdc.generate_data(DC_1, CACHE_NAME, 0, DATA_SIZE, backups=1) - - mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) - - mdc.check_data(DC_1, CACHE_NAME, 0, DATA_SIZE) - - mdc.stop_servers() - - @cluster(num_nodes=7) - @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=50) - def test_distribution_survives_node_loss(self, ignite_version, cross_dc_latency_ms): - """ - With backups=3 (two copies per DC) losing one node of a DC leaves at least one - copy of every partition in that DC, and after the baseline change and rebalance - the filter restores the exact two-copies-per-DC placement on the surviving nodes. - """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc={DC_1: 1, DC_2: 0}) - - with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms): - mdc.start_servers() - - mdc.generate_data(DC_1, CACHE_NAME, 0, DATA_SIZE, backups=3) - - mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=2) - - mdc.servers[DC_2].stop_node(mdc.servers[DC_2].nodes[-1]) - - # Immediately after the loss the surviving copies must still cover both DCs - # (at least one OWNING copy of every partition per DC). - mdc.verify_cache_distribution(CACHE_NAME) - - mdc.check_data(DC_1, CACHE_NAME, 0, DATA_SIZE) - - # Adopt the new topology as baseline and let rebalance restore the full - # two-copies-per-DC placement on the remaining 3 + 2 nodes. - control = mdc.control(DC_1) - - control.set_baseline(control.cluster_state().topology_version) - - mdc.servers[DC_1].await_rebalance() - mdc.servers[DC_2].await_rebalance() - - mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=2) - - mdc.check_data(DC_1, CACHE_NAME, 0, DATA_SIZE) - - mdc.stop_servers() diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/latency_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/latency_test.py deleted file mode 100644 index c7dbe569239e1..0000000000000 --- a/modules/ducktests/tests/ignitetest/tests/mdc/latency_test.py +++ /dev/null @@ -1,113 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Crude cross-DC latency effects. - -With a large netem delay between the DCs the average synchronous operation latency -(measured inside the load application as wall time / operations) becomes a reliable -proxy for "did the operation cross the DC boundary": - -* GET with ``readFromBackup=true`` (MDC-aware local reads): every partition has a copy - in the client's DC, so reads are served locally and the average stays far below the - configured delay - i.e. read latency is NOT affected by the cross-DC link. -* GET with ``readFromBackup=false``: roughly half the keys have a remote primary, each - such read pays the full cross-DC round trip, so the average grows well above the - local case. -* PUT with ``FULL_SYNC``: unavoidably pays the cross-DC price - the write is acknowledged - only after the backup in the other DC responds, so the average exceeds the one-way delay. - -All thresholds are deliberately loose (2x+ margins): this is a demonstration of the -effect, not a benchmark. -""" -from ducktape.mark import matrix - -from ignitetest.services.mdc.mdc_cluster import MdcCluster, cross_dc_network, DC_1 -from ignitetest.utils import cluster, ignite_versions -from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH - -CACHE_LOCAL_READS = "mdc-lat-rfb" -CACHE_REMOTE_READS = "mdc-lat-norfb" - -KEYS = 2048 - -LOCAL_GET_ITERS = 300 -REMOTE_GET_ITERS = 60 -PUT_ITERS = 30 - -PUT_OFFSET = 1_000_000 - - -class MdcLatencyTest(IgniteTest): - """ - Tests for cross-DC latency effects on cache reads and writes. - """ - @cluster(num_nodes=6) - @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[50]) - def test_cross_dc_latency_effects(self, ignite_version, cross_dc_latency_ms): - """ - Local reads are unaffected by the cross-DC delay, non-local reads and FULL_SYNC - writes pay it. - """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=2, runners_per_dc={DC_1: 1}) - - with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms): - mdc.start_servers() - - # Both caches: backups=1 (a copy of every partition in each DC), FULL_SYNC. - # They differ only in readFromBackup - the switch enabling MDC-local reads. - mdc.generate_data(DC_1, CACHE_LOCAL_READS, 0, KEYS, backups=1) - mdc.generate_data(DC_1, CACHE_REMOTE_READS, 0, KEYS, backups=1, readFromBackup=False) - - svc = mdc.run_load(DC_1, "GET", CACHE_LOCAL_READS, "latLocalGet", - keyFrom=0, keyTo=KEYS, iterations=LOCAL_GET_ITERS) - - avg_local_get = mdc.result_float(svc, "latLocalGetAvgOpMs") - max_local_get = mdc.result_float(svc, "latLocalGetMaxOpMs") - - svc = mdc.run_load(DC_1, "GET", CACHE_REMOTE_READS, "latRemoteGet", - keyFrom=0, keyTo=KEYS, iterations=REMOTE_GET_ITERS) - - avg_remote_get = mdc.result_float(svc, "latRemoteGetAvgOpMs") - max_remote_get = mdc.result_float(svc, "latRemoteGetMaxOpMs") - - svc = mdc.run_load(DC_1, "PUT", CACHE_LOCAL_READS, "latPut", - keyFrom=PUT_OFFSET, iterations=PUT_ITERS) - - avg_put = mdc.result_float(svc, "latPutAvgOpMs") - min_put = mdc.result_float(svc, "latPutMinOpMs") - - self.logger.info(f"Cross-DC latency effects [delayMs={cross_dc_latency_ms}, " - f"avgLocalGetMs={avg_local_get}, avgRemoteGetMs={avg_remote_get}, " - f"avgPutMs={avg_put}]") - - # Local reads: served by the same-DC copy, so the cross-DC delay never applies. - assert avg_local_get < cross_dc_latency_ms * 0.1 and max_local_get < cross_dc_latency_ms, \ - f"MDC-aware local reads should not pay the cross-DC latency " \ - f"[avgMs={avg_local_get}, maxMs={max_local_get}, delayMs={cross_dc_latency_ms}]" - - assert avg_remote_get > cross_dc_latency_ms * 0.9 and max_remote_get > cross_dc_latency_ms, \ - f"Reads without readFromBackup should pay the cross-DC latency " \ - f"[avgMs={avg_remote_get}, maxMs={max_remote_get}, delayMs={cross_dc_latency_ms}]" - - # FULL_SYNC writes: the backup in the other DC must acknowledge each update, - # so a synchronous put cannot possibly complete under the one-way delay. - assert avg_put > 2 * cross_dc_latency_ms and min_put > 2 * cross_dc_latency_ms, \ - f"FULL_SYNC writes should pay the cross-DC latency " \ - f"[avgMs={avg_put}, minMs={min_put}, delayMs={cross_dc_latency_ms}]" - - mdc.stop_servers() From 0fa5de66ac673532f338069b4c0b1ac9856d6854 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 14 Jul 2026 19:46:36 +0300 Subject: [PATCH 19/44] IGNITE-27833 WIP --- .../tests/mdc/MdcCacheAwareApplication.java | 5 +- .../internal/ducktest/utils/OpStats.java | 2 +- modules/ducktests/tests/docker/Dockerfile | 50 ++++++++--------- .../ignitetest/services/mdc/mdc_cluster.py | 53 ++++++++----------- .../utils/ignite_configuration/__init__.py | 1 + .../ignite_configuration/communication.py | 4 +- .../templates/ignite_configuration_macro.j2 | 1 + .../tests/mdc/partition_load_test.py | 7 +-- .../tests/mdc/partition_resilience_test.py | 34 +++++++++--- .../ignitetest/tests/mdc/thin_client_test.py | 4 +- 10 files changed, 88 insertions(+), 73 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java index e181289a1ba54..11766e97cbf12 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java @@ -89,7 +89,7 @@ protected IgniteCache mdcSqlCache(JsonNode jNode) { cacheCfg.setQueryEntities(Collections.singletonList( new QueryEntity(Integer.class, Integer.class).setTableName(SQL_TABLE))); - return ignite.getOrCreateCache(cacheCfg); // todo: check query engine + return ignite.getOrCreateCache(cacheCfg); } /** @@ -157,7 +157,8 @@ protected static > E getEnum(JsonNode jNode, String fieldName, return dfltVal; try { return Enum.valueOf(dfltVal.getDeclaringClass(), field.asText().toUpperCase()); - } catch (IllegalArgumentException e) { + } + catch (IllegalArgumentException e) { return dfltVal; // Fallback if string doesn't match any enum constant } } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java index 206bcf28362e4..7f54c0967a40d 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java @@ -46,4 +46,4 @@ public double avgNs() { public double tps() { return totalNs > 0 ? cnt * 1e9 / totalNs : -1; } -} \ No newline at end of file +} diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index db6f4222f0ec4..b8271f4a22ba4 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -126,31 +126,31 @@ RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" ARG APACHE_ARCHIVE="https://archive.apache.org/dist/" -# Install Ignite -RUN for v in "2.7.6" "2.17.0"; \ - do cd /opt; \ - curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\ - unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v; \ - done \ - && rm /opt/apache-ignite-*-bin.zip - -# Install Zookeeper -ARG ZOOKEEPER_VERSION="3.5.8" -ARG ZOOKEEPER_NAME="zookeeper-$ZOOKEEPER_VERSION" -ARG ZOOKEEPER_RELEASE_NAME="apache-$ZOOKEEPER_NAME-bin" -ARG ZOOKEEPER_RELEASE_ARTIFACT="$ZOOKEEPER_RELEASE_NAME.tar.gz" -RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT \ - && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT \ - && mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME - -# Install Kafka -ARG KAFKA_VERSION="3.9.1" -ARG KAFKA_NAME="kafka" -ARG KAFKA_RELEASE_NAME="${KAFKA_NAME}_2.13-$KAFKA_VERSION" -ARG KAFKA_RELEASE_ARTIFACT="$KAFKA_RELEASE_NAME.tgz" -RUN cd /opt && curl -O $APACHE_ARCHIVE/kafka/$KAFKA_VERSION/$KAFKA_RELEASE_ARTIFACT \ - && tar xvf $KAFKA_RELEASE_ARTIFACT && rm $KAFKA_RELEASE_ARTIFACT \ - && mv /opt/$KAFKA_RELEASE_NAME /opt/$KAFKA_NAME-$KAFKA_VERSION +## Install Ignite +#RUN for v in "2.7.6" "2.17.0"; \ +# do cd /opt; \ +# curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\ +# unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v; \ +# done \ +# && rm /opt/apache-ignite-*-bin.zip +# +## Install Zookeeper +#ARG ZOOKEEPER_VERSION="3.5.8" +#ARG ZOOKEEPER_NAME="zookeeper-$ZOOKEEPER_VERSION" +#ARG ZOOKEEPER_RELEASE_NAME="apache-$ZOOKEEPER_NAME-bin" +#ARG ZOOKEEPER_RELEASE_ARTIFACT="$ZOOKEEPER_RELEASE_NAME.tar.gz" +#RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT \ +# && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT \ +# && mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME +# +## Install Kafka +#ARG KAFKA_VERSION="3.9.1" +#ARG KAFKA_NAME="kafka" +#ARG KAFKA_RELEASE_NAME="${KAFKA_NAME}_2.13-$KAFKA_VERSION" +#ARG KAFKA_RELEASE_ARTIFACT="$KAFKA_RELEASE_NAME.tgz" +#RUN cd /opt && curl -O $APACHE_ARCHIVE/kafka/$KAFKA_VERSION/$KAFKA_RELEASE_ARTIFACT \ +# && tar xvf $KAFKA_RELEASE_ARTIFACT && rm $KAFKA_RELEASE_ARTIFACT \ +# && mv /opt/$KAFKA_RELEASE_NAME /opt/$KAFKA_NAME-$KAFKA_VERSION # Install Jmxterm ARG JMXTERM_NAME="jmxterm" diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py index 07e43079a662d..d942b00f32961 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -33,8 +33,8 @@ from ignitetest.services.network_group.configuration import NetworkGroupStore, CrossNetworkGroupConfiguration from ignitetest.services.network_group.manager import NetworkGroupManager from ignitetest.services.utils.control_utility import ControlUtility -from ignitetest.services.utils.ignite_configuration import IgniteConfiguration -from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, TcpCommunicationSpi +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster, from_ignite_services from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration from ignitetest.utils.version import IgniteVersion @@ -52,9 +52,9 @@ LOAD_APP = _APP_PKG + "MdcContinuousLoadApplication" THIN_LOAD_APP = _APP_PKG + "MdcThinClientLoadApplication" -# Suspicious server log patterns: none of them is expected in any MDC scenario, todo: Replace with log checks or control.sh commands +# Suspicious server log patterns: none of them is expected in any MDC scenario, # partitioned or not. Matched against the node console capture. -LRT_PATTERN = "Found long running transaction" +LRT_PATTERN = "long running transactions" PME_FREEZE_PATTERN = "Failed to wait for partition map exchange" LOST_PARTITIONS_PATTERN = "Detected lost partitions" @@ -89,12 +89,18 @@ class MdcCluster: """ def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, int]] = 3, runners_per_dc: Union[int, Dict[str, int]] = 1, - loaders_per_dc: Union[int, Dict[str, int]] = 0, # todo: не понял зачем это нужно - client_connector: bool = False): + loaders_per_dc: Union[int, Dict[str, int]] = 0, + client_connector: bool = False, + network_timeout: int = 5_000, + tcp_connect_timeout: int = 5_000): self.test_context = test.test_context self.logger = test.logger - cfg_kwargs = {"version": IgniteVersion(ignite_version)} + cfg_kwargs = { + "version": IgniteVersion(ignite_version), + "network_timeout": network_timeout, + "communication_spi": TcpCommunicationSpi(connect_timeout=tcp_connect_timeout) + } if client_connector: cfg_kwargs["client_connector_configuration"] = ClientConnectorConfiguration() @@ -122,6 +128,14 @@ def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, i # subsequent ones preserve work dirs (and logs - hence unique result prefixes). self._started_apps = set() + def sync_service_discovery(self): + all_servers = list(self.servers.values()) + + discovery_spi = from_ignite_services(all_servers) + + for dc_key, service in self.servers.items(): + service.config = service.config._replace(discovery_spi=discovery_spi) + def _app_service(self, dc: str) -> IgniteApplicationService: client_cfg = self.ignite_config._replace(client_mode=True, discovery_spi=from_ignite_cluster(self.servers[dc])) @@ -239,8 +253,6 @@ def _first_start(self, svc) -> bool: return first - # ------------------------------------------------------- domain applications - def generate_data(self, dc: str, cache_name: str, from_idx: int, to_idx: int, backups: int, main_dc: str = DC_1, **cache_params) -> IgniteApplicationService: """ @@ -381,9 +393,7 @@ def verify_servers_log_clean(self): """ for pattern in (LRT_PATTERN, PME_FREEZE_PATTERN, LOST_PARTITIONS_PATTERN): for dc, svc in self.servers.items(): - hits = svc.check_event_absent(pattern) - - assert hits == 0, f"Suspicious events found in {dc} server logs [pattern='{pattern}', hits={hits}]" + svc.check_event_absent(pattern) def verify_no_hanging_txs(self, dc: str = DC_1): """ @@ -450,25 +460,6 @@ def dc_of(copy): layout_hint=f"DC attribute: {dc_attr}, expected DCs: {sorted(expected_dcs)}") -def assert_cross_dc_distribution(distribution, dc_addresses, owning_only=True, copies_per_dc=None): - """ - Same assertion, but DC membership is derived from node IP addresses. - - :param dc_addresses: Dict: DC name -> collection of node IP addresses of that DC, - e.g. {"DC1": ["192.168.64.9", ...], "DC2": [...]}. - """ - dc_addresses = {dc: set(addrs) for dc, addrs in dc_addresses.items()} - - def dc_of(copy): - for dc, addrs in dc_addresses.items(): - if any(addr in addrs for addr in copy.node_addresses): - return dc - return None - - _assert_cross_dc(distribution, set(dc_addresses), dc_of, owning_only, copies_per_dc, - layout_hint=f"DC layout: { {dc: sorted(addrs) for dc, addrs in dc_addresses.items()} }") - - def _assert_cross_dc(distribution, expected_dcs, dc_of, owning_only, copies_per_dc, layout_hint): violations = [] diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py index 1fdec71e3b4cb..6c9ea5b03f686 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py @@ -74,6 +74,7 @@ class IgniteConfiguration(NamedTuple): auto_activation_enabled: bool = None transaction_configuration: TransactionConfiguration = None sql_configuration: Bean = None + network_timeout: int = 5_000 def prepare_ssl(self, test_globals, shared_root): """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py index 528c8f9bbca29..ee092651b7df9 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py @@ -53,7 +53,8 @@ def __init__(self, connections_per_node: int = None, use_paired_connections: bool = None, message_queue_limit: int = None, - unacknowledged_messages_buffer_size: int = None): + unacknowledged_messages_buffer_size: int = None, + connect_timeout: int = 5_000): self.local_port = local_port self.local_port_range = local_port_range self.idle_connection_timeout: int = idle_connection_timeout @@ -63,6 +64,7 @@ def __init__(self, self.use_paired_connections: bool = use_paired_connections self.message_queue_limit: int = message_queue_limit self.unacknowledged_messages_buffer_size: int = unacknowledged_messages_buffer_size + self.connect_timeout: int = connect_timeout @property def class_name(self): diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/ignite_configuration_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/ignite_configuration_macro.j2 index fd1cbef5b60ec..c4953903afc38 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/ignite_configuration_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/ignite_configuration_macro.j2 @@ -38,6 +38,7 @@ + diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py index f22e77ce1c085..71b413f8fb900 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py @@ -66,7 +66,6 @@ # Boundary tolerance for the background load: a transient error window is possible at # the instant the partition is enabled/healed, but must stay marginal. -BG_MAX_ERR_RATIO = 0.01 BG_MAX_STALL_MS = 60_000 @@ -154,10 +153,8 @@ def test_load_through_partition(self, ignite_version, cross_dc_latency_ms): ops = mdc.result_int(svc, f"bg{dc}OpsCnt") errs = mdc.result_int(svc, f"bg{dc}ErrCnt") max_stall = mdc.result_int(svc, f"bg{dc}MaxStallMs") - min_window = mdc.result_int(svc, f"bg{dc}MinWindowOps") - self.logger.info(f"Background GET load [dc={dc}, ops={ops}, errs={errs}, " - f"maxStallMs={max_stall}, minWindowOps={min_window}]") + self.logger.info(f"Background GET load [dc={dc}, ops={ops}, errs={errs}, maxStallMs={max_stall}]") assert ops > 0, f"Background get load performed no operations [dc={dc}]" @@ -191,7 +188,7 @@ def test_load_through_partition(self, ignite_version, cross_dc_latency_ms): @cluster(num_nodes=8) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[20], cross_dc_loss=[0.1]) + @matrix(cross_dc_latency_ms=[20], cross_dc_loss=[0.01]) def test_degraded_link_without_partition(self, ignite_version, cross_dc_latency_ms, cross_dc_loss): """ A slow, lossy WAN that stays connected ("slow WAN" as opposed to "broken WAN"): diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index b1dc3c4ecce60..9882020f88250 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -41,6 +41,7 @@ BLIP_SETTLE_SECS = 5 BLIPS = 10 +BG_MAX_STALL_MS = 500 class MdcPartitionResilienceTest(IgniteTest): """ @@ -48,14 +49,14 @@ class MdcPartitionResilienceTest(IgniteTest): """ @cluster(num_nodes=12) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[20]) + @matrix(cross_dc_latency_ms=[100]) def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms): """ The canonical split-brain lifecycle: partition -> split into two healthy half-rings (DC1 active, DC2 read-only) -> all data readable everywhere -> heal -> DC2 rejoins via restart -> writes restored everywhere, distribution and consistency verified. """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=5, runners_per_dc=1) + mdc = MdcCluster(self, ignite_version, srv_per_dc=5, runners_per_dc=1, network_timeout=10_000, tcp_connect_timeout=10_000) with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() @@ -97,7 +98,7 @@ def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency @cluster(num_nodes=6) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=20) + @matrix(cross_dc_latency_ms=[25, 50, 100]) def test_main_dc_loss_and_return(self, ignite_version, cross_dc_latency_ms): """ The inverse of the canonical scenario: the MAIN data center goes down entirely. @@ -127,6 +128,8 @@ def test_main_dc_loss_and_return(self, ignite_version, cross_dc_latency_ms): # ...but the surviving DC is read-only while the main DC is invisible. mdc.check_put_admissibility(DC_2, CACHE_NAME, False) + mdc.sync_service_discovery() + mdc.servers[DC_1].start(clean=False) mdc.servers[DC_1].await_rebalance() @@ -142,16 +145,16 @@ def test_main_dc_loss_and_return(self, ignite_version, cross_dc_latency_ms): mdc.stop_servers() - @cluster(num_nodes=8) + @cluster(num_nodes=10) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[20]) + @matrix(cross_dc_latency_ms=[25, 50, 100]) def test_short_partition_blips_do_not_split(self, ignite_version, cross_dc_latency_ms): """ A flapping WAN link: several short (below the failure detection timeout) full cross-DC connectivity drops. The cluster must NOT split: after the blips it is still one ACTIVE cluster, all data is intact and both DCs accept writes. """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc=1) + mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc=1, loaders_per_dc=1) with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() @@ -159,6 +162,12 @@ def test_short_partition_blips_do_not_split(self, ignite_version, cross_dc_laten mdc.generate_data(DC_1, CACHE_NAME, 0, 100, backups=BACKUPS) mdc.generate_data(DC_2, CACHE_NAME, 100, 200, backups=BACKUPS) + for dc in (DC_1, DC_2): + mdc.start_loader(dc, {"mode": "GET", "cacheName": CACHE_NAME, + "keyFrom": 0, "keyTo": 200, + "tolerateErrors": True, "opPauseMs": 5, + "resultPrefix": f"bg{dc}"}) + for i in range(BLIPS): self.logger.info(f"Network blip {i + 1}/{BLIPS}") @@ -172,6 +181,19 @@ def test_short_partition_blips_do_not_split(self, ignite_version, cross_dc_laten mdc.verify_whole_cluster_healthy() + for dc in (DC_1, DC_2): + svc = mdc.stop_loader(dc) + + ops = mdc.result_int(svc, f"bg{dc}OpsCnt") + errs = mdc.result_int(svc, f"bg{dc}ErrCnt") + max_stall = mdc.result_int(svc, f"bg{dc}MaxStallMs") + + self.logger.info(f"Background GET load [dc={dc}, ops={ops}, errs={errs}, maxStallMs={max_stall}]") + + assert ops > 0, f"Background get load performed no operations [dc={dc}]" + assert errs == 0, f"Background get load errors exceed the boundary tolerance [dc={dc}, ops={ops}, errs={errs}]" + assert max_stall < BG_MAX_STALL_MS, f"Background get load stalled for too long [dc={dc}, maxStallMs={max_stall}]" + mdc.check_data(DC_1, CACHE_NAME, 0, 200) mdc.check_data(DC_2, CACHE_NAME, 0, 200) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py index 9e3991fef9c83..dcf7deb6a80d8 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py @@ -36,8 +36,8 @@ from ducktape.mark import matrix from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.mdc.mdc_cluster import MdcCluster, dc_jvm_opts, DC_1, DC_2, cross_dc_network, THIN_LOAD_APP from ignitetest.services.utils.ignite_configuration import IgniteThinClientConfiguration -from ignitetest.tests.mdc.fixture import DC_1, DC_2, THIN_LOAD_APP, MdcCluster, cross_dc_network, dc_jvm_opts from ignitetest.utils import cluster, ignite_versions from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, IgniteVersion @@ -63,7 +63,7 @@ class MdcThinClientTest(IgniteTest): """ @cluster(num_nodes=8) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[200]) + @matrix(cross_dc_latency_ms=[20]) def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_dc_latency_ms): """ DC-pinned thin client reads locally (latency far below the cross-DC delay), From 1ef3b8526e7549ec032082ed9bd6f7317eade58d Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 14 Jul 2026 20:07:17 +0300 Subject: [PATCH 20/44] IGNITE-27833 WIP --- .../tests/ignitetest/tests/mdc/partition_resilience_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index 9882020f88250..486ab981b6429 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -49,14 +49,14 @@ class MdcPartitionResilienceTest(IgniteTest): """ @cluster(num_nodes=12) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[100]) + @matrix(cross_dc_latency_ms=[25, 50, 100]) def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms): """ The canonical split-brain lifecycle: partition -> split into two healthy half-rings (DC1 active, DC2 read-only) -> all data readable everywhere -> heal -> DC2 rejoins via restart -> writes restored everywhere, distribution and consistency verified. """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=5, runners_per_dc=1, network_timeout=10_000, tcp_connect_timeout=10_000) + mdc = MdcCluster(self, ignite_version, srv_per_dc=5, runners_per_dc=1, network_timeout=20_000, tcp_connect_timeout=10_000) with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() From 334e37dc70ff33c22a30f984823df5936475bedc Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 14 Jul 2026 21:06:05 +0300 Subject: [PATCH 21/44] IGNITE-27833 WIP --- .../ignitetest/tests/mdc/thin_client_test.py | 79 +++++++------------ 1 file changed, 27 insertions(+), 52 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py index dcf7deb6a80d8..1c368baa7b88c 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py @@ -17,11 +17,8 @@ Thin client data-center-aware routing. Every thin client is configured with the addresses of ALL server nodes of BOTH DCs. -A client pinned to a data center (via the ``-DIGNITE_DATA_CENTER_ID=...`` JVM system -property, per the MDC docs) must route partition-aware reads to nodes of its own DC: -with a large cross-DC netem delay its average read latency stays small, while an -unpinned client's reads hit remote primaries for about half the keys and pay the -cross-DC round trip. Latency is the routing proxy - no metrics infrastructure needed. +A client pinned to a data center must route partition-aware reads to nodes of its own DC: +with a large cross-DC netem delay its average read latency stays small. The partition part verifies the thin client experience of a split-brain: a client pinned to the main-DC half keeps writing; a client pinned to the read-only half still @@ -47,14 +44,13 @@ KEYS = 100 PINNED_GET_ITERS = 200 -UNPINNED_GET_ITERS = 60 PUT_ITERS = 30 OFFSET_DURING = 1_000_000 OFFSET_REJECTED_PROBES = 9_000_000 OFFSET_AFTER = 2_000_000 -SPLIT_SETTLE_SECS = 10 +SPLIT_SETTLE_SECS = 15 class MdcThinClientTest(IgniteTest): @@ -63,12 +59,11 @@ class MdcThinClientTest(IgniteTest): """ @cluster(num_nodes=8) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[20]) + @matrix(cross_dc_latency_ms=[25, 50, 100]) def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_dc_latency_ms): """ - DC-pinned thin client reads locally (latency far below the cross-DC delay), - an unpinned client pays the cross-DC price; through a partition the pinned - clients behave like their half-ring: main half writes, read-only half reads + DC-pinned thin client reads locally (latency far below the cross-DC delay); + through a partition the pinned clients behave like their half-ring: main half writes, read-only half reads but rejects writes, and writes resume after the heal. """ mdc = MdcCluster(self, ignite_version, srv_per_dc=2, runners_per_dc={DC_1: 1}, @@ -76,62 +71,44 @@ def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_ # All thin clients get the full address list of both DCs: DC preference must come # from routing, not from the address list. - pinned_dc1 = self._thin_client(mdc, dc_jvm_opts(DC_1)) - pinned_dc2 = self._thin_client(mdc, dc_jvm_opts(DC_2)) - unpinned = self._thin_client(mdc, []) + cli_dc1 = self._thin_client(mdc, dc_jvm_opts(DC_1)) + cli_dc2 = self._thin_client(mdc, dc_jvm_opts(DC_2)) - # Register the thin client hosts into the network groups, so netem and the - # partition apply to them. The unpinned client observes from DC1's side. - mdc.register(DC_1, pinned_dc1) - mdc.register(DC_2, pinned_dc2) - mdc.register(DC_1, unpinned) + # Register the thin client hosts into the network groups, so netem and the partition apply to them. + mdc.register(DC_1, cli_dc1) + mdc.register(DC_2, cli_dc2) first_start = set() - def run_thin(svc, params): - svc.params = params + def run_thin(cli_svc, params): + cli_svc.params = params - svc.start(clean=id(svc) not in first_start) + cli_svc.start(clean=id(cli_svc) not in first_start) - first_start.add(id(svc)) + first_start.add(id(cli_svc)) - svc.wait() - svc.stop() + cli_svc.wait() + cli_svc.stop() - return svc + return cli_svc with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() - # readFromBackup=true + FULL_SYNC (the fixture defaults) are exactly the - # documented preconditions of thin client partition-aware local reads. mdc.generate_data(DC_1, CACHE_NAME, 0, KEYS, backups=1) - # ---- DC-aware routing: latency comparison. - svc = run_thin(pinned_dc1, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, + svc = run_thin(cli_dc1, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, "keyTo": KEYS, "iterations": PINNED_GET_ITERS, "resultPrefix": "pinnedGet"}) - avg_pinned = mdc.result_float(svc, "pinnedGetAvgOpMs") + avg_cli_dc_1 = mdc.result_float(svc, "pinnedGetAvgOpMs") - svc = run_thin(unpinned, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, - "keyTo": KEYS, "iterations": UNPINNED_GET_ITERS, - "resultPrefix": "freeGet"}) + self.logger.info(f"Thin client routing latency [delayMs={cross_dc_latency_ms}, getAvgOpMs={avg_cli_dc_1}]") - avg_unpinned = mdc.result_float(svc, "freeGetAvgOpMs") - - self.logger.info(f"Thin client routing latency [delayMs={cross_dc_latency_ms}, " - f"pinnedAvgMs={avg_pinned}, unpinnedAvgMs={avg_unpinned}]") - - assert avg_pinned < cross_dc_latency_ms * 0.5, \ + assert avg_cli_dc_1 < cross_dc_latency_ms, \ f"DC-pinned thin client reads should be served locally " \ - f"[avgMs={avg_pinned}, delayMs={cross_dc_latency_ms}]" - - assert avg_unpinned > avg_pinned * 2, \ - f"Unpinned thin client reads should pay the cross-DC latency " \ - f"[unpinnedAvgMs={avg_unpinned}, pinnedAvgMs={avg_pinned}]" + f"[avgMs={avg_cli_dc_1}, delayMs={cross_dc_latency_ms}]" - # ---- Thin clients through a partition. net.enable_network_partition(DC_1, DC_2) sleep(SPLIT_SETTLE_SECS) @@ -139,13 +116,13 @@ def run_thin(svc, params): mdc.verify_split_brain() # The client pinned to the main half keeps writing... - run_thin(pinned_dc1, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_DURING, + run_thin(cli_dc1, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_DURING, "keyTo": OFFSET_DURING + PUT_ITERS, "iterations": PUT_ITERS, "resultPrefix": "duringPut"}) # ...while the client pinned to the read-only half still reads cleanly # (its DC1 addresses are unreachable, so it falls back to DC2 nodes)... - svc = run_thin(pinned_dc2, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, + svc = run_thin(cli_dc2, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, "keyTo": KEYS, "iterations": PINNED_GET_ITERS, "resultPrefix": "roGet"}) @@ -153,22 +130,20 @@ def run_thin(svc, params): "Thin client reads in the read-only DC must be clean" # ...and has every write rejected by the topology validator. - run_thin(pinned_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, + run_thin(cli_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_REJECTED_PROBES, "keyTo": OFFSET_REJECTED_PROBES + PUT_ITERS, "iterations": PUT_ITERS, "expectAdmissible": False, "resultPrefix": "roPut"}) - # ---- Heal, rejoin, thin client writes resume in the former read-only DC. net.disable_network_partition(DC_1, DC_2) mdc.restart(DC_2) - run_thin(pinned_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_AFTER, + run_thin(cli_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_AFTER, "keyTo": OFFSET_AFTER + PUT_ITERS, "iterations": PUT_ITERS, "resultPrefix": "afterPut"}) - # Everything the thin clients acknowledged is readable via the thick client. mdc.check_data(DC_1, CACHE_NAME, OFFSET_DURING, OFFSET_DURING + PUT_ITERS) mdc.check_data(DC_1, CACHE_NAME, OFFSET_AFTER, OFFSET_AFTER + PUT_ITERS) From cab011d79e188b381d41b35f36a4d84a30ad9547 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 15 Jul 2026 12:36:03 +0300 Subject: [PATCH 22/44] IGNITE-27833 WIP --- .../mdc/MdcDataGeneratorApplication.java | 44 ++- .../ignitetest/services/mdc/mdc_cluster.py | 13 +- .../tests/mdc/partition_load_test.py | 264 ------------------ .../tests/mdc/partition_resilience_test.py | 8 +- .../ignitetest/tests/mdc/thin_client_test.py | 4 +- 5 files changed, 49 insertions(+), 284 deletions(-) delete mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java index e8727dfddc6f1..b0be33b4e123d 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java @@ -17,28 +17,52 @@ public class MdcDataGeneratorApplication extends MdcCacheAwareApplication { int from = jNode.path("from").asInt(0); int to = jNode.path("to").asInt(10_000); int batchSize = jNode.path("batchSize").asInt(1_024); + boolean sqlMode = jNode.path("sqlMode").asBoolean(false); markInitialized(); waitForActivation(); - IgniteCache cache = mdcCache(jNode); + IgniteCache cache = null; + IgniteCache sqlCache = null; - log.info("Data generation started [dc=" + dcId() + ", cache=" + cache.getName() + + if (sqlMode) + sqlCache = mdcSqlCache(jNode); + else + cache = mdcCache(jNode); + + log.info("Data generation started [dc=" + dcId() + ", sqlMode=" + sqlMode + ", from=" + from + ", to=" + to + "]"); - Map batch = new TreeMap<>(); + if (sqlMode) { + Map batch = new TreeMap<>(); - for (int i = from; i < to && !terminated(); i++) { - batch.put(i, new IndexedDataRecord(i)); + for (int i = from; i < to && !terminated(); i++) { + batch.put(i, i); - if (batch.size() >= batchSize) { - cache.putAll(batch); - batch.clear(); + if (batch.size() >= batchSize) { + sqlCache.putAll(batch); + batch.clear(); + } } + + if (!batch.isEmpty() && !terminated()) + sqlCache.putAll(batch); } + else { + Map batch = new TreeMap<>(); + + for (int i = from; i < to && !terminated(); i++) { + batch.put(i, new IndexedDataRecord(i)); - if (!batch.isEmpty() && !terminated()) - cache.putAll(batch); + if (batch.size() >= batchSize) { + cache.putAll(batch); + batch.clear(); + } + } + + if (!batch.isEmpty() && !terminated()) + cache.putAll(batch); + } log.info("Data generation finished [dc=" + dcId() + ", entries=" + (to - from) + "]"); diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py index d942b00f32961..afad737106de8 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -42,7 +42,12 @@ DC_2 = "DC2" DCS = (DC_1, DC_2) +IGNITE_STARTUP_TIMEOUT_SEC = 90 + DATA_CENTER_ATTR = "IGNITE_DATA_CENTER_ID" +IGNITE_SQL_RETRY_TIMEOUT_ATTR = "IGNITE_SQL_RETRY_TIMEOUT" + +IGNITE_SQL_RETRY_TIMEOUT_MS = 1_000 _APP_PKG = "org.apache.ignite.internal.ducktest.tests.mdc." @@ -63,7 +68,7 @@ def dc_jvm_opts(dc: str) -> List[str]: """ :return: JVM options assigning a node to the given data center. """ - return [f"-D{DATA_CENTER_ATTR}={dc}"] + return [f"-D{DATA_CENTER_ATTR}={dc}", f"-D{IGNITE_SQL_RETRY_TIMEOUT_ATTR}={IGNITE_SQL_RETRY_TIMEOUT_MS}"] def _per_dc(value: Union[int, Dict[str, int]]) -> Dict[str, int]: @@ -110,7 +115,7 @@ def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, i self.srv_per_dc = _per_dc(srv_per_dc) self.servers: Dict[str, IgniteService] = { - dc: IgniteService(self.test_context, self.ignite_config, num_nodes=num, jvm_opts=dc_jvm_opts(dc)) + dc: IgniteService(self.test_context, self.ignite_config, num_nodes=num, jvm_opts=dc_jvm_opts(dc), startup_timeout_sec=IGNITE_STARTUP_TIMEOUT_SEC) for dc, num in self.srv_per_dc.items() if num > 0} self.runners: Dict[str, List[IgniteApplicationService]] = { @@ -254,14 +259,14 @@ def _first_start(self, svc) -> bool: return first def generate_data(self, dc: str, cache_name: str, from_idx: int, to_idx: int, backups: int, - main_dc: str = DC_1, **cache_params) -> IgniteApplicationService: + main_dc: str = DC_1, sql_mode: bool = False, **cache_params) -> IgniteApplicationService: """ Creates the MDC cache (if absent) and populates keys ``[from_idx, to_idx)``. Extra cache parameters (``atomicity``, ``syncMode``, ``readFromBackup``, ``partitions``, ...) are passed through to the cache configuration builder. """ params = {"cacheName": cache_name, "backups": backups, "mainDc": main_dc, - "from": from_idx, "to": to_idx, **cache_params} + "from": from_idx, "to": to_idx, "sqlMode": sql_mode, **cache_params} return self.run_app(dc, GENERATOR_APP, params) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py deleted file mode 100644 index 71b413f8fb900..0000000000000 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_load_test.py +++ /dev/null @@ -1,264 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Continuous cache/tx/SQL load through a network partition. - -The availability claims verified here: - -* GET load (cache API) runs in BOTH data centers from before the partition, through the - partition and its healing, in a single application run - so any exception crossing a - phase boundary fails the test on loader stop. Per-window throughput sampling and the - max-stall metric additionally show that availability never dropped for long. -* PUT / transactional / SQL DML load succeeds in the main-DC half in every phase, is - rejected (and only rejected) in the read-only half during the partition, and resumes - everywhere after healing. -* Everything that was acknowledged as written is readable afterwards from the OPPOSITE - data center ("as much as we succeeded to put, we get back"). -* Negative invariants: no long running transactions, no PME freeze, no lost partitions, - no hanging transactions, and idle_verify finds no conflicts after the recovery. -""" -from time import sleep - -from ducktape.mark import matrix - -from ignitetest.services.mdc.mdc_cluster import MdcCluster, cross_dc_network, DC_1, DC_2 -from ignitetest.utils import cluster, ignite_versions -from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH - -ATOMIC_CACHE = "mdc-atomic" -TX_CACHE = "mdc-tx" -SQL_CACHE = "mdc-sql" - -# Pre-loaded data set, written half from each DC. -GEN_SIZE = 1_000 - -# Non-intersecting key offsets for the write bursts of each phase. -OFFSET_BEFORE_DC1 = 1_000_000 -OFFSET_BEFORE_DC2 = 1_500_000 -OFFSET_DURING_DC1 = 2_000_000 -OFFSET_AFTER_DC1 = 3_000_000 -OFFSET_AFTER_DC2 = 3_500_000 - -# Probe keys of the expected-to-be-rejected bursts (never intersect readable data). -OFFSET_REJECTED_PROBES = 9_000_000 - -PUT_BURST = 200 -TX_BURST = 60 -SQL_BURST = 100 -GET_BURST = 500 -REJECTED_BURST = 30 - -SPLIT_SETTLE_SECS = 10 - -# Boundary tolerance for the background load: a transient error window is possible at -# the instant the partition is enabled/healed, but must stay marginal. -BG_MAX_STALL_MS = 60_000 - - -class MdcPartitionLoadTest(IgniteTest): - """ - Tests for continuous load through an MDC network partition. - """ - @cluster(num_nodes=10) - @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[20]) - def test_load_through_partition(self, ignite_version, cross_dc_latency_ms): - """ - Background gets in both DCs across the whole scenario; put/tx/sql bursts before, - during and after the partition; cross-DC readback of every acknowledged write. - """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc=1, loaders_per_dc=1) - - with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: - mdc.start_servers() - - mdc.generate_data(DC_1, ATOMIC_CACHE, 0, GEN_SIZE // 2, backups=1) - mdc.generate_data(DC_2, ATOMIC_CACHE, GEN_SIZE // 2, GEN_SIZE, backups=1) - - # Background gets: a single run per DC spanning every phase. Any exception - # surfaces on stop_loader(); transient boundary errors are counted, not thrown. - for dc in (DC_1, DC_2): - mdc.start_loader(dc, {"mode": "GET", "cacheName": ATOMIC_CACHE, - "keyFrom": 0, "keyTo": GEN_SIZE, - "tolerateErrors": True, "opPauseMs": 5, - "resultPrefix": f"bg{dc}"}) - - # written: (cache, kind, written_from_dc, offset, count) - written = [] - - # ---- Phase 1: before the partition, both DCs accept writes. - written += self._admissible_write_bursts(mdc, DC_1, "before", OFFSET_BEFORE_DC1) - written.append(self._admissible_put_burst(mdc, DC_2, "before", OFFSET_BEFORE_DC2)) - - before_sql = next(entry for entry in written if entry[0] == SQL_CACHE) - - net.enable_network_partition(DC_1, DC_2) - - sleep(SPLIT_SETTLE_SECS) - - mdc.verify_split_brain() - - # ---- Phase 2: during the partition. - # The main-DC half keeps accepting the full write load. - written += self._admissible_write_bursts(mdc, DC_1, "during", OFFSET_DURING_DC1) - - # The read-only half serves reads cleanly - cache API and SQL... - svc = mdc.run_load(DC_2, "GET", ATOMIC_CACHE, "duringDC2Get", - keyFrom=0, keyTo=GEN_SIZE, iterations=GET_BURST) - - assert mdc.result_int(svc, "duringDC2GetErrCnt") == 0, "Reads in the read-only DC must be clean" - - svc = mdc.run_load(DC_2, "SQL_SELECT", SQL_CACHE, "duringDC2Sql", - keyFrom=before_sql[3], keyTo=before_sql[3] + before_sql[4], - iterations=before_sql[4]) - - assert mdc.result_int(svc, "duringDC2SqlErrCnt") == 0, "SQL reads in the read-only DC must be clean" - - # ...and rejects every write - plain, transactional and SQL DML. - mdc.run_load(DC_2, "PUT", ATOMIC_CACHE, "duringDC2Put", keyFrom=OFFSET_REJECTED_PROBES, - iterations=REJECTED_BURST, expectAdmissible=False) - - mdc.run_load(DC_2, "TX_PUT", TX_CACHE, "duringDC2Tx", keyFrom=OFFSET_REJECTED_PROBES, - iterations=REJECTED_BURST, expectAdmissible=False) - - mdc.run_load(DC_2, "SQL_PUT", SQL_CACHE, "duringDC2SqlPut", keyFrom=OFFSET_REJECTED_PROBES, - iterations=REJECTED_BURST, expectAdmissible=False) - - # ---- Phase 3: heal, rejoin the read-only half, writes resume everywhere. - net.disable_network_partition(DC_1, DC_2) - - mdc.restart(DC_2) - - written += self._admissible_write_bursts(mdc, DC_1, "after", OFFSET_AFTER_DC1) - written.append(self._admissible_put_burst(mdc, DC_2, "after", OFFSET_AFTER_DC2)) - - # ---- Background loads: no exception on stop, availability never dropped. - for dc in (DC_1, DC_2): - svc = mdc.stop_loader(dc) - - ops = mdc.result_int(svc, f"bg{dc}OpsCnt") - errs = mdc.result_int(svc, f"bg{dc}ErrCnt") - max_stall = mdc.result_int(svc, f"bg{dc}MaxStallMs") - - self.logger.info(f"Background GET load [dc={dc}, ops={ops}, errs={errs}, maxStallMs={max_stall}]") - - assert ops > 0, f"Background get load performed no operations [dc={dc}]" - - assert errs == 0, f"Background get load errors exceed the boundary tolerance [dc={dc}, ops={ops}, errs={errs}]" - - assert max_stall < BG_MAX_STALL_MS, f"Background get load stalled for too long [dc={dc}, maxStallMs={max_stall}]" - - # ---- Readback: every acknowledged write is readable from the OPPOSITE DC. - for idx, (cache, kind, src_dc, offset, cnt) in enumerate(written): - assert cnt > 0, f"Write burst acknowledged nothing [cache={cache}, offset={offset}]" - - dst_dc = DC_2 if src_dc == DC_1 else DC_1 - - if kind == "sql": - svc = mdc.run_load(dst_dc, "SQL_SELECT", cache, f"rb{idx}{dst_dc}", - keyFrom=offset, keyTo=offset + cnt, iterations=cnt) - - assert mdc.result_int(svc, f"rb{idx}{dst_dc}ErrCnt") == 0, \ - f"SQL readback failed [cache={cache}, offset={offset}, cnt={cnt}, dc={dst_dc}]" - else: - mdc.check_data(dst_dc, cache, offset, offset + cnt) - - # ---- Negative invariants. - mdc.verify_no_hanging_txs() - - mdc.control(DC_1).idle_verify(",".join([ATOMIC_CACHE, TX_CACHE, SQL_CACHE])) - - mdc.verify_servers_log_clean() - - mdc.stop_servers() - - @cluster(num_nodes=8) - @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[20], cross_dc_loss=[0.01]) - def test_degraded_link_without_partition(self, ignite_version, cross_dc_latency_ms, cross_dc_loss): - """ - A slow, lossy WAN that stays connected ("slow WAN" as opposed to "broken WAN"): - the cluster must NOT split, and cache, transactional and SQL load must complete - cleanly in both DCs (slower, but without a single error) - TCP absorbs the loss. - """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc=1) - - with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms, loss=cross_dc_loss): - mdc.start_servers() - - mdc.generate_data(DC_1, ATOMIC_CACHE, 0, GEN_SIZE // 2, backups=1) - mdc.generate_data(DC_2, ATOMIC_CACHE, GEN_SIZE // 2, GEN_SIZE, backups=1) - - for dc, offset in ((DC_1, OFFSET_BEFORE_DC1), (DC_2, OFFSET_BEFORE_DC2)): - svc = mdc.run_load(dc, "GET", ATOMIC_CACHE, f"deg{dc}Get", - keyFrom=0, keyTo=GEN_SIZE, iterations=GET_BURST) - - assert mdc.result_int(svc, f"deg{dc}GetErrCnt") == 0, f"Degraded-link reads must be clean [dc={dc}]" - - self._admissible_put_burst(mdc, dc, "deg", offset) - - self._admissible_burst(mdc, DC_1, "TX_PUT", TX_CACHE, "degDC1Tx", OFFSET_DURING_DC1, TX_BURST, - atomicity="TRANSACTIONAL") - - mdc.verify_whole_cluster_healthy() - - mdc.verify_cache_distribution(ATOMIC_CACHE, copies_per_dc=1) - - mdc.control(DC_1).idle_verify(",".join([ATOMIC_CACHE, TX_CACHE])) - - mdc.verify_servers_log_clean() - - mdc.stop_servers() - - def _admissible_write_bursts(self, mdc: MdcCluster, dc: str, phase: str, offset: int): - """ - Runs the full trio of write bursts (cache API put, transactional put, SQL DML) - expected to fully succeed, and returns the written ranges for later readback. - """ - return [ - self._admissible_burst(mdc, dc, "PUT", ATOMIC_CACHE, f"{phase}{dc}Put", offset, PUT_BURST), - self._admissible_burst(mdc, dc, "TX_PUT", TX_CACHE, f"{phase}{dc}Tx", offset, TX_BURST, - atomicity="TRANSACTIONAL"), - self._admissible_burst(mdc, dc, "SQL_PUT", SQL_CACHE, f"{phase}{dc}Sql", offset, SQL_BURST), - ] - - def _admissible_put_burst(self, mdc: MdcCluster, dc: str, phase: str, offset: int): - """ - Runs a single cache API put burst expected to fully succeed. - """ - return self._admissible_burst(mdc, dc, "PUT", ATOMIC_CACHE, f"{phase}{dc}Put", offset, PUT_BURST) - - @staticmethod - def _admissible_burst(mdc: MdcCluster, dc: str, mode: str, cache: str, prefix: str, - offset: int, iterations: int, **cache_params): - """ - Runs one write burst that must succeed completely: the load application fails fast - on any exception, and the acknowledged count must match the requested iterations. - - :return: (cache, kind, dc, offset, count) tuple for the readback stage. - """ - svc = mdc.run_load(dc, mode, cache, prefix, keyFrom=offset, iterations=iterations, - createCache=True, backups=1, mainDc=DC_1, **cache_params) - - ops = mdc.result_int(svc, f"{prefix}OpsCnt") - - assert ops == iterations, \ - f"Write burst is incomplete [dc={dc}, mode={mode}, cache={cache}, exp={iterations}, ops={ops}]" - - kind = "sql" if mode.startswith("SQL") else "data" - - return cache, kind, dc, offset, ops diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index 486ab981b6429..86300a318d72f 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -22,7 +22,7 @@ """ from time import sleep -from ducktape.mark import matrix +from ducktape.mark import parametrize from ignitetest.services.mdc.mdc_cluster import MdcCluster, cross_dc_network, DC_1, DC_2 from ignitetest.utils import cluster, ignite_versions @@ -49,7 +49,7 @@ class MdcPartitionResilienceTest(IgniteTest): """ @cluster(num_nodes=12) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[25, 50, 100]) + @parametrize(cross_dc_latency_ms=100) def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency_ms): """ The canonical split-brain lifecycle: partition -> split into two healthy half-rings @@ -98,7 +98,7 @@ def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency @cluster(num_nodes=6) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[25, 50, 100]) + @parametrize(cross_dc_latency_ms=100) def test_main_dc_loss_and_return(self, ignite_version, cross_dc_latency_ms): """ The inverse of the canonical scenario: the MAIN data center goes down entirely. @@ -147,7 +147,7 @@ def test_main_dc_loss_and_return(self, ignite_version, cross_dc_latency_ms): @cluster(num_nodes=10) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[25, 50, 100]) + @parametrize(cross_dc_latency_ms=100) def test_short_partition_blips_do_not_split(self, ignite_version, cross_dc_latency_ms): """ A flapping WAN link: several short (below the failure detection timeout) full diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py index 1c368baa7b88c..ddde6d0cfa996 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py @@ -30,7 +30,7 @@ """ from time import sleep -from ducktape.mark import matrix +from ducktape.mark import parametrize from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.mdc.mdc_cluster import MdcCluster, dc_jvm_opts, DC_1, DC_2, cross_dc_network, THIN_LOAD_APP @@ -59,7 +59,7 @@ class MdcThinClientTest(IgniteTest): """ @cluster(num_nodes=8) @ignite_versions(str(DEV_BRANCH)) - @matrix(cross_dc_latency_ms=[25, 50, 100]) + @parametrize(cross_dc_latency_ms=100) def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_dc_latency_ms): """ DC-pinned thin client reads locally (latency far below the cross-DC delay); From aee65b494c90e377b7e76e9799a15a31168cb69d Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 15 Jul 2026 14:16:14 +0300 Subject: [PATCH 23/44] IGNITE-27833 WIP --- .../mdc/MdcContinuousLoadApplication.java | 20 ++- .../tests/mdc/transactional_partition_test.py | 136 ++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java index d572148f9fc7d..e981b720c6b1a 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java @@ -42,6 +42,11 @@ *

  • {@code tolerateErrors} - if {@code true}, operation failures are counted instead of * failing fast. Intended for background loads crossing a partition boundary, where a * short transient error window is possible;
    • + *
  • {@code stopOnError} - if {@code true}, the load stops gracefully on the very first + * operation failure (rather than failing the application), records the number of + * successful operations and a {@code StoppedOnError} flag, and finishes. Takes precedence + * over {@code tolerateErrors}. Intended for a load that must be cut off by the first + * exception a network partition triggers;
    • *
  • {@code opPauseMs} - pause between operations, default 0;
    • *
  • {@code resultPrefix} - prefix for recorded results, so that several runs reusing one * service produce uniquely named results;
    • @@ -86,6 +91,7 @@ private enum Mode { boolean expectAdmissible = jNode.path("expectAdmissible").asBoolean(true); boolean tolerateErrors = jNode.path("tolerateErrors").asBoolean(false); + boolean stopOnError = jNode.path("stopOnError").asBoolean(false); long opPauseMs = jNode.path("opPauseMs").asLong(0); @@ -122,6 +128,8 @@ private enum Mode { long opsCnt = 0; long errCnt = 0; + boolean stoppedOnError = false; + OpStats stats = new OpStats(); long maxStallMs = 0; @@ -210,6 +218,15 @@ private enum Mode { if (writeMode && !expectAdmissible) log.info("Write rejected as expected [dc=" + dcId() + ", key=" + key + ", msg=" + e.getMessage() + "]"); + else if (stopOnError) { + log.info("Operation failed, cutting the load on first error [dc=" + dcId() + ", mode=" + mode + + ", key=" + key + ", succeeded=" + opsCnt + ", msg=" + e.getMessage() + "]"); + + errCnt++; + stoppedOnError = true; + + break; + } else if (!tolerateErrors) throw new IllegalStateException("Operation failed [dc=" + dcId() + ", mode=" + mode + ", key=" + key + "]", e); @@ -251,6 +268,7 @@ else if (!tolerateErrors) recordResult(pfx + "OpsCnt", String.valueOf(opsCnt)); recordResult(pfx + "ErrCnt", String.valueOf(errCnt)); + recordResult(pfx + "StoppedOnError", String.valueOf(stoppedOnError)); recordResult(pfx + "DurationMs", String.valueOf(durationMs)); recordResult(pfx + "AvgOpMs", fmtMs(stats.avgNs())); @@ -262,7 +280,7 @@ else if (!tolerateErrors) recordResult(pfx + "MaxStallMs", String.valueOf(maxStallMs)); log.info("MDC load finished [dc=" + dcId() + ", mode=" + mode + ", ops=" + opsCnt + - ", errs=" + errCnt + ", durationMs=" + durationMs + + ", errs=" + errCnt + ", stoppedOnError=" + stoppedOnError + ", durationMs=" + durationMs + ", avgOpMs=" + fmtMs(stats.avgNs()) + ", maxOpMs=" + fmtMs(stats.maxNs()) + ", maxStallMs=" + maxStallMs + "]"); diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py new file mode 100644 index 0000000000000..af7192378e65a --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +MDC transactional load through a cross-DC network partition. + +A TRANSACTIONAL cache spans both data centers: with backups=1 and the +MdcAffinityBackupFilter every partition owns exactly one copy per DC, so every +implicit-transaction write (a plain put on a transactional cache) must reach a +node in the other DC. A continuous single-threaded insert load runs from the main +DC; the instant the DCs are partitioned the next commit cannot reach all partition +copies and fails with a cache exception. The load cuts itself off on that first +exception and records how many inserts had succeeded. + +After the split settles the test asserts that the cluster really split-brained, +that the load stopped because of the partition (not because it ran out of work), +and - the point of the scenario - that the aborted implicit transactions left +nothing hanging on either half-ring and no suspicious entries in the server logs. + +Data accessibility during the split is deliberately NOT checked: a transactional +cache needs all partition copies available, which a split-brained half-ring cannot +offer. +""" +from time import sleep + +from ducktape.mark import parametrize + +from ignitetest.services.mdc.mdc_cluster import MdcCluster, cross_dc_network, DC_1, DC_2 +from ignitetest.utils import cluster, ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH + +CACHE_NAME = "mdc-tx-load" + +BACKUPS = 1 + +# Small seed so the cross-DC cache distribution can be verified before the split. +SEED_KEYS = 100 + +# Fresh, disjoint key range for the continuous insert load: the load advances the key +# on every success, so [LOAD_KEY_FROM, LOAD_KEY_FROM + successfulInserts) gets inserted. +LOAD_KEY_FROM = 1_000_000 +LOAD_KEY_TO = 100_000_000 + +# Let the load accumulate successful inserts before the DCs are cut apart. +LOAD_WARMUP_SECS = 5 + +# Time for discovery to detect the partition and for both half-rings to complete PME +# and account for the split. +SPLIT_SETTLE_SECS = 15 + + +class MdcTransactionalPartitionTest(IgniteTest): + """ + Transactional load resilience to a cross-DC network partition. + """ + @cluster(num_nodes=6) + @ignite_versions(str(DEV_BRANCH)) + @parametrize(cross_dc_latency_ms=20) + def test_transactional_load_cut_on_partition(self, ignite_version, cross_dc_latency_ms): + """ + Continuous implicit-transaction insert load from the main DC is cut off by the first + cache exception the cross-DC partition triggers; afterwards no transaction is left + hanging on either half-ring and the server logs are clean. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=2, + runners_per_dc={DC_1: 1}, loaders_per_dc={DC_1: 1}) + + with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: + mdc.start_servers() + + # Create the TRANSACTIONAL cache and seed it, so every partition owns one copy + # per DC - the reason an implicit-transaction write must touch both DCs. + mdc.generate_data(DC_1, CACHE_NAME, 0, SEED_KEYS, backups=BACKUPS, atomicity="TRANSACTIONAL") + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) + + # Continuous single-threaded implicit-transaction insert load from the main DC. + # It stops on the very first exception (stopOnError) instead of failing the app, + # recording how many inserts had succeeded up to that point. + mdc.start_loader(DC_1, { + "mode": "PUT", + "cacheName": CACHE_NAME, + "keyFrom": LOAD_KEY_FROM, + "keyTo": LOAD_KEY_TO, + "stopOnError": True, + "resultPrefix": "txLoad" + }) + + sleep(LOAD_WARMUP_SECS) + + net.enable_network_partition(DC_1, DC_2) + + # The load hits its first exception here and cuts itself off; give discovery and + # PME time to detect the split and settle into two independent half-rings. + sleep(SPLIT_SETTLE_SECS) + + mdc.verify_split_brain() + + # The load has already finished on its own - this just collects its results. + svc = mdc.stop_loader(DC_1) + + inserts = mdc.result_int(svc, "txLoadOpsCnt") + stopped_on_error = svc.extract_result("txLoadStoppedOnError") + + self.logger.info(f"Transactional load cut by the partition " + f"[insertsBeforeSplit={inserts}, stoppedOnError={stopped_on_error}]") + + assert inserts > 0, "The transactional load performed no successful inserts before the split" + + assert stopped_on_error == "true", \ + "The transactional load must be cut off by the partition's first cache exception " \ + f"[stoppedOnError={stopped_on_error}, inserts={inserts}]" + + # The point of the scenario: the aborted implicit transactions leave nothing + # hanging on either half-ring... + mdc.verify_no_hanging_txs(DC_1) + mdc.verify_no_hanging_txs(DC_2) + + # ...and neither half-ring logged a hung PME, a long running transaction or a + # lost partition. + mdc.verify_servers_log_clean() + + mdc.stop_servers() From d381a49aeb62b63b52e33a380bcf09119d758944 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 15 Jul 2026 15:46:00 +0300 Subject: [PATCH 24/44] IGNITE-27833 WIP --- .../services/utils/control_utility.py | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index c8f6cbe19bad4..aad6d9a2bfdfd 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -38,6 +38,9 @@ class ControlUtility: """ BASE_COMMAND = "control.sh" + # Column separator used by control.sh --system-view (SystemViewCommand.COLUMN_SEPARATOR). + SYSTEM_VIEW_COLUMN_SEPARATOR = " " + def __init__(self, cluster, ssl_params=None, username=None, password=None): self._cluster = cluster self.logger = cluster.context.logger @@ -258,6 +261,118 @@ def __parse_cache_distribution(output, user_attributes=None): return CacheDistribution(groups=groups) + def system_view(self, name, node=None, node_id=None, node_ids=None, all_nodes=False): + """ + Prints the content of a system view (``control.sh --system-view``). + + :param name: System view name. Both the SQL ("CACHES") and Java ("caches") styles are accepted. + :param node: Node to run control.sh on (a random alive node if None). + :param node_id: Single node id to read the view from (discouraged upstream, prefer + ``node_ids`` or ``all_nodes``). + :param node_ids: List of node ids to read the view from. + :param all_nodes: Read the view from all nodes. + :return: Dict mapping node id (str) to a list of rows; each row is a dict keyed by the + view column name (e.g. ``row["CACHE_NAME"]``). + """ + cmd = f"--system-view {name}" + + if all_nodes: + cmd += " --all-nodes" + elif node_ids: + cmd += f" --node-ids {','.join(node_ids)}" + elif node_id: + cmd += f" --node-id {node_id}" + + return self.__parse_system_view(self.__run(cmd, node=node), name) + + def caches(self, cache_name=None, node=None, node_id=None, node_ids=None, all_nodes=False): + """ + Reads the CACHES system view and returns its rows. + + Each row is a dict keyed by the view column names, e.g. ``CACHE_NAME``, ``CACHE_ID``, + ``CACHE_GROUP_ID``, ``CACHE_GROUP_NAME``, ``CACHE_TYPE``, ``CACHE_MODE``, + ``ATOMICITY_MODE``, ``BACKUPS``. + + :param cache_name: If set, only rows whose ``CACHE_NAME`` equals it are returned. + :return: List of rows (dicts). CACHES is cluster-wide, so by default the rows of a + single node are returned to avoid duplicates; when ``node_ids`` or + ``all_nodes`` is requested the rows of every queried node are concatenated. + """ + by_node = self.system_view("CACHES", node=node, node_id=node_id, node_ids=node_ids, all_nodes=all_nodes) + + if node_ids or all_nodes: + rows = [row for node_rows in by_node.values() for row in node_rows] + else: + rows = next(iter(by_node.values()), []) + + if cache_name is not None: + rows = [row for row in rows if row.get("CACHE_NAME") == cache_name] + + return rows + + @staticmethod + def __parse_system_view(output, name): + """ + Parses ``control.sh --system-view`` output. + + The command prints one block per node:: + + Results from node with ID: + --- + COL_A COL_B ... + valA valB ... + --- + + Columns are padded to their widest value and joined by the 4-space column separator; + cells never hold an empty value (nulls print as "null"), so splitting on the separator + and dropping the padding gaps recovers the columns - the same approach the upstream + SystemViewCommandTest uses. + """ + if "No system view with specified name was found" in output: + raise AssertionError(f"No system view named '{name}' was found:\n{output}") + + node_pattern = re.compile(r"Results from node with ID: (?P\S+)") + sep = ControlUtility.SYSTEM_VIEW_COLUMN_SEPARATOR + + result = {} + node_id = None + header = None + # idle -> await_open (saw the node header) -> body (between the two '---' delimiters). + state = "idle" + + for line in output.splitlines(): + stripped = line.strip() + + match = node_pattern.match(stripped) + if match: + node_id = match.group("node_id") + result[node_id] = [] + header = None + state = "await_open" + continue + + if state == "await_open": + if stripped == "---": + state = "body" + continue + + if state == "body": + if stripped == "---": + state = "idle" + continue + + if not stripped: + continue + + cells = [cell.strip() for cell in line.split(sep) if cell.strip()] + + if header is None: + header = cells + else: + result[node_id].append(dict(zip(header, cells))) + + return result + def check_consistency(self, args): """ Consistency check. From 6509df66e5250ddc1e804f0a8b5af45165e9b3e1 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 15 Jul 2026 17:58:09 +0300 Subject: [PATCH 25/44] IGNITE-27833 WIP --- .../mdc/MdcContinuousLoadApplication.java | 4 +- .../services/utils/control_utility.py | 10 +-- .../tests/mdc/transactional_partition_test.py | 76 ++++++++++--------- 3 files changed, 47 insertions(+), 43 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java index e981b720c6b1a..bb4f522e7e5d3 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java @@ -219,8 +219,8 @@ private enum Mode { log.info("Write rejected as expected [dc=" + dcId() + ", key=" + key + ", msg=" + e.getMessage() + "]"); else if (stopOnError) { - log.info("Operation failed, cutting the load on first error [dc=" + dcId() + ", mode=" + mode + - ", key=" + key + ", succeeded=" + opsCnt + ", msg=" + e.getMessage() + "]"); + log.warn("Operation failed, cutting the load on first error [dc=" + dcId() + ", mode=" + mode + + ", key=" + key + ", succeeded=" + opsCnt + ", msg=" + e.getMessage() + "]", e); errCnt++; stoppedOnError = true; diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index aad6d9a2bfdfd..daa0481c376ab 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -38,9 +38,6 @@ class ControlUtility: """ BASE_COMMAND = "control.sh" - # Column separator used by control.sh --system-view (SystemViewCommand.COLUMN_SEPARATOR). - SYSTEM_VIEW_COLUMN_SEPARATOR = " " - def __init__(self, cluster, ssl_params=None, username=None, password=None): self._cluster = cluster self.logger = cluster.context.logger @@ -285,7 +282,7 @@ def system_view(self, name, node=None, node_id=None, node_ids=None, all_nodes=Fa return self.__parse_system_view(self.__run(cmd, node=node), name) - def caches(self, cache_name=None, node=None, node_id=None, node_ids=None, all_nodes=False): + def system_view_caches(self, cache_name=None, node=None, node_id=None, node_ids=None, all_nodes=False): """ Reads the CACHES system view and returns its rows. @@ -325,14 +322,13 @@ def __parse_system_view(output, name): Columns are padded to their widest value and joined by the 4-space column separator; cells never hold an empty value (nulls print as "null"), so splitting on the separator - and dropping the padding gaps recovers the columns - the same approach the upstream - SystemViewCommandTest uses. + and dropping the padding gaps recovers the columns. """ if "No system view with specified name was found" in output: raise AssertionError(f"No system view named '{name}' was found:\n{output}") node_pattern = re.compile(r"Results from node with ID: (?P\S+)") - sep = ControlUtility.SYSTEM_VIEW_COLUMN_SEPARATOR + sep = " " result = {} node_id = None diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py index af7192378e65a..edf87e49d7131 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py @@ -46,16 +46,16 @@ BACKUPS = 1 -# Small seed so the cross-DC cache distribution can be verified before the split. -SEED_KEYS = 100 - # Fresh, disjoint key range for the continuous insert load: the load advances the key # on every success, so [LOAD_KEY_FROM, LOAD_KEY_FROM + successfulInserts) gets inserted. -LOAD_KEY_FROM = 1_000_000 -LOAD_KEY_TO = 100_000_000 +LOAD_KEY_FROM_DC_1 = 1_000_000 +LOAD_KEY_TO_DC_1 = 10_000_000 + +LOAD_KEY_FROM_DC_2 = 10_000_000 +LOAD_KEY_TO_DC_2 = 20_000_000 # Let the load accumulate successful inserts before the DCs are cut apart. -LOAD_WARMUP_SECS = 5 +LOAD_WARMUP_SECS = 15 # Time for discovery to detect the partition and for both half-rings to complete PME # and account for the split. @@ -66,38 +66,36 @@ class MdcTransactionalPartitionTest(IgniteTest): """ Transactional load resilience to a cross-DC network partition. """ - @cluster(num_nodes=6) + @cluster(num_nodes=8) @ignite_versions(str(DEV_BRANCH)) - @parametrize(cross_dc_latency_ms=20) + @parametrize(cross_dc_latency_ms=100) def test_transactional_load_cut_on_partition(self, ignite_version, cross_dc_latency_ms): """ Continuous implicit-transaction insert load from the main DC is cut off by the first cache exception the cross-DC partition triggers; afterwards no transaction is left hanging on either half-ring and the server logs are clean. """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=2, - runners_per_dc={DC_1: 1}, loaders_per_dc={DC_1: 1}) + mdc = MdcCluster(self, ignite_version, srv_per_dc=2, runners_per_dc=1, loaders_per_dc=1, network_timeout=20_000, tcp_connect_timeout=10_000) with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() - # Create the TRANSACTIONAL cache and seed it, so every partition owns one copy - # per DC - the reason an implicit-transaction write must touch both DCs. - mdc.generate_data(DC_1, CACHE_NAME, 0, SEED_KEYS, backups=BACKUPS, atomicity="TRANSACTIONAL") - - mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) - # Continuous single-threaded implicit-transaction insert load from the main DC. # It stops on the very first exception (stopOnError) instead of failing the app, # recording how many inserts had succeeded up to that point. - mdc.start_loader(DC_1, { - "mode": "PUT", - "cacheName": CACHE_NAME, - "keyFrom": LOAD_KEY_FROM, - "keyTo": LOAD_KEY_TO, - "stopOnError": True, - "resultPrefix": "txLoad" - }) + for dc, offset_from, to in [(DC_1, LOAD_KEY_FROM_DC_1, LOAD_KEY_TO_DC_1), (DC_2, LOAD_KEY_FROM_DC_2, LOAD_KEY_TO_DC_2)]: + mdc.start_loader(dc, { + "mode": "TX_PUT", + "cacheName": CACHE_NAME, + "keyFrom": offset_from, + "keyTo": to, + "stopOnError": True, + "resultPrefix": f"txLoad{dc}", + "createCache": True, + "backups": BACKUPS, + "atomicity": "TRANSACTIONAL", + "mainDc": DC_1, + }) sleep(LOAD_WARMUP_SECS) @@ -110,19 +108,16 @@ def test_transactional_load_cut_on_partition(self, ignite_version, cross_dc_late mdc.verify_split_brain() # The load has already finished on its own - this just collects its results. - svc = mdc.stop_loader(DC_1) - - inserts = mdc.result_int(svc, "txLoadOpsCnt") - stopped_on_error = svc.extract_result("txLoadStoppedOnError") + for dc in [DC_1, DC_2]: + svc = mdc.stop_loader(dc) - self.logger.info(f"Transactional load cut by the partition " - f"[insertsBeforeSplit={inserts}, stoppedOnError={stopped_on_error}]") + inserts = mdc.result_int(svc, f"txLoad{dc}OpsCnt") + errors = mdc.result_int(svc, f"txLoad{dc}ErrCnt") - assert inserts > 0, "The transactional load performed no successful inserts before the split" + self.logger.info(f"Transactional load cut by the partition " + f"[txLoad{dc}OpsCnt={inserts}, txLoad{dc}ErrCnt={errors}]") - assert stopped_on_error == "true", \ - "The transactional load must be cut off by the partition's first cache exception " \ - f"[stoppedOnError={stopped_on_error}, inserts={inserts}]" + assert inserts > 0, "The transactional load performed no successful inserts" # The point of the scenario: the aborted implicit transactions leave nothing # hanging on either half-ring... @@ -133,4 +128,17 @@ def test_transactional_load_cut_on_partition(self, ignite_version, cross_dc_late # lost partition. mdc.verify_servers_log_clean() + net.disable_network_partition(DC_1, DC_2) + + mdc.restart(DC_2) + + mdc.check_put_admissibility(DC_1, CACHE_NAME, True, key_offset=100_000_000) + mdc.check_put_admissibility(DC_2, CACHE_NAME, True, key_offset=101_000_000) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) + + mdc.control(DC_1).idle_verify(CACHE_NAME) + + mdc.verify_servers_log_clean() + mdc.stop_servers() From 77994507fd06f542e0a8c6df531a6c075f7b5e7f Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 15 Jul 2026 17:58:27 +0300 Subject: [PATCH 26/44] IGNITE-27833 WIP --- modules/ducktests/tests/docker/Dockerfile | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index b8271f4a22ba4..db6f4222f0ec4 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -126,31 +126,31 @@ RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" ARG APACHE_ARCHIVE="https://archive.apache.org/dist/" -## Install Ignite -#RUN for v in "2.7.6" "2.17.0"; \ -# do cd /opt; \ -# curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\ -# unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v; \ -# done \ -# && rm /opt/apache-ignite-*-bin.zip -# -## Install Zookeeper -#ARG ZOOKEEPER_VERSION="3.5.8" -#ARG ZOOKEEPER_NAME="zookeeper-$ZOOKEEPER_VERSION" -#ARG ZOOKEEPER_RELEASE_NAME="apache-$ZOOKEEPER_NAME-bin" -#ARG ZOOKEEPER_RELEASE_ARTIFACT="$ZOOKEEPER_RELEASE_NAME.tar.gz" -#RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT \ -# && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT \ -# && mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME -# -## Install Kafka -#ARG KAFKA_VERSION="3.9.1" -#ARG KAFKA_NAME="kafka" -#ARG KAFKA_RELEASE_NAME="${KAFKA_NAME}_2.13-$KAFKA_VERSION" -#ARG KAFKA_RELEASE_ARTIFACT="$KAFKA_RELEASE_NAME.tgz" -#RUN cd /opt && curl -O $APACHE_ARCHIVE/kafka/$KAFKA_VERSION/$KAFKA_RELEASE_ARTIFACT \ -# && tar xvf $KAFKA_RELEASE_ARTIFACT && rm $KAFKA_RELEASE_ARTIFACT \ -# && mv /opt/$KAFKA_RELEASE_NAME /opt/$KAFKA_NAME-$KAFKA_VERSION +# Install Ignite +RUN for v in "2.7.6" "2.17.0"; \ + do cd /opt; \ + curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\ + unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v; \ + done \ + && rm /opt/apache-ignite-*-bin.zip + +# Install Zookeeper +ARG ZOOKEEPER_VERSION="3.5.8" +ARG ZOOKEEPER_NAME="zookeeper-$ZOOKEEPER_VERSION" +ARG ZOOKEEPER_RELEASE_NAME="apache-$ZOOKEEPER_NAME-bin" +ARG ZOOKEEPER_RELEASE_ARTIFACT="$ZOOKEEPER_RELEASE_NAME.tar.gz" +RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT \ + && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT \ + && mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME + +# Install Kafka +ARG KAFKA_VERSION="3.9.1" +ARG KAFKA_NAME="kafka" +ARG KAFKA_RELEASE_NAME="${KAFKA_NAME}_2.13-$KAFKA_VERSION" +ARG KAFKA_RELEASE_ARTIFACT="$KAFKA_RELEASE_NAME.tgz" +RUN cd /opt && curl -O $APACHE_ARCHIVE/kafka/$KAFKA_VERSION/$KAFKA_RELEASE_ARTIFACT \ + && tar xvf $KAFKA_RELEASE_ARTIFACT && rm $KAFKA_RELEASE_ARTIFACT \ + && mv /opt/$KAFKA_RELEASE_NAME /opt/$KAFKA_NAME-$KAFKA_VERSION # Install Jmxterm ARG JMXTERM_NAME="jmxterm" From 3f15bf69f06d8cc66b9cedd6863708d32edd1378 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 15 Jul 2026 19:40:51 +0300 Subject: [PATCH 27/44] IGNITE-27833 WIP --- .../ducktest/tests/dto/IndexedDataRecord.java | 17 ++++++++ .../tests/mdc/MdcCacheAwareApplication.java | 41 +++++++++++++++++-- .../mdc/MdcContinuousLoadApplication.java | 19 ++++++++- .../tests/mdc/MdcDataCheckerApplication.java | 17 ++++++++ .../mdc/MdcDataGeneratorApplication.java | 17 ++++++++ ...MdcPutAdmissibilityCheckerApplication.java | 17 ++++++++ .../mdc/MdcThinClientLoadApplication.java | 20 ++++++++- .../internal/ducktest/utils/OpStats.java | 17 ++++++++ .../ignite/internal/ducktest/utils/Utils.java | 17 ++++++++ .../tests/ignitetest/services/ignite_app.py | 12 +++--- .../tests/ignitetest/services/mdc/__init__.py | 2 +- .../ignitetest/services/mdc/mdc_cluster.py | 6 ++- .../services/network_group/__init__.py | 2 +- .../services/network_group/configuration.py | 3 +- .../services/network_group/manager.py | 2 +- .../tests/mdc/partition_resilience_test.py | 9 ++-- .../tests/mdc/transactional_partition_test.py | 6 ++- 17 files changed, 201 insertions(+), 23 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dto/IndexedDataRecord.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dto/IndexedDataRecord.java index 00250c3b3444e..f479d663a31cf 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dto/IndexedDataRecord.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dto/IndexedDataRecord.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.ducktest.tests.dto; import java.util.Arrays; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java index 11766e97cbf12..4459990515a4b 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java @@ -1,7 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.ducktest.tests.mdc; import java.util.Collections; import java.util.HashSet; +import java.util.Locale; import java.util.Set; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.IgniteCache; @@ -149,17 +167,34 @@ protected static String dcId() { return IgniteSystemProperties.getString(IGNITE_DATA_CENTER_ID); } - /** */ + /** + * Parses an optional enum-valued field. A missing, null or unrecognized value falls back + * to {@code dfltVal}. + */ protected static > E getEnum(JsonNode jNode, String fieldName, E dfltVal) { JsonNode field = jNode.path(fieldName); if (field.isMissingNode() || field.isNull()) return dfltVal; + try { - return Enum.valueOf(dfltVal.getDeclaringClass(), field.asText().toUpperCase()); + return Enum.valueOf(dfltVal.getDeclaringClass(), field.asText().toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { - return dfltVal; // Fallback if string doesn't match any enum constant + return dfltVal; // Fallback if string doesn't match any enum constant. } } + + /** + * Parses a required enum-valued field. Throws if the field is missing, null or not a valid + * constant of {@code cls} (case-insensitively) - a typo must fail the run, not pick a default. + */ + protected static > E getEnum(JsonNode jNode, String fieldName, Class cls) { + JsonNode field = jNode.path(fieldName); + + if (field.isMissingNode() || field.isNull()) + throw new IllegalArgumentException("Missing required enum field '" + fieldName + "'"); + + return Enum.valueOf(cls, field.asText().toUpperCase(Locale.ROOT)); + } } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java index bb4f522e7e5d3..ae0bba0841d15 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.ducktest.tests.mdc; import java.util.List; @@ -80,7 +97,7 @@ private enum Mode { /** {@inheritDoc} */ @Override public void run(JsonNode jNode) throws Exception { - Mode mode = Mode.valueOf(jNode.get("mode").asText().toUpperCase()); + Mode mode = getEnum(jNode, "mode", Mode.class); String cacheName = jNode.path("cacheName").asText(DFLT_CACHE_NAME); boolean createCache = jNode.path("createCache").asBoolean(false); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java index 6de7cb544d2ee..23e7cf7764725 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.ducktest.tests.mdc; import com.fasterxml.jackson.databind.JsonNode; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java index b0be33b4e123d..6b1881f887199 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.ducktest.tests.mdc; import java.util.Map; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java index 6ec4be1cda6a0..cbf50ab398c09 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.ducktest.tests.mdc; import javax.cache.CacheException; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java index 7f1e67ee4bf2f..cc0fb7c771224 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.ducktest.tests.mdc; +import java.util.Locale; import javax.cache.CacheException; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.client.ClientCache; @@ -27,7 +45,7 @@ public class MdcThinClientLoadApplication extends IgniteAwareApplication { /** {@inheritDoc} */ @Override public void run(JsonNode jNode) throws Exception { - String mode = jNode.get("mode").asText().toUpperCase(); + String mode = jNode.get("mode").asText().toUpperCase(Locale.ROOT); String cacheName = jNode.get("cacheName").asText(); int keyFrom = jNode.path("keyFrom").asInt(0); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java index 7f54c0967a40d..331cb5b915737 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/OpStats.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.ducktest.utils; /** Accumulates latency of completed operations. */ diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java index d9f61e5bfc675..1befd8498a1b1 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.ducktest.utils; import java.util.Locale; diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 268a63354dc0f..833cb2ed63034 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -44,26 +44,26 @@ def __init__(self, context, config, java_class_name=None, num_nodes=1, params="" self.params = params def await_started(self, nodes=None): - super().await_started() + super().await_started(nodes) - self.__check_status(self.APP_INIT_EVT_MSG, timeout=self.startup_timeout_sec) + self.__check_status(self.APP_INIT_EVT_MSG, timeout=self.startup_timeout_sec, nodes=nodes) def await_stopped(self): super().await_stopped() self.__check_status(self.APP_FINISH_EVT_MSG) - def __check_status(self, desired, timeout=1): - self.await_event("%s\\|%s" % (desired, self.APP_BROKEN_EVT_MSG), timeout, from_the_beginning=True) + def __check_status(self, desired, timeout=1, nodes=None): + self.await_event("%s\\|%s" % (desired, self.APP_BROKEN_EVT_MSG), timeout, nodes=nodes, from_the_beginning=True) try: - self.await_event(self.APP_BROKEN_EVT_MSG, 1, from_the_beginning=True) + self.await_event(self.APP_BROKEN_EVT_MSG, 1, nodes=nodes, from_the_beginning=True) raise IgniteExecutionException("Java application execution failed. %s" % self.extract_result("ERROR")) except TimeoutError: pass try: - self.await_event(desired, 1, from_the_beginning=True) + self.await_event(desired, 1, nodes=nodes, from_the_beginning=True) except Exception: raise Exception("Java application execution failed.") from None diff --git a/modules/ducktests/tests/ignitetest/services/mdc/__init__.py b/modules/ducktests/tests/ignitetest/services/mdc/__init__.py index ba4f3680ef3e1..052a4803df6a2 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/__init__.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/__init__.py @@ -15,4 +15,4 @@ """ Multi DC Cluster Service -""" \ No newline at end of file +""" diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py index afad737106de8..cac0039b556f1 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -115,7 +115,8 @@ def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, i self.srv_per_dc = _per_dc(srv_per_dc) self.servers: Dict[str, IgniteService] = { - dc: IgniteService(self.test_context, self.ignite_config, num_nodes=num, jvm_opts=dc_jvm_opts(dc), startup_timeout_sec=IGNITE_STARTUP_TIMEOUT_SEC) + dc: IgniteService(self.test_context, self.ignite_config, num_nodes=num, jvm_opts=dc_jvm_opts(dc), + startup_timeout_sec=IGNITE_STARTUP_TIMEOUT_SEC) for dc, num in self.srv_per_dc.items() if num > 0} self.runners: Dict[str, List[IgniteApplicationService]] = { @@ -444,7 +445,8 @@ def cross_dc_network(logger, mdc: MdcCluster, delay_ms: Optional[int] = None, return NetworkGroupManager(logger, store, mdc.network_registry()) -def assert_cross_dc_distribution_by_attribute(distribution, dc_attr, expected_dcs, owning_only=True, copies_per_dc=None): +def assert_cross_dc_distribution_by_attribute(distribution, dc_attr, expected_dcs, owning_only=True, + copies_per_dc=None): """ Asserts that every partition of every cache group has at least one copy in every DC, using a node attribute (requested via --user-attributes) as the DC marker. diff --git a/modules/ducktests/tests/ignitetest/services/network_group/__init__.py b/modules/ducktests/tests/ignitetest/services/network_group/__init__.py index 321d706c84f58..867bd40a8353d 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/__init__.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/__init__.py @@ -18,4 +18,4 @@ This module provides tools for simulating complex network topologies by defining traffic impairments between logical groups. -""" \ No newline at end of file +""" diff --git a/modules/ducktests/tests/ignitetest/services/network_group/configuration.py b/modules/ducktests/tests/ignitetest/services/network_group/configuration.py index 7f3fde990f5f9..3fbf382b8c35f 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/configuration.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/configuration.py @@ -24,7 +24,6 @@ class CrossNetworkGroupConfiguration: """ delay: Optional[str] = None # tcset time expression, e.g. "100ms" loss: Optional[float] = None # fraction in [0.0, 1.0], e.g. 0.1 (10%) - rate: str = "1gbit" # Default to high-speed interface def __post_init__(self): if self.loss is not None and not 0.0 <= self.loss <= 1.0: @@ -62,4 +61,4 @@ def get_config(self, src_group: str, dst_group: str) -> Optional[CrossNetworkGro """ :return: :class:`CrossNetworkGroupConfiguration` for traffic from src to dst or None if not defined. """ - return self.matrix.get(src_group, {}).get(dst_group) \ No newline at end of file + return self.matrix.get(src_group, {}).get(dst_group) diff --git a/modules/ducktests/tests/ignitetest/services/network_group/manager.py b/modules/ducktests/tests/ignitetest/services/network_group/manager.py index ac985ec89ca6d..a55f9a7b69418 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/manager.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/manager.py @@ -52,7 +52,7 @@ class NetworkGroupManager: Deploys and tears down traffic-control rules between logical node groups, and toggles full network partitions between them at test time. - Baseline impairments (delay/loss/rate) are deployed once via tcset. + Baseline impairments (delay/loss) are deployed once via tcset. Partitions are layered on top as per-pair iptables DROP chains. The netem rules are never touched by a partition, so healing is a pure chain flush that automatically restores the originally deployed impairments. diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index 86300a318d72f..bb8827267b65b 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -56,7 +56,8 @@ def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency (DC1 active, DC2 read-only) -> all data readable everywhere -> heal -> DC2 rejoins via restart -> writes restored everywhere, distribution and consistency verified. """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=5, runners_per_dc=1, network_timeout=20_000, tcp_connect_timeout=10_000) + mdc = MdcCluster(self, ignite_version, srv_per_dc=5, runners_per_dc=1, + network_timeout=20_000, tcp_connect_timeout=10_000) with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() @@ -191,8 +192,10 @@ def test_short_partition_blips_do_not_split(self, ignite_version, cross_dc_laten self.logger.info(f"Background GET load [dc={dc}, ops={ops}, errs={errs}, maxStallMs={max_stall}]") assert ops > 0, f"Background get load performed no operations [dc={dc}]" - assert errs == 0, f"Background get load errors exceed the boundary tolerance [dc={dc}, ops={ops}, errs={errs}]" - assert max_stall < BG_MAX_STALL_MS, f"Background get load stalled for too long [dc={dc}, maxStallMs={max_stall}]" + assert errs == 0, \ + f"Background get load errors exceed the boundary tolerance [dc={dc}, ops={ops}, errs={errs}]" + assert max_stall < BG_MAX_STALL_MS, \ + f"Background get load stalled for too long [dc={dc}, maxStallMs={max_stall}]" mdc.check_data(DC_1, CACHE_NAME, 0, 200) mdc.check_data(DC_2, CACHE_NAME, 0, 200) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py index edf87e49d7131..73364e2a55221 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py @@ -75,7 +75,8 @@ def test_transactional_load_cut_on_partition(self, ignite_version, cross_dc_late cache exception the cross-DC partition triggers; afterwards no transaction is left hanging on either half-ring and the server logs are clean. """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=2, runners_per_dc=1, loaders_per_dc=1, network_timeout=20_000, tcp_connect_timeout=10_000) + mdc = MdcCluster(self, ignite_version, srv_per_dc=2, runners_per_dc=1, loaders_per_dc=1, + network_timeout=20_000, tcp_connect_timeout=10_000) with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() @@ -83,7 +84,8 @@ def test_transactional_load_cut_on_partition(self, ignite_version, cross_dc_late # Continuous single-threaded implicit-transaction insert load from the main DC. # It stops on the very first exception (stopOnError) instead of failing the app, # recording how many inserts had succeeded up to that point. - for dc, offset_from, to in [(DC_1, LOAD_KEY_FROM_DC_1, LOAD_KEY_TO_DC_1), (DC_2, LOAD_KEY_FROM_DC_2, LOAD_KEY_TO_DC_2)]: + for dc, offset_from, to in [(DC_1, LOAD_KEY_FROM_DC_1, LOAD_KEY_TO_DC_1), + (DC_2, LOAD_KEY_FROM_DC_2, LOAD_KEY_TO_DC_2)]: mdc.start_loader(dc, { "mode": "TX_PUT", "cacheName": CACHE_NAME, From 419f51d49d61d05070adcf7817f92179266da2f1 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 15 Jul 2026 20:39:42 +0300 Subject: [PATCH 28/44] IGNITE-27833 WIP --- .../internal/ducktest/tests/mdc/LoadMode.java | 46 ++++ .../tests/mdc/MdcCacheAwareApplication.java | 44 +--- .../mdc/MdcContinuousLoadApplication.java | 217 +++++++++--------- .../tests/mdc/MdcDataCheckerApplication.java | 2 +- .../mdc/MdcDataGeneratorApplication.java | 58 ++--- ...MdcPutAdmissibilityCheckerApplication.java | 93 -------- .../mdc/MdcThinClientLoadApplication.java | 10 +- .../ignite/internal/ducktest/utils/Utils.java | 39 +++- .../ducktests/tests/docker/requirements.txt | 2 +- .../ignitetest/services/mdc/mdc_cluster.py | 58 +++-- .../services/network_group/manager.py | 12 +- .../services/network_group/tc_rule_args.py | 8 - .../tests/mdc/partition_resilience_test.py | 1 + .../ignitetest/tests/mdc/thin_client_test.py | 52 ++--- 14 files changed, 296 insertions(+), 346 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/LoadMode.java delete mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/LoadMode.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/LoadMode.java new file mode 100644 index 0000000000000..a6f4bc2b546be --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/LoadMode.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.mdc; + +/** Operation modes shared by the MDC load applications. */ +public enum LoadMode { + /** Cache API reads with value verification. */ + GET, + + /** Cache API writes. */ + PUT, + + /** Transactional cache API writes (requires a TRANSACTIONAL cache). */ + TX_PUT, + + /** SQL reads with value verification (requires an SQL-enabled cache). */ + SQL_SELECT, + + /** SQL DML writes (requires an SQL-enabled cache). */ + SQL_PUT; + + /** @return Whether the mode writes to the cache. */ + public boolean isWrite() { + return this == PUT || this == TX_PUT || this == SQL_PUT; + } + + /** @return Whether the mode operates through SQL. */ + public boolean isSql() { + return this == SQL_SELECT || this == SQL_PUT; + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java index 4459990515a4b..a17b07df56bd2 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java @@ -19,7 +19,6 @@ import java.util.Collections; import java.util.HashSet; -import java.util.Locale; import java.util.Set; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.IgniteCache; @@ -39,6 +38,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; +import static org.apache.ignite.internal.ducktest.utils.Utils.getEnum; /** * Base class for MDC test applications. @@ -54,11 +54,12 @@ *
  • {@code datacenters} - full DC set for majority-based validation (odd DC count mode), * takes precedence over {@code mainDc};
    • *
  • {@code dcsNum} - number of data centers, default 2;
    • + *
  • {@code cacheMode} - {@link CacheMode}, default {@code PARTITIONED};
    • *
  • {@code atomicity} - {@link CacheAtomicityMode}, default {@code ATOMIC};
    • - *
  • {@code syncMode} - {@link CacheWriteSynchronizationMode}, default {@code FULL_SYNC}. + *
  • {@code writeSync} - {@link CacheWriteSynchronizationMode}, default {@code FULL_SYNC}. * Note: MDC-aware local reads require a mode other than {@code PRIMARY_SYNC};
    • *
  • {@code readFromBackup} - default {@code true}, required for DC-local reads;
    • - *
  • {@code partitions} - affinity partitions number, default 32.
    • + *
  • {@code partitions} - affinity partitions number, default 512.
    • * */ public abstract class MdcCacheAwareApplication extends IgniteAwareApplication { @@ -71,8 +72,8 @@ public abstract class MdcCacheAwareApplication extends IgniteAwareApplication { /** */ protected static final CacheMode DFLT_CACHE_MODE = PARTITIONED; - /** */ - protected static final int DFLT_BACKUPS = 0; + /** One backup: with the default 2 DCs, {@code (backups + 1) / dcsNum} = 1 copy per DC. */ + protected static final int DFLT_BACKUPS = 1; /** */ protected static final int DFLT_DCS_NUM = 2; @@ -156,7 +157,7 @@ protected CacheConfiguration mdcCacheConfiguration(JsonNode jNod * @param cacheName Cache name. * @return Existing cache. The cache must have been created by the generator beforehand. */ - protected IgniteCache mdcCache(String cacheName) { + protected IgniteCache existingCache(String cacheName) { return ignite.cache(cacheName); } @@ -166,35 +167,4 @@ protected IgniteCache mdcCache(String cacheName) { protected static String dcId() { return IgniteSystemProperties.getString(IGNITE_DATA_CENTER_ID); } - - /** - * Parses an optional enum-valued field. A missing, null or unrecognized value falls back - * to {@code dfltVal}. - */ - protected static > E getEnum(JsonNode jNode, String fieldName, E dfltVal) { - JsonNode field = jNode.path(fieldName); - - if (field.isMissingNode() || field.isNull()) - return dfltVal; - - try { - return Enum.valueOf(dfltVal.getDeclaringClass(), field.asText().toUpperCase(Locale.ROOT)); - } - catch (IllegalArgumentException e) { - return dfltVal; // Fallback if string doesn't match any enum constant. - } - } - - /** - * Parses a required enum-valued field. Throws if the field is missing, null or not a valid - * constant of {@code cls} (case-insensitively) - a typo must fail the run, not pick a default. - */ - protected static > E getEnum(JsonNode jNode, String fieldName, Class cls) { - JsonNode field = jNode.path(fieldName); - - if (field.isMissingNode() || field.isNull()) - throw new IllegalArgumentException("Missing required enum field '" + fieldName + "'"); - - return Enum.valueOf(cls, field.asText().toUpperCase(Locale.ROOT)); - } } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java index ae0bba0841d15..1b2568bd2f800 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java @@ -32,13 +32,14 @@ import org.apache.ignite.transactions.TransactionIsolation; import static org.apache.ignite.internal.ducktest.utils.Utils.fmtMs; +import static org.apache.ignite.internal.ducktest.utils.Utils.getEnum; import static org.apache.ignite.internal.ducktest.utils.Utils.timed; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; /** * Universal MDC load application. Runs a single-threaded synchronous load of the - * requested {@link Mode} either for a fixed number of iterations (a "burst") or until + * requested {@link LoadMode} either for a fixed number of iterations (a "burst") or until * externally terminated (a "background" load spanning several test phases, e.g. a * network partition and its healing). *

    @@ -71,33 +72,36 @@ * */ public class MdcContinuousLoadApplication extends MdcCacheAwareApplication { - /** Load modes. */ - private enum Mode { - /** Cache API reads with value verification. */ - GET, + /** */ + public static final TransactionConcurrency DFLT_TX_CONCURRENCY = PESSIMISTIC; - /** Cache API writes. */ - PUT, + /** */ + public static final TransactionIsolation DFLT_TX_ISOLATION = REPEATABLE_READ; - /** Transactional cache API writes (requires a TRANSACTIONAL cache). */ - TX_PUT, + /** Cache for the cache API modes ({@code null} in SQL modes). */ + private IgniteCache cache; - /** SQL reads with value verification (requires an SQL-enabled cache). */ - SQL_SELECT, + /** Cache for the SQL modes ({@code null} in cache API modes). */ + private IgniteCache sqlCache; - /** SQL DML writes (requires an SQL-enabled cache). */ - SQL_PUT - } + /** Compiled DML statement for {@link LoadMode#SQL_PUT}. */ + private String mergeSql; + + /** Compiled query for {@link LoadMode#SQL_SELECT}. */ + private String selectSql; + + /** Latency of successful operations. */ + private final OpStats stats = new OpStats(); /** */ - public static final TransactionConcurrency DFLT_TX_CONCURRENCY = PESSIMISTIC; + private TransactionConcurrency txConcurrency; /** */ - public static final TransactionIsolation DFLT_TX_ISOLATION = REPEATABLE_READ; + private TransactionIsolation txIsolation; /** {@inheritDoc} */ @Override public void run(JsonNode jNode) throws Exception { - Mode mode = getEnum(jNode, "mode", Mode.class); + LoadMode mode = getEnum(jNode, "mode", LoadMode.class); String cacheName = jNode.path("cacheName").asText(DFLT_CACHE_NAME); boolean createCache = jNode.path("createCache").asBoolean(false); @@ -114,29 +118,19 @@ private enum Mode { String pfx = jNode.path("resultPrefix").asText(""); - TransactionConcurrency txConcurrency = getEnum(jNode, "txConcurrency", DFLT_TX_CONCURRENCY); - TransactionIsolation txIsolation = getEnum(jNode, "txIsolation", DFLT_TX_ISOLATION); + txConcurrency = getEnum(jNode, "txConcurrency", DFLT_TX_CONCURRENCY); + txIsolation = getEnum(jNode, "txIsolation", DFLT_TX_ISOLATION); markInitialized(); waitForActivation(); - boolean sqlMode = mode == Mode.SQL_SELECT || mode == Mode.SQL_PUT; - boolean writeMode = mode == Mode.PUT || mode == Mode.TX_PUT || mode == Mode.SQL_PUT; - - IgniteCache cache0 = null; - IgniteCache sqlCache0 = null; - - if (sqlMode) - sqlCache0 = createCache ? mdcSqlCache(jNode) : ignite.cache(cacheName); + if (mode.isSql()) + sqlCache = createCache ? mdcSqlCache(jNode) : ignite.cache(cacheName); else - cache0 = createCache ? mdcCache(jNode) : ignite.cache(cacheName); + cache = createCache ? mdcCache(jNode) : ignite.cache(cacheName); - // Effectively-final copies for use inside timed lambdas. - IgniteCache cache = cache0; - IgniteCache sqlCache = sqlCache0; - - String mergeSql = String.format("MERGE INTO \"%s\".%s(_KEY, _VAL) VALUES(?, ?)", cacheName, SQL_TABLE); - String selectSql = String.format("SELECT _VAL FROM \"%s\".%s WHERE _KEY = ?", cacheName, SQL_TABLE); + mergeSql = String.format("MERGE INTO \"%s\".%s(_KEY, _VAL) VALUES(?, ?)", cacheName, SQL_TABLE); + selectSql = String.format("SELECT _VAL FROM \"%s\".%s WHERE _KEY = ?", cacheName, SQL_TABLE); log.info("MDC load started [dc=" + dcId() + ", mode=" + mode + ", cache=" + cacheName + ", keyFrom=" + keyFrom + ", keyTo=" + keyTo + ", iterations=" + iterations + @@ -147,92 +141,23 @@ private enum Mode { boolean stoppedOnError = false; - OpStats stats = new OpStats(); - long maxStallMs = 0; long startTs = System.currentTimeMillis(); long lastOkTs = startTs; - int key0 = keyFrom; + int key = keyFrom; while (!terminated() && (iterations == 0 || opsCnt + errCnt < iterations)) { - int key = key0; - boolean ok; try { - switch (mode) { - case GET: { - IndexedDataRecord val = timed(stats, () -> cache.get(key)); - - ok = val != null && val.equals(new IndexedDataRecord(key)); - - if (!ok) - log.error("Read entry is missed or corrupted [dc=" + dcId() + ", key=" + key + - ", val=" + val + "]"); - - break; - } - - case PUT: { - IndexedDataRecord val = new IndexedDataRecord(key); - - timed(stats, () -> cache.put(key, val)); - - ok = true; - - break; - } - - case TX_PUT: { - IndexedDataRecord val = new IndexedDataRecord(key); - - timed(stats, () -> { - try (Transaction tx = ignite.transactions().txStart(txConcurrency, txIsolation)) { - cache.put(key, val); - - tx.commit(); - } - }); - - ok = true; - - break; - } - - case SQL_PUT: { - SqlFieldsQuery qry = new SqlFieldsQuery(mergeSql).setArgs(key, key); - - timed(stats, () -> sqlCache.query(qry).getAll()); - - ok = true; - - break; - } - - case SQL_SELECT: { - SqlFieldsQuery qry = new SqlFieldsQuery(selectSql).setArgs(key); - - List> rows = timed(stats, () -> sqlCache.query(qry).getAll()); - - ok = !rows.isEmpty() && Objects.equals(rows.get(0).get(0), key); - - if (!ok) - log.error("SQL row is missed or corrupted [dc=" + dcId() + ", key=" + key + - ", rows=" + rows + "]"); - - break; - } - - default: - throw new IllegalArgumentException("Unknown mode: " + mode); - } + ok = doOperation(mode, key); } catch (CacheException | IgniteException e) { ok = false; - if (writeMode && !expectAdmissible) + if (mode.isWrite() && !expectAdmissible) log.info("Write rejected as expected [dc=" + dcId() + ", key=" + key + ", msg=" + e.getMessage() + "]"); else if (stopOnError) { @@ -265,11 +190,11 @@ else if (!tolerateErrors) // Writes advance on success only, so [keyFrom, keyFrom + opsCnt) is guaranteed written. // The inadmissible-probe mode advances always to probe distinct keys. Reads cycle the range. - if (ok || (writeMode && !expectAdmissible)) { - key0++; + if (ok || (mode.isWrite() && !expectAdmissible)) { + key++; - if (key0 >= keyTo) - key0 = keyFrom; + if (key >= keyTo) + key = keyFrom; } if (opPauseMs > 0) @@ -278,7 +203,7 @@ else if (!tolerateErrors) long durationMs = System.currentTimeMillis() - startTs; - if (writeMode && !expectAdmissible && opsCnt > 0) { + if (mode.isWrite() && !expectAdmissible && opsCnt > 0) { throw new IllegalStateException("Write load is admissible while expected to be inadmissible [dc=" + dcId() + ", mode=" + mode + ", succeeded=" + opsCnt + ", rejected=" + errCnt + "]"); } @@ -303,4 +228,74 @@ else if (!tolerateErrors) markFinished(); } + + /** + * Executes a single operation of the given mode against the given key. Throws a + * {@link CacheException} or {@link IgniteException} on operation failure (e.g. a write + * rejected by the topology validator). + * + * @return {@code true} if the operation succeeded and (for reads) returned the expected value. + */ + private boolean doOperation(LoadMode mode, int key) { + switch (mode) { + case GET: { + IndexedDataRecord val = timed(stats, () -> cache.get(key)); + + boolean ok = val != null && val.equals(new IndexedDataRecord(key)); + + if (!ok) + log.error("Read entry is missed or corrupted [dc=" + dcId() + ", key=" + key + + ", val=" + val + "]"); + + return ok; + } + + case PUT: { + IndexedDataRecord val = new IndexedDataRecord(key); + + timed(stats, () -> cache.put(key, val)); + + return true; + } + + case TX_PUT: { + IndexedDataRecord val = new IndexedDataRecord(key); + + timed(stats, () -> { + try (Transaction tx = ignite.transactions().txStart(txConcurrency, txIsolation)) { + cache.put(key, val); + + tx.commit(); + } + }); + + return true; + } + + case SQL_PUT: { + SqlFieldsQuery qry = new SqlFieldsQuery(mergeSql).setArgs(key, key); + + timed(stats, () -> sqlCache.query(qry).getAll()); + + return true; + } + + case SQL_SELECT: { + SqlFieldsQuery qry = new SqlFieldsQuery(selectSql).setArgs(key); + + List> rows = timed(stats, () -> sqlCache.query(qry).getAll()); + + boolean ok = !rows.isEmpty() && Objects.equals(rows.get(0).get(0), key); + + if (!ok) + log.error("SQL row is missed or corrupted [dc=" + dcId() + ", key=" + key + + ", rows=" + rows + "]"); + + return ok; + } + + default: + throw new IllegalArgumentException("Unknown mode: " + mode); + } + } } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java index 23e7cf7764725..2232a5b59f51a 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataCheckerApplication.java @@ -37,7 +37,7 @@ public class MdcDataCheckerApplication extends MdcCacheAwareApplication { markInitialized(); waitForActivation(); - IgniteCache cache = mdcCache(cacheName); + IgniteCache cache = existingCache(cacheName); log.info("Data check started [dc=" + dcId() + ", cache=" + cache.getName() + ", from=" + from + ", to=" + to + "]"); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java index 6b1881f887199..e73dbfb66af0e 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java @@ -19,6 +19,7 @@ import java.util.Map; import java.util.TreeMap; +import java.util.function.IntFunction; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.IgniteInterruptedCheckedException; @@ -26,7 +27,8 @@ /** * Populates the MDC cache with a deterministic data set: keys in {@code [from, to)}, - * values {@code IndexedDataRecord(key)}. Run it before the network partition. + * values {@code IndexedDataRecord(key)} (or the key itself in {@code sqlMode}). + * Run it before the network partition. */ public class MdcDataGeneratorApplication extends MdcCacheAwareApplication { /** {@inheritDoc} */ @@ -39,50 +41,36 @@ public class MdcDataGeneratorApplication extends MdcCacheAwareApplication { markInitialized(); waitForActivation(); - IgniteCache cache = null; - IgniteCache sqlCache = null; - - if (sqlMode) - sqlCache = mdcSqlCache(jNode); - else - cache = mdcCache(jNode); - log.info("Data generation started [dc=" + dcId() + ", sqlMode=" + sqlMode + ", from=" + from + ", to=" + to + "]"); - if (sqlMode) { - Map batch = new TreeMap<>(); - - for (int i = from; i < to && !terminated(); i++) { - batch.put(i, i); + if (sqlMode) + load(mdcSqlCache(jNode), from, to, batchSize, i -> i); + else + load(mdcCache(jNode), from, to, batchSize, IndexedDataRecord::new); - if (batch.size() >= batchSize) { - sqlCache.putAll(batch); - batch.clear(); - } - } + log.info("Data generation finished [dc=" + dcId() + ", entries=" + (to - from) + "]"); - if (!batch.isEmpty() && !terminated()) - sqlCache.putAll(batch); - } - else { - Map batch = new TreeMap<>(); + markFinished(); + } - for (int i = from; i < to && !terminated(); i++) { - batch.put(i, new IndexedDataRecord(i)); + /** + * Streams keys {@code [from, to)} into the cache in {@code putAll} batches, deriving + * each value from its key. + */ + private void load(IgniteCache cache, int from, int to, int batchSize, IntFunction valFn) { + Map batch = new TreeMap<>(); - if (batch.size() >= batchSize) { - cache.putAll(batch); - batch.clear(); - } - } + for (int i = from; i < to && !terminated(); i++) { + batch.put(i, valFn.apply(i)); - if (!batch.isEmpty() && !terminated()) + if (batch.size() >= batchSize) { cache.putAll(batch); + batch.clear(); + } } - log.info("Data generation finished [dc=" + dcId() + ", entries=" + (to - from) + "]"); - - markFinished(); + if (!batch.isEmpty() && !terminated()) + cache.putAll(batch); } } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java deleted file mode 100644 index cbf50ab398c09..0000000000000 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcPutAdmissibilityCheckerApplication.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.ducktest.tests.mdc; - -import javax.cache.CacheException; -import com.fasterxml.jackson.databind.JsonNode; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.internal.IgniteInterruptedCheckedException; -import org.apache.ignite.internal.ducktest.tests.dto.IndexedDataRecord; - -/** - * Checks whether the put load is admissible or inadmissible for the DC this client - * belongs to. - *

    - * With {@code expectAdmissible=true} every probe put must succeed (primary DC). - * With {@code expectAdmissible=false} every probe put must fail with a - * {@link CacheException} thrown by the topology validator (read-only DC). - *

    - * Probe keys start at {@code keyOffset} (defaults to 1_000_000) so they never - * intersect with the data set verified by {@link MdcDataCheckerApplication}. - */ -public class MdcPutAdmissibilityCheckerApplication extends MdcCacheAwareApplication { - /** {@inheritDoc} */ - @Override public void run(JsonNode jNode) throws IgniteInterruptedCheckedException { - String cacheName = jNode.path("cacheName").asText(DFLT_CACHE_NAME); - boolean expectAdmissible = jNode.get("expectAdmissible").asBoolean(); - int probes = jNode.path("probes").asInt(100); - int keyOffset = jNode.path("keyOffset").asInt(1_000_000); - - markInitialized(); - waitForActivation(); - - IgniteCache cache = mdcCache(cacheName); - - log.info("Put admissibility check started [dc=" + dcId() + ", cache=" + cache.getName() + - ", expectAdmissible=" + expectAdmissible + ", probes=" + probes + "]"); - - int succeeded = 0; - int rejected = 0; - - for (int i = 0; i < probes && !terminated(); i++) { - int key = keyOffset + i; - - try { - cache.put(key, new IndexedDataRecord(key)); - - succeeded++; - - if (!expectAdmissible) - log.error("Put unexpectedly succeeded in read-only DC [dc=" + dcId() + ", key=" + key + "]"); - } - catch (CacheException e) { - rejected++; - - if (expectAdmissible) - log.error("Put unexpectedly rejected [dc=" + dcId() + ", key=" + key + "]", e); - else - log.info("Put rejected as expected [dc=" + dcId() + ", key=" + key + - ", msg=" + e.getMessage() + "]"); - } - } - - if (expectAdmissible && rejected > 0) { - throw new IllegalStateException("Put load is inadmissible while expected to be admissible [dc=" + - dcId() + ", succeeded=" + succeeded + ", rejected=" + rejected + "]"); - } - - if (!expectAdmissible && succeeded > 0) { - throw new IllegalStateException("Put load is admissible while expected to be inadmissible [dc=" + - dcId() + ", succeeded=" + succeeded + ", rejected=" + rejected + "]"); - } - - log.info("Put admissibility check passed [dc=" + dcId() + - ", succeeded=" + succeeded + ", rejected=" + rejected + "]"); - - markFinished(); - } -} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java index cc0fb7c771224..e854cfc49a0d2 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.ducktest.tests.mdc; -import java.util.Locale; import javax.cache.CacheException; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.client.ClientCache; @@ -27,6 +26,7 @@ import org.apache.ignite.internal.ducktest.utils.OpStats; import static org.apache.ignite.internal.ducktest.utils.Utils.fmtMs; +import static org.apache.ignite.internal.ducktest.utils.Utils.getEnum; import static org.apache.ignite.internal.ducktest.utils.Utils.timed; /** @@ -45,14 +45,18 @@ public class MdcThinClientLoadApplication extends IgniteAwareApplication { /** {@inheritDoc} */ @Override public void run(JsonNode jNode) throws Exception { - String mode = jNode.get("mode").asText().toUpperCase(Locale.ROOT); + LoadMode mode = getEnum(jNode, "mode", LoadMode.class); + + if (mode != LoadMode.GET && mode != LoadMode.PUT) + throw new IllegalArgumentException("Unsupported thin client mode: " + mode); + String cacheName = jNode.get("cacheName").asText(); int keyFrom = jNode.path("keyFrom").asInt(0); int keyTo = jNode.path("keyTo").asInt(Integer.MAX_VALUE); long iterations = jNode.path("iterations").asLong(100); - boolean put = "PUT".equals(mode); + boolean put = mode == LoadMode.PUT; boolean expectAdmissible = jNode.path("expectAdmissible").asBoolean(true); String pfx = jNode.path("resultPrefix").asText(""); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java index 1befd8498a1b1..97873529b00aa 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java @@ -19,9 +19,15 @@ import java.util.Locale; import java.util.function.Supplier; +import com.fasterxml.jackson.databind.JsonNode; -/** */ +/** Small helpers shared by the ducktest applications. */ public final class Utils { + /** */ + private Utils() { + // No-op. + } + /** * Runs the operation, records its latency into {@code stats} and returns its result. * If the operation throws, nothing is recorded and the exception propagates. @@ -52,4 +58,35 @@ public static void timed(OpStats stats, Runnable op) { public static String fmtMs(double ns) { return String.format(Locale.US, "%.3f", ns < 0 ? -1.0 : ns / 1e6); } + + /** + * Parses an optional enum-valued field. A missing, null or unrecognized value falls back + * to {@code dfltVal}. + */ + public static > E getEnum(JsonNode jNode, String fieldName, E dfltVal) { + JsonNode field = jNode.path(fieldName); + + if (field.isMissingNode() || field.isNull()) + return dfltVal; + + try { + return Enum.valueOf(dfltVal.getDeclaringClass(), field.asText().toUpperCase(Locale.ROOT)); + } + catch (IllegalArgumentException e) { + return dfltVal; // Fallback if string doesn't match any enum constant. + } + } + + /** + * Parses a required enum-valued field. Throws if the field is missing, null or not a valid + * constant of {@code cls} (case-insensitively) - a typo must fail the run, not pick a default. + */ + public static > E getEnum(JsonNode jNode, String fieldName, Class cls) { + JsonNode field = jNode.path(fieldName); + + if (field.isMissingNode() || field.isNull()) + throw new IllegalArgumentException("Missing required enum field '" + fieldName + "'"); + + return Enum.valueOf(cls, field.asText().toUpperCase(Locale.ROOT)); + } } diff --git a/modules/ducktests/tests/docker/requirements.txt b/modules/ducktests/tests/docker/requirements.txt index c9f9a003dcf2a..469c95f7b1a21 100644 --- a/modules/ducktests/tests/docker/requirements.txt +++ b/modules/ducktests/tests/docker/requirements.txt @@ -16,4 +16,4 @@ filelock==3.8.2 ducktape==0.13.0 looseversion==1.3.0 -tcconfig==0.30.1 \ No newline at end of file +tcconfig==0.30.1 diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py index cac0039b556f1..1508c08e31b7e 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -53,7 +53,6 @@ GENERATOR_APP = _APP_PKG + "MdcDataGeneratorApplication" DATA_CHECKER_APP = _APP_PKG + "MdcDataCheckerApplication" -PUT_CHECKER_APP = _APP_PKG + "MdcPutAdmissibilityCheckerApplication" LOAD_APP = _APP_PKG + "MdcContinuousLoadApplication" THIN_LOAD_APP = _APP_PKG + "MdcThinClientLoadApplication" @@ -134,12 +133,17 @@ def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, i # subsequent ones preserve work dirs (and logs - hence unique result prefixes). self._started_apps = set() - def sync_service_discovery(self): - all_servers = list(self.servers.values()) + # Admissibility checks run on reusable services, so each check needs a unique result prefix. + self._adm_checks = 0 - discovery_spi = from_ignite_services(all_servers) + def sync_service_discovery(self): + """ + Points every server service at a discovery SPI covering all DCs (used to let a + stopped DC rejoin the full cluster on restart). + """ + discovery_spi = from_ignite_services(list(self.servers.values())) - for dc_key, service in self.servers.items(): + for service in self.servers.values(): service.config = service.config._replace(discovery_spi=discovery_spi) def _app_service(self, dc: str) -> IgniteApplicationService: @@ -150,8 +154,8 @@ def _app_service(self, dc: str) -> IgniteApplicationService: def register(self, dc: str, service): """ Registers an extra service (e.g. a thin client app) into a DC's network group, - so netem impairments and partitions apply to it. Must be called before the - :class:`NetworkGroupManager` is entered. + so netem impairments and partitions apply to it. Must be called before + :func:`cross_dc_network` snapshots the registry into a :class:`NetworkGroupManager`. """ self.extras[dc].append(service) @@ -215,9 +219,18 @@ def run_app(self, dc: str, java_class: str, params: dict, runner: int = 0) -> Ig Runs a run-to-completion application on one of the DC's reusable runner services and returns the service (for ``extract_result``). """ - svc = self.runners[dc][runner] + return self.run_service(self.runners[dc][runner], params, java_class=java_class) + + def run_service(self, svc: IgniteApplicationService, params: dict, + java_class: str = None) -> IgniteApplicationService: + """ + Runs any reusable run-to-completion application service (a runner, a registered + thin client, ...): the first start is clean, subsequent starts preserve work dirs. + Returns the service (for ``extract_result``). + """ + if java_class is not None: + svc.java_class_name = java_class - svc.java_class_name = java_class svc.params = params svc.start(clean=self._first_start(svc)) @@ -263,7 +276,7 @@ def generate_data(self, dc: str, cache_name: str, from_idx: int, to_idx: int, ba main_dc: str = DC_1, sql_mode: bool = False, **cache_params) -> IgniteApplicationService: """ Creates the MDC cache (if absent) and populates keys ``[from_idx, to_idx)``. - Extra cache parameters (``atomicity``, ``syncMode``, ``readFromBackup``, + Extra cache parameters (``atomicity``, ``writeSync``, ``readFromBackup``, ``partitions``, ...) are passed through to the cache configuration builder. """ params = {"cacheName": cache_name, "backups": backups, "mainDc": main_dc, @@ -271,28 +284,37 @@ def generate_data(self, dc: str, cache_name: str, from_idx: int, to_idx: int, ba return self.run_app(dc, GENERATOR_APP, params) - def check_data(self, dc: str, cache_name: str, from_idx: int, to_idx: int) -> IgniteApplicationService: + def check_data(self, dc: str, cache_name: str, from_idx: int, to_idx: int) -> Optional[IgniteApplicationService]: """ Verifies that every key in ``[from_idx, to_idx)`` is readable and holds the expected value, from a client in the given DC. + + :return: The service that ran the check, or None for an empty range. """ if to_idx <= from_idx: self.logger.debug(f"Nothing to check [cache={cache_name}, from={from_idx}, to={to_idx}]") - return self.runners[dc][0] + return None params = {"cacheName": cache_name, "from": from_idx, "to": to_idx} return self.run_app(dc, DATA_CHECKER_APP, params) def check_put_admissibility(self, dc: str, cache_name: str, admissible: bool, - key_offset: int = 1_000_000) -> IgniteApplicationService: + key_offset: int = 1_000_000, probes: int = 100) -> IgniteApplicationService: """ Verifies that put load from the given DC is admissible (primary DC visible) or - rejected by the topology validator (read-only DC). + rejected by the topology validator (read-only DC). A PUT burst of the load + application: an admissible check fails fast on the first rejected put, an + inadmissible check fails if any of the probe puts succeeds. + + Probe keys start at ``key_offset`` (defaults to 1_000_000) so they never intersect + with the data set verified by ``check_data``. """ - params = {"cacheName": cache_name, "expectAdmissible": admissible, "keyOffset": key_offset} + self._adm_checks += 1 - return self.run_app(dc, PUT_CHECKER_APP, params) + return self.run_load(dc, "PUT", cache_name, f"admCheck{self._adm_checks}", + keyFrom=key_offset, keyTo=key_offset + probes, + iterations=probes, expectAdmissible=admissible) def run_load(self, dc: str, mode: str, cache_name: str, result_prefix: str, runner: int = 0, **params) -> IgniteApplicationService: @@ -398,7 +420,7 @@ def verify_servers_log_clean(self): were detected, no PME hang and no lost partitions were reported. """ for pattern in (LRT_PATTERN, PME_FREEZE_PATTERN, LOST_PARTITIONS_PATTERN): - for dc, svc in self.servers.items(): + for svc in self.servers.values(): svc.check_event_absent(pattern) def verify_no_hanging_txs(self, dc: str = DC_1): @@ -407,6 +429,8 @@ def verify_no_hanging_txs(self, dc: str = DC_1): """ txs = self.control(dc).tx() + # ControlUtility.tx() returns a list of parsed transactions, or the raw command + # output (a str) when nothing parsed - i.e. when there are no transactions. assert not isinstance(txs, list) or not txs, f"No active transactions expected [txs={txs}]" @staticmethod diff --git a/modules/ducktests/tests/ignitetest/services/network_group/manager.py b/modules/ducktests/tests/ignitetest/services/network_group/manager.py index a55f9a7b69418..bc1033452227e 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/manager.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/manager.py @@ -266,7 +266,7 @@ def _log_network(self, log_tag: str): """ self.logger.debug(f"Network State Overview: [START][{log_tag}]") - node_to_status_map = {} + node_statuses = [] for group, services in self.network_group_registry.items(): for svc in services: @@ -285,12 +285,12 @@ def _log_network(self, log_tag: str): partition_str = self._format_partition_drops( self._parse_partition_drops(iptables_lines)) - node_status = f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : " \ - f"{constraints}{targets_str}{partition_str}" + node_statuses.append(f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : " + f"{constraints}{targets_str}{partition_str}") - node_to_status_map.update({id(node): node_status}) - - for node_id, node_status in node_to_status_map.items(): + # The per-node SSH probes above flood the debug log with their own command output. + # Collect first, print contiguously after: the overview must stay readable as one block. + for node_status in node_statuses: self.logger.debug(node_status) self.logger.debug(f"Network State Overview: [END][{log_tag}]") diff --git a/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py b/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py index c7b028c5def76..d5a8d7f0db753 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/tc_rule_args.py @@ -27,7 +27,6 @@ # tcset rule actions. ACTION_OVERWRITE = "--overwrite" ACTION_ADD = "--add" -ACTION_CHANGE = "--change" # -w: wait on the xtables lock instead of failing IPTABLES = "sudo iptables -w" @@ -76,13 +75,6 @@ def to_tcdel_all_cmd(interface: str) -> str: return f"{SUDO_PREFIX} tcdel {interface} --all" -def to_tcdel_ip_cmd(interface: str, dst_host_or_ip: str) -> str: - """ - Compiles the absolute clear command for destination host. - """ - return f"{SUDO_PREFIX} tcdel {interface} --peer {dst_host_or_ip}" - - def partition_chain_name(group_a: str, group_b: str) -> str: """ Deterministic, order-independent iptables chain name for a group pair. diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index bb8827267b65b..89d9b9db28335 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -43,6 +43,7 @@ BG_MAX_STALL_MS = 500 + class MdcPartitionResilienceTest(IgniteTest): """ Tests for cluster network partition resilience in MultiDC. diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py index ddde6d0cfa996..cca1ec0532c31 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py @@ -37,7 +37,7 @@ from ignitetest.services.utils.ignite_configuration import IgniteThinClientConfiguration from ignitetest.utils import cluster, ignite_versions from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH, IgniteVersion +from ignitetest.utils.version import DEV_BRANCH CACHE_NAME = "mdc-thin" @@ -78,28 +78,14 @@ def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_ mdc.register(DC_1, cli_dc1) mdc.register(DC_2, cli_dc2) - first_start = set() - - def run_thin(cli_svc, params): - cli_svc.params = params - - cli_svc.start(clean=id(cli_svc) not in first_start) - - first_start.add(id(cli_svc)) - - cli_svc.wait() - cli_svc.stop() - - return cli_svc - with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() mdc.generate_data(DC_1, CACHE_NAME, 0, KEYS, backups=1) - svc = run_thin(cli_dc1, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, - "keyTo": KEYS, "iterations": PINNED_GET_ITERS, - "resultPrefix": "pinnedGet"}) + svc = mdc.run_service(cli_dc1, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, + "keyTo": KEYS, "iterations": PINNED_GET_ITERS, + "resultPrefix": "pinnedGet"}) avg_cli_dc_1 = mdc.result_float(svc, "pinnedGetAvgOpMs") @@ -116,33 +102,33 @@ def run_thin(cli_svc, params): mdc.verify_split_brain() # The client pinned to the main half keeps writing... - run_thin(cli_dc1, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_DURING, - "keyTo": OFFSET_DURING + PUT_ITERS, "iterations": PUT_ITERS, - "resultPrefix": "duringPut"}) + mdc.run_service(cli_dc1, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_DURING, + "keyTo": OFFSET_DURING + PUT_ITERS, "iterations": PUT_ITERS, + "resultPrefix": "duringPut"}) # ...while the client pinned to the read-only half still reads cleanly # (its DC1 addresses are unreachable, so it falls back to DC2 nodes)... - svc = run_thin(cli_dc2, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, - "keyTo": KEYS, "iterations": PINNED_GET_ITERS, - "resultPrefix": "roGet"}) + svc = mdc.run_service(cli_dc2, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, + "keyTo": KEYS, "iterations": PINNED_GET_ITERS, + "resultPrefix": "roGet"}) assert mdc.result_int(svc, "roGetErrCnt") == 0, \ "Thin client reads in the read-only DC must be clean" # ...and has every write rejected by the topology validator. - run_thin(cli_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, - "keyFrom": OFFSET_REJECTED_PROBES, - "keyTo": OFFSET_REJECTED_PROBES + PUT_ITERS, - "iterations": PUT_ITERS, "expectAdmissible": False, - "resultPrefix": "roPut"}) + mdc.run_service(cli_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, + "keyFrom": OFFSET_REJECTED_PROBES, + "keyTo": OFFSET_REJECTED_PROBES + PUT_ITERS, + "iterations": PUT_ITERS, "expectAdmissible": False, + "resultPrefix": "roPut"}) net.disable_network_partition(DC_1, DC_2) mdc.restart(DC_2) - run_thin(cli_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_AFTER, - "keyTo": OFFSET_AFTER + PUT_ITERS, "iterations": PUT_ITERS, - "resultPrefix": "afterPut"}) + mdc.run_service(cli_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_AFTER, + "keyTo": OFFSET_AFTER + PUT_ITERS, "iterations": PUT_ITERS, + "resultPrefix": "afterPut"}) mdc.check_data(DC_1, CACHE_NAME, OFFSET_DURING, OFFSET_DURING + PUT_ITERS) mdc.check_data(DC_1, CACHE_NAME, OFFSET_AFTER, OFFSET_AFTER + PUT_ITERS) @@ -157,7 +143,7 @@ def _thin_client(self, mdc: MdcCluster, jvm_opts) -> IgniteApplicationService: return IgniteApplicationService( self.test_context, IgniteThinClientConfiguration(addresses=mdc.thin_client_addresses(), - version=IgniteVersion(str(DEV_BRANCH))), + version=mdc.ignite_config.version), java_class_name=THIN_LOAD_APP, num_nodes=1, jvm_opts=jvm_opts) From 76199ff1cbdb873b8c1b0fec1639e4cd12df8168 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 16 Jul 2026 10:02:49 +0300 Subject: [PATCH 29/44] IGNITE-27833 [ducktests] Add Multi-DC Integration Test for Cross-DC Network Partition Resilience --- .../services/utils/control_utility.py | 111 ------------------ 1 file changed, 111 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index daa0481c376ab..c8f6cbe19bad4 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -258,117 +258,6 @@ def __parse_cache_distribution(output, user_attributes=None): return CacheDistribution(groups=groups) - def system_view(self, name, node=None, node_id=None, node_ids=None, all_nodes=False): - """ - Prints the content of a system view (``control.sh --system-view``). - - :param name: System view name. Both the SQL ("CACHES") and Java ("caches") styles are accepted. - :param node: Node to run control.sh on (a random alive node if None). - :param node_id: Single node id to read the view from (discouraged upstream, prefer - ``node_ids`` or ``all_nodes``). - :param node_ids: List of node ids to read the view from. - :param all_nodes: Read the view from all nodes. - :return: Dict mapping node id (str) to a list of rows; each row is a dict keyed by the - view column name (e.g. ``row["CACHE_NAME"]``). - """ - cmd = f"--system-view {name}" - - if all_nodes: - cmd += " --all-nodes" - elif node_ids: - cmd += f" --node-ids {','.join(node_ids)}" - elif node_id: - cmd += f" --node-id {node_id}" - - return self.__parse_system_view(self.__run(cmd, node=node), name) - - def system_view_caches(self, cache_name=None, node=None, node_id=None, node_ids=None, all_nodes=False): - """ - Reads the CACHES system view and returns its rows. - - Each row is a dict keyed by the view column names, e.g. ``CACHE_NAME``, ``CACHE_ID``, - ``CACHE_GROUP_ID``, ``CACHE_GROUP_NAME``, ``CACHE_TYPE``, ``CACHE_MODE``, - ``ATOMICITY_MODE``, ``BACKUPS``. - - :param cache_name: If set, only rows whose ``CACHE_NAME`` equals it are returned. - :return: List of rows (dicts). CACHES is cluster-wide, so by default the rows of a - single node are returned to avoid duplicates; when ``node_ids`` or - ``all_nodes`` is requested the rows of every queried node are concatenated. - """ - by_node = self.system_view("CACHES", node=node, node_id=node_id, node_ids=node_ids, all_nodes=all_nodes) - - if node_ids or all_nodes: - rows = [row for node_rows in by_node.values() for row in node_rows] - else: - rows = next(iter(by_node.values()), []) - - if cache_name is not None: - rows = [row for row in rows if row.get("CACHE_NAME") == cache_name] - - return rows - - @staticmethod - def __parse_system_view(output, name): - """ - Parses ``control.sh --system-view`` output. - - The command prints one block per node:: - - Results from node with ID: - --- - COL_A COL_B ... - valA valB ... - --- - - Columns are padded to their widest value and joined by the 4-space column separator; - cells never hold an empty value (nulls print as "null"), so splitting on the separator - and dropping the padding gaps recovers the columns. - """ - if "No system view with specified name was found" in output: - raise AssertionError(f"No system view named '{name}' was found:\n{output}") - - node_pattern = re.compile(r"Results from node with ID: (?P\S+)") - sep = " " - - result = {} - node_id = None - header = None - # idle -> await_open (saw the node header) -> body (between the two '---' delimiters). - state = "idle" - - for line in output.splitlines(): - stripped = line.strip() - - match = node_pattern.match(stripped) - if match: - node_id = match.group("node_id") - result[node_id] = [] - header = None - state = "await_open" - continue - - if state == "await_open": - if stripped == "---": - state = "body" - continue - - if state == "body": - if stripped == "---": - state = "idle" - continue - - if not stripped: - continue - - cells = [cell.strip() for cell in line.split(sep) if cell.strip()] - - if header is None: - header = cells - else: - result[node_id].append(dict(zip(header, cells))) - - return result - def check_consistency(self, args): """ Consistency check. From 9a9591d53b0816dd2e255a504d10526b2b581c54 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 16 Jul 2026 10:10:58 +0300 Subject: [PATCH 30/44] IGNITE-27833 Bump ducktest checkstyle to python 3.9 --- .github/workflows/commit-check.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/commit-check.yml b/.github/workflows/commit-check.yml index 88e1356b09a91..f7c948f89d762 100644 --- a/.github/workflows/commit-check.yml +++ b/.github/workflows/commit-check.yml @@ -158,7 +158,6 @@ jobs: fail-fast: false matrix: cfg: - - { python: "3.8", toxenv: "py38" } - { python: "3.9", toxenv: "py39" } - { python: "3.9", toxenv: "codestyle" } steps: From 662c2dc6e146af14de8878e2d089381c5fda791d Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Fri, 17 Jul 2026 00:34:01 +0300 Subject: [PATCH 31/44] IGNITE-27833 minor fixes --- .../mdc/MdcContinuousLoadApplication.java | 32 +++++-------------- .../mdc/MdcThinClientLoadApplication.java | 16 ++++------ .../ignite/internal/ducktest/utils/Utils.java | 12 ++----- .../services/utils/control_utility.py | 5 +-- .../ignite_configuration/communication.py | 2 +- .../ignitetest/tests/mdc/thin_client_test.py | 8 +++-- modules/ducktests/tests/tox.ini | 2 +- 7 files changed, 26 insertions(+), 51 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java index 1b2568bd2f800..ab109cb1e58c3 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java @@ -149,15 +149,15 @@ public class MdcContinuousLoadApplication extends MdcCacheAwareApplication { int key = keyFrom; while (!terminated() && (iterations == 0 || opsCnt + errCnt < iterations)) { - boolean ok; + boolean ok = true; try { - ok = doOperation(mode, key); + doOperation(mode, key); } catch (CacheException | IgniteException e) { ok = false; - if (mode.isWrite() && !expectAdmissible) + if (!expectAdmissible) log.info("Write rejected as expected [dc=" + dcId() + ", key=" + key + ", msg=" + e.getMessage() + "]"); else if (stopOnError) { @@ -233,29 +233,21 @@ else if (!tolerateErrors) * Executes a single operation of the given mode against the given key. Throws a * {@link CacheException} or {@link IgniteException} on operation failure (e.g. a write * rejected by the topology validator). - * - * @return {@code true} if the operation succeeded and (for reads) returned the expected value. */ - private boolean doOperation(LoadMode mode, int key) { + private void doOperation(LoadMode mode, int key) { switch (mode) { case GET: { IndexedDataRecord val = timed(stats, () -> cache.get(key)); - boolean ok = val != null && val.equals(new IndexedDataRecord(key)); - - if (!ok) - log.error("Read entry is missed or corrupted [dc=" + dcId() + ", key=" + key + + if (val == null || !val.equals(new IndexedDataRecord(key))) + throw new IgniteException("Read entry is missed or corrupted [dc=" + dcId() + ", key=" + key + ", val=" + val + "]"); - - return ok; } case PUT: { IndexedDataRecord val = new IndexedDataRecord(key); timed(stats, () -> cache.put(key, val)); - - return true; } case TX_PUT: { @@ -268,16 +260,12 @@ private boolean doOperation(LoadMode mode, int key) { tx.commit(); } }); - - return true; } case SQL_PUT: { SqlFieldsQuery qry = new SqlFieldsQuery(mergeSql).setArgs(key, key); timed(stats, () -> sqlCache.query(qry).getAll()); - - return true; } case SQL_SELECT: { @@ -285,13 +273,9 @@ private boolean doOperation(LoadMode mode, int key) { List> rows = timed(stats, () -> sqlCache.query(qry).getAll()); - boolean ok = !rows.isEmpty() && Objects.equals(rows.get(0).get(0), key); - - if (!ok) - log.error("SQL row is missed or corrupted [dc=" + dcId() + ", key=" + key + + if (rows.isEmpty() || !Objects.equals(rows.get(0).get(0), key)) + throw new IgniteException("SQL row is missed or corrupted [dc=" + dcId() + ", key=" + key + ", rows=" + rows + "]"); - - return ok; } default: diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java index e854cfc49a0d2..dc81338f5e990 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java @@ -19,6 +19,7 @@ import javax.cache.CacheException; import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteException; import org.apache.ignite.client.ClientCache; import org.apache.ignite.client.ClientException; import org.apache.ignite.internal.ducktest.tests.dto.IndexedDataRecord; @@ -81,29 +82,26 @@ public class MdcThinClientLoadApplication extends IgniteAwareApplication { for (long i = 0; i < iterations && !terminated(); i++) { int key = key0; - boolean ok; + boolean ok = true; try { if (put) { IndexedDataRecord val = new IndexedDataRecord(key); timed(stats, () -> cache.put(key, val)); - - ok = true; } else { IndexedDataRecord val = timed(stats, () -> cache.get(key)); - ok = val != null && val.equals(new IndexedDataRecord(key)); - - if (!ok) - log.error("Read entry is missed or corrupted [key=" + key + ", val=" + val + "]"); + if (val == null || !val.equals(new IndexedDataRecord(key))) + throw new IgniteException("Read entry is missed or corrupted [key=" + key + + ", val=" + val + "]"); } } - catch (ClientException | CacheException e) { + catch (ClientException | CacheException | IgniteException e) { ok = false; - if (put && !expectAdmissible) + if (!expectAdmissible) log.info("Put rejected as expected [key=" + key + ", msg=" + e.getMessage() + "]"); else throw new IllegalStateException("Operation failed [mode=" + mode + ", key=" + key + "]", e); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java index 97873529b00aa..c32de2e40b664 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java @@ -59,22 +59,14 @@ public static String fmtMs(double ns) { return String.format(Locale.US, "%.3f", ns < 0 ? -1.0 : ns / 1e6); } - /** - * Parses an optional enum-valued field. A missing, null or unrecognized value falls back - * to {@code dfltVal}. - */ + /** Parses an optional enum-valued field. Defaults only on missing/null. */ public static > E getEnum(JsonNode jNode, String fieldName, E dfltVal) { JsonNode field = jNode.path(fieldName); if (field.isMissingNode() || field.isNull()) return dfltVal; - try { - return Enum.valueOf(dfltVal.getDeclaringClass(), field.asText().toUpperCase(Locale.ROOT)); - } - catch (IllegalArgumentException e) { - return dfltVal; // Fallback if string doesn't match any enum constant. - } + return Enum.valueOf(dfltVal.getDeclaringClass(), field.asText().toUpperCase(Locale.ROOT)); } /** diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index c8f6cbe19bad4..7005436ed3c1b 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -244,7 +244,7 @@ def __parse_cache_distribution(output, user_attributes=None): attrs = {} if user_attributes and match.group("attr_values") is not None: values = [v.strip() for v in match.group("attr_values").split(",")] - attrs = dict(zip(user_attributes, values)) + attrs = dict(zip(sorted(user_attributes), values)) copy = PartitionCopy(node_id=match.group("node_id"), primary=match.group("primary") == "P", @@ -492,9 +492,6 @@ def __parse_cluster_state(output): coordinator = next((node for node in baseline if node.order == order), None) - if coordinator is None: - raise AssertionError(f"Coordinator with order={order} is not found in baseline [baseline={baseline}]") - return ClusterState(state=state, topology_version=topology, baseline=baseline, coordinator=coordinator) def __run(self, cmd, node=None): diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py index ee092651b7df9..e00a261b58905 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py @@ -54,7 +54,7 @@ def __init__(self, use_paired_connections: bool = None, message_queue_limit: int = None, unacknowledged_messages_buffer_size: int = None, - connect_timeout: int = 5_000): + connect_timeout: int = None): self.local_port = local_port self.local_port_range = local_port_range self.idle_connection_timeout: int = idle_connection_timeout diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py index cca1ec0532c31..f54af09cd016d 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py @@ -57,7 +57,7 @@ class MdcThinClientTest(IgniteTest): """ Tests for thin client DC-aware routing and behavior through a partition. """ - @cluster(num_nodes=8) + @cluster(num_nodes=7) @ignite_versions(str(DEV_BRANCH)) @parametrize(cross_dc_latency_ms=100) def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_dc_latency_ms): @@ -88,13 +88,17 @@ def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_ "resultPrefix": "pinnedGet"}) avg_cli_dc_1 = mdc.result_float(svc, "pinnedGetAvgOpMs") + err_cnt_cli_dc_1 = mdc.result_int(svc, "pinnedGetErrCnt") - self.logger.info(f"Thin client routing latency [delayMs={cross_dc_latency_ms}, getAvgOpMs={avg_cli_dc_1}]") + self.logger.info(f"Thin client routing latency [delayMs={cross_dc_latency_ms}, getAvgOpMs={avg_cli_dc_1}, " + f"getErrCnt={err_cnt_cli_dc_1}]") assert avg_cli_dc_1 < cross_dc_latency_ms, \ f"DC-pinned thin client reads should be served locally " \ f"[avgMs={avg_cli_dc_1}, delayMs={cross_dc_latency_ms}]" + assert err_cnt_cli_dc_1 == 0, f"Expected 0 errors for DC-pinned thin client reads, but found {err_cnt_cli_dc_1}" + net.enable_network_partition(DC_1, DC_2) sleep(SPLIT_SETTLE_SECS) diff --git a/modules/ducktests/tests/tox.ini b/modules/ducktests/tests/tox.ini index e0fa4a74df8aa..5c598aa55f8fb 100644 --- a/modules/ducktests/tests/tox.ini +++ b/modules/ducktests/tests/tox.ini @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. [tox] -envlist = codestyle, py{38,39,310,311,312,313} +envlist = codestyle, py{39,310,311,312,313} skipsdist = True [testenv] From 09f8b90b3b118a1608c578d6f4397688dd788a0b Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Fri, 17 Jul 2026 00:59:13 +0300 Subject: [PATCH 32/44] IGNITE-27833 discovery SPI documented --- .../ignitetest/services/mdc/mdc_cluster.py | 17 ++++++++++++++--- .../tests/mdc/partition_resilience_test.py | 3 +++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py index 1508c08e31b7e..38d69109fda8b 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -34,7 +34,8 @@ from ignitetest.services.network_group.manager import NetworkGroupManager from ignitetest.services.utils.control_utility import ControlUtility from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, TcpCommunicationSpi -from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster, from_ignite_services +from ignitetest.services.utils.ignite_configuration.discovery import TcpDiscoverySpi, from_ignite_cluster, \ + from_ignite_services from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration from ignitetest.utils.version import IgniteVersion @@ -100,8 +101,15 @@ def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, i self.test_context = test.test_context self.logger = test.logger + # A single discovery SPI (hence a single ip finder) shared by both DCs' server + # services is what makes the two DCs form ONE cluster: prepare_on_start() + # memoizes the addresses of the first started DC into the shared ip finder, so + # the second DC discovers through the first DC's nodes, and restart() re-joins + # the same way. Restarting the first started DC itself is the one case this + # breaks - see sync_service_discovery(). cfg_kwargs = { "version": IgniteVersion(ignite_version), + "discovery_spi": TcpDiscoverySpi(), "network_timeout": network_timeout, "communication_spi": TcpCommunicationSpi(connect_timeout=tcp_connect_timeout) } @@ -138,8 +146,11 @@ def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, i def sync_service_discovery(self): """ - Points every server service at a discovery SPI covering all DCs (used to let a - stopped DC rejoin the full cluster on restart). + Points every server service at a discovery SPI covering all DCs. + + Required before restarting the FIRST started DC: the shared ip finder holds only + that DC's addresses, so after a full stop its nodes would seed off themselves and + form a separate cluster instead of rejoining the surviving DC. """ discovery_spi = from_ignite_services(list(self.servers.values())) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index 89d9b9db28335..666992d417ef1 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -130,6 +130,9 @@ def test_main_dc_loss_and_return(self, ignite_version, cross_dc_latency_ms): # ...but the surviving DC is read-only while the main DC is invisible. mdc.check_put_admissibility(DC_2, CACHE_NAME, False) + # The shared ip finder memoized only DC1's (first started) addresses, so a + # restarted DC1 would seed off itself and form a separate cluster. Point + # discovery at both DCs so it rejoins through the surviving DC2. mdc.sync_service_discovery() mdc.servers[DC_1].start(clean=False) From d14ab0535f7b4501f8c94998e56745e75ec29640 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Fri, 17 Jul 2026 01:18:55 +0300 Subject: [PATCH 33/44] IGNITE-27833 codestyle --- .../ducktests/tests/ignitetest/tests/mdc/thin_client_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py index f54af09cd016d..776b61f79b052 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py @@ -97,7 +97,8 @@ def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_ f"DC-pinned thin client reads should be served locally " \ f"[avgMs={avg_cli_dc_1}, delayMs={cross_dc_latency_ms}]" - assert err_cnt_cli_dc_1 == 0, f"Expected 0 errors for DC-pinned thin client reads, but found {err_cnt_cli_dc_1}" + assert err_cnt_cli_dc_1 == 0, (f"Expected 0 errors for DC-pinned thin client reads, " + f"but found {err_cnt_cli_dc_1}") net.enable_network_partition(DC_1, DC_2) From 69b6538c0fc7f131c2001a282e721673a70c9a89 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Fri, 17 Jul 2026 11:16:15 +0300 Subject: [PATCH 34/44] IGNITE-27833 codestyle --- .github/workflows/commit-check.yml | 1 + .../mdc/MdcContinuousLoadApplication.java | 10 ++++ modules/ducktests/tests/docker/Dockerfile | 50 +++++++++---------- .../ducktests/tests/docker/requirements.txt | 2 +- .../tests/mdc/partition_resilience_test.py | 3 +- modules/ducktests/tests/tox.ini | 2 +- 6 files changed, 40 insertions(+), 28 deletions(-) diff --git a/.github/workflows/commit-check.yml b/.github/workflows/commit-check.yml index f7c948f89d762..88e1356b09a91 100644 --- a/.github/workflows/commit-check.yml +++ b/.github/workflows/commit-check.yml @@ -158,6 +158,7 @@ jobs: fail-fast: false matrix: cfg: + - { python: "3.8", toxenv: "py38" } - { python: "3.9", toxenv: "py39" } - { python: "3.9", toxenv: "codestyle" } steps: diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java index ab109cb1e58c3..5fc1d33194a24 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java @@ -242,12 +242,16 @@ private void doOperation(LoadMode mode, int key) { if (val == null || !val.equals(new IndexedDataRecord(key))) throw new IgniteException("Read entry is missed or corrupted [dc=" + dcId() + ", key=" + key + ", val=" + val + "]"); + + break; } case PUT: { IndexedDataRecord val = new IndexedDataRecord(key); timed(stats, () -> cache.put(key, val)); + + break; } case TX_PUT: { @@ -260,12 +264,16 @@ private void doOperation(LoadMode mode, int key) { tx.commit(); } }); + + break; } case SQL_PUT: { SqlFieldsQuery qry = new SqlFieldsQuery(mergeSql).setArgs(key, key); timed(stats, () -> sqlCache.query(qry).getAll()); + + break; } case SQL_SELECT: { @@ -276,6 +284,8 @@ private void doOperation(LoadMode mode, int key) { if (rows.isEmpty() || !Objects.equals(rows.get(0).get(0), key)) throw new IgniteException("SQL row is missed or corrupted [dc=" + dcId() + ", key=" + key + ", rows=" + rows + "]"); + + break; } default: diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index db6f4222f0ec4..ffacf36255ae6 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -126,31 +126,31 @@ RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" ARG APACHE_ARCHIVE="https://archive.apache.org/dist/" -# Install Ignite -RUN for v in "2.7.6" "2.17.0"; \ - do cd /opt; \ - curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\ - unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v; \ - done \ - && rm /opt/apache-ignite-*-bin.zip - -# Install Zookeeper -ARG ZOOKEEPER_VERSION="3.5.8" -ARG ZOOKEEPER_NAME="zookeeper-$ZOOKEEPER_VERSION" -ARG ZOOKEEPER_RELEASE_NAME="apache-$ZOOKEEPER_NAME-bin" -ARG ZOOKEEPER_RELEASE_ARTIFACT="$ZOOKEEPER_RELEASE_NAME.tar.gz" -RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT \ - && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT \ - && mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME - -# Install Kafka -ARG KAFKA_VERSION="3.9.1" -ARG KAFKA_NAME="kafka" -ARG KAFKA_RELEASE_NAME="${KAFKA_NAME}_2.13-$KAFKA_VERSION" -ARG KAFKA_RELEASE_ARTIFACT="$KAFKA_RELEASE_NAME.tgz" -RUN cd /opt && curl -O $APACHE_ARCHIVE/kafka/$KAFKA_VERSION/$KAFKA_RELEASE_ARTIFACT \ - && tar xvf $KAFKA_RELEASE_ARTIFACT && rm $KAFKA_RELEASE_ARTIFACT \ - && mv /opt/$KAFKA_RELEASE_NAME /opt/$KAFKA_NAME-$KAFKA_VERSION +## Install Ignite +#RUN for v in "2.7.6" "2.17.0"; \ +# do cd /opt; \ +# curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\ +# unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v; \ +# done \ +# && rm /opt/apache-ignite-*-bin.zip + +## Install Zookeeper +#ARG ZOOKEEPER_VERSION="3.5.8" +#ARG ZOOKEEPER_NAME="zookeeper-$ZOOKEEPER_VERSION" +#ARG ZOOKEEPER_RELEASE_NAME="apache-$ZOOKEEPER_NAME-bin" +#ARG ZOOKEEPER_RELEASE_ARTIFACT="$ZOOKEEPER_RELEASE_NAME.tar.gz" +#RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT \ +# && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT \ +# && mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME +# +## Install Kafka +#ARG KAFKA_VERSION="3.9.1" +#ARG KAFKA_NAME="kafka" +#ARG KAFKA_RELEASE_NAME="${KAFKA_NAME}_2.13-$KAFKA_VERSION" +#ARG KAFKA_RELEASE_ARTIFACT="$KAFKA_RELEASE_NAME.tgz" +#RUN cd /opt && curl -O $APACHE_ARCHIVE/kafka/$KAFKA_VERSION/$KAFKA_RELEASE_ARTIFACT \ +# && tar xvf $KAFKA_RELEASE_ARTIFACT && rm $KAFKA_RELEASE_ARTIFACT \ +# && mv /opt/$KAFKA_RELEASE_NAME /opt/$KAFKA_NAME-$KAFKA_VERSION # Install Jmxterm ARG JMXTERM_NAME="jmxterm" diff --git a/modules/ducktests/tests/docker/requirements.txt b/modules/ducktests/tests/docker/requirements.txt index 469c95f7b1a21..6f96304e5b22c 100644 --- a/modules/ducktests/tests/docker/requirements.txt +++ b/modules/ducktests/tests/docker/requirements.txt @@ -16,4 +16,4 @@ filelock==3.8.2 ducktape==0.13.0 looseversion==1.3.0 -tcconfig==0.30.1 +tcconfig==0.29.1 diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index 666992d417ef1..53c4ef0a969bd 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -159,7 +159,8 @@ def test_short_partition_blips_do_not_split(self, ignite_version, cross_dc_laten cross-DC connectivity drops. The cluster must NOT split: after the blips it is still one ACTIVE cluster, all data is intact and both DCs accept writes. """ - mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc=1, loaders_per_dc=1) + mdc = MdcCluster(self, ignite_version, srv_per_dc=3, runners_per_dc=1, loaders_per_dc=1, + network_timeout=20_000, tcp_connect_timeout=10_000) with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() diff --git a/modules/ducktests/tests/tox.ini b/modules/ducktests/tests/tox.ini index 5c598aa55f8fb..e0fa4a74df8aa 100644 --- a/modules/ducktests/tests/tox.ini +++ b/modules/ducktests/tests/tox.ini @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. [tox] -envlist = codestyle, py{39,310,311,312,313} +envlist = codestyle, py{38,39,310,311,312,313} skipsdist = True [testenv] From 22e2ea9e9ea3b4ae2fc467e20e572b76c08d06f8 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 20 Jul 2026 18:40:02 +0300 Subject: [PATCH 35/44] IGNITE-27833 Rename MDC load flags to remove negated boolean options Replace the always-negated expectAdmissible and tolerateErrors profile options with inadmissible and continueOnError, and reorder the exception handling into a negation-free error-policy ladder (inadmissible -> stopOnError -> continueOnError -> fail). Behavior and precedence are unchanged. --- .../mdc/MdcContinuousLoadApplication.java | 32 +++++++++---------- .../mdc/MdcThinClientLoadApplication.java | 10 +++--- .../ignitetest/services/mdc/mdc_cluster.py | 2 +- .../tests/mdc/partition_resilience_test.py | 2 +- .../ignitetest/tests/mdc/thin_client_test.py | 2 +- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java index 5fc1d33194a24..d00486fa08ecd 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java @@ -54,17 +54,17 @@ * write modes advance sequentially from {@code keyFrom} on every success, so on * completion the keys {@code [keyFrom, keyFrom + opsCnt)} are guaranteed written; *

  • {@code iterations} - number of operations; {@code 0} means "run until terminated";
    • - *
  • {@code expectAdmissible} - write modes only. {@code true}: writes must succeed - * (default); {@code false}: every write must be rejected by the topology validator + *
  • {@code inadmissible} - write modes only. {@code false} (default): writes must + * succeed; {@code true}: every write must be rejected by the topology validator * (read-only DC), any success fails the application;
    • - *
  • {@code tolerateErrors} - if {@code true}, operation failures are counted instead of - * failing fast. Intended for background loads crossing a partition boundary, where a - * short transient error window is possible;
    • *
  • {@code stopOnError} - if {@code true}, the load stops gracefully on the very first * operation failure (rather than failing the application), records the number of * successful operations and a {@code StoppedOnError} flag, and finishes. Takes precedence - * over {@code tolerateErrors}. Intended for a load that must be cut off by the first + * over {@code continueOnError}. Intended for a load that must be cut off by the first * exception a network partition triggers;
    • + *
  • {@code continueOnError} - if {@code true}, operation failures are counted instead of + * failing fast. Intended for background loads crossing a partition boundary, where a + * short transient error window is possible;
    • *
  • {@code opPauseMs} - pause between operations, default 0;
    • *
  • {@code resultPrefix} - prefix for recorded results, so that several runs reusing one * service produce uniquely named results;
    • @@ -110,8 +110,8 @@ public class MdcContinuousLoadApplication extends MdcCacheAwareApplication { int keyTo = jNode.path("keyTo").asInt(Integer.MAX_VALUE); long iterations = jNode.path("iterations").asLong(0); - boolean expectAdmissible = jNode.path("expectAdmissible").asBoolean(true); - boolean tolerateErrors = jNode.path("tolerateErrors").asBoolean(false); + boolean inadmissible = jNode.path("inadmissible").asBoolean(false); + boolean continueOnError = jNode.path("continueOnError").asBoolean(false); boolean stopOnError = jNode.path("stopOnError").asBoolean(false); long opPauseMs = jNode.path("opPauseMs").asLong(0); @@ -134,7 +134,7 @@ public class MdcContinuousLoadApplication extends MdcCacheAwareApplication { log.info("MDC load started [dc=" + dcId() + ", mode=" + mode + ", cache=" + cacheName + ", keyFrom=" + keyFrom + ", keyTo=" + keyTo + ", iterations=" + iterations + - ", expectAdmissible=" + expectAdmissible + ", tolerateErrors=" + tolerateErrors + "]"); + ", inadmissible=" + inadmissible + ", continueOnError=" + continueOnError + "]"); long opsCnt = 0; long errCnt = 0; @@ -157,7 +157,7 @@ public class MdcContinuousLoadApplication extends MdcCacheAwareApplication { catch (CacheException | IgniteException e) { ok = false; - if (!expectAdmissible) + if (inadmissible) log.info("Write rejected as expected [dc=" + dcId() + ", key=" + key + ", msg=" + e.getMessage() + "]"); else if (stopOnError) { @@ -169,12 +169,12 @@ else if (stopOnError) { break; } - else if (!tolerateErrors) - throw new IllegalStateException("Operation failed [dc=" + dcId() + ", mode=" + mode + - ", key=" + key + "]", e); - else + else if (continueOnError) log.warn("Operation failed, tolerated [dc=" + dcId() + ", mode=" + mode + ", key=" + key + ", msg=" + e.getMessage() + "]"); + else + throw new IllegalStateException("Operation failed [dc=" + dcId() + ", mode=" + mode + + ", key=" + key + "]", e); } if (ok) { @@ -190,7 +190,7 @@ else if (!tolerateErrors) // Writes advance on success only, so [keyFrom, keyFrom + opsCnt) is guaranteed written. // The inadmissible-probe mode advances always to probe distinct keys. Reads cycle the range. - if (ok || (mode.isWrite() && !expectAdmissible)) { + if (ok || (mode.isWrite() && inadmissible)) { key++; if (key >= keyTo) @@ -203,7 +203,7 @@ else if (!tolerateErrors) long durationMs = System.currentTimeMillis() - startTs; - if (mode.isWrite() && !expectAdmissible && opsCnt > 0) { + if (mode.isWrite() && inadmissible && opsCnt > 0) { throw new IllegalStateException("Write load is admissible while expected to be inadmissible [dc=" + dcId() + ", mode=" + mode + ", succeeded=" + opsCnt + ", rejected=" + errCnt + "]"); } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java index dc81338f5e990..2eaf491796e79 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java @@ -39,7 +39,7 @@ * delay, DC-local routing yields a small average, cross-DC routing a large one. *

    * Parameters: {@code mode} ({@code GET}/{@code PUT}), {@code cacheName}, {@code keyFrom}, - * {@code keyTo}, {@code iterations}, {@code expectAdmissible} (PUT only), + * {@code keyTo}, {@code iterations}, {@code inadmissible} (PUT only), * {@code resultPrefix}. *

    */ @@ -58,7 +58,7 @@ public class MdcThinClientLoadApplication extends IgniteAwareApplication { long iterations = jNode.path("iterations").asLong(100); boolean put = mode == LoadMode.PUT; - boolean expectAdmissible = jNode.path("expectAdmissible").asBoolean(true); + boolean inadmissible = jNode.path("inadmissible").asBoolean(false); String pfx = jNode.path("resultPrefix").asText(""); @@ -68,7 +68,7 @@ public class MdcThinClientLoadApplication extends IgniteAwareApplication { log.info("MDC thin client load started [mode=" + mode + ", cache=" + cacheName + ", keyFrom=" + keyFrom + ", keyTo=" + keyTo + ", iterations=" + iterations + - ", expectAdmissible=" + expectAdmissible + "]"); + ", inadmissible=" + inadmissible + "]"); long opsCnt = 0; long errCnt = 0; @@ -101,7 +101,7 @@ public class MdcThinClientLoadApplication extends IgniteAwareApplication { catch (ClientException | CacheException | IgniteException e) { ok = false; - if (!expectAdmissible) + if (inadmissible) log.info("Put rejected as expected [key=" + key + ", msg=" + e.getMessage() + "]"); else throw new IllegalStateException("Operation failed [mode=" + mode + ", key=" + key + "]", e); @@ -122,7 +122,7 @@ public class MdcThinClientLoadApplication extends IgniteAwareApplication { long durationMs = System.currentTimeMillis() - startTs; - if (put && !expectAdmissible && opsCnt > 0) { + if (put && inadmissible && opsCnt > 0) { throw new IllegalStateException("Put load is admissible while expected to be inadmissible " + "[succeeded=" + opsCnt + ", rejected=" + errCnt + "]"); } diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py index 38d69109fda8b..2b3d7f20d5dd2 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -325,7 +325,7 @@ def check_put_admissibility(self, dc: str, cache_name: str, admissible: bool, return self.run_load(dc, "PUT", cache_name, f"admCheck{self._adm_checks}", keyFrom=key_offset, keyTo=key_offset + probes, - iterations=probes, expectAdmissible=admissible) + iterations=probes, inadmissible=not admissible) def run_load(self, dc: str, mode: str, cache_name: str, result_prefix: str, runner: int = 0, **params) -> IgniteApplicationService: diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index 53c4ef0a969bd..2648fa693d794 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -171,7 +171,7 @@ def test_short_partition_blips_do_not_split(self, ignite_version, cross_dc_laten for dc in (DC_1, DC_2): mdc.start_loader(dc, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, "keyTo": 200, - "tolerateErrors": True, "opPauseMs": 5, + "continueOnError": True, "opPauseMs": 5, "resultPrefix": f"bg{dc}"}) for i in range(BLIPS): diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py index 776b61f79b052..da75d779746d2 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py @@ -124,7 +124,7 @@ def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_ mdc.run_service(cli_dc2, {"mode": "PUT", "cacheName": CACHE_NAME, "keyFrom": OFFSET_REJECTED_PROBES, "keyTo": OFFSET_REJECTED_PROBES + PUT_ITERS, - "iterations": PUT_ITERS, "expectAdmissible": False, + "iterations": PUT_ITERS, "inadmissible": True, "resultPrefix": "roPut"}) net.disable_network_partition(DC_1, DC_2) From c043aa3a50a22d35e26847d5cbc700252cc018d9 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 20 Jul 2026 18:44:51 +0300 Subject: [PATCH 36/44] IGNITE-27833 minor fixes --- .../ducktest/tests/mdc/MdcDataGeneratorApplication.java | 2 +- .../java/org/apache/ignite/internal/ducktest/utils/Utils.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java index e73dbfb66af0e..b64b75b823d68 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java @@ -28,7 +28,7 @@ /** * Populates the MDC cache with a deterministic data set: keys in {@code [from, to)}, * values {@code IndexedDataRecord(key)} (or the key itself in {@code sqlMode}). - * Run it before the network partition. + * Runs it until the network partition event. */ public class MdcDataGeneratorApplication extends MdcCacheAwareApplication { /** {@inheritDoc} */ diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java index c32de2e40b664..25324348bdc4f 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/Utils.java @@ -59,7 +59,7 @@ public static String fmtMs(double ns) { return String.format(Locale.US, "%.3f", ns < 0 ? -1.0 : ns / 1e6); } - /** Parses an optional enum-valued field. Defaults only on missing/null. */ + /** Parses an optional enum-valued field. Defaults only on missing/null. */ public static > E getEnum(JsonNode jNode, String fieldName, E dfltVal) { JsonNode field = jNode.path(fieldName); From a683873774b17b530a79012a45a6b87087b6482c Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 22 Jul 2026 14:12:30 +0300 Subject: [PATCH 37/44] IGNITE-27833 fix baseline tests & codestyle nits --- .../tests/mdc/MdcDataGeneratorApplication.java | 2 +- .../tests/ignitetest/services/mdc/mdc_cluster.py | 3 ++- .../tests/control_utility/baseline_test.py | 12 ++++++------ .../tests/ignitetest/tests/index_rebuild_test.py | 4 ++-- .../tests/mdc/transactional_partition_test.py | 10 +++++----- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java index b64b75b823d68..335efb52b8183 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcDataGeneratorApplication.java @@ -28,7 +28,7 @@ /** * Populates the MDC cache with a deterministic data set: keys in {@code [from, to)}, * values {@code IndexedDataRecord(key)} (or the key itself in {@code sqlMode}). - * Runs it until the network partition event. + * Run it to completion before enabling the network partition. */ public class MdcDataGeneratorApplication extends MdcCacheAwareApplication { /** {@inheritDoc} */ diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py index 2b3d7f20d5dd2..0a32ea29ae444 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -62,6 +62,7 @@ LRT_PATTERN = "long running transactions" PME_FREEZE_PATTERN = "Failed to wait for partition map exchange" LOST_PARTITIONS_PATTERN = "Detected lost partitions" +ASSERTION_ERROR_PATTERN = "AssertionError" def dc_jvm_opts(dc: str) -> List[str]: @@ -430,7 +431,7 @@ def verify_servers_log_clean(self): Verifies the negative invariants on all server nodes: no long running transactions were detected, no PME hang and no lost partitions were reported. """ - for pattern in (LRT_PATTERN, PME_FREEZE_PATTERN, LOST_PARTITIONS_PATTERN): + for pattern in (LRT_PATTERN, PME_FREEZE_PATTERN, LOST_PARTITIONS_PATTERN, ASSERTION_ERROR_PATTERN): for svc in self.servers.values(): svc.check_event_absent(pattern) diff --git a/modules/ducktests/tests/ignitetest/tests/control_utility/baseline_test.py b/modules/ducktests/tests/ignitetest/tests/control_utility/baseline_test.py index d9211786707ed..6bab99208caf7 100644 --- a/modules/ducktests/tests/ignitetest/tests/control_utility/baseline_test.py +++ b/modules/ducktests/tests/ignitetest/tests/control_utility/baseline_test.py @@ -63,8 +63,8 @@ def test_baseline_set(self, ignite_version): # Set baseline using topology version. new_node = self.__start_ignite_nodes(ignite_version, 1, join_cluster=servers) - _, version, _ = control_utility.cluster_state() - control_utility.set_baseline(version) + cluster_state = control_utility.cluster_state() + control_utility.set_baseline(cluster_state.topology_version) blt_size += 1 baseline = control_utility.baseline() @@ -125,15 +125,15 @@ def test_activate_deactivate(self, ignite_version): control_utility.activate() - state, _, _ = control_utility.cluster_state() + cluster_state = control_utility.cluster_state() - assert state.lower() == 'active', 'Unexpected state %s' % state + assert cluster_state.state.lower() == 'active', 'Unexpected state %s' % cluster_state.state control_utility.deactivate() - state, _, _ = control_utility.cluster_state() + cluster_state = control_utility.cluster_state() - assert state.lower() == 'inactive', 'Unexpected state %s' % state + assert cluster_state.state.lower() == 'inactive', 'Unexpected state %s' % cluster_state.state @cluster(num_nodes=NUM_NODES) @ignore_if(lambda version, globals: version < V_2_8_0) diff --git a/modules/ducktests/tests/ignitetest/tests/index_rebuild_test.py b/modules/ducktests/tests/ignitetest/tests/index_rebuild_test.py index 18b73446a4b80..d78ff64135a83 100644 --- a/modules/ducktests/tests/ignitetest/tests/index_rebuild_test.py +++ b/modules/ducktests/tests/ignitetest/tests/index_rebuild_test.py @@ -80,8 +80,8 @@ def test_index_bin_rebuild(self, ignite_version, backups, cache_count, entry_cou control_utility.disable_baseline_auto_adjust() - _, version, _ = control_utility.cluster_state() - control_utility.set_baseline(version) + cluster_state = control_utility.cluster_state() + control_utility.set_baseline(cluster_state.topology_version) preload_time = preload_data( self.test_context, diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py index 73364e2a55221..187453bbe18d0 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/transactional_partition_test.py @@ -18,7 +18,7 @@ A TRANSACTIONAL cache spans both data centers: with backups=1 and the MdcAffinityBackupFilter every partition owns exactly one copy per DC, so every -implicit-transaction write (a plain put on a transactional cache) must reach a +explicit-transaction write (a plain put on a transactional cache) must reach a node in the other DC. A continuous single-threaded insert load runs from the main DC; the instant the DCs are partitioned the next commit cannot reach all partition copies and fails with a cache exception. The load cuts itself off on that first @@ -26,7 +26,7 @@ After the split settles the test asserts that the cluster really split-brained, that the load stopped because of the partition (not because it ran out of work), -and - the point of the scenario - that the aborted implicit transactions left +and - the point of the scenario - that the aborted explicit transactions left nothing hanging on either half-ring and no suspicious entries in the server logs. Data accessibility during the split is deliberately NOT checked: a transactional @@ -71,7 +71,7 @@ class MdcTransactionalPartitionTest(IgniteTest): @parametrize(cross_dc_latency_ms=100) def test_transactional_load_cut_on_partition(self, ignite_version, cross_dc_latency_ms): """ - Continuous implicit-transaction insert load from the main DC is cut off by the first + Continuous explicit-transaction insert load from the main DC is cut off by the first cache exception the cross-DC partition triggers; afterwards no transaction is left hanging on either half-ring and the server logs are clean. """ @@ -81,7 +81,7 @@ def test_transactional_load_cut_on_partition(self, ignite_version, cross_dc_late with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() - # Continuous single-threaded implicit-transaction insert load from the main DC. + # Continuous single-threaded explicit-transaction insert load from the main DC. # It stops on the very first exception (stopOnError) instead of failing the app, # recording how many inserts had succeeded up to that point. for dc, offset_from, to in [(DC_1, LOAD_KEY_FROM_DC_1, LOAD_KEY_TO_DC_1), @@ -121,7 +121,7 @@ def test_transactional_load_cut_on_partition(self, ignite_version, cross_dc_late assert inserts > 0, "The transactional load performed no successful inserts" - # The point of the scenario: the aborted implicit transactions leave nothing + # The point of the scenario: the aborted explicit transactions leave nothing # hanging on either half-ring... mdc.verify_no_hanging_txs(DC_1) mdc.verify_no_hanging_txs(DC_2) From bd980df28481febf2f528d783a1a04b21aaab911 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 22 Jul 2026 14:59:53 +0300 Subject: [PATCH 38/44] IGNITE-27833 ignite log event trigger fixed --- .../ducktests/tests/ignitetest/services/mdc/mdc_cluster.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py index 0a32ea29ae444..176f6ad37d89c 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -64,6 +64,9 @@ LOST_PARTITIONS_PATTERN = "Detected lost partitions" ASSERTION_ERROR_PATTERN = "AssertionError" +# Every log of a server node, including the ones rotated by a restart. +ALL_LOGS_GLOB = "ignite*.log*" + def dc_jvm_opts(dc: str) -> List[str]: """ @@ -433,7 +436,7 @@ def verify_servers_log_clean(self): """ for pattern in (LRT_PATTERN, PME_FREEZE_PATTERN, LOST_PARTITIONS_PATTERN, ASSERTION_ERROR_PATTERN): for svc in self.servers.values(): - svc.check_event_absent(pattern) + svc.check_event_absent(pattern, log_file=ALL_LOGS_GLOB) def verify_no_hanging_txs(self, dc: str = DC_1): """ From abd76f376140e37ebe1b2a894ede835a268232a4 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 22 Jul 2026 15:01:09 +0300 Subject: [PATCH 39/44] IGNITE-27833 optimize network state logging --- .../services/network_group/check_manager.py | 141 ++++++++++++++++++ .../services/network_group/manager.py | 102 ++++++++++--- 2 files changed, 220 insertions(+), 23 deletions(-) create mode 100644 modules/ducktests/tests/checks/services/network_group/check_manager.py diff --git a/modules/ducktests/tests/checks/services/network_group/check_manager.py b/modules/ducktests/tests/checks/services/network_group/check_manager.py new file mode 100644 index 0000000000000..115c641d277cd --- /dev/null +++ b/modules/ducktests/tests/checks/services/network_group/check_manager.py @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks the batched network probe NetworkGroupManager._log_network issues. + +The probe replaces three per-node SSH round-trips with a single command, so what +matters is that its combined output still splits back into exactly the three +sections the separate commands produced, and that an incomplete answer degrades +into a poorer overview instead of an exception - this runs inside an active +network partition, where nothing may fail the test around it. +""" + +import pytest + +from ignitetest.services.network_group.manager import NetworkGroupManager, PROBE_SECTION_SEPARATOR +from ignitetest.services.network_group.tc_rule_args import partition_chain_name + +INTERFACE = "eth0" + +CHAIN = partition_chain_name("DC1", "DC2") + +QDISC_SECTION = "qdisc netem 8001: root refcnt 2 limit 1000 delay 100ms loss 5%" + +# 'c0a80105' is 192.168.1.5 as tc reports it in a u32 destination match. +FILTER_SECTION = "filter parent 8001: protocol ip pref 1 u32 chain 0\n" \ + " match c0a80105/ffffffff at 16" + +IPTABLES_SECTION = "\n".join([ + "-P INPUT ACCEPT", + f"-N {CHAIN}", + f"-A {CHAIN} -s 192.168.1.5/32 -j DROP", + f"-A {CHAIN} -d 192.168.1.5/32 -j DROP" +]) + + +def _probe(*sections): + return f"\n{PROBE_SECTION_SEPARATOR}\n".join(sections) + + +# What each section renders as in the overview when the node did not answer with it. +NEUTRAL_RENDERINGS = [ + (NetworkGroupManager._parse_qdisc_constraints, "noqueue"), + (NetworkGroupManager._parse_filter_destinations, []), + (lambda lines: NetworkGroupManager._format_partition_drops( + NetworkGroupManager._parse_partition_drops(lines)), "") +] + + +class CheckNetworkProbe: + """ + Checks the composition and the parsing of the batched network probe. + """ + def check_probe_cmd_delimits_every_section(self, monkeypatch): + """The probe asks for all three dumps in one command, separator between each.""" + monkeypatch.setattr(NetworkGroupManager, "_get_default_network_interface", + lambda self, node: INTERFACE) + + cmd = NetworkGroupManager(logger=None, network_group_store=None, + network_group_registry={})._to_network_probe_cmd(node=None) + + assert cmd.count(PROBE_SECTION_SEPARATOR) == 2, "Three sections need exactly two separators" + + assert f"tc qdisc show dev {INTERFACE}" in cmd + assert f"tc filter show dev {INTERFACE}" in cmd + assert "iptables -S" in cmd + + # Unconditional chaining: one unavailable dump must not swallow the ones after it. + assert "&&" not in cmd + + def check_splits_into_the_three_sections(self): + """A complete probe yields exactly what the three separate commands produced.""" + qdisc, flt, iptables = NetworkGroupManager._split_probe_output( + _probe(QDISC_SECTION, FILTER_SECTION, IPTABLES_SECTION)) + + assert qdisc == QDISC_SECTION.splitlines() + assert flt == FILTER_SECTION.splitlines() + assert iptables == IPTABLES_SECTION.splitlines() + + def check_parses_an_active_partition(self): + """The deployed impairment and the fully cut peer both survive the round trip.""" + qdisc, flt, iptables = NetworkGroupManager._split_probe_output( + _probe(QDISC_SECTION, FILTER_SECTION, IPTABLES_SECTION)) + + assert NetworkGroupManager._parse_qdisc_constraints(qdisc) == "netem(delay: 100ms, loss: 5%)" + assert NetworkGroupManager._parse_filter_destinations(flt) == ["192.168.1.5"] + + drops = NetworkGroupManager._format_partition_drops( + NetworkGroupManager._parse_partition_drops(iptables)) + + assert drops == f" | partition: {CHAIN} <-X-> [192.168.1.5]" + + def check_parses_a_healed_link(self): + """After a heal the netem impairment is still reported, the drops are gone.""" + qdisc, _, iptables = NetworkGroupManager._split_probe_output( + _probe(QDISC_SECTION, FILTER_SECTION, "-P INPUT ACCEPT")) + + assert NetworkGroupManager._parse_qdisc_constraints(qdisc) == "netem(delay: 100ms, loss: 5%)" + assert NetworkGroupManager._format_partition_drops( + NetworkGroupManager._parse_partition_drops(iptables)) == "" + + def check_flags_a_half_applied_partition(self): + """A one-way drop is called out - the reason the iptables dump is worth keeping.""" + _, _, iptables = NetworkGroupManager._split_probe_output( + _probe(QDISC_SECTION, FILTER_SECTION, f"-A {CHAIN} -d 192.168.1.5/32 -j DROP")) + + drops = NetworkGroupManager._format_partition_drops( + NetworkGroupManager._parse_partition_drops(iptables)) + + assert drops == f" | partition: {CHAIN} out-X only [192.168.1.5]" + + @pytest.mark.parametrize(["probe_output", "unanswered"], [ + ("", [0, 1, 2]), + (QDISC_SECTION, [1, 2]), + (_probe(QDISC_SECTION, FILTER_SECTION), [2]), + (_probe("", "", ""), [0, 1, 2]) + ]) + def check_incomplete_probe_degrades(self, probe_output, unanswered): + """A node that answers partially yields a poorer overview, never an exception.""" + sections = NetworkGroupManager._split_probe_output(probe_output) + + assert len(sections) == 3, "Callers unpack three sections regardless of what came back" + + for idx in unanswered: + assert sections[idx] == [], f"Section {idx} went unanswered, it must come back empty" + + render, neutral = NEUTRAL_RENDERINGS[idx] + + assert render(sections[idx]) == neutral, f"Section {idx} must render as its neutral form" diff --git a/modules/ducktests/tests/ignitetest/services/network_group/manager.py b/modules/ducktests/tests/ignitetest/services/network_group/manager.py index bc1033452227e..3f90b7f9450a1 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/manager.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/manager.py @@ -43,6 +43,10 @@ # Upper bound on concurrent SSH sessions used to apply tc rules cluster-wide. MAX_PARALLEL_SSH_SESSIONS = 16 +# Separates the qdisc, filter and iptables sections in the output of the batched +# network probe issued by _log_network. +PROBE_SECTION_SEPARATOR = "=== ignitetest network probe section ===" + # A rule spec: (src_group, dst_group, action, config). RuleSpec = Tuple[str, str, str, CrossNetworkGroupConfiguration] @@ -234,6 +238,27 @@ def run(task): self.logger.debug(f"[{tag}] tc rules applied on {len(tasks)} node(s) in {monotonic() - started:.2f}s") + def _ssh_output_parallel(self, tasks: List[Tuple[object, str]], tag: str) -> List[str]: + """ + Executes one command per node concurrently and returns the collected + outputs in task order. + """ + if not tasks: + return [] + + started = monotonic() + + def run(task): + node, cmd = task + return self._get_ssh_output(node, cmd) + + with ThreadPoolExecutor(max_workers=min(MAX_PARALLEL_SSH_SESSIONS, len(tasks))) as pool: + outputs = list(pool.map(run, tasks)) + + self.logger.debug(f"[{tag}] probed {len(tasks)} node(s) in {monotonic() - started:.2f}s") + + return outputs + def _prefetch_network_interfaces(self): """ Warms up the per-node network interface cache in parallel, so that @@ -261,32 +286,41 @@ def _iter_all_nodes(self) -> Iterator: def _log_network(self, log_tag: str): """ - Logs a concise, structured overview of the active traffic control - queuing disciplines (qdiscs) and routing filters across all cluster nodes. + Logs a concise, structured overview of the active traffic control queuing + disciplines (qdiscs), routing filters and partition drops across all cluster nodes. + + Every node is probed with a single batched command and all nodes are probed + concurrently. This runs right after a partition is toggled, so it has to cost + one round-trip cluster-wide: probing node by node stretches a partition well + past the outage the test asked for and can push a short blip over the failure + detection timeout it is meant to stay under. """ self.logger.debug(f"Network State Overview: [START][{log_tag}]") + entries = [(group, svc, node) + for group, services in self.network_group_registry.items() + for svc in services + for node in svc.nodes] + + probes = self._ssh_output_parallel([(node, self._to_network_probe_cmd(node)) for _, _, node in entries], + tag="PROBE") + node_statuses = [] - for group, services in self.network_group_registry.items(): - for svc in services: - for node in svc.nodes: - qdisc_output = self._exec_tc_show_command(node, "qdisc") - filter_output = self._exec_tc_show_command(node, "filter") + for (group, svc, node), probe in zip(entries, probes): + qdisc_lines, filter_lines, iptables_lines = self._split_probe_output(probe) - dst_ips = self._parse_filter_destinations(filter_output) - constraints = self._parse_qdisc_constraints(qdisc_output) + dst_ips = self._parse_filter_destinations(filter_lines) + constraints = self._parse_qdisc_constraints(qdisc_lines) - targets_str = f" -> to [{', '.join(dst_ips)}]" if dst_ips and constraints != "noqueue" else "" - node_ip = socket.gethostbyname(node.account.externally_routable_ip) + targets_str = f" -> to [{', '.join(dst_ips)}]" if dst_ips and constraints != "noqueue" else "" + node_ip = socket.gethostbyname(node.account.externally_routable_ip) - iptables_lines = self._get_ssh_output( - node, "sudo iptables -S 2>/dev/null || true").splitlines() - partition_str = self._format_partition_drops( - self._parse_partition_drops(iptables_lines)) + partition_str = self._format_partition_drops( + self._parse_partition_drops(iptables_lines)) - node_statuses.append(f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : " - f"{constraints}{targets_str}{partition_str}") + node_statuses.append(f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : " + f"{constraints}{targets_str}{partition_str}") # The per-node SSH probes above flood the debug log with their own command output. # Collect first, print contiguously after: the overview must stay readable as one block. @@ -295,18 +329,40 @@ def _log_network(self, log_tag: str): self.logger.debug(f"Network State Overview: [END][{log_tag}]") - def _exec_tc_show_command(self, node, sub_system: str) -> List[str]: + def _to_network_probe_cmd(self, node) -> str: """ - Executes a 'tc show' shell query command on a remote node over SSH - and decodes the terminal response cleanly into strings. + Builds the single command that dumps everything the overview needs from a + node - the netem qdiscs, the u32 filters and the iptables rules - as three + PROBE_SECTION_SEPARATOR delimited sections. + + The sections are chained unconditionally: this is a debug dump, so a node + that cannot answer one of the probes degrades to an incomplete overview + instead of failing the test around it. """ interface = self._get_default_network_interface(node) - cmd = f"sudo tc {sub_system} show dev {interface}" + return " ; ".join([ + f"sudo tc qdisc show dev {interface} 2>/dev/null", + f"echo '{PROBE_SECTION_SEPARATOR}'", + f"sudo tc filter show dev {interface} 2>/dev/null", + f"echo '{PROBE_SECTION_SEPARATOR}'", + "sudo iptables -S 2>/dev/null || true" + ]) + + @staticmethod + def _split_probe_output(probe_output: str) -> Tuple[List[str], List[str], List[str]]: + """ + Splits a combined probe output back into its qdisc, filter and iptables line + lists. Sections a node did not answer with come back empty, rendering as a + plain 'noqueue' with no partition suffix. + """ + sections = probe_output.split(PROBE_SECTION_SEPARATOR) + + sections += [""] * (3 - len(sections)) - raw_bytes = node.account.ssh_output(cmd, allow_fail=False) + qdisc_lines, filter_lines, iptables_lines = (section.strip().splitlines() for section in sections[:3]) - return raw_bytes.decode(sys.getdefaultencoding()).splitlines() + return qdisc_lines, filter_lines, iptables_lines @staticmethod def _parse_filter_destinations(filter_lines: List[str]) -> List[str]: From 4640cf8fa2bf9eb28aaf8ebb7b89aba900b3f90a Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 22 Jul 2026 15:10:42 +0300 Subject: [PATCH 40/44] IGNITE-27833 mainDC failover added --- .../tests/mdc/MdcCacheAwareApplication.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java index a17b07df56bd2..447486faa9b75 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcCacheAwareApplication.java @@ -50,7 +50,8 @@ *

      *
    • {@code cacheName} - cache name;
      • *
    • {@code backups} - number of backups; {@code (backups + 1)} must be divisible by {@code dcsNum};
      • - *
    • {@code mainDc} - main data center for the topology validator (2 DC mode);
      • + *
    • {@code mainDc} - main data center for the topology validator (2 DC mode); required, and must be + * non-empty, unless {@code datacenters} is given;
      • *
    • {@code datacenters} - full DC set for majority-based validation (odd DC count mode), * takes precedence over {@code mainDc};
      • *
    • {@code dcsNum} - number of data centers, default 2;
      • @@ -137,8 +138,14 @@ protected CacheConfiguration mdcCacheConfiguration(JsonNode jNod topValidator.setDatacenters(dcs); } - else - topValidator.setMainDatacenter(jNode.path("mainDc").asText()); + else { + String mainDc = jNode.hasNonNull("mainDc") ? jNode.get("mainDc").asText().trim() : ""; + + if (mainDc.isEmpty()) + throw new IllegalArgumentException("Either 'datacenters' or a non-empty 'mainDc' must be specified."); + + topValidator.setMainDatacenter(mainDc); + } return new CacheConfiguration() .setName(cacheName) From e96dd5c85022b1e7f3965084d783ba5e604b6ea7 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 22 Jul 2026 15:16:44 +0300 Subject: [PATCH 41/44] IGNITE-27833 get load fixed --- .../ducktest/tests/mdc/MdcContinuousLoadApplication.java | 4 +++- .../ducktest/tests/mdc/MdcThinClientLoadApplication.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java index d00486fa08ecd..6ec52a5570475 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcContinuousLoadApplication.java @@ -157,7 +157,9 @@ public class MdcContinuousLoadApplication extends MdcCacheAwareApplication { catch (CacheException | IgniteException e) { ok = false; - if (inadmissible) + // A read reaching here is a missed or corrupted entry, never a rejected write: + // 'inadmissible' must not excuse it. + if (mode.isWrite() && inadmissible) log.info("Write rejected as expected [dc=" + dcId() + ", key=" + key + ", msg=" + e.getMessage() + "]"); else if (stopOnError) { diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java index 2eaf491796e79..f2c6bbea5ac39 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/mdc/MdcThinClientLoadApplication.java @@ -101,7 +101,9 @@ public class MdcThinClientLoadApplication extends IgniteAwareApplication { catch (ClientException | CacheException | IgniteException e) { ok = false; - if (inadmissible) + // A read reaching here is a missed or corrupted entry, never a rejected write: + // 'inadmissible' must not excuse it. + if (put && inadmissible) log.info("Put rejected as expected [key=" + key + ", msg=" + e.getMessage() + "]"); else throw new IllegalStateException("Operation failed [mode=" + mode + ", key=" + key + "]", e); From 2e7f92303d571c8004b1cb3da24f6e3b270e2c34 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 22 Jul 2026 15:20:27 +0300 Subject: [PATCH 42/44] IGNITE-27833 fix pinned client with main DC test --- .../ignitetest/tests/mdc/thin_client_test.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py index da75d779746d2..1fce04e526c53 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/thin_client_test.py @@ -18,7 +18,8 @@ Every thin client is configured with the addresses of ALL server nodes of BOTH DCs. A client pinned to a data center must route partition-aware reads to nodes of its own DC: -with a large cross-DC netem delay its average read latency stays small. +with a large cross-DC netem delay its average read latency stays small. This is asserted +from both DCs, so no routing that fixes on a single DC can satisfy it. The partition part verifies the thin client experience of a split-brain: a client pinned to the main-DC half keeps writing; a client pinned to the read-only half still @@ -62,7 +63,7 @@ class MdcThinClientTest(IgniteTest): @parametrize(cross_dc_latency_ms=100) def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_dc_latency_ms): """ - DC-pinned thin client reads locally (latency far below the cross-DC delay); + Each DC-pinned thin client reads locally (latency far below the cross-DC delay); through a partition the pinned clients behave like their half-ring: main half writes, read-only half reads but rejects writes, and writes resume after the heal. """ @@ -83,22 +84,23 @@ def test_thin_client_dc_aware_routing_and_partition(self, ignite_version, cross_ mdc.generate_data(DC_1, CACHE_NAME, 0, KEYS, backups=1) - svc = mdc.run_service(cli_dc1, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, + for dc, cli in [(DC_1, cli_dc1), (DC_2, cli_dc2)]: + svc = mdc.run_service(cli, {"mode": "GET", "cacheName": CACHE_NAME, "keyFrom": 0, "keyTo": KEYS, "iterations": PINNED_GET_ITERS, - "resultPrefix": "pinnedGet"}) + "resultPrefix": f"pinnedGet{dc}"}) - avg_cli_dc_1 = mdc.result_float(svc, "pinnedGetAvgOpMs") - err_cnt_cli_dc_1 = mdc.result_int(svc, "pinnedGetErrCnt") + avg_ms = mdc.result_float(svc, f"pinnedGet{dc}AvgOpMs") + err_cnt = mdc.result_int(svc, f"pinnedGet{dc}ErrCnt") - self.logger.info(f"Thin client routing latency [delayMs={cross_dc_latency_ms}, getAvgOpMs={avg_cli_dc_1}, " - f"getErrCnt={err_cnt_cli_dc_1}]") + self.logger.info(f"Thin client routing latency [dc={dc}, delayMs={cross_dc_latency_ms}, " + f"getAvgOpMs={avg_ms}, getErrCnt={err_cnt}]") - assert avg_cli_dc_1 < cross_dc_latency_ms, \ - f"DC-pinned thin client reads should be served locally " \ - f"[avgMs={avg_cli_dc_1}, delayMs={cross_dc_latency_ms}]" + assert avg_ms < cross_dc_latency_ms, \ + f"DC-pinned thin client reads should be served locally " \ + f"[dc={dc}, avgMs={avg_ms}, delayMs={cross_dc_latency_ms}]" - assert err_cnt_cli_dc_1 == 0, (f"Expected 0 errors for DC-pinned thin client reads, " - f"but found {err_cnt_cli_dc_1}") + assert err_cnt == 0, \ + f"Expected 0 errors for DC-pinned thin client reads [dc={dc}, errCnt={err_cnt}]" net.enable_network_partition(DC_1, DC_2) From 95dab09453de7b01815503b1107ae968cf5b9195 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 22 Jul 2026 15:24:09 +0300 Subject: [PATCH 43/44] IGNITE-27833 change settle timing --- .../tests/ignitetest/tests/mdc/partition_resilience_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index 2648fa693d794..2ac3929805558 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -34,7 +34,7 @@ BACKUPS = 1 # Time for discovery to detect the partition and for both half-rings to complete PME. -SPLIT_SETTLE_SECS = 10 +SPLIT_SETTLE_SECS = 20 # A blip must stay well below the failure detection timeout (10s by default). BLIP_SECS = 1 From d3cb1a10cf41d1fdefa8d65a06f854e14a4bd090 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 22 Jul 2026 15:58:38 +0300 Subject: [PATCH 44/44] IGNITE-27833 user attributes parsing fixed --- .../checks/utils/check_cache_distribution.py | 128 ++++++++++++++++++ .../services/utils/control_utility.py | 30 +++- 2 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 modules/ducktests/tests/checks/utils/check_cache_distribution.py diff --git a/modules/ducktests/tests/checks/utils/check_cache_distribution.py b/modules/ducktests/tests/checks/utils/check_cache_distribution.py new file mode 100644 index 0000000000000..5eb580de60ca4 --- /dev/null +++ b/modules/ducktests/tests/checks/utils/check_cache_distribution.py @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks parsing of 'control.sh --cache distribution' output. + +The user attribute values in a row are emitted in the iteration order of the node's +attribute map on the control.sh side, which matches neither the --user-attributes +argument order nor alphabetical order. The header line is printed from the keys of +that same map, so these checks pin down that the names come from the header and not +from the requested list. +""" + +import pytest + +from ignitetest.services.utils.control_utility import ControlUtility + +HEADER = "[groupId,partition,nodeId,primary,state,updateCounter,partitionSize,nodeAddresses" + +GROUP = "[next group: id=-1368047377, name=mdc-cache]" + + +def _parse(output, user_attributes=None): + # pylint: disable=protected-access + return ControlUtility._ControlUtility__parse_cache_distribution(output, user_attributes) + + +def _copies(distribution, group="mdc-cache", partition=0): + return distribution.groups[group].partitions[partition] + + +class CheckCacheDistribution: + """ + Checks the distribution output parser. + """ + def check_no_user_attributes(self): + """Without --user-attributes the header has no trailing names and rows carry none.""" + distribution = _parse("\n".join([ + HEADER + "]", + GROUP, + "-1368047377,0,a1b2c3d4,P,OWNING,42,100,[127.0.0.1, 10.0.0.1]" + ])) + + copy = _copies(distribution)[0] + + assert copy.node_id == "a1b2c3d4" + assert copy.primary + assert copy.state == "OWNING" + assert copy.update_counter == 42 + assert copy.partition_size == 100 + assert copy.node_addresses == ["127.0.0.1", "10.0.0.1"] + assert copy.user_attributes == {} + + def check_single_user_attribute(self): + """The one-attribute case all current callers use.""" + distribution = _parse("\n".join([ + HEADER + ",IGNITE_DATA_CENTER_ID]", + GROUP, + "-1368047377,0,a1b2c3d4,P,OWNING,42,100,[127.0.0.1],DC1", + "-1368047377,0,e5f6a7b8,B,OWNING,42,100,[10.0.0.1],DC2" + ]), user_attributes=["IGNITE_DATA_CENTER_ID"]) + + primary, backup = _copies(distribution) + + assert primary.user_attributes == {"IGNITE_DATA_CENTER_ID": "DC1"} + assert backup.user_attributes == {"IGNITE_DATA_CENTER_ID": "DC2"} + + def check_attribute_names_follow_the_header_not_the_request(self): + """ + The regression the header-driven parse exists for: the map order on the control.sh + side is neither the requested order nor alphabetical, so zipping the values against + a sorted request list mis-assigns every value. + """ + distribution = _parse("\n".join([ + HEADER + ",ZONE,DC]", + GROUP, + "-1368047377,0,a1b2c3d4,P,OWNING,42,100,[127.0.0.1],east,DC1" + ]), user_attributes=["DC", "ZONE"]) + + assert _copies(distribution)[0].user_attributes == {"ZONE": "east", "DC": "DC1"} + + def check_missing_attribute_value_keeps_columns_aligned(self): + """A node lacking an attribute prints an empty field, it does not drop the column.""" + distribution = _parse("\n".join([ + HEADER + ",ZONE,DC]", + GROUP, + "-1368047377,0,a1b2c3d4,P,OWNING,42,100,[127.0.0.1],,DC1" + ]), user_attributes=["DC", "ZONE"]) + + assert _copies(distribution)[0].user_attributes == {"ZONE": "", "DC": "DC1"} + + def check_requested_attribute_absent_from_header_fails(self): + """A silently dropped --user-attributes argument must not parse as success.""" + with pytest.raises(AssertionError, match="missing from the distribution output"): + _parse("\n".join([ + HEADER + ",DC]", + GROUP, + "-1368047377,0,a1b2c3d4,P,OWNING,42,100,[127.0.0.1],DC1" + ]), user_attributes=["DC", "ZONE"]) + + def check_multiple_groups_and_partitions(self): + """Rows are filed under the group header that precedes them.""" + distribution = _parse("\n".join([ + HEADER + ",DC]", + GROUP, + "-1368047377,0,a1b2c3d4,P,OWNING,42,100,[127.0.0.1],DC1", + "-1368047377,1,e5f6a7b8,P,OWNING,7,50,[10.0.0.1],DC2", + "[next group: id=42, name=other-cache]", + "42,0,a1b2c3d4,P,MOVING,1,10,[127.0.0.1],DC1" + ]), user_attributes=["DC"]) + + assert sorted(distribution.groups) == ["mdc-cache", "other-cache"] + assert sorted(distribution.groups["mdc-cache"].partitions) == [0, 1] + + assert _copies(distribution, partition=1)[0].update_counter == 7 + assert _copies(distribution, "other-cache")[0].state == "MOVING" diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index 7005436ed3c1b..8ace03db5ae8d 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -213,8 +213,19 @@ def cache_distribution(self, node_id=None, cache_names=None, user_attributes=Non def __parse_cache_distribution(output, user_attributes=None): group_pattern = re.compile(r"\[next group: id=(?P-?\d+), name=(?P[^\]]+)\]") - # Trailing attribute values appear after nodeAddresses, comma-separated, - # in the same order they were passed to --user-attributes. + # Column header, carrying the attribute names after nodeAddresses, e.g. + # [groupId,...,nodeAddresses,IGNITE_DATA_CENTER_ID]. + header_pattern = re.compile( + r"^\[groupId,partition,nodeId,primary,state,updateCounter,partitionSize,nodeAddresses" + r"(?P(?:,[^,\]]*)*)\]$") + + # Trailing attribute values appear after nodeAddresses, comma-separated, in the iteration + # order of the node's attribute map as deserialized in the control.sh JVM - which is + # neither the --user-attributes order nor alphabetical. CacheDistributionTaskResult#print() + # builds the header from the keys of that very map and each row from its values, so the + # header is the only reliable source of the names. Every requested attribute is always put + # (CacheDistributionTask), so a value missing on a node prints as an empty field rather + # than shifting the columns. row_pattern = re.compile(r"(?P-?\d+)," r"(?P\d+)," r"(?P[0-9a-fA-F]+)," @@ -227,10 +238,21 @@ def __parse_cache_distribution(output, user_attributes=None): groups = {} cur_group = None + attr_names = [] for line in output.splitlines(): line = line.strip() + match = header_pattern.match(line) + if match: + attr_names = [name.strip() for name in match.group("attr_names").split(",") if name.strip()] + + assert not user_attributes or sorted(attr_names) == sorted(user_attributes), \ + f"Requested user attributes are missing from the distribution output " \ + f"[requested={sorted(user_attributes)}, reported={sorted(attr_names)}]" + + continue + match = group_pattern.search(line) if match: cur_group = CacheGroupDistribution(group_id=int(match.group("group_id")), @@ -242,9 +264,9 @@ def __parse_cache_distribution(output, user_attributes=None): match = row_pattern.match(line) if match and cur_group is not None: attrs = {} - if user_attributes and match.group("attr_values") is not None: + if attr_names and match.group("attr_values") is not None: values = [v.strip() for v in match.group("attr_values").split(",")] - attrs = dict(zip(sorted(user_attributes), values)) + attrs = dict(zip(attr_names, values)) copy = PartitionCopy(node_id=match.group("node_id"), primary=match.group("primary") == "P",