Merge branch 'main' into konflux/mintmaker/main/konflux-base_image-quay.io-aipcc-base-images-cuda-12.9-el9.6

Author: atheo89

.cursor/rules/test-conventions.mdc

@@ -24,6 +24,8 @@ alwaysApply: false
 | pytest | Test runner for all Python tests (subtests support is built in since pytest 9.0) |
 | pytest-cov | Coverage (XML + terminal) |
 | allure-pytest | Issue tracking + step decoration |
+| hypothesis | Property-based tests for pure helpers (`tests/unit/test_property_helpers.py`) |
+| crosshair | Optional SMT backend for Hypothesis (`make test-crosshair`; not default CI) |
 | testcontainers | Container lifecycle for integration tests |
 | docker (Python) | Low-level container operations |
 | pyfakefs | Filesystem mocking for unit tests |

.cursor/skills/arm64-rosa-gpu-smoke/SKILL.md

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Run ODH tests/manual GPU notebooks on all ARM CUDA images.
+# See ../SKILL.md Phase 3b. Uses non-interactive exec (oc exec -q equivalent).
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+REPO_ROOT="$(cd "$(dirname "$0")/../../../.." && pwd)"
+cd "$REPO_ROOT"
+exec uv run "$ROOT/scripts/gpu-manual-tests.py" "$@"

.cursor/skills/arm64-rosa-gpu-smoke/scripts/gpu-manual-tests.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+# GPU smoke via Pod (no Notebook CR / RHOAI required). Run after nvidia.com/gpu allocatable.
+set -euo pipefail
+
+: "${NS:?Set NS to a unique, dedicated namespace — this is a shared account, never default to a personal name}"
+: "${TAG:=rhoai-3.6-ea.1}"
+: "${PULL_SECRET:=rhoai-pull}"
+: "${TIMEOUT:=900}"
+: "${CLUSTER_CONTEXT:?Set CLUSTER_CONTEXT to the exact kubeconfig context (e.g. \$(oc config current-context) captured right after login) — never rely on the ambient current-context, which another process on this machine can change mid-session}"
+
+IMG="${1:?usage: $0 <full-image-ref>}"
+
+# NS/PULL_SECRET/IMG are spliced directly into the YAML heredoc below with
+# no serialization — reject anything that could break out of its scalar
+# context before that happens. NS/PULL_SECRET are also constrained to
+# valid Kubernetes names, which they need to be anyway. IMG must be a
+# single image-reference token — whitespace and "#" both open room for
+# a YAML comment or truncated value (e.g. "img # pytorch" would silently
+# truncate the interpolated image to "img").
+for _var_name in NS PULL_SECRET; do
+  _val="${!_var_name}"
+  [[ "$_val" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]] || { echo "ERROR: $_var_name '$_val' is not a valid Kubernetes name" >&2; exit 1; }
+done
+[[ "$IMG" =~ ^[^[:space:]#]+$ ]] || { echo "ERROR: IMG must be a single image-reference token (no whitespace or '#')" >&2; exit 1; }
+
+hash_cmd() { command -v sha256sum >/dev/null 2>&1 && sha256sum || shasum -a 256; }
+timeout_cmd() { command -v timeout >/dev/null 2>&1 && echo timeout || command -v gtimeout >/dev/null 2>&1 && echo gtimeout || { echo "ERROR: need GNU timeout (brew install coreutils for gtimeout on macOS)" >&2; exit 1; }; }
+POD="gpu-smoke-$(printf '%s' "$IMG" | hash_cmd | cut -c1-16)-${RANDOM}${RANDOM}"
+
+if [[ "$IMG" == *"-runtime-"* ]]; then
+  IS_RUNTIME=1
+else
+  IS_RUNTIME=0
+fi
+
+if [[ "$IMG" == *"pytorch"* ]]; then
+  LIB=torch
+elif [[ "$IMG" == *"tensorflow"* ]]; then
+  LIB=tensorflow
+elif [[ "$IMG" == *"minimal-cuda"* ]]; then
+  LIB=minimal
+else
+  echo "Unknown image type for $IMG" >&2
+  exit 1
+fi
+
+cleanup() {
+  oc --context "$CLUSTER_CONTEXT" delete pod "$POD" -n "$NS" --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || true
+  sleep 3
+}
+trap cleanup EXIT
+
+CMD='["sleep","infinity"]'
+if [[ "$IS_RUNTIME" -eq 0 ]]; then
+  CMD='null'
+fi
+
+cat <<EOF | oc --context "$CLUSTER_CONTEXT" apply -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: $POD
+  namespace: $NS
+spec:
+  restartPolicy: Never
+  automountServiceAccountToken: false
+  nodeSelector:
+    kubernetes.io/arch: arm64
+  tolerations:
+  - key: nvidia.com/gpu
+    operator: Exists
+    effect: NoSchedule
+  imagePullSecrets:
+  - name: $PULL_SECRET
+  containers:
+  - name: smoke
+    image: $IMG
+$(if [[ "$IS_RUNTIME" -eq 1 ]]; then
+cat <<INNER
+    command: ["sleep", "infinity"]
+INNER
+fi)
+    resources:
+      limits:
+        nvidia.com/gpu: "1"
+      requests:
+        nvidia.com/gpu: "1"
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop: ["ALL"]
+      runAsNonRoot: true
+      seccompProfile:
+        type: RuntimeDefault
+    volumeMounts:
+    - name: workspace
+      mountPath: /opt/app-root/src
+  volumes:
+  - name: workspace
+    emptyDir: {}
+EOF
+
+echo "==> Waiting for pod $POD (image pull may take several minutes)..."
+if ! oc --context "$CLUSTER_CONTEXT" wait --for=condition=Ready "pod/$POD" -n "$NS" --timeout="${TIMEOUT}s"; then
+  echo "FAIL: pod not ready" >&2
+  oc --context "$CLUSTER_CONTEXT" describe pod "$POD" -n "$NS" | tail -20
+  exit 1
+fi
+SCHEDULED_NODE=$(oc --context "$CLUSTER_CONTEXT" get pod "$POD" -n "$NS" -o jsonpath='{.spec.nodeName}')
+echo "==> Scheduled on node: $SCHEDULED_NODE"
+
+case "$LIB" in
+  torch)
+    # if/raise, not assert — assert is a no-op under PYTHONOPTIMIZE, and a
+    # SMOKE_PASS that can lie is worse than one that's merely verbose
+    PY='import platform, torch
+if platform.machine() != "aarch64": raise SystemExit(platform.machine())
+if not torch.cuda.is_available(): raise SystemExit("cuda not available")
+print("device:", torch.cuda.get_device_name(0))
+print("sm:", torch.cuda.get_device_capability())
+x = torch.randn(1024, 1024, device="cuda")
+print("matmul_ok:", float((x @ x).mean().item()))
+print("SMOKE_PASS")'
+    ;;
+  tensorflow)
+    PY='import os, platform, tensorflow as tf
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+if platform.machine() != "aarch64": raise SystemExit(platform.machine())
+gpus = tf.config.list_physical_devices("GPU")
+print("gpus:", gpus)
+if not gpus: raise SystemExit("no GPU devices")
+tf.config.set_soft_device_placement(False)
+with tf.device("/GPU:0"):
+    x = tf.random.uniform([1024, 1024])
+    result = tf.matmul(x, x)
+if "GPU" not in result.device: raise SystemExit(result.device)
+print("SMOKE_PASS")'
+    ;;
+  minimal)
+    PY='import platform, subprocess
+if platform.machine() != "aarch64": raise SystemExit(platform.machine())
+out = subprocess.check_output(["nvidia-smi", "-L"], text=True)
+print(out.strip())
+if "GPU" not in out: raise SystemExit("no GPU in nvidia-smi -L output")
+print("SMOKE_PASS")'
+    ;;
+esac
+
+echo "==> Running GPU check ($LIB)..."
+"$(timeout_cmd)" "${EXEC_TIMEOUT:-$TIMEOUT}s" oc --context "$CLUSTER_CONTEXT" exec -n "$NS" "$POD" -c smoke -- python -c "$PY"
+echo "==> PASS $IMG"

.cursor/skills/arm64-rosa-gpu-smoke/scripts/gpu-manual-tests.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Shared by rosa-hcp-provision and arm64-rosa-gpu-smoke: build a
+# dockerconfigjson pull secret from interactively-entered credentials and
+# create it in the cluster. Never accepts a credential as a CLI argument
+# (would leak via shell history / `ps`) and never reads
+# ~/.docker/config.json automatically (would silently harvest whatever
+# credential happens to be cached there).
+#
+# Usage: CLUSTER_CONTEXT=<ctx> create-pull-secret.sh <secret-name> <namespace> <registry-host-group> [<registry-host-group> ...]
+#
+# Each <registry-host-group> is one or more comma-separated auth keys that
+# share a single prompted credential, e.g. "quay.io,quay.io/rhoai" writes
+# the same auth under both keys (useful when a client does exact-key
+# lookup rather than prefix matching on the dockerconfigjson). Prompts for
+# a username, and skips the whole group entirely (no entry in the secret)
+# if the username is left blank — this is how a caller can conditionally
+# omit a registry it doesn't need (e.g. registry.redhat.io when only a
+# quay.io credential is available), without any special-cased flag. If a
+# username is given, the password is required non-empty.
+#
+# Requires CLUSTER_CONTEXT in the environment and passes it explicitly on
+# every oc call — never relies on / mutates the ambient current-context.
+# `oc config use-context` changes shared, machine-wide kubeconfig state; a
+# concurrent process changing it between the caller's setup and this
+# script's execution would otherwise create the secret in the wrong
+# cluster (see rosa-hcp-provision/SKILL.md's "always pass --context" rule).
+set -euo pipefail
+
+: "${CLUSTER_CONTEXT:?Set CLUSTER_CONTEXT to the exact kubeconfig context of the target cluster}"
+
+if [ "$#" -lt 3 ]; then
+  echo "Usage: CLUSTER_CONTEXT=<ctx> $0 <secret-name> <namespace> <registry-host-group> [<registry-host-group> ...]" >&2
+  exit 2
+fi
+
+SECRET_NAME="$1"
+NAMESPACE="$2"
+shift 2
+
+SECRET_FILE=$(umask 077 && mktemp)
+AUTH_FILE=""
+cleanup() { rm -f "$SECRET_FILE" "${AUTH_FILE:-}"; }
+trap cleanup EXIT
+
+AUTHS_JSON="{}"
+ANY_HOST_CONFIGURED=false
+
+for GROUP in "$@"; do
+  read -r -p "Username for ${GROUP} (leave blank to skip this registry): " REG_USER
+  if [ -z "$REG_USER" ]; then
+    echo "Skipping ${GROUP} (no username given)" >&2
+    continue
+  fi
+  read -rs -p "Password/token for ${GROUP}: " REG_PASS; echo
+  if [ -z "$REG_PASS" ]; then
+    echo "ERROR: a username was given for ${GROUP} but the password was empty" >&2
+    exit 1
+  fi
+  AUTH_FILE=$(umask 077 && mktemp)
+  printf '%s' "${REG_USER}:${REG_PASS}" | base64 | tr -d '\n' > "$AUTH_FILE"
+  IFS=',' read -ra HOST_KEYS <<< "$GROUP"
+  for HOST in "${HOST_KEYS[@]}"; do
+    # --rawfile, not --arg $AUTH — --arg would put the (base64-encoded,
+    # still sensitive) credential into jq's own process argument list.
+    AUTHS_JSON=$(printf '%s' "$AUTHS_JSON" | jq --arg host "$HOST" --rawfile auth "$AUTH_FILE" \
+      '.[$host] = {"auth": $auth}')
+  done
+  rm -f "$AUTH_FILE"
+  ANY_HOST_CONFIGURED=true
+  unset REG_USER REG_PASS AUTH_FILE
+done
+
+if [ "$ANY_HOST_CONFIGURED" != true ]; then
+  echo "ERROR: no registry host was configured (every username was left blank)" >&2
+  exit 1
+fi
+
+printf '%s' "$AUTHS_JSON" | jq '{"auths": .}' > "$SECRET_FILE"
+
+oc --context "$CLUSTER_CONTEXT" create secret generic "$SECRET_NAME" -n "$NAMESPACE" \
+  --from-file=.dockerconfigjson="$SECRET_FILE" \
+  --type=kubernetes.io/dockerconfigjson \
+  --dry-run=client -o yaml | oc --context "$CLUSTER_CONTEXT" apply -f -
+
+rm -f "$SECRET_FILE"
+echo "Created/updated secret ${SECRET_NAME} in namespace ${NAMESPACE}" >&2

.cursor/skills/arm64-rosa-gpu-smoke/scripts/gpu-smoke-pod.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# Shared by rosa-hcp-provision docs: after creating a Subscription with
+# installPlanApproval: Manual, find the InstallPlan for an exact CSV
+# name (never `tail -1`/label-selector matching, which can grab an
+# unrelated or older InstallPlan when more than one exists in the
+# namespace), approve it, and wait for that CSV to succeed.
+#
+# Usage: CLUSTER_CONTEXT=<ctx> wait-for-csv.sh <namespace> <csv-name>
+#
+# Requires CLUSTER_CONTEXT in the environment and passes it explicitly on
+# every oc call — never relies on the ambient current-context, which is
+# shared, mutable, machine-wide state a concurrent process could change
+# between the caller's setup and this script's execution (see
+# rosa-hcp-provision/SKILL.md's "always pass --context" rule). Assumes the
+# Subscription referencing <csv-name> has already been applied (OLM
+# creates the InstallPlan asynchronously afterward, so this script polls
+# rather than querying once).
+set -euo pipefail
+
+: "${CLUSTER_CONTEXT:?Set CLUSTER_CONTEXT to the exact kubeconfig context of the target cluster}"
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: CLUSTER_CONTEXT=<ctx> $0 <namespace> <csv-name>" >&2
+  exit 2
+fi
+
+NAMESPACE="$1"
+CSV_NAME="$2"
+
+INSTALLPLAN=""
+for i in $(seq 1 12); do
+  INSTALLPLAN=$(oc --context "$CLUSTER_CONTEXT" get installplan -n "$NAMESPACE" -o json | \
+    jq -r --arg csv "$CSV_NAME" \
+      '.items[] | select(.spec.clusterServiceVersionNames | index($csv)) | .metadata.name')
+  MATCH_COUNT=$(printf '%s\n' "$INSTALLPLAN" | grep -c . || true)
+  if [ "$MATCH_COUNT" -eq 1 ]; then
+    break
+  fi
+  echo "Waiting for exactly one InstallPlan for ${CSV_NAME} (attempt ${i}/12, found ${MATCH_COUNT})..." >&2
+  sleep 5
+done
+
+MATCH_COUNT=$(printf '%s\n' "$INSTALLPLAN" | grep -c . || true)
+if [ "$MATCH_COUNT" -ne 1 ]; then
+  echo "ERROR: expected exactly one InstallPlan for ${CSV_NAME} in ${NAMESPACE}, found ${MATCH_COUNT}" >&2
+  exit 1
+fi
+
+oc --context "$CLUSTER_CONTEXT" patch installplan "$INSTALLPLAN" -n "$NAMESPACE" --type merge -p '{"spec":{"approved":true}}'
+
+# OLM creates the CSV object asynchronously after the InstallPlan is
+# approved. Poll for it to exist before starting the Succeeded-vs-Failed
+# race below — some oc/kubectl client versions return NotFound immediately
+# for `oc wait` against a not-yet-existing resource instead of waiting for
+# it to appear, which would otherwise let the race start handicapped.
+CSV_EXISTS=false
+for i in $(seq 1 12); do
+  if oc --context "$CLUSTER_CONTEXT" get csv "$CSV_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then
+    CSV_EXISTS=true
+    break
+  fi
+  echo "Waiting for csv/${CSV_NAME} to be created (attempt ${i}/12)..." >&2
+  sleep 5
+done
+if [ "$CSV_EXISTS" != true ]; then
+  echo "ERROR: csv/${CSV_NAME} in ${NAMESPACE} was not created within 60s of InstallPlan approval" >&2
+  exit 1
+fi
+
+# Race Succeeded against Failed instead of a single `--for=Succeeded` wait.
+# A CSV has a well-known binary failure signal (status.phase can reach
+# "Failed", distinct from "Succeeded") — unlike most resources (e.g. a
+# Pod, which has no single "give up early" condition), so this doesn't
+# need a heuristic: whichever phase is reached first is authoritative, and
+# the other wait is killed instead of blocking for the rest of the
+# 300s timeout after the outcome is already known. Portable to macOS's
+# stock bash 3.2 (no `wait -n`, which needs bash 4.3+) via a small polling
+# loop over two flag files instead.
+RESULT_DIR=$(mktemp -d)
+SUCCEEDED_PID=""
+FAILED_PID=""
+trap 'rm -rf "$RESULT_DIR"; kill "$SUCCEEDED_PID" "$FAILED_PID" 2>/dev/null || true' EXIT
+
+(oc --context "$CLUSTER_CONTEXT" wait --for=jsonpath='{.status.phase}'=Succeeded "csv/${CSV_NAME}" -n "$NAMESPACE" --timeout=300s >/dev/null 2>&1 \
+  && touch "$RESULT_DIR/succeeded") &
+SUCCEEDED_PID=$!
+
+(oc --context "$CLUSTER_CONTEXT" wait --for=jsonpath='{.status.phase}'=Failed "csv/${CSV_NAME}" -n "$NAMESPACE" --timeout=300s >/dev/null 2>&1 \
+  && touch "$RESULT_DIR/failed") &
+FAILED_PID=$!
+
+while true; do
+  if [ -f "$RESULT_DIR/succeeded" ]; then
+    kill "$FAILED_PID" 2>/dev/null || true
+    echo "csv/${CSV_NAME} in ${NAMESPACE} reached Succeeded" >&2
+    exit 0
+  fi
+  if [ -f "$RESULT_DIR/failed" ]; then
+    kill "$SUCCEEDED_PID" 2>/dev/null || true
+    echo "ERROR: csv/${CSV_NAME} in ${NAMESPACE} reached Failed" >&2
+    oc --context "$CLUSTER_CONTEXT" get csv "$CSV_NAME" -n "$NAMESPACE" -o jsonpath='{.status.message}{"\n"}' >&2 || true
+    exit 1
+  fi
+  if ! kill -0 "$SUCCEEDED_PID" 2>/dev/null && ! kill -0 "$FAILED_PID" 2>/dev/null; then
+    echo "ERROR: csv/${CSV_NAME} in ${NAMESPACE} reached neither Succeeded nor Failed within 300s" >&2
+    exit 1
+  fi
+  sleep 2
+done