Merge branch 'main' into m/confra-endpt

Author: webern

.gitignore

@@ -10,6 +10,7 @@ dhat-heap.json
 .DS_Store
 dist/
 checks_venv/
+.venv/
 # JavaScript-y things related to documentation generation.
 yarn.lock
 /.vitepress/

.gitlab/benchmark.yml

@@ -8,8 +8,30 @@
   - export SMP_ECR_HOST="${SMP_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"
   - export ADP_IMG_BASE="${SMP_ECR_HOST}/${SMP_TEAM_ID}-saluki:adp-${CI_PIPELINE_ID}"
   - export ADP_CHECKS_IMG_BASE="${SMP_ECR_HOST}/${SMP_TEAM_ID}-saluki:adp-checks-${CI_PIPELINE_ID}"
+  - export SMP_IMG_BASE="${SMP_ECR_HOST}/${SMP_TEAM_ID}-saluki:agent-adp-${CI_PIPELINE_ID}"
   - git fetch --verbose --deepen=999 origin ${CI_COMMIT_BRANCH} main
 
+# Layer the ADP image at ${ADP_IMG} into the converged Datadog Agent image and push it as
+# ${SMP_IMG}. Both the baseline and comparison build jobs end with this.
+#
+# SMP fixes one target image per job, and the tag filtering experiments need the Core Agent in the
+# picture: a `metric_tag_filterlist` is Agent configuration, and only reaches ADP over the config
+# stream. So every experiment runs against the converged image; the ones that don't care about the
+# Core Agent simply override the command to run ADP by itself, as they always have.
+.build-smp-target-image: &build-smp-target-image
+  - docker buildx build
+    --file ./docker/Dockerfile.datadog-agent
+    --tag ${SMP_IMG}
+    --build-arg DD_AGENT_IMAGE=registry.datadoghq.com/agent:${PUBLIC_DD_AGENT_VERSION}
+    --build-arg ADP_IMAGE=${ADP_IMG}
+    --label git.repository=${CI_PROJECT_NAME}
+    --label git.branch=${CI_COMMIT_REF_NAME}
+    --label git.commit=${CI_COMMIT_SHA}
+    --label ci.pipeline_id=${CI_PIPELINE_ID}
+    --label ci.job_id=${CI_JOB_ID}
+    --push
+    .
+
 build-adp-baseline-image:
   extends: .build-common-variables
   stage: benchmark
@@ -50,17 +72,22 @@ build-adp-baseline-image:
       --build-arg SPDX_LICENSES_IMAGE=${SPDX_LICENSES_IMAGE}
       --build-arg APP_IMAGE=${GBI_BASE_IMAGE}
       --build-arg APP_GIT_HASH=${BASELINE_ADP_SHA}
-      --build-arg INTERNAL_BUILD=true
       --label git.repository=${CI_PROJECT_NAME}
       --label git.branch=${CI_COMMIT_REF_NAME}
       --label git.commit=${CI_COMMIT_SHA}
       --label ci.pipeline_id=${CI_PIPELINE_ID}
       --label ci.job_id=${CI_JOB_ID}
       --push
       .
+    # Bundle that ADP build into the converged Datadog Agent image, which is the image SMP
+    # actually runs.
+    - export ADP_IMG="${BASELINE_ADP_IMG}"
+    - export SMP_IMG="${SMP_IMG_BASE}-${BASELINE_ADP_SHA}"
+    - *build-smp-target-image
     # Generate our image SHA/path for the runner job to use.
     - echo "BASELINE_ADP_SHA=${BASELINE_ADP_SHA}" >> smp-vars.env
     - echo "BASELINE_ADP_IMG=${BASELINE_ADP_IMG}" >> smp-vars.env
+    - echo "BASELINE_SMP_IMG=${SMP_IMG}" >> smp-vars.env
   artifacts:
     reports:
       dotenv: smp-vars.env
@@ -102,17 +129,22 @@ build-adp-comparison-image:
       --build-arg SPDX_LICENSES_IMAGE=${SPDX_LICENSES_IMAGE}
       --build-arg APP_IMAGE=${GBI_BASE_IMAGE}
       --build-arg APP_GIT_HASH=${COMPARISON_ADP_SHA}
-      --build-arg INTERNAL_BUILD=true
       --label git.repository=${CI_PROJECT_NAME}
       --label git.branch=${CI_COMMIT_REF_NAME}
       --label git.commit=${CI_COMMIT_SHA}
       --label ci.pipeline_id=${CI_PIPELINE_ID}
       --label ci.job_id=${CI_JOB_ID}
       --push
       .
+    # Bundle that ADP build into the converged Datadog Agent image, which is the image SMP
+    # actually runs.
+    - export ADP_IMG="${COMPARISON_ADP_IMG}"
+    - export SMP_IMG="${SMP_IMG_BASE}-${COMPARISON_ADP_SHA}"
+    - *build-smp-target-image
     # Generate our image SHA/path for the runner job to use.
     - echo "COMPARISON_ADP_SHA=${COMPARISON_ADP_SHA}" >> smp-vars.env
     - echo "COMPARISON_ADP_IMG=${COMPARISON_ADP_IMG}" >> smp-vars.env
+    - echo "COMPARISON_SMP_IMG=${SMP_IMG}" >> smp-vars.env
   artifacts:
     reports:
       dotenv: smp-vars.env
@@ -149,8 +181,8 @@ run-benchmarks-adp:
     - ./smp --team-id ${SMP_TEAM_ID} --aws-named-profile ${AWS_NAMED_PROFILE}
       job submit
       --warmup-seconds 0
-      --baseline-image ${BASELINE_ADP_IMG}
-      --comparison-image ${COMPARISON_ADP_IMG}
+      --baseline-image ${BASELINE_SMP_IMG}
+      --comparison-image ${COMPARISON_SMP_IMG}
       --baseline-sha ${BASELINE_ADP_SHA}
       --comparison-sha ${COMPARISON_ADP_SHA}
       --target-config-dir ./test/smp/regression/adp/quality-gates/
@@ -241,8 +273,8 @@ run-benchmarks-adp-full:
     - ./smp --team-id ${SMP_TEAM_ID} --aws-named-profile ${AWS_NAMED_PROFILE}
       job submit
       --warmup-seconds 0
-      --baseline-image ${BASELINE_ADP_IMG}
-      --comparison-image ${COMPARISON_ADP_IMG}
+      --baseline-image ${BASELINE_SMP_IMG}
+      --comparison-image ${COMPARISON_SMP_IMG}
       --baseline-sha ${BASELINE_ADP_SHA}
       --comparison-sha ${COMPARISON_ADP_SHA}
       --target-config-dir ./test/smp/regression/adp/full/

Makefile

@@ -171,7 +171,6 @@ build-adp-image-base:
 		--build-arg "APP_IDENTIFIER=$(ADP_APP_IDENTIFIER)" \
 		--build-arg "APP_VERSION=$(ADP_APP_VERSION)" \
 		--build-arg "APP_GIT_HASH=$(ADP_APP_GIT_HASH)" \
-		--build-arg "INTERNAL_BUILD=true" \
 		--file ./docker/Dockerfile.agent-data-plane \
 		.
 
@@ -842,15 +841,17 @@ endif
 	--inlined-functions true --timeline --upload-period 10 --preset cpu_live_heap \
 	target/release/agent-data-plane run
 
+SMP_GENERATOR := test/smp/regression/adp/generate_experiments.py
+
 .PHONY: generate-smp-experiments
 generate-smp-experiments: ## Generates SMP experiment configs from experiments.yaml
 	@echo "[*] Generating SMP experiment configurations..."
-	@python3 test/smp/regression/adp/generate_experiments.py
+	@$(PYTHON) $(SMP_GENERATOR)
 
 .PHONY: check-smp-experiments
 check-smp-experiments: ## Verifies SMP experiment configs are up-to-date (CI)
 	@echo "[*] Checking SMP experiment configurations..."
-	@python3 test/smp/regression/adp/generate_experiments.py --check
+	@$(PYTHON) $(SMP_GENERATOR) --check
 
 .PHONY: profile-run-smp-experiment
 profile-run-smp-experiment: ## Runs a specific SMP experiment for Saluki
@@ -925,6 +926,28 @@ run-docs: ## Runs a local development server for documentation
 
 ##@ Utility
 
+# Interpreter used by targets that run our Python tooling: the local virtualenv when there is one,
+# otherwise whatever `python3` resolves to. That keeps local development on the pinned versions in
+# requirements.txt -- some of these scripts generate committed files that CI compares
+# byte-for-byte -- without making the virtualenv mandatory, since CI's build image pre-installs
+# the same requirements system-wide.
+#
+# Deliberately recursive (`=`, not `:=`) so the choice is made when a recipe runs rather than when
+# the makefile is parsed, and `make ensure-python-venv generate-smp-experiments` works in one go.
+# There is no separate "activate the virtualenv" target because make gives every recipe line its
+# own shell; naming the interpreter is the equivalent, and activation does nothing more than put
+# this same binary first on PATH.
+VENV_DIR := .venv
+VENV_PYTHON := $(VENV_DIR)/bin/python
+PYTHON_REQUIREMENTS := requirements.txt
+PYTHON = $(if $(wildcard $(VENV_PYTHON)),$(VENV_PYTHON),python3)
+
+.PHONY: ensure-python-venv
+ensure-python-venv: ## Creates the virtualenv that Python tooling prefers, or updates an existing one
+	@echo "[*] Installing Python tooling dependencies into $(VENV_DIR)..."
+	@python3 -m venv $(VENV_DIR)
+	@$(VENV_PYTHON) -m pip install --quiet --disable-pip-version-check --requirement $(PYTHON_REQUIREMENTS)
+
 .PHONY: update-protos
 update-protos: ## Updates all vendored Protocol Buffers definitions from their source repositories
 	@DD_AGENT_GIT_TAG=$(PROTOBUF_SRC_REPO_DD_AGENT) \

ci/images/definitions/build/Dockerfile

@@ -39,7 +39,26 @@ RUN --mount=type=bind,from=ci-scripts,target=/scripts \
     apt-get update && \
     apt-get install -y --no-install-recommends \
       build-essential software-properties-common curl ca-certificates git gnupg2 \
-      lsb-release make cmake unzip gcc g++ binutils jq bc bzip2 ninja-build python3 python3-pip python3-yaml && \
+      lsb-release make cmake unzip gcc g++ binutils jq bc bzip2 ninja-build python3 python3-pip \
+      python3-venv && \
+    /scripts/clean-temporary-caches.sh
+
+# Pre-install the Python dependencies our tooling needs (requirements.txt), so jobs don't pay to
+# download them on every run. They go to the system interpreter, which is what the Makefile falls
+# back to when there is no local virtualenv, and the versions stay pinned to the same file
+# developers install from.
+#
+# Two flags earn their place here:
+#
+# - `--break-system-packages`, because the Ubuntu base marks its interpreter as externally
+#   managed. In a single-purpose CI image that is exactly what we want.
+# - `--ignore-installed`, because the base ships some of these as dpkg packages (PyYAML, today).
+#   pip cannot uninstall those -- they have no RECORD file -- and fails the build rather than
+#   upgrading them. This skips the removal and installs our pinned versions alongside, into
+#   /usr/local/lib, which precedes the distro's dist-packages on `sys.path` and so wins at import.
+RUN --mount=type=bind,source=requirements.txt,target=/tmp/requirements.txt \
+    --mount=type=bind,from=ci-scripts,target=/scripts \
+    pip3 install --break-system-packages --ignore-installed --requirement /tmp/requirements.txt && \
     /scripts/clean-temporary-caches.sh
 
 # Install the Docker CLI directly, as well as the CI-specific credential helper.

ci/tooling/maybe-profile.sh

@@ -54,7 +54,6 @@ ARG APP_GIT_HASH=""
 ARG APP_VERSION=""
 ARG APP_BUILD_TIME=""
 ARG DD_AGENT_VERSION=""
-ARG INTERNAL_BUILD=""
 ARG BUILDCACHE_REMOTE=
 ARG BUILDCACHE_DEBUG=
 
@@ -100,12 +99,8 @@ RUN --mount=type=bind,source=docker/scripts,target=/scripts \
 # and so on.
 FROM ${BUILD_IMAGE} AS final-prep
 
-ARG TARGETARCH
-ARG INTERNAL_BUILD=""
 ARG SPDX_LICENSES_VERSION=3.28.0
 ENV BUILD_ROOT=/rootfs
-ENV TARGETARCH=${TARGETARCH}
-ENV INTERNAL_BUILD=${INTERNAL_BUILD}
 ENV SPDX_LICENSES_VERSION=${SPDX_LICENSES_VERSION}
 
 RUN --mount=type=bind,source=docker/scripts,target=/scripts \

docker/Dockerfile.agent-data-plane

@@ -0,0 +1,12 @@
+# Python dependencies for this repository's tooling.
+#
+# `make ensure-python-venv` installs these into a local virtualenv, which the targets that run
+# Python scripts prefer over the system interpreter. CI's build image pre-installs this same file
+# system-wide instead, so its jobs don't download anything at run time.
+#
+# Versions are pinned because some of that tooling generates committed files that CI compares
+# byte-for-byte (see test/smp/regression/adp/generate_experiments.py): an unpinned YAML emitter or
+# template engine could reformat its output and fail a check for reasons that have nothing to do
+# with the change under review.
+Jinja2==3.1.6
+PyYAML==6.0.3