Build, test, and publish Python distributions

Fix nested request reply service feedback #780

Workflow file for this run

	name: Build, test, and publish Python distributions

	on:
	push:
	branches:
	- "**"
	tags:
	- "v_..*"
	pull_request:
	workflow_dispatch:

	permissions:
	actions: read
	contents: read

	jobs:
	validate-release:
	name: Validate release metadata
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	persist-credentials: false
	- uses: actions/setup-python@v6
	with:
	python-version: "3.12"
	- name: Validate package metadata
	run: \|
	python -m pip install --upgrade trove-classifiers
	python python/tests/test_packaging.py
	- name: Validate release tag
	if: github.ref_type == 'tag'
	shell: python
	env:
	RELEASE_TAG: ${{ github.ref_name }}
	run: \|
	from urllib.error import HTTPError
	from urllib.request import urlopen
	import os
	import re

	tag = os.environ["RELEASE_TAG"]
	match = re.fullmatch(r"v_(\d+\.\d+\.\d+)", tag)
	if match is None:
	raise SystemExit(f"release tag must match v_x.x.x, got {tag!r}")

	# The TAG is the version authority: the project version in
	# pyproject.toml is ignored for releases (the publish job restamps
	# the tested artifacts' metadata to the tag version).
	version = match.group(1)

	try:
	with urlopen(f"https://pypi.org/pypi/hg_cpp/{version}/json"):
	pass
	except HTTPError as error:
	if error.code != 404:
	raise
	else:
	raise SystemExit(f"hg_cpp {version} already exists on PyPI")

	print(f"Validated new hg_cpp release {version}")

	reuse-build:
	name: Find tested distributions for this commit
	needs: validate-release
	runs-on: ubuntu-latest
	outputs:
	run-id: ${{ steps.find.outputs.run-id }}
	steps:
	- name: Find successful build by commit SHA
	id: find
	if: github.ref_type == 'tag'
	uses: actions/github-script@v8
	with:
	script: \|
	const requiredArtifacts = new Set([
	"distribution-sdist",
	"distribution-wheel-macos-26",
	"distribution-wheel-ubuntu-latest",
	"distribution-wheel-windows-latest",
	]);
	const runs = await github.paginate(
	github.rest.actions.listWorkflowRuns,
	{
	owner: context.repo.owner,
	repo: context.repo.repo,
	workflow_id: "build.yml",
	event: "push",
	head_sha: context.sha,
	status: "success",
	per_page: 100,
	},
	);

	for (const run of runs) {
	if (run.head_sha !== context.sha \|\| run.id === context.runId) {
	continue;
	}

	const artifacts = await github.paginate(
	github.rest.actions.listWorkflowRunArtifacts,
	{
	owner: context.repo.owner,
	repo: context.repo.repo,
	run_id: run.id,
	per_page: 100,
	},
	);
	const availableArtifacts = new Set(
	artifacts
	.filter((artifact) => !artifact.expired)
	.map((artifact) => artifact.name),
	);
	if ([...requiredArtifacts].every((name) => availableArtifacts.has(name))) {
	core.notice(`Reusing tested distributions from workflow run ${run.id}`);
	core.setOutput("run-id", String(run.id));
	return;
	}
	}

	core.notice(`No reusable distributions found for commit ${context.sha}`);
	core.setOutput("run-id", "");

	build-wheel:
	name: Build ${{ matrix.os }} wheel
	needs:
	- validate-release
	- reuse-build
	if: needs.reuse-build.outputs.run-id == ''
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- os: ubuntu-latest
	parallel: "2"
	cmake_generator: Ninja
	cmake_args: -DHGRAPH_WARNINGS_AS_ERRORS=ON
	- os: windows-latest
	parallel: "2"
	cmake_generator: Visual Studio 18 2026
	cmake_args: -DHGRAPH_WARNINGS_AS_ERRORS=OFF
	- os: macos-26
	parallel: "1"
	cmake_generator: Ninja
	cmake_args: -DHGRAPH_WARNINGS_AS_ERRORS=OFF -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0

	steps:
	- uses: actions/checkout@v6
	with:
	persist-credentials: false
	- uses: actions/setup-python@v6
	with:
	python-version: "3.12"
	- name: Configure current Clang on macOS
	if: runner.os == 'macOS'
	shell: bash
	run: \|
	echo "CC=/usr/bin/clang" >> "$GITHUB_ENV"
	echo "CXX=/usr/bin/clang++" >> "$GITHUB_ENV"
	echo "MACOSX_DEPLOYMENT_TARGET=15.0" >> "$GITHUB_ENV"
	/usr/bin/clang++ --version
	- name: Install wheel build tools
	run: python -m pip install --upgrade build abi3audit
	- name: Build stable ABI wheel
	env:
	CMAKE_GENERATOR: ${{ matrix.cmake_generator }}
	CMAKE_ARGS: ${{ matrix.cmake_args }}
	CMAKE_BUILD_PARALLEL_LEVEL: ${{ matrix.parallel }}
	run: python -m build --wheel
	- name: Repair Linux wheel
	if: runner.os == 'Linux'
	run: \|
	python -m pip install --upgrade auditwheel
	mkdir wheelhouse
	auditwheel repair \
	--exclude libarrow.so.2400 \
	--exclude libarrow_compute.so.2400 \
	--wheel-dir wheelhouse \
	dist/*.whl
	rm dist/*.whl
	mv wheelhouse/*.whl dist/
	- name: Verify wheel ABI
	shell: python
	run: \|
	from pathlib import Path
	import subprocess
	import sys

	wheels = list(Path("dist").glob("*.whl"))
	assert len(wheels) == 1, wheels
	wheel = wheels[0]
	assert wheel.name.startswith("hg_cpp-"), wheel.name
	assert "-cp312-abi3-" in wheel.name, wheel.name
	if sys.platform == "darwin":
	assert "-macosx_15_0_arm64.whl" in wheel.name, wheel.name
	elif sys.platform == "win32":
	assert wheel.name.endswith("-win_amd64.whl"), wheel.name
	else:
	assert wheel.name.endswith("_x86_64.whl"), wheel.name
	subprocess.check_call(["abi3audit", "--strict", str(wheel)])
	- uses: actions/upload-artifact@v7
	with:
	name: distribution-wheel-${{ matrix.os }}
	path: dist/*.whl
	if-no-files-found: error

	build-sdist:
	name: Build source distribution
	needs:
	- validate-release
	- reuse-build
	if: needs.reuse-build.outputs.run-id == ''
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	persist-credentials: false
	- uses: actions/setup-python@v6
	with:
	python-version: "3.12"
	- name: Build source distribution
	run: \|
	python -m pip install --upgrade build
	python -m build --sdist
	- name: Verify source distribution
	shell: python
	run: \|
	from pathlib import Path
	import tomllib

	version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
	archives = list(Path("dist").glob("*.tar.gz"))
	assert len(archives) == 1, archives
	assert archives[0].name == f"hg_cpp-{version}.tar.gz", archives[0].name
	- uses: actions/upload-artifact@v7
	with:
	name: distribution-sdist
	path: dist/*.tar.gz
	if-no-files-found: error

	test-wheel:
	name: Test ${{ matrix.os }} / Python ${{ matrix.python-version }}
	needs:
	- reuse-build
	- build-wheel
	if: needs.reuse-build.outputs.run-id == ''
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os:
	- ubuntu-latest
	- windows-latest
	- macos-26
	python-version:
	- "3.12"
	- "3.13"
	- "3.14"

	steps:
	- uses: actions/checkout@v6
	with:
	persist-credentials: false
	- uses: actions/setup-python@v6
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install uv
	run: python -m pip install --upgrade uv
	- uses: actions/download-artifact@v7
	with:
	name: distribution-wheel-${{ matrix.os }}
	path: dist
	- name: Sync test dependencies
	run: uv sync --frozen --all-extras --all-groups --no-install-project --python ${{ matrix.python-version }}
	- name: Install wheel
	shell: python
	run: \|
	from pathlib import Path
	import subprocess
	import sys

	wheels = list(Path("dist").glob("*.whl"))
	assert len(wheels) == 1, wheels
	venv_python = Path(".venv") / (
	"Scripts/python.exe" if sys.platform == "win32" else "bin/python"
	)
	subprocess.check_call([
	"uv", "pip", "install", "--python", str(venv_python), str(wheels[0]),
	])
	- name: Confirm installed distribution
	run: uv run --no-sync python -c "import importlib.metadata as m; import _hgraph; print(m.version('hg_cpp'), _hgraph.__file__)"
	- name: Run Python compatibility suite
	run: uv run --no-sync python -m pytest python/tests -q -m "not wip"

	native-cpp:
	name: Native C++ / ${{ matrix.os }}
	needs:
	- validate-release
	- reuse-build
	if: needs.reuse-build.outputs.run-id == ''
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- os: ubuntu-latest
	parallel: "2"
	- os: macos-26
	parallel: "1"

	steps:
	- uses: actions/checkout@v6
	with:
	persist-credentials: false
	- uses: actions/setup-python@v6
	with:
	python-version: "3.12"
	- name: Configure current Clang on macOS
	if: runner.os == 'macOS'
	shell: bash
	run: \|
	echo "CC=/usr/bin/clang" >> "$GITHUB_ENV"
	echo "CXX=/usr/bin/clang++" >> "$GITHUB_ENV"
	/usr/bin/clang++ --version
	- name: Install native build dependencies
	run: python -m pip install --upgrade "cmake>=3.26" ninja "pyarrow>=24,<25"
	- name: Expose Arrow runtime libraries
	shell: python
	run: \|
	import os
	from pathlib import Path
	import pyarrow

	with open(os.environ["GITHUB_PATH"], "a", encoding="utf-8") as path_file:
	path_file.write(str(Path(pyarrow.__file__).resolve().parent) + "\n")
	- name: Configure native C++ tests
	run: cmake -S . -B build-native -GNinja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON -DHGRAPH_BUILD_PYTHON_BINDINGS=OFF -DHGRAPH_ENABLE_PYTHON_USER_NODES=OFF -DHGRAPH_ENABLE_IDE_PYTHON_HEADER_HINTS=OFF -DHGRAPH_USE_PYARROW_ARROW=ON -DHGRAPH_FETCH_SIMDJSON=ON -DHGRAPH_ENABLE_DEBUGGER_SMOKE_TESTS=ON -DHGRAPH_WARNINGS_AS_ERRORS=ON
	- name: Build native C++ tests
	run: cmake --build build-native --parallel ${{ matrix.parallel }}
	- name: Run native C++ tests
	run: ctest --test-dir build-native --output-on-failure --parallel ${{ matrix.parallel }}

	native-install:
	name: Native C++ / Linux shared install
	needs:
	- validate-release
	- reuse-build
	if: needs.reuse-build.outputs.run-id == ''
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	persist-credentials: false
	- uses: mamba-org/setup-micromamba@v3
	with:
	environment-name: hgraph-native
	cache-environment: true
	create-args: >-
	cmake>=3.26
	ninja
	libarrow=24
	libarrow-compute=24
	fmt
	spdlog>=1.15
	simdjson>=4.5
	- name: Configure shared C++ tests
	shell: micromamba-shell {0}
	run: >-
	cmake -S . -B build-shared -G Ninja
	-DCMAKE_BUILD_TYPE=Release
	-DCMAKE_PREFIX_PATH="$CONDA_PREFIX"
	-DBUILD_TESTING=ON
	-DHGRAPH_BUILD_SHARED=ON
	-DHGRAPH_BUILD_PYTHON_BINDINGS=OFF
	-DHGRAPH_ENABLE_PYTHON_USER_NODES=OFF
	-DHGRAPH_ENABLE_IDE_PYTHON_HEADER_HINTS=OFF
	-DHGRAPH_ENABLE_DEBUGGER_SMOKE_TESTS=ON
	-DHGRAPH_WARNINGS_AS_ERRORS=ON
	- name: Build shared C++ tests
	shell: micromamba-shell {0}
	run: cmake --build build-shared --parallel 2
	- name: Run shared C++ tests
	shell: micromamba-shell {0}
	run: ctest --test-dir build-shared --output-on-failure --parallel 2
	- name: Install shared C++ package
	shell: micromamba-shell {0}
	run: cmake --install build-shared --prefix "$GITHUB_WORKSPACE/install-shared"
	- name: Configure installed-package consumer
	shell: micromamba-shell {0}
	run: >-
	cmake -S tests/install_consumer -B build-consumer -G Ninja
	-DCMAKE_BUILD_TYPE=Release
	-DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/install-shared;$CONDA_PREFIX"
	- name: Build and test installed-package consumer
	shell: micromamba-shell {0}
	run: \|
	cmake --build build-consumer --parallel 2
	LD_LIBRARY_PATH="$CONDA_PREFIX/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \
	ctest --test-dir build-consumer --output-on-failure

	publish:
	name: Publish hg_cpp to PyPI
	if: >-
	always() &&
	startsWith(github.ref, 'refs/tags/v_') &&
	needs.validate-release.result == 'success' &&
	needs.reuse-build.result == 'success' &&
	(
	needs.reuse-build.outputs.run-id != '' \|\|
	(
	needs.build-sdist.result == 'success' &&
	needs.test-wheel.result == 'success' &&
	needs.native-cpp.result == 'success' &&
	needs.native-install.result == 'success'
	)
	)
	needs:
	- validate-release
	- reuse-build
	- build-sdist
	- test-wheel
	- native-cpp
	- native-install
	runs-on: ubuntu-latest
	environment:
	name: release
	url: https://pypi.org/p/hg_cpp
	permissions:
	actions: read
	id-token: write
	steps:
	- uses: actions/download-artifact@v7
	with:
	pattern: distribution-*
	path: dist
	merge-multiple: true
	github-token: ${{ github.token }}
	run-id: ${{ needs.reuse-build.outputs.run-id \|\| github.run_id }}
	- name: Restamp distributions to the tag version
	shell: python
	env:
	RELEASE_TAG: ${{ github.ref_name }}
	run: \|
	# The TAG is the version authority. The distributions were built
	# (and fully tested) at whatever version pyproject.toml carried -
	# rewriting the version is a METADATA-ONLY operation, so the
	# expensive compiled artifacts are reused byte-identical.
	import base64
	import csv
	import hashlib
	import io
	import os
	import re
	import tarfile
	import zipfile
	from pathlib import Path

	version = os.environ["RELEASE_TAG"].removeprefix("v_")

	def record_hash(data: bytes) -> str:
	digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest())
	return "sha256=" + digest.decode().rstrip("=")

	def restamp_wheel(path: Path) -> None:
	name, old_version, rest = path.name.split("-", 2)
	if old_version == version:
	print(f"{path.name}: already at {version}")
	return
	old_info = f"{name}-{old_version}.dist-info"
	new_info = f"{name}-{version}.dist-info"
	entries = {}
	with zipfile.ZipFile(path) as archive:
	for item in archive.infolist():
	entries[item.filename] = archive.read(item.filename)
	renamed = {}
	for item_name, data in entries.items():
	new_name = item_name.replace(old_info + "/", new_info + "/")
	if new_name == f"{new_info}/METADATA":
	text = data.decode()
	text, count = re.subn(rf"(?m)^Version: {re.escape(old_version)}$",
	f"Version: {version}", text, count=1)
	assert count == 1, "wheel METADATA Version line not found"
	data = text.encode()
	renamed[new_name] = data
	record_name = f"{new_info}/RECORD"
	rows = []
	for row in csv.reader(io.StringIO(renamed[record_name].decode())):
	if not row:
	continue
	entry = row[0].replace(old_info + "/", new_info + "/")
	if entry == record_name:
	rows.append([entry, "", ""])
	else:
	data = renamed[entry]
	rows.append([entry, record_hash(data), str(len(data))])
	buffer = io.StringIO(newline="")
	csv.writer(buffer, lineterminator="\n").writerows(rows)
	renamed[record_name] = buffer.getvalue().encode()
	new_path = path.with_name(path.name.replace(f"-{old_version}-", f"-{version}-", 1))
	with zipfile.ZipFile(new_path, "w", zipfile.ZIP_DEFLATED) as archive:
	for item_name, data in renamed.items():
	archive.writestr(item_name, data)
	if new_path != path:
	path.unlink()
	print(f"{path.name} -> {new_path.name}")

	def restamp_sdist(path: Path) -> None:
	name, old_version = path.name.removesuffix(".tar.gz").rsplit("-", 1)
	if old_version == version:
	print(f"{path.name}: already at {version}")
	return
	old_root = f"{name}-{old_version}"
	new_root = f"{name}-{version}"
	members = []
	with tarfile.open(path, "r:gz") as archive:
	for member in archive.getmembers():
	data = archive.extractfile(member).read() if member.isfile() else None
	member.name = member.name.replace(old_root, new_root, 1)
	# long (>100 char) paths ride PAX 'path' headers which
	# would override the renamed member.name at write time.
	member.pax_headers.pop("path", None)
	member.pax_headers.pop("linkpath", None)
	if member.name == f"{new_root}/PKG-INFO":
	text = data.decode()
	text, count = re.subn(rf"(?m)^Version: {re.escape(old_version)}$",
	f"Version: {version}", text, count=1)
	assert count == 1, "sdist PKG-INFO Version line not found"
	data = text.encode()
	elif member.name == f"{new_root}/pyproject.toml":
	text = data.decode()
	text, count = re.subn(r'(?m)^version = "[^"]+"$',
	f'version = "{version}"', text, count=1)
	assert count == 1, "sdist pyproject.toml version not found"
	data = text.encode()
	if data is not None:
	member.size = len(data)
	members.append((member, data))
	new_path = path.with_name(f"{new_root}.tar.gz")
	with tarfile.open(new_path, "w:gz") as archive:
	for member, data in members:
	archive.addfile(member, io.BytesIO(data) if data is not None else None)
	if new_path != path:
	path.unlink()
	print(f"{path.name} -> {new_path.name}")

	for wheel in sorted(Path("dist").glob("*.whl")):
	restamp_wheel(wheel)
	for sdist in sorted(Path("dist").glob("*.tar.gz")):
	restamp_sdist(sdist)
	- name: Publish package distributions to PyPI
	uses: pypa/gh-action-pypi-publish@release/v1
	with:
	packages-dir: dist

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Fix nested request reply service feedback #780

Workflow file

Fix nested request reply service feedback #780

Uh oh!

Workflow file for this run