Tests

feat(console): show all-agent LLM and tool-call trend on Token Usage #8482

Workflow file for this run

	name: Tests

	on:
	push:
	branches: [main, master, dev, develop]
	paths:
	- 'src/**'
	- 'tests/**'
	- 'pyproject.toml'
	- 'setup.py'
	- 'deploy/Dockerfile'
	- '.github/workflows/tests.yml'
	pull_request:
	branches: [main, master, dev, develop]
	workflow_dispatch:
	inputs:
	integration_marker:
	description: >-
	Pytest marker expression for the integration tier. Leave blank
	to run the FULL integration suite (the default for every event).
	Fill in to narrow a manual run, e.g. "integration and p0",
	"integration and (p0 or p1)".
	required: false
	default: ''

	# Coverage data is collected on a single matrix entry per test class
	# (ubuntu-latest + python 3.13 for lower tracer overhead)
	# instead of being re-run from scratch in coverage-report. The data files
	# (.coverage.unit / .coverage.contract / .coverage.integration plus their
	# cobertura xml) are uploaded as short-lived artifacts and consumed by the
	# coverage-report job, which only combines and renders — it does not run
	# pytest itself anymore.

	jobs:
	spam-gate:
	name: PR Spam Gate
	if: github.event_name == 'pull_request'
	uses: ./.github/workflows/pr-spam-gate.yml
	with:
	author: ${{ github.event.pull_request.user.login }}

	# Replaces the former `on.pull_request.paths` filter: the workflow now runs
	# (and reports a status) on every PR so `Test Summary` can be a required
	# check, but docs-only PRs skip the approval gate and every test tier.
	changes:
	name: Detect code changes
	runs-on: ubuntu-latest
	outputs:
	code: ${{ github.event_name != 'pull_request' && 'true' \|\| steps.filter.outputs.code }}
	steps:
	- uses: actions/checkout@v4
	if: github.event_name == 'pull_request'
	- uses: dorny/paths-filter@v3
	if: github.event_name == 'pull_request'
	id: filter
	with:
	filters: \|
	code:
	- 'src/**'
	- 'tests/**'
	- 'pyproject.toml'
	- 'setup.py'
	- '.github/workflows/tests.yml'

	approval-gate:
	name: Maintainer Approval
	needs: [spam-gate, changes]
	if: \|
	always() &&
	needs.changes.outputs.code == 'true' &&
	(needs.spam-gate.result == 'skipped' \|\| needs.spam-gate.outputs.blocked != 'true')
	runs-on: ubuntu-latest
	environment: maintainer-approved
	steps:
	- name: Approval granted
	run: echo "Approved by maintainer"

	unit-tests:
	name: Unit Tests - py${{ matrix.python-version }} - ${{ matrix.os }}
	needs: approval-gate
	if: \|
	always() &&
	needs.approval-gate.result == 'success'
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.11", "3.13"]
	os: [ubuntu-latest]
	include:
	- os: macos-latest
	python-version: "3.11"
	- os: windows-latest
	python-version: "3.11"

	steps:
	- uses: actions/checkout@v4

	- name: Install Linux isolation dependency
	if: \|
	runner.os == 'Linux' &&
	matrix.python-version == '3.11'
	shell: bash
	run: \|
	sudo apt-get update
	sudo apt-get install -y bubblewrap
	if sysctl kernel.apparmor_restrict_unprivileged_userns \
	>/dev/null 2>&1; then
	sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
	fi

	- name: Set up Node.js (for console build)
	uses: actions/setup-node@v4
	with:
	node-version: '20'
	cache: 'npm'
	cache-dependency-path: console/package-lock.json

	- name: Build console frontend
	shell: bash
	env:
	NODE_OPTIONS: "--max-old-space-size=8192"
	run: \|
	cd console && npm ci && npm run build

	- name: Copy console build into package
	shell: bash
	run: \|
	rm -rf src/qwenpaw/console/*
	mkdir -p src/qwenpaw/console
	cp -R console/dist/* src/qwenpaw/console/

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	cache: 'pip'

	- name: Install dependencies
	shell: bash
	run: \|
	python -m pip install --upgrade pip
	pip install -e ".[dev,test,full]"

	- name: Run unit tests
	shell: bash
	env:
	COVERAGE_FILE: .coverage.unit
	run: \|
	# Coverage is collected only on the ubuntu/py3.13 entry so the
	# data file can be uploaded for coverage-report. Other matrix
	# entries only verify cross-platform compatibility.
	if [ "${{ matrix.os }}" = "ubuntu-latest" ] && \
	[ "${{ matrix.python-version }}" = "3.13" ]; then
	pytest tests/unit -v \
	--cov=src/qwenpaw \
	--cov-report=xml:coverage.unit.xml
	else
	pytest tests/unit -v
	fi

	- name: Run Hub Local runtime E2E
	if: matrix.python-version == '3.11'
	shell: bash
	env:
	QWENPAW_LOCAL_RUNTIME_E2E: '1'
	run: \|
	python -m pip install --no-deps --force-reinstall .
	pytest tests/e2e/test_hub_local_runtime.py -v

	- name: Upload unit coverage data
	if: \|
	matrix.os == 'ubuntu-latest' &&
	matrix.python-version == '3.13'
	uses: actions/upload-artifact@v4
	with:
	name: coverage-data-unit
	path: \|
	.coverage.unit
	coverage.unit.xml
	retention-days: 1
	include-hidden-files: true

	contract-tests:
	name: Contract Tests - py${{ matrix.python-version }} - ${{ matrix.os }}
	needs: approval-gate
	if: \|
	always() &&
	needs.approval-gate.result == 'success'
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.11", "3.13"]
	os: [ubuntu-latest]
	include:
	- os: macos-latest
	python-version: "3.11"
	- os: windows-latest
	python-version: "3.11"

	steps:
	- uses: actions/checkout@v4

	- name: Set up Node.js (for console build)
	uses: actions/setup-node@v4
	with:
	node-version: '20'
	cache: 'npm'
	cache-dependency-path: console/package-lock.json

	- name: Build console frontend
	shell: bash
	env:
	NODE_OPTIONS: "--max-old-space-size=8192"
	run: \|
	cd console && npm ci && npm run build

	- name: Copy console build into package
	shell: bash
	run: \|
	rm -rf src/qwenpaw/console/*
	mkdir -p src/qwenpaw/console
	cp -R console/dist/* src/qwenpaw/console/

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	cache: 'pip'

	- name: Install dependencies
	shell: bash
	run: \|
	python -m pip install --upgrade pip
	pip install -e ".[dev,test,full]"

	- name: Run contract tests
	shell: bash
	env:
	COVERAGE_FILE: .coverage.contract
	run: \|
	# Same conditional-coverage pattern as unit-tests.
	if [ "${{ matrix.os }}" = "ubuntu-latest" ] && \
	[ "${{ matrix.python-version }}" = "3.13" ]; then
	pytest tests/contract -v \
	--cov=src/qwenpaw \
	--cov-report=xml:coverage.contract.xml \
	--cov-fail-under=0
	else
	pytest tests/contract -v
	fi

	- name: Upload contract coverage data
	if: \|
	matrix.os == 'ubuntu-latest' &&
	matrix.python-version == '3.13'
	uses: actions/upload-artifact@v4
	with:
	name: coverage-data-contract
	path: \|
	.coverage.contract
	coverage.contract.xml
	retention-days: 1
	include-hidden-files: true

	integrated-tests:
	name: Integrated Tests - py${{ matrix.python-version }} - ${{ matrix.os }}
	needs: approval-gate
	if: \|
	always() &&
	needs.approval-gate.result == 'success'
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.11", "3.13"]
	os: [ubuntu-latest]
	include:
	- os: macos-latest
	python-version: "3.11"
	- os: windows-latest
	python-version: "3.11"

	steps:
	- uses: actions/checkout@v4

	- name: Set up Node.js (for console build)
	uses: actions/setup-node@v4
	with:
	node-version: '20'
	cache: 'npm'
	cache-dependency-path: console/package-lock.json

	- name: Build console frontend
	shell: bash
	env:
	NODE_OPTIONS: "--max-old-space-size=8192"
	run: \|
	cd console && npm ci && npm run build

	- name: Copy console build into package
	shell: bash
	run: \|
	rm -rf src/qwenpaw/console/*
	mkdir -p src/qwenpaw/console
	cp -R console/dist/* src/qwenpaw/console/

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	cache: 'pip'

	- name: Install dependencies
	shell: bash
	run: \|
	python -m pip install --upgrade pip
	# The macOS runner ships setuptools 65.5.0, which pip's resolver
	# upgrades to the latest release (84.x) while backtracking the
	# dependency graph. setuptools >= 82 no longer ships
	# pkg_resources.declare_namespace, which lark-oapi's namespace
	# packages still call at import time, so the Feishu mock IM
	# integration tests crash with AttributeError on macOS. Pin
	# setuptools <82 here — the pin must ride in the SAME pip command
	# as the install: a separate `pip install "setuptools<82"` step
	# beforehand gets upgraded away by this resolution again
	# (reproduced with pip 26.2.1; see CI run 31571533395).
	if [ "${{ runner.os }}" = "macOS" ]; then
	pip install -e ".[dev,test,full]" "setuptools<82"
	else
	pip install -e ".[dev,test,full]"
	fi

	# tests/integration/browser carries the integration marker but no
	# priority marker, so any expression without a p0/p1 filter (a
	# workflow_dispatch of "integration", say) selects it and needs a real
	# Chromium. Same command as the e2e workflows.
	- name: Install Playwright browser
	shell: bash
	run: \|
	playwright install chromium --with-deps

	- name: Check if integrated tests exist
	id: check-integrated
	shell: bash
	run: \|
	if [ -d "tests/integration" ] && compgen -G "tests/integration/*.py" > /dev/null; then
	echo "has_tests=true" >> "$GITHUB_OUTPUT"
	else
	echo "has_tests=false" >> "$GITHUB_OUTPUT"
	fi

	- name: Determine pytest marker expression
	id: marker
	if: steps.check-integrated.outputs.has_tests == 'true'
	shell: bash
	env:
	DISPATCH_MARKER: ${{ inputs.integration_marker }}
	run: \|
	# Manual dispatch override -> use whatever the maintainer typed.
	# Everything else (PR gate / push) -> FULL integration suite.
	# The PR gate is the only layer that reliably runs (post-merge
	# push runs wait on the maintainer-approved environment), so
	# problems are blocked here rather than detected after merge.
	if [ -n "${DISPATCH_MARKER}" ]; then
	EXPR="${DISPATCH_MARKER}"
	else
	EXPR="integration"
	fi
	echo "expr=$EXPR" >> "$GITHUB_OUTPUT"
	echo "Selected marker expression: $EXPR"

	- name: Run integrated tests
	if: steps.check-integrated.outputs.has_tests == 'true'
	shell: bash
	env:
	# Subprocess coverage on the ubuntu/py3.11 entry only. conftest.py
	# treats empty / missing as off, so the other matrix entries do
	# not pay the tracer overhead. Multi-platform coverage is opt-in
	# via full-tests-nightly.yml dispatch (coverage_platforms input).
	QWENPAW_INTEGRATION_COVERAGE: ${{ (matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11') && '1' \|\| '' }}
	# Windows/macOS runners are slower and IO-bound; under xdist
	# parallel the default HTTP timeouts are too tight and
	# intermittently surface as ``httpx.ReadTimeout`` (e.g. the real
	# plugin install on macOS). Lift the floor on non-Linux runners.
	QWENPAW_INTEGRATION_HTTP_TIMEOUT: ${{ matrix.os != 'ubuntu-latest' && '120' \|\| '' }}
	run: \|
	if [ -n "$QWENPAW_INTEGRATION_COVERAGE" ]; then
	# Subprocess coverage entry. Parent process must not carry
	# --cov, hence --no-cov here.
	pytest tests/integration -v --no-cov \
	-n auto --dist=loadscope --timeout=300 \
	-m "${{ steps.marker.outputs.expr }}"
	cp .integration_coverage/integration_subproc \
	.coverage.integration
	# `coverage xml` honours fail_under and exits 2 when below;
	# tolerate that — the combined value is what matters.
	coverage xml --data-file=.coverage.integration \
	-o coverage.integration.xml \|\| [ "$?" -eq 2 ]
	else
	pytest tests/integration -v \
	-n auto --dist=loadscope --timeout=300 \
	-m "${{ steps.marker.outputs.expr }}"
	fi

	- name: Upload integration coverage data
	if: \|
	steps.check-integrated.outputs.has_tests == 'true' &&
	matrix.os == 'ubuntu-latest' &&
	matrix.python-version == '3.11'
	uses: actions/upload-artifact@v4
	with:
	name: coverage-data-integration
	path: \|
	.coverage.integration
	coverage.integration.xml
	retention-days: 1
	include-hidden-files: true

	coverage-report:
	name: Coverage Report
	needs: [approval-gate, unit-tests, contract-tests, integrated-tests]
	if: \|
	always() &&
	needs.approval-gate.result == 'success'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	pull-requests: write
	steps:
	- uses: actions/checkout@v4

	- name: Set up Python 3.12
	uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	cache: 'pip'

	- name: Install coverage tool
	shell: bash
	run: \|
	python -m pip install --upgrade pip
	# Only the coverage tool is needed for combine / report; no need
	# to install the full project (the per-tier .coverage data files
	# arrive via download-artifact below).
	pip install coverage

	- name: Download coverage data artifacts
	uses: actions/download-artifact@v4
	with:
	pattern: coverage-data-*
	merge-multiple: true

	- name: Inspect downloaded coverage data
	shell: bash
	run: \|
	ls -la .coverage* coverage.*.xml 2>&1 \| head -30 \|\| true

	- name: Combine all coverage data
	shell: bash
	run: \|
	# Collect all coverage data files: unit, contract, and integration.
	coverage combine \
	.coverage.unit .coverage.contract .coverage.integration
	# Tolerate fail-under (exit 2): combined number is what matters
	# and reflected in the summary regardless.
	coverage xml -o coverage.combined.xml \|\| [ "$?" -eq 2 ]
	coverage html -d htmlcov-combined \|\| [ "$?" -eq 2 ]
	echo "Combined coverage report:"
	coverage report --skip-covered --fail-under=0
	coverage json -o coverage.combined.json \|\| [ "$?" -eq 2 ]

	- name: Build coverage summary table
	id: cov_summary
	shell: bash
	run: \|
	# Best-effort number per tier; missing/malformed data => "n/a".
	set +e

	# Read line-rate from cobertura xml directly to avoid the
	# coverage source-filter mismatch that would otherwise produce
	# "No data to report" on the per-tier data files.
	get_pct() {
	python3 -c 'import sys, xml.etree.ElementTree as ET; print(round(float(ET.parse(sys.argv[1]).getroot().attrib.get("line-rate", "0")) * 100))' "$1" 2>/dev/null \|\| echo "n/a"
	}
	UNIT=$(get_pct coverage.unit.xml)
	CONTRACT=$(get_pct coverage.contract.xml)
	INTEGRATION=$(get_pct coverage.integration.xml)
	COMBINED=$(get_pct coverage.combined.xml)
	echo "UNIT=${UNIT:-n/a}"
	echo "CONTRACT=${CONTRACT:-n/a}"
	echo "INTEGRATION=${INTEGRATION:-n/a}"
	echo "COMBINED=${COMBINED:-n/a}"

	{
	echo "## 📊 Coverage report"
	echo ""
	echo "\| Test category \| Coverage \|"
	echo "\|---\|---\|"
	echo "\| Unit \| ${UNIT:-n/a}% \|"
	echo "\| Contract \| ${CONTRACT:-n/a}% \|"
	echo "\| Integration \| ${INTEGRATION:-n/a}% \|"
	echo "\| Combined \| ${COMBINED:-n/a}% \|"
	echo ""
	echo "> Combined = \`coverage combine\` of unit + contract + integration (subprocess mode)."
	echo "> HTML reports under workflow artifact \`coverage-reports\` (\`htmlcov-combined\` / \`htmlcov-integration\`)."
	echo "> Coverage is collected only on the ubuntu/py3.13 matrix entry; other matrix entries verify cross-platform compatibility without the tracer overhead."
	} > coverage_summary.md

	cat coverage_summary.md >> "$GITHUB_STEP_SUMMARY"

	- name: Sticky coverage PR comment
	if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
	uses: actions/github-script@v7
	with:
	script: \|
	const fs = require('fs');
	const body = fs.readFileSync('coverage_summary.md', 'utf8');
	const marker = '<!-- qwenpaw-coverage-summary -->';
	const finalBody = `${marker}\n${body}`;
	const comments = await github.paginate(
	github.rest.issues.listComments,
	{
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	},
	);
	const existing = comments.find(
	(c) => c.body && c.body.includes(marker),
	);
	if (existing) {
	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: existing.id,
	body: finalBody,
	});
	} else {
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	body: finalBody,
	});
	}

	- name: Upload coverage artifacts
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: coverage-reports
	path: \|
	coverage.unit.xml
	coverage.contract.xml
	coverage.integration.xml
	coverage.combined.xml
	htmlcov-combined/
	htmlcov-integration/
	retention-days: 7

	test-summary:
	name: Test Summary
	needs: [changes, approval-gate, unit-tests, contract-tests, integrated-tests, coverage-report]
	# Fail-closed gate. This job ALWAYS runs (no path condition) so the
	# required-check context can never be satisfied by an accidental skip.
	# Three-state decision:
	# 1. change detection did not succeed -> red (its `code` output is
	# unreliable when the detection job fails/is cancelled, so the
	# gate must close rather than open);
	# 2. detection succeeded and reported a docs-only PR -> explicit
	# green (instead of skipping, which a ruleset cannot distinguish
	# from a bypass);
	# 3. code change -> approval plus EVERY test tier must be strictly
	# `success`; failure/cancelled/skipped are all rejected.
	# coverage-report is intentionally observed, not enforced: a coverage
	# tooling outage must not block an otherwise green PR.
	if: always()
	runs-on: ubuntu-latest
	steps:
	- name: Check test results
	shell: bash
	run: \|
	echo "Changes detection: ${{ needs.changes.result }}"
	echo "Approval gate: ${{ needs.approval-gate.result }}"
	echo "Unit tests: ${{ needs.unit-tests.result }}"
	echo "Contract tests: ${{ needs.contract-tests.result }}"
	echo "Integrated tests: ${{ needs.integrated-tests.result }}"
	echo "Coverage report: ${{ needs.coverage-report.result }}"

	if [ "${{ needs.changes.result }}" != "success" ]; then
	echo "❌ Change detection did not succeed (${{ needs.changes.result }}) — gate closed, refusing untested merge"
	exit 1
	fi

	if [ "${{ needs.changes.outputs.code }}" != "true" ]; then
	echo "✅ Docs-only change — no test tiers required"
	exit 0
	fi

	if [ "${{ needs.approval-gate.result }}" != "success" ]; then
	echo "❌ Approval not granted"
	exit 1
	fi

	if [ "${{ needs.unit-tests.result }}" != "success" ] \|\| \
	[ "${{ needs.contract-tests.result }}" != "success" ] \|\| \
	[ "${{ needs.integrated-tests.result }}" != "success" ]; then
	echo "❌ Every test tier must be success (failure/cancelled/skipped are all rejected)"
	exit 1
	fi

	echo "✅ All tests passed"

Navigation Menu

Uh oh!

feat(console): show all-agent LLM and tool-call trend on Token Usage #8482

Workflow file

feat(console): show all-agent LLM and tool-call trend on Token Usage #8482

Uh oh!

Workflow file for this run