fix(ci): close silent-skip gap in integration split (review #7293)

Reviewer correctly flagged that 22 integration test functions carry
the integration marker but no p0/p1/p2 priority marker, so the three
shard expressions silently excluded them (browser real-page journeys,
session isolation, token rotation, WS auth, ACP MCP scope, Windows
native host repair).

Fix (both suggested approaches combined):
1. Assign priority markers to all 22 unclassified tests (19 -> p1,
   3 -> p2), so the three priority shards cover the full suite again.
2. Add a fourth 'fallback' shard with expression
   'integration and not (p0 or p1 or p2)': any future unclassified
   test still RUNS (never silently skipped) instead of vanishing.
3. Add a fail-closed guard: the ubuntu fallback shard fails the run if
   it collects anything, forcing new tests to get a priority marker.
4. Tolerate empty fallback: pytest exit 5 (no tests collected) is
   expected there; coverage upload uses if-no-files-found: ignore and
   the combine step picks up whichever shard files exist.

AST audit after fix: 0 unclassified integration tests.

署名：秦琼·CIOps@QPQAT

Author: yutai78786

.github/workflows/tests.yml

@@ -276,7 +276,7 @@ jobs:
       matrix:
         python-version: ["3.11", "3.13"]
         os: [ubuntu-latest]
-        shard: [p0, p1, p2]
+        shard: [p0, p1, p2, fallback]
         include:
           - os: macos-latest
             python-version: "3.11"
@@ -287,6 +287,9 @@ jobs:
           - os: macos-latest
             python-version: "3.11"
             shard: p2
+          - os: macos-latest
+            python-version: "3.11"
+            shard: fallback
           - os: windows-latest
             python-version: "3.11"
             shard: p0
@@ -296,6 +299,9 @@ jobs:
           - os: windows-latest
             python-version: "3.11"
             shard: p2
+          - os: windows-latest
+            python-version: "3.11"
+            shard: fallback
 
     steps:
       - uses: actions/checkout@v4
@@ -381,16 +387,42 @@ jobs:
           if [ -n "${DISPATCH_MARKER}" ]; then
             EXPR="${DISPATCH_MARKER}"
           else
-            # Split by shard for parallel execution
+            # Split by shard for parallel execution. The fallback shard
+            # catches any integration test that lacks a p0/p1/p2 marker
+            # so it can never be silently skipped.
             case "${{ matrix.shard }}" in
               p0) EXPR="integration and p0" ;;
               p1) EXPR="integration and p1" ;;
               p2) EXPR="integration and p2" ;;
+              fallback) EXPR="integration and not (p0 or p1 or p2)" ;;
             esac
           fi
           echo "expr=$EXPR" >> "$GITHUB_OUTPUT"
           echo "Selected marker expression: $EXPR"
 
+      - name: Fail on unclassified integration tests
+        if: |
+          steps.check-integrated.outputs.has_tests == 'true' &&
+          matrix.shard == 'fallback' &&
+          matrix.os == 'ubuntu-latest' &&
+          matrix.python-version == '3.11'
+        shell: bash
+        run: |
+          # Guard: every integration test must carry a priority marker.
+          # If the fallback shard collects anything, a new unclassified
+          # test slipped in -- fail loudly instead of silently running
+          # it outside the three priority shards.
+          UNCLASSIFIED=$(python -m pytest tests/integration --collect-only -q \
+            -m "integration and not (p0 or p1 or p2)" 2>/dev/null \
+            | grep -c "::" || true)
+          if [ "${UNCLASSIFIED}" -gt 0 ]; then
+            echo "::error::${UNCLASSIFIED} integration test(s) lack a p0/p1/p2 priority marker. Assign one so the test joins a priority shard."
+            python -m pytest tests/integration --collect-only -q \
+              -m "integration and not (p0 or p1 or p2)" 2>/dev/null | grep "::" || true
+            exit 1
+          fi
+          echo "No unclassified integration tests."
+
       - name: Run integrated tests
         if: steps.check-integrated.outputs.has_tests == 'true'
         shell: bash
@@ -412,16 +444,30 @@ jobs:
             pytest tests/integration -v --no-cov \
               -n auto --dist=loadscope --timeout=300 \
               -m "${{ steps.marker.outputs.expr }}"
-            cp .integration_coverage/integration_subproc \
-              .coverage.integration.${{ matrix.shard }}
-            # `coverage xml` honours fail_under and exits 2 when below;
-            # tolerate that — the combined value is what matters.
-            coverage xml --data-file=.coverage.integration.${{ matrix.shard }} \
-              -o coverage.integration.${{ matrix.shard }}.xml || [ "$?" -eq 2 ]
+            PYTEST_RC=$?
+            # exit 5 = no tests collected: expected for the fallback
+            # shard when every integration test carries a priority
+            # marker. Any other nonzero code still fails.
+            if [ "$PYTEST_RC" -ne 0 ] && [ "$PYTEST_RC" -ne 5 ]; then
+              exit "$PYTEST_RC"
+            fi
+            if [ -f .integration_coverage/integration_subproc ]; then
+              cp .integration_coverage/integration_subproc \
+                .coverage.integration.${{ matrix.shard }}
+              # `coverage xml` honours fail_under and exits 2 when below;
+              # tolerate that — the combined value is what matters.
+              coverage xml --data-file=.coverage.integration.${{ matrix.shard }} \
+                -o coverage.integration.${{ matrix.shard }}.xml || [ "$?" -eq 2 ]
+            fi
           else
             pytest tests/integration -v \
               -n auto --dist=loadscope --timeout=300 \
-              -m "${{ steps.marker.outputs.expr }}"
+              -m "${{ steps.marker.outputs.expr }}" || {
+                RC=$?
+                # exit 5 = no tests collected: expected for the
+                # fallback shard when nothing is unclassified.
+                [ "$RC" -eq 5 ] || exit "$RC"
+              }
           fi
 
       - name: Upload integration coverage data
@@ -437,6 +483,10 @@ jobs:
             coverage.integration.${{ matrix.shard }}.xml
           retention-days: 1
           include-hidden-files: true
+          # The fallback shard produces no data file when every
+          # integration test carries a priority marker; a missing
+          # artifact there is expected, not an error.
+          if-no-files-found: ignore
 
   coverage-report:
     name: Coverage Report
@@ -480,13 +530,16 @@ jobs:
       - name: Combine all coverage data
         shell: bash
         run: |
-          # First, combine the three integration shards into one
-          coverage combine --data-file=.coverage.integration \
-            .coverage.integration.p0 \
-            .coverage.integration.p1 \
-            .coverage.integration.p2
-          coverage xml --data-file=.coverage.integration \
-            -o coverage.integration.xml || [ "$?" -eq 2 ]
+          # First, combine the integration shards into one. The fallback
+          # shard only produces a data file when it actually ran tests
+          # (i.e. some integration test lacked a priority marker), so
+          # include whichever shard files exist.
+          SHARDS=$(ls .coverage.integration.* 2>/dev/null || true)
+          if [ -n "$SHARDS" ]; then
+            coverage combine --data-file=.coverage.integration $SHARDS
+            coverage xml --data-file=.coverage.integration \
+              -o coverage.integration.xml || [ "$?" -eq 2 ]
+          fi
 
           # Then, combine all three tiers: unit, contract, and integration.
           coverage combine \

tests/integration/browser/test_browser_tool_e2e.py

@@ -3,6 +3,7 @@
 
 from collections.abc import AsyncGenerator
 
+import pytest
 import pytest_asyncio
 from agentscope.message import ToolResultState
 
@@ -26,6 +27,7 @@ async def reset_kernel() -> AsyncGenerator[None, None]:
     kernel._MANAGER = None  # pylint: disable=protected-access
 
 
+@pytest.mark.p1
 async def test_browser_tool_drives_a_real_page(fixture_url: str) -> None:
     code = (
         "browser = await Browser.connect(identity='guest')\n"

tests/integration/browser/test_e2e_isolation.py

@@ -21,6 +21,7 @@ def request(session_id: str, code: str) -> ExecRequest:
     )
 
 
+@pytest.mark.p1
 async def test_two_incognito_sessions_are_isolated(fixture_url: str) -> None:
     plane = SubprocessPlane()
     first_code = (
@@ -55,6 +56,7 @@ async def test_two_incognito_sessions_are_isolated(fixture_url: str) -> None:
             await link.close_all()
 
 
+@pytest.mark.p1
 async def test_same_session_id_is_isolated_by_workspace(
     fixture_url: str,
 ) -> None:
@@ -118,6 +120,7 @@ async def test_same_session_id_is_isolated_by_workspace(
     "contexts",
     [("profile", "incognito"), ("incognito", "profile")],
 )
+@pytest.mark.p1
 async def test_profile_and_incognito_have_independent_process_cells(
     contexts: tuple[str, str],
     tmp_path,

tests/integration/browser/test_execution_lifecycle_e2e.py

@@ -9,6 +9,7 @@
 import os
 import time
 
+import pytest
 from fastapi import FastAPI
 
 from qwenpaw.app._app import _start_browser_runtime, _stop_browser_runtime
@@ -30,6 +31,7 @@ def _request(
     )
 
 
+@pytest.mark.p1
 async def test_worker_is_reused_then_reclaimed() -> None:
     plane = SubprocessPlane()
     request = _request("reuse", "session", "import os\nreturn os.getpid()")
@@ -46,6 +48,7 @@ async def test_worker_is_reused_then_reclaimed() -> None:
         await plane.discard_all_workers()
 
 
+@pytest.mark.p1
 async def test_sibling_sessions_run_without_serializing() -> None:
     """Sibling sessions must run in parallel, not queue behind one lock.
 
@@ -103,6 +106,7 @@ async def test_sibling_sessions_run_without_serializing() -> None:
         await plane.discard_all_workers()
 
 
+@pytest.mark.p1
 async def test_timeout_reclaims_only_the_affected_worker() -> None:
     plane = SubprocessPlane(exec_timeout_seconds=5.0)
     runtime = KernelRuntime(plane=plane)
@@ -128,6 +132,7 @@ async def test_timeout_reclaims_only_the_affected_worker() -> None:
         await plane.discard_all_workers()
 
 
+@pytest.mark.p1
 async def test_runtime_shutdown_reclaims_real_workers() -> None:
     plane = SubprocessPlane()
     runtime = KernelRuntime(plane=plane)

tests/integration/browser/test_playwright_provider_e2e.py

@@ -19,6 +19,7 @@ def _locator_spec(method: str, *args: str, **kwargs: str) -> list[dict]:
     ]
 
 
+@pytest.mark.p1
 async def test_real_provider_isolates_sessions_and_profile(
     fixture_url: str,
     tmp_path,
@@ -62,6 +63,7 @@ async def test_real_provider_isolates_sessions_and_profile(
         await link.close_all()
 
 
+@pytest.mark.p1
 async def test_real_provider_observes_locates_and_acts(
     fixture_url: str,
 ) -> None:
@@ -106,6 +108,7 @@ async def test_real_provider_observes_locates_and_acts(
         await link.close_all()
 
 
+@pytest.mark.p1
 async def test_real_provider_rejects_unsupported_context_and_method() -> None:
     link = PlaywrightControlLink()
     try:

tests/integration/browser/test_token_rotation.py

@@ -36,13 +36,15 @@ def bridge_config(
     return config_path
 
 
+@pytest.mark.p1
 def test_rotation_recovers(bridge_config: Path) -> None:
     _write_config(bridge_config, "token-a")
     assert ws_handler._expected_token() == "token-a"
     _write_config(bridge_config, "token-b")  # plugin repair(reset) rotates
     assert ws_handler._expected_token() == "token-b"
 
 
+@pytest.mark.p1
 def test_missing_file_falls_back_to_cache(bridge_config: Path) -> None:
     _write_config(bridge_config, "token-a")
     assert ws_handler._expected_token() == "token-a"
@@ -51,6 +53,7 @@ def test_missing_file_falls_back_to_cache(bridge_config: Path) -> None:
     assert not bridge_config.exists()  # fallback must not rewrite or rotate
 
 
+@pytest.mark.p1
 def test_bootstrap_generates_once(bridge_config: Path) -> None:
     token = ws_handler._expected_token()
     assert token
@@ -72,6 +75,7 @@ def websocket_client(
     return TestClient(app)
 
 
+@pytest.mark.p1
 def test_ws_handshake_after_rotation(
     websocket_client: TestClient,
     bridge_config: Path,

tests/integration/browser/test_ws_bridge_auth.py

@@ -26,6 +26,7 @@ def websocket_client(monkeypatch: pytest.MonkeyPatch) -> TestClient:
     return TestClient(app)
 
 
+@pytest.mark.p1
 def test_nm_bridge_denies_wrong_token(websocket_client: TestClient) -> None:
     with pytest.raises(WebSocketDenialResponse) as denied:
         with websocket_client.websocket_connect(
@@ -36,6 +37,7 @@ def test_nm_bridge_denies_wrong_token(websocket_client: TestClient) -> None:
     assert denied.value.status_code == 401
 
 
+@pytest.mark.p1
 def test_nm_bridge_accepts_correct_token(websocket_client: TestClient) -> None:
     with websocket_client.websocket_connect("/ws/chrome?token=expected-token"):
         pass

tests/integration/test_acp_mcp_driver_flow.py

@@ -38,6 +38,7 @@ async def request_approval(
         self.contexts.append(context)
 
 
+@pytest.mark.p1
 @pytest.mark.asyncio
 async def test_acp_mcp_card_discovers_approves_and_invokes_tool(
     tmp_path: Path,
@@ -108,6 +109,7 @@ async def test_acp_mcp_card_discovers_approves_and_invokes_tool(
     assert await manager.card_store.list_paths() == []
 
 
+@pytest.mark.p1
 @pytest.mark.asyncio
 async def test_acp_mcp_tool_cannot_be_invoked_without_session_scope(
     tmp_path: Path,

tests/integration/test_chrome_native_host_install.py

@@ -100,6 +100,7 @@ def test_successful_install_records_a_passing_probe(
     assert result["installed"] is True
 
 
+@pytest.mark.p2
 def test_non_reset_repair_preserves_existing_bridge_token(
     isolated_home: Path,
     monkeypatch: pytest.MonkeyPatch,
@@ -140,13 +141,15 @@ def test_non_reset_repair_preserves_existing_bridge_token(
         ),
     ],
 )
+@pytest.mark.p2
 def test_windows_batch_path_literal_normalizes_and_escapes(
     source: str,
     expected: str,
 ) -> None:
     assert extension_setup._windows_batch_path_literal(source) == expected
 
 
+@pytest.mark.p2
 def test_windows_launcher_uses_cmd_safe_path_literals(
     isolated_home: Path,
     monkeypatch: pytest.MonkeyPatch,