ai-agents-for-beginners/.github/workflows/smoke-test.yml at main · microsoft/ai-agents-for-beginners · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
name: Smoke-test hosted agents

# Validates that a deployed Microsoft Foundry hosted agent actually answers,
# using the AI Smoke Test action: https://github.com/marketplace/actions/ai-smoke-test
#
# This is a lightweight, cheap post-deploy gate that complements (does not replace)
# the full evaluation pipeline taught in Lesson 10 and Lesson 16. It POSTs the
# prompts in the selected tests/lesson-XX-smoke-tests.json catalog to the agent's
# Responses endpoint and asserts on the returned text. See tests/README.md for the
# catalog-to-lesson-to-agent mapping.
#
# Run it manually from the Actions tab (workflow_dispatch) and supply your
# Foundry project endpoint and hosted agent name. Configure these repository
# secrets first (Settings -> Secrets and variables -> Actions):
#   AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID
# The federated identity needs the "Azure AI User" role at Foundry project scope.

on:
  workflow_dispatch:
    inputs:
      tests_file:
        description: "Which lesson's smoke-test catalog to run"
        required: true
        type: choice
        default: tests/lesson-16-smoke-tests.json
        options:
          - tests/lesson-01-smoke-tests.json
          - tests/lesson-04-smoke-tests.json
          - tests/lesson-05-smoke-tests.json
          - tests/lesson-16-smoke-tests.json
      project_endpoint:
        description: "Foundry project endpoint, e.g. https://<account>.services.ai.azure.com/api/projects/<project>"
        required: true
        type: string
      agent_name:
        description: "Deployed hosted agent name (must match the catalog's lesson; comma-separate for multiple)"
        required: true
        type: string
        default: ContosoSupportAgent

permissions:
  id-token: write   # required for azure/login OIDC
  contents: read

jobs:
  smoke:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

      - name: Azure login (OIDC)
        uses: azure/login@7184910d9eb2b1c5e48f7073824a90609bb9b6d6 # v2.3.1
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      - name: Smoke-test hosted agent
        uses: JFolberth/ai-smoketest@v1
        with:
          project_endpoint: ${{ inputs.project_endpoint }}
          agent_name: ${{ inputs.agent_name }}
          tests_file: ${{ inputs.tests_file }}