PR Pipeline - Code Coverage and Triggers

eng/pipelines/pr/jobs/test-buildproj-job.yml

@@ -18,6 +18,10 @@ parameters:
   - name: buildSuffix
     type: string
 
+  # Name of the artifact to upload the test results (including code coverage) to.
+  - name: testResultsArtifactName
+    type: string
+
   # Dotnet CLI verbosity level.
   - name: dotnetVerbosity
     type: string
@@ -78,6 +82,10 @@ jobs:
   - job: "test_${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}"
     displayName: "${{ parameters.testDisplayName }}_${{ parameters.platformDisplayName }}"
 
+    variables:
+      - name: testResultsPath
+        value: "$(REPO_ROOT)/test_results/${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}"
+
     pool:
       name: ${{ parameters.poolName }}
       demands:
@@ -107,3 +115,13 @@ jobs:
             -p:BuildNumber='$(Build.BuildNumber)'
             -p:BuildSuffix='${{ parameters.buildSuffix }}'
             -p:TestFramework=${{ parameters.platformDotnet }}
+            -p:TestResultsFolderPath=${{ variables.testResultsPath }}
+
+      # Publish test results
+      - template: /eng/pipelines/pr/steps/publish-test-results-step.yml@self
+        parameters:
+          buildConfiguration: ${{ parameters.buildConfiguration }}
+          platformDisplayName: ${{ parameters.platformDisplayName }}
+          testDisplayName: ${{ parameters.testDisplayName }}
+          testResultsArtifactName: ${{ parameters.testResultsArtifactName }}
+          testResultsPath: ${{ variables.testResultsPath }}

eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml

@@ -33,6 +33,10 @@ parameters:
   - name: stageNameSecrets
     type: string
 
+  # Name of the artifact to upload the test results (including code coverage) to.
+  - name: testResultsArtifactName
+    type: string
+
   # Platform Parameters ====================================================
 
   # Display name of the platform. This will be formatted into the job's official name as well as
@@ -202,3 +206,11 @@ jobs:
             -p:TestFramework=${{ parameters.platformDotnet }}
             -p:TestSet=${{ parameters.testSet }}
 
+      # Publish test results
+      - template: /eng/pipelines/pr/steps/publish-test-results-step.yml@self
+        parameters:
+          buildConfiguration: ${{ parameters.buildConfiguration }}
+          platformDisplayName: ${{ parameters.platformDisplayName }}
+          testDisplayName: "sqlclient_manual_${{ parameters.configDisplayName}}_${{ parameters.testSet }}"
+          testResultsArtifactName: ${{ parameters.testResultsArtifactName }}
+          testResultsPath: ${{ variables.testResultsPath }}

eng/pipelines/pr/pr-pipeline.yml

@@ -4,8 +4,56 @@
 # See the LICENSE file in the project root for more information.                #
 #################################################################################
 
+# This pipeline builds and tests all products that we release. It is intended to run as validation
+# for every PR. The set of tests it executes is not exhaustive, as many tests require special
+# environments to provide complete coverage. Instead, this PR strikes a balance between complete
+# confidence and speed of PR validation.
+#
+# It is triggered by pushes to PRs that target the main, dev/*, feat/*, and release/* branches in
+# GitHub. The dev/* pattern includes dev/automation/* branches created by AI agents.
+#
+# It maps to the "sqlclient-pr" pipeline in the Public ADO project:
+#    https://sqlclientdrivers.visualstudio.com/public/_build?definitionId=2281
+# And the "pr-pipeline" pipeline in the internal ADO project:
+#    https://dev.azure.com/SqlClientDrivers/ADO.Net/_build?definitionId=2271
+
+# Set the pipeline run name to the day-of-year and the daily run counter.
 name: $(DayOfYear)$(Rev:rr)
 
+# Trigger PR validation runs for all pushes to PRs that target the specified branches.
+# https://learn.microsoft.com/en-us/azure/devops/pipelines/repos/github?view=azure-devops&tabs=yaml#pr-triggers
+pr:
+  branches:
+    include:
+      # GitHub repo branch targets that will trigger PR validation builds.
+      - dev/*
+      - feat/*
+      - main
+      - release/*
+
+  paths:
+    include:
+      - .azuredevops/*
+      - .config/*
+      - doc/*
+      - eng/pipelines/*
+      - src/*
+      - tools/*
+      - azurepipelines-coverage.yml
+      - build.proj
+      - Directory.Packages.props
+      - dotnet-tools.json
+      - global.json
+      - NuGet.config
+    exclude:
+      - eng/pipelines/ci/*
+      - eng/pipelines/kerberos/*
+      - eng/pipelines/onebranch/*
+      - eng/pipelines/stress/*
+
+# Do not trigger commit or schedule runs for this pipeline.
+trigger: none
+
 parameters:
   # General Parameters =====================================================
 
@@ -21,63 +69,68 @@ parameters:
       - detailed
       - diagnostic
 
+  - name: platforms
+    type: object
+    default:
+      - displayName: "windows_net462"
+        dotnet: "net462"
+        image: "ADO-MMS22-SQL22"
+        operatingSystem: "Windows"
+      - displayName: "windows_net8"
+        dotnet: "net8.0"
+        image: "ADO-MMS22-SQL22"
+        operatingSystem: "Windows"
+      - displayName: "windows_net9"
+        dotnet: "net9.0"
+        image: "ADO-MMS22-SQL22"
+        operatingSystem: "Windows"
+      - displayName: "windows_net10"
+        dotnet: "net10.0"
+        image: "ADO-MMS22-SQL22"
+        operatingSystem: "Windows"
+
+      - displayName: "linux_net8"
+        dotnet: "net8.0"
+        image: "ADO-UB22-SQL22"
+        operatingSystem: "Linux"
+      - displayName: "linux_net9"
+        dotnet: "net9.0"
+        image: "ADO-UB22-SQL22"
+        operatingSystem: "Linux"
+      - displayName: "linux_net10"
+        dotnet: "net10.0"
+        image: "ADO-UB22-SQL22"
+        operatingSystem: "Linux"
+
 variables:
   - template: /eng/pipelines/common/variables/common-variables.yml@self
   - template: /eng/pipelines/pr/variables/pr-variables.yml@self
 
 stages:
   # Stage 1a: Build and pack all projects in the repository
-  - template: /eng/pipelines/pr/stages/pack-stage.yml
+  - template: /eng/pipelines/pr/stages/pack-stage.yml@self
     parameters:
       buildConfiguration: Debug
       buildSuffix: pr
       stageName: ${{ variables.stageNamePack }}
 
   # Stage 1b: Generate secrets
-  - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml
+  - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml@self
     parameters:
       stageName: ${{ variables.stageNameSecrets }}
 
   # Stage 2: Execute tests and collect code coverage
-  - template: /eng/pipelines/pr/stages/test-stages.yml
+  - template: /eng/pipelines/pr/stages/test-stages.yml@self
     parameters:
       buildConfiguration: Debug
       buildSuffix: pr
       dotnetVerbosity: ${{ parameters.dotnetVerbosity }}
       poolName: $(PoolNameDefault)
       stageNamePack: ${{ variables.stageNamePack }}
       stageNameSecrets: ${{ variables.stageNameSecrets }}
+      testResultsArtifactName: ${{ variables.testResultsArtifactName }}
 
-      platforms:
-        - displayName: "windows_net462"
-          dotnet: "net462"
-          image: "ADO-MMS22-SQL22"
-          operatingSystem: "Windows"
-        - displayName: "windows_net8"
-          dotnet: "net8.0"
-          image: "ADO-MMS22-SQL22"
-          operatingSystem: "Windows"
-        - displayName: "windows_net9"
-          dotnet: "net9.0"
-          image: "ADO-MMS22-SQL22"
-          operatingSystem: "Windows"
-        - displayName: "windows_net10"
-          dotnet: "net10.0"
-          image: "ADO-MMS22-SQL22"
-          operatingSystem: "Windows"
-
-        - displayName: "linux_net8"
-          dotnet: "net8.0"
-          image: "ADO-UB22-SQL22"
-          operatingSystem: "Linux"
-        - displayName: "linux_net9"
-          dotnet: "net9.0"
-          image: "ADO-UB22-SQL22"
-          operatingSystem: "Linux"
-        - displayName: "linux_net10"
-          dotnet: "net10.0"
-          image: "ADO-UB22-SQL22"
-          operatingSystem: "Linux"
+      platforms: ${{ parameters.platforms }}
 
       manualTestAzureKeyVaultUrl: $(AzureKeyVaultUrl)
       manualTestAzureKeyVaultTenantId: $(AzureKeyVaultTenantId)
@@ -91,4 +144,9 @@ stages:
       manualTestUserManagedIdentityClientId: $(UserManagedIdentityClientId)
 
   # Stage 3: Collect code coverage
-  # @TODO:
+  - template: /eng/pipelines/pr/stages/collect-coverage-stage.yml@self
+    parameters:
+      codeCovApiToken: $(CodeCovApiToken)
+      dependsOn:
+        - ${{ each platform in parameters.platforms }}:
+            - "test_${{ platform.displayName }}"

eng/pipelines/pr/stages/collect-coverage-stage.yml

@@ -0,0 +1,128 @@
+#################################################################################
+# Licensed to the .NET Foundation under one or more agreements.                 #
+# The .NET Foundation licenses this file to you under the MIT license.          #
+# See the LICENSE file in the project root for more information.                #
+#################################################################################
+
+parameters:
+
+  # API token used to access the CodeCov API
+  - name: codeCovApiToken
+    type: string
+
+  # Names of the test stages that must complete before collecting coverage.
+  - name: dependsOn
+    type: object
+    default: []
+
+stages:
+  - stage: collect_code_coverage
+    displayName: "Collect code coverage"
+    dependsOn: ${{ parameters.dependsOn }}
+
+    jobs:
+      - job: collect_code_coverage
+        displayName: "Collect Code Coverage"
+
+        pool:
+          vmImage: ubuntu-latest
+
+        variables:
+
+          # Set up a temporary directory that is cleaned up after each job run. This helps avoid
+          # disk space issues on pooled agents that may run many jobs before being retired.
+          - name: workingDir
+            value: "$(Agent.TempDirectory)/coverage"
+
+        steps:
+
+          # Part 1) Merge coverage reports =================================
+
+          # Install .NET SDK
+          - template: /eng/pipelines/common/steps/install-dotnet.yml@self
+
+          # Install additional dotnet tools
+          #
+          # We must work around a bug in the dotnet CLI that prevents tool installs when multiple
+          # project/solution files are present in the working directory. At various times, multiple
+          # project files have existed in the root of the repository. See SDK issue:
+          #   https://github.com/dotnet/sdk/issues/9623
+          #
+          # However, we must respect the nuget.config to only download packages from governed
+          # feeds. Thus, we copy the nuget.config from the root of the repo to a temp directory
+          # then run dotnet tool installation from that folder.
+          - task: CopyFiles@2
+            displayName: Copy nuget.config for tool installs
+            inputs:
+              contents: "$(REPO_ROOT)/nuget.config"
+              targetFolder: $(Agent.TempDirectory)
+
+          - task: DotNetCoreCLI@2
+            displayName: Install dotnet-coverage
+            inputs:
+              command: custom
+              custom: tool
+              workingDirectory: $(Agent.TempDirectory)
+              arguments: install --global dotnet-coverage --version 18.3.2
+
+          # Download coverage reports from the test jobs.
+          # NOTE: The artifact name is not specified, so *all* artifacts of the build will be
+          #    checked for the coverage reports.
+          - task: DownloadPipelineArtifact@2
+            displayName: Download coverage reports
+            inputs:
+              itemPattern: '**/*.coverage'
+              targetPath: "${{ variables.workingDir }}/originals"
+
+          # Merge original files into a single Cobertura XML file
+          - script: >-
+              dotnet-coverage merge "${{ variables.workingDir }}/originals/**/*.coverage"
+              --output "${{ variables.workingDir }}/merge/cobertura.xml"
+              --output-format cobertura
+              --log-file "${{ variables.workingDir }}/merge/merge.log"
+              --log-level Verbose
+            displayName: Merge coverage files
+
+          # Part 2) Publish merged results to the pipeline results/artifacts
+
+          # Publish the merged coverage file as a pipeline artifact
+          - task: PublishPipelineArtifact@1
+            displayName: Publish coverage artifact
+            inputs:
+              artifact: merged_coverage
+              targetPath: "${{ variables.workingDir }}/merge"
+
+          # Publish the merged coverage file as coverage results so they can be viewed in ADO UI.
+          - task: PublishCodeCoverageResults@2
+            displayName: Publish coverage results
+            inputs:
+              summaryFileLocation: "${{ variables.workingDir }}/merge/cobertura.xml"
+
+          # Part 3) Publish merged results to CodeCov
+
+          # Download the CodeCov CLI
+          # @TODO: should this be using something more governed?
+          - script: |
+              curl -o "${{ variables.workingDir }}/codecov" https://cli.codecov.io/latest/linux/codecov
+              chmod +x "${{ variables.workingDir }}/codecov"
+            displayName: Download CodeCov CLI
+
+          # Upload the report
+          # NOTE: pipeline name is used as "flag" (-F) to distinguish reports from other pipelines.
+          # NOTE: Override repository metadata because this ADO pipeline uploads coverage for the
+          #     GitHub repository. CodeCov otherwise derives an invalid slug from the ADO metadata.
+          # NOTE: CodeCov's CLI requires that "options" go before the "command"
+          #     https://docs.codecov.com/docs/the-codecov-cli
+          - script: >-
+              "${{ variables.workingDir }}/codecov"
+              --verbose
+              upload-process
+              --fail-on-error
+              --git-service github
+              --slug dotnet/SqlClient
+              -f "${{ variables.workingDir }}/merge/cobertura.xml"
+              -F $(Build.DefinitionName)
+            displayName: Upload coverage to CodeCov
+            env:
+              CODECOV_TOKEN: ${{ parameters.codeCovApiToken }}
+