From 963bccc7ff12f9e090f9284799761c340ae73014 Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Wed, 3 Jun 2026 15:20:23 -0500 Subject: [PATCH 01/17] First crack at adding test result publishing to non-manual test jobs. # Conflicts: # eng/pipelines/pr/jobs/test-buildproj-job.yml # eng/pipelines/pr/stages/test-stages.yml --- eng/pipelines/pr/jobs/test-buildproj-job.yml | 24 +++++++ eng/pipelines/pr/pr-pipeline.yml | 1 + eng/pipelines/pr/stages/test-stages.yml | 14 ++++ .../pr/steps/publish-test-results-step.yml | 68 +++++++++++++++++++ eng/pipelines/pr/variables/pr-variables.yml | 3 + 5 files changed, 110 insertions(+) create mode 100644 eng/pipelines/pr/steps/publish-test-results-step.yml diff --git a/eng/pipelines/pr/jobs/test-buildproj-job.yml b/eng/pipelines/pr/jobs/test-buildproj-job.yml index ec2d292ec8..cc1e239161 100644 --- a/eng/pipelines/pr/jobs/test-buildproj-job.yml +++ b/eng/pipelines/pr/jobs/test-buildproj-job.yml @@ -18,6 +18,11 @@ parameters: - name: buildSuffix type: string + # True to emit debug information and steps. + - name: debug + type: boolean + default: false + # Dotnet CLI verbosity level. - name: dotnetVerbosity type: string @@ -74,10 +79,18 @@ parameters: type: string default: '' + # Name of the artifact to upload the test results (including code coverage) to. + - name: testResultsArtifactName + type: string + jobs: - job: "test_${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}" displayName: "${{ parameters.testDisplayName }}_${{ parameters.platformDisplayName }}" + variables: + - name: testResultsPath + value: "$(REPO_ROOT)/test_results/${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}" + pool: name: ${{ parameters.poolName }} demands: @@ -107,3 +120,14 @@ jobs: -p:BuildNumber='$(Build.BuildNumber)' -p:BuildSuffix='${{ parameters.buildSuffix }}' -p:TestFramework=${{ parameters.platformDotnet }} + -p:TestResultsFolderPath=${{ variables.testResultsPath }} + + # Publish test results + - template: /eng/pipelines/pr/steps/publish-test-results-step.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + platformDisplayName: ${{ parameters.platformDisplayName }} + testDisplayName: ${{ parameters.testDisplayName }} + testResultsArtifactName: ${{ parameters.testResultsArtifactName }} + testResultsPath: ${{ variables.testResultsPath }} diff --git a/eng/pipelines/pr/pr-pipeline.yml b/eng/pipelines/pr/pr-pipeline.yml index daf393e502..48fb2e229a 100644 --- a/eng/pipelines/pr/pr-pipeline.yml +++ b/eng/pipelines/pr/pr-pipeline.yml @@ -47,6 +47,7 @@ stages: poolName: $(PoolNameDefault) stageNamePack: ${{ variables.stageNamePack }} stageNameSecrets: ${{ variables.stageNameSecrets }} + testResultsArtifactName: ${{ variables.testResultsArtifactName }} platforms: - displayName: "windows_net462" diff --git a/eng/pipelines/pr/stages/test-stages.yml b/eng/pipelines/pr/stages/test-stages.yml index 3a1e2c2011..346519f6d8 100644 --- a/eng/pipelines/pr/stages/test-stages.yml +++ b/eng/pipelines/pr/stages/test-stages.yml @@ -37,6 +37,10 @@ parameters: - name: poolName type: string + # Name of the pool to use for jobs that require customized VM images. + - name: poolName + type: string + # Name of the build/pack stage that this stage will depend on. - name: stageNamePack type: string @@ -45,6 +49,12 @@ parameters: - name: stageNameSecrets type: string + # Name of the artifact to upload the test results (including code coverage) to. + # Note: This artifact will be used to store results from all jobs. Ensure that the folders used + # to store the results is unique to the test run. + - name: testResultsArtifactName + type: string + # Manual Test Configuration Parameters ================================== # Azure Key Vault tenant ID that will be set in the config.jsonc file for sqlclient manual tests. @@ -129,6 +139,7 @@ stages: platformDotnet: ${{ platform.dotnet }} platformImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} + testResultsArtifactName: ${{ parameters.testResultsArtifactName }} packageShortName: "Abstractions" testDisplayName: "abstractions" @@ -143,6 +154,7 @@ stages: platformDotnet: ${{ platform.dotnet }} platformImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} + testResultsArtifactName: ${{ parameters.testResultsArtifactName }} packageShortName: "Azure" testDisplayName: "azure" @@ -157,6 +169,7 @@ stages: platformDotnet: ${{ platform.dotnet }} platformImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} + testResultsArtifactName: ${{ parameters.testResultsArtifactName }} packageShortName: "SqlClient" testDisplayName: "sqlclient_functional" @@ -250,6 +263,7 @@ stages: platformDotnet: ${{ platform.dotnet }} platformImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} + testResultsArtifactName: ${{ parameters.testResultsArtifactName }} packageShortName: "SqlClient" testDisplayName: "sqlclient_unit" diff --git a/eng/pipelines/pr/steps/publish-test-results-step.yml b/eng/pipelines/pr/steps/publish-test-results-step.yml new file mode 100644 index 0000000000..4699213102 --- /dev/null +++ b/eng/pipelines/pr/steps/publish-test-results-step.yml @@ -0,0 +1,68 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Publishes the test result files from the job to the pipeline and uploads the test results +# (including code coverage results) to a pipeline artifact. + +parameters: + + # Configuration to use to build the project. + - name: buildConfiguration + type: string + values: + - Debug + - Release + + # True to emit debug information and steps. + - name: debug + type: boolean + default: false + + # Display name of the platform. This will be formatted into the test result name. As such, only + # alphanumeric and _ characters are permitted. + - name: platformDisplayName + type: string + + # Display name of the test target that is being executed. This will be formatted into the test + # result name. As such, only alphanumeric and _ characters are permitted. + - name: testDisplayName + type: string + + # Name of the artifact to upload the test results (including code coverage) to. + - name: testResultsArtifactName + type: string + + # Absolute path to the test results folder. This folder will be published to the artifact defined + # by ${{ testResultsArtifactName }} + - name: testResultsPath + type: string + +steps: + # Show the files in the test results directory + - ${{ if eq(parameters.debug, true) }}: + script: tree /a /f ${{ parameters.testResultsPath }} + displayName: Output Test Result Tree + condition: succeededOrFailed() + + # Publish the test results (trx files) to the pipeline + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + buildConfiguration: ${{ parameters.buildConfiguration }} + testResultsFiles: | + ${{ parameters.testResultsPath }}/*.trx + ${{ parameters.testResultsPath }}/**/*.coverage + testRunTitle: "${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}" + condition: succeededOrFailed() + + # Publish the test results as artifacts for the pipeline + - task: PublishPipelineArtifact + displayName: 'Publish Test Artifacts' + inputs: + artifact: ${{ parameters.testResultsArtifactName }} + targetPath: ${{ parameters.testResultsPath }} + condition: succeededOrFailed() + diff --git a/eng/pipelines/pr/variables/pr-variables.yml b/eng/pipelines/pr/variables/pr-variables.yml index a9da73850e..6675323f9e 100644 --- a/eng/pipelines/pr/variables/pr-variables.yml +++ b/eng/pipelines/pr/variables/pr-variables.yml @@ -29,3 +29,6 @@ variables: - name: stageNameSecrets value: "secrets_stage" + + - name: testResultsArtifactName + value: "test_results" From 1ce8fca640644ac5223ad728d7751bd10313bb3f Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 8 Jun 2026 11:38:12 -0500 Subject: [PATCH 02/17] If statement syntax? --- eng/pipelines/pr/steps/publish-test-results-step.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/pr/steps/publish-test-results-step.yml b/eng/pipelines/pr/steps/publish-test-results-step.yml index 4699213102..93e3e4669b 100644 --- a/eng/pipelines/pr/steps/publish-test-results-step.yml +++ b/eng/pipelines/pr/steps/publish-test-results-step.yml @@ -43,9 +43,9 @@ parameters: steps: # Show the files in the test results directory - ${{ if eq(parameters.debug, true) }}: - script: tree /a /f ${{ parameters.testResultsPath }} - displayName: Output Test Result Tree - condition: succeededOrFailed() + - script: tree /a /f ${{ parameters.testResultsPath }} + displayName: Output Test Result Tree + condition: succeededOrFailed() # Publish the test results (trx files) to the pipeline - task: PublishTestResults@2 From 3e28bcd82a365a462d57af4d47b143f0df0568d7 Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 8 Jun 2026 11:39:01 -0500 Subject: [PATCH 03/17] Ok forget the if statement. --- eng/pipelines/pr/steps/publish-test-results-step.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/pr/steps/publish-test-results-step.yml b/eng/pipelines/pr/steps/publish-test-results-step.yml index 93e3e4669b..8e52bcd34b 100644 --- a/eng/pipelines/pr/steps/publish-test-results-step.yml +++ b/eng/pipelines/pr/steps/publish-test-results-step.yml @@ -42,10 +42,9 @@ parameters: steps: # Show the files in the test results directory - - ${{ if eq(parameters.debug, true) }}: - - script: tree /a /f ${{ parameters.testResultsPath }} - displayName: Output Test Result Tree - condition: succeededOrFailed() + - script: tree /a /f ${{ parameters.testResultsPath }} + displayName: Output Test Result Tree + condition: succeededOrFailed() # Publish the test results (trx files) to the pipeline - task: PublishTestResults@2 From 824f0d210bc68962dd60dfad83931d59dc2ee725 Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 8 Jun 2026 11:42:28 -0500 Subject: [PATCH 04/17] Version --- eng/pipelines/pr/steps/publish-test-results-step.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/pr/steps/publish-test-results-step.yml b/eng/pipelines/pr/steps/publish-test-results-step.yml index 8e52bcd34b..429ec1d130 100644 --- a/eng/pipelines/pr/steps/publish-test-results-step.yml +++ b/eng/pipelines/pr/steps/publish-test-results-step.yml @@ -58,7 +58,7 @@ steps: condition: succeededOrFailed() # Publish the test results as artifacts for the pipeline - - task: PublishPipelineArtifact + - task: PublishPipelineArtifact@1 displayName: 'Publish Test Artifacts' inputs: artifact: ${{ parameters.testResultsArtifactName }} From 7bd1466f10bc5944e7329ff0cd4de471b8d5a0dc Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 8 Jun 2026 12:32:18 -0500 Subject: [PATCH 05/17] Unique job names... could be better, but let's start here. --- eng/pipelines/pr/steps/publish-test-results-step.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/pr/steps/publish-test-results-step.yml b/eng/pipelines/pr/steps/publish-test-results-step.yml index 429ec1d130..c38022e506 100644 --- a/eng/pipelines/pr/steps/publish-test-results-step.yml +++ b/eng/pipelines/pr/steps/publish-test-results-step.yml @@ -61,7 +61,7 @@ steps: - task: PublishPipelineArtifact@1 displayName: 'Publish Test Artifacts' inputs: - artifact: ${{ parameters.testResultsArtifactName }} + artifact: ${{ parameters.testResultsArtifactName }}_$(System.JobId) targetPath: ${{ parameters.testResultsPath }} condition: succeededOrFailed() From 7bc3a203dcaee43c02a794dd341848bdb128d9ec Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 8 Jun 2026 13:35:25 -0500 Subject: [PATCH 06/17] Tree in linux? --- eng/pipelines/pr/steps/publish-test-results-step.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/pr/steps/publish-test-results-step.yml b/eng/pipelines/pr/steps/publish-test-results-step.yml index c38022e506..17dc3aac72 100644 --- a/eng/pipelines/pr/steps/publish-test-results-step.yml +++ b/eng/pipelines/pr/steps/publish-test-results-step.yml @@ -42,7 +42,7 @@ parameters: steps: # Show the files in the test results directory - - script: tree /a /f ${{ parameters.testResultsPath }} + - pwsh: Show-Tree ${{ parameters.testResultsPath }} displayName: Output Test Result Tree condition: succeededOrFailed() From 3388570a64219c59bb88bfbaa55102113e59e9a6 Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 8 Jun 2026 16:55:56 -0500 Subject: [PATCH 07/17] Ok forget it, we don't need do emit the tree.... --- eng/pipelines/pr/steps/publish-test-results-step.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/eng/pipelines/pr/steps/publish-test-results-step.yml b/eng/pipelines/pr/steps/publish-test-results-step.yml index 17dc3aac72..222dea298b 100644 --- a/eng/pipelines/pr/steps/publish-test-results-step.yml +++ b/eng/pipelines/pr/steps/publish-test-results-step.yml @@ -16,11 +16,6 @@ parameters: - Debug - Release - # True to emit debug information and steps. - - name: debug - type: boolean - default: false - # Display name of the platform. This will be formatted into the test result name. As such, only # alphanumeric and _ characters are permitted. - name: platformDisplayName @@ -41,11 +36,6 @@ parameters: type: string steps: - # Show the files in the test results directory - - pwsh: Show-Tree ${{ parameters.testResultsPath }} - displayName: Output Test Result Tree - condition: succeededOrFailed() - # Publish the test results (trx files) to the pipeline - task: PublishTestResults@2 displayName: 'Publish Test Results' From 412f2e054e77ef34590bc3a545978f5e5db14cd5 Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 8 Jun 2026 17:00:45 -0500 Subject: [PATCH 08/17] Wire up test result uploading to sqlclient manual test jobs --- eng/pipelines/pr/jobs/test-buildproj-job.yml | 12 +++--------- eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml | 13 +++++++++++++ eng/pipelines/pr/stages/test-stages.yml | 3 +++ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/pr/jobs/test-buildproj-job.yml b/eng/pipelines/pr/jobs/test-buildproj-job.yml index cc1e239161..67923afb37 100644 --- a/eng/pipelines/pr/jobs/test-buildproj-job.yml +++ b/eng/pipelines/pr/jobs/test-buildproj-job.yml @@ -18,10 +18,9 @@ parameters: - name: buildSuffix type: string - # True to emit debug information and steps. - - name: debug - type: boolean - default: false + # Name of the artifact to upload the test results (including code coverage) to. + - name: testResultsArtifactName + type: string # Dotnet CLI verbosity level. - name: dotnetVerbosity @@ -79,10 +78,6 @@ parameters: type: string default: '' - # Name of the artifact to upload the test results (including code coverage) to. - - name: testResultsArtifactName - type: string - jobs: - job: "test_${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}" displayName: "${{ parameters.testDisplayName }}_${{ parameters.platformDisplayName }}" @@ -126,7 +121,6 @@ jobs: - template: /eng/pipelines/pr/steps/publish-test-results-step.yml@self parameters: buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} platformDisplayName: ${{ parameters.platformDisplayName }} testDisplayName: ${{ parameters.testDisplayName }} testResultsArtifactName: ${{ parameters.testResultsArtifactName }} diff --git a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml index 9ca2c7c555..98336a540f 100644 --- a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml +++ b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml @@ -33,6 +33,10 @@ parameters: - name: stageNameSecrets type: string + # Name of the artifact to upload the test results (including code coverage) to. + - name: testResultsArtifactName + type: string + # Platform Parameters ==================================================== # Display name of the platform. This will be formatted into the job's official name as well as @@ -202,3 +206,12 @@ jobs: -p:TestFramework=${{ parameters.platformDotnet }} -p:TestSet=${{ parameters.testSet }} + # Publish test results + - template: /eng/pipelines/pr/steps/publish-test-results-step.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + platformDisplayName: ${{ parameters.platformDisplayName }} + testDisplayName: ${{ parameters.testDisplayName }} + testResultsArtifactName: ${{ parameters.testResultsArtifactName }} + testResultsPath: ${{ variables.testResultsPath }} + diff --git a/eng/pipelines/pr/stages/test-stages.yml b/eng/pipelines/pr/stages/test-stages.yml index 346519f6d8..8766379131 100644 --- a/eng/pipelines/pr/stages/test-stages.yml +++ b/eng/pipelines/pr/stages/test-stages.yml @@ -182,6 +182,7 @@ stages: buildSuffix: ${{ parameters.buildSuffix }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} stageNameSecrets: ${{ parameters.stageNameSecrets }} + testResultsArtifactName: ${{ parameters.testResultsArtifactName }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} @@ -208,6 +209,7 @@ stages: buildSuffix: ${{ parameters.buildSuffix }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} stageNameSecrets: ${{ parameters.stageNameSecrets }} + testResultsArtifactName: ${{ parameters.testResultsArtifactName }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} @@ -234,6 +236,7 @@ stages: buildSuffix: ${{ parameters.buildSuffix }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} stageNameSecrets: ${{ parameters.stageNameSecrets }} + testResultsArtifactName: ${{ parameters.testResultsArtifactName }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} From 9818712800ec235b0c7c9efb8b67c741364f7af6 Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 8 Jun 2026 17:22:02 -0500 Subject: [PATCH 09/17] Give a test display name to sql manual tests --- eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml index 98336a540f..0b4ca4b3b0 100644 --- a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml +++ b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml @@ -211,7 +211,7 @@ jobs: parameters: buildConfiguration: ${{ parameters.buildConfiguration }} platformDisplayName: ${{ parameters.platformDisplayName }} - testDisplayName: ${{ parameters.testDisplayName }} + testDisplayName: "sqlclient_manual_${{ parameters.configDisplayName}}_${{ parameters.testSet }}" testResultsArtifactName: ${{ parameters.testResultsArtifactName }} testResultsPath: ${{ variables.testResultsPath }} From 800a688063fca9582d7634e85f4b65b8f8d6785c Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 8 Jun 2026 17:52:28 -0500 Subject: [PATCH 10/17] First attempt at code coverage report publishing --- .../pr/jobs/test-sqlclientmanual-job.yml | 1 - .../pr/stages/collect-coverage-stage.yml | 117 ++++++++++++++++++ eng/pipelines/pr/variables/pr-variables.yml | 1 + 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 eng/pipelines/pr/stages/collect-coverage-stage.yml diff --git a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml index 0b4ca4b3b0..313ce5fb44 100644 --- a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml +++ b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml @@ -214,4 +214,3 @@ jobs: testDisplayName: "sqlclient_manual_${{ parameters.configDisplayName}}_${{ parameters.testSet }}" testResultsArtifactName: ${{ parameters.testResultsArtifactName }} testResultsPath: ${{ variables.testResultsPath }} - diff --git a/eng/pipelines/pr/stages/collect-coverage-stage.yml b/eng/pipelines/pr/stages/collect-coverage-stage.yml new file mode 100644 index 0000000000..15bd48662e --- /dev/null +++ b/eng/pipelines/pr/stages/collect-coverage-stage.yml @@ -0,0 +1,117 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +parameters: + + # API token used to access the CodeCov API + - name: codeCovApiToken + type: string + +stages: + - stage: collect_code_coverage + displayName: "Collect code coverage" + dependsOn: + # @TODO: + + jobs: + - job: collect_code_coverage + displayName: "Collect Code Coverage" + + pool: + vmImage: ubuntu-latest + + variables: + + # Set up a temporary directory that is cleaned up after each job run. This helps avoid + # disk space issues on pooled agents that may run many jobs before being retired. + - name: workingDir + value: "$(Agent.TempDirectory)/coverage" + + steps: + + # Part 1) Merge coverage reports ================================= + + # Install .NET SDK + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + + # Install additional dotnet tools + # + # We must work around a bug in the dotnet CLI that prevents tool installs when multiple + # project/solution files are present in the working directory. At various times, multiple + # project files have existed in the root of the repository. See SDK issue: + # https://github.com/dotnet/sdk/issues/9623 + # + # However, we must respect the nuget.config to only download packages from governed + # feeds. Thus, we copy the nuget.config from the root of the repo to a temp directory + # then run dotnet tool installation from that folder. + - task: CopyFiles@2 + displayName: Copy nuget.config for tool installs + inputs: + contents: "$(REPO_ROOT)/nuget.config" + targetFolder: $(Agent.TempDirectory) + + - task: DotNetCoreCLI@2 + displayName: Install dotnet-coverage + inputs: + command: custom + custom: tool + workingDirectory: $(Agent.TempDirectory) + arguments: install --global dotnet-coverage --version 18.3.2 + + # Download coverage reports from the test jobs. + # NOTE: The artifact name is not specified, so *all* artifacts of the build will be + # checked for the coverage reports. + - task: DownloadPipelineArtifact@2 + displayName: Download coverage reports + inputs: + itemPattern: '**/*.coverage' + targetPath: "${{ variables.workingDir }}/originals" + + # Merge original files into a single Cobertura XML file + - script: >- + dotnet-coverage merge "${{ variables.workingDir }}/originals/**/*.coverage" + --output "${{ variables.workingDir }}/merge/cobertura.xml" + --output-format cobertura + --log-file "${{ variables.workingDir }}/merge/merge.log" + --log-level Verbose + displayName: Merge coverage files + + # Part 2) Publish merged results to the pipeline results/artifacts + + # Publish the merged coverage file as a pipeline artifact + - task: PublishPipelineArtifact@1 + displayName: Publish coverage artifact + inputs: + artifact: merged_coverage + targetPath: "${{ variables.workingDir }}/merge" + + # Publish the merged coverage file as coverage results so they can be viewed in ADO UI. + - task: PublishCodeCoverageResults@2 + displayName: Publish coverage results + inputs: + summaryFileLocation: "${{ variables.workingDir }}/merge/cobertura.xml" + + # Part 3) Publish merged results to CodeCov + + # Download the CodeCov CLI + # @TODO: should this be using something more governed? + - script: | + curl -o "${{ variables.workingDir }}/codecov" https://cli.codecov.io/latest/linux/codecov + chmod +x "${{ variables.workingDir }}/codecov" + displayName: Download CodeCov CLI + + # Upload the report + # NOTE: pipeline name is used as "flag" (-F) to distinguish reports from other pipelines. + - script: >- + "${{ variables.workingDir }}/codecov" + upload-process + --verbose + --fail-on-error + -f "${{ variables.workingDir }}/merge/cobertura.xml" + -t ${{ parameters.codeCovApiToken }} + -F $(Build.DefinitionName) + displayName: Upload coverage to CodeCov + diff --git a/eng/pipelines/pr/variables/pr-variables.yml b/eng/pipelines/pr/variables/pr-variables.yml index 6675323f9e..c100dc92ce 100644 --- a/eng/pipelines/pr/variables/pr-variables.yml +++ b/eng/pipelines/pr/variables/pr-variables.yml @@ -13,6 +13,7 @@ variables: # # AzureKeyVaultTenantId # AzureKeyVaultUrl + # CodeCovApiToken # ConnectionStringNp_Azure_ManagedIdentity # ConnectionStringNp_LocalhostDefault_UsernamePassword # ConnectionStringTcp_Azure_ManagedIdentity From 7056130f9c720cd5d4c09c56500a1c1b06724e4a Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 8 Jun 2026 18:06:19 -0500 Subject: [PATCH 11/17] :robot: Code coverage takes a dynamic dependency on the platforms to ensure all tests are first. --- eng/pipelines/pr/pr-pipeline.yml | 78 +++++++++++-------- .../pr/stages/collect-coverage-stage.yml | 8 +- 2 files changed, 50 insertions(+), 36 deletions(-) diff --git a/eng/pipelines/pr/pr-pipeline.yml b/eng/pipelines/pr/pr-pipeline.yml index 48fb2e229a..3cf180ca11 100644 --- a/eng/pipelines/pr/pr-pipeline.yml +++ b/eng/pipelines/pr/pr-pipeline.yml @@ -4,6 +4,40 @@ # See the LICENSE file in the project root for more information. # ################################################################################# +parameters: + - name: platforms + type: object + default: + - displayName: "windows_net462" + dotnet: "net462" + image: "ADO-MMS22-SQL22" + operatingSystem: "Windows" + - displayName: "windows_net8" + dotnet: "net8.0" + image: "ADO-MMS22-SQL22" + operatingSystem: "Windows" + - displayName: "windows_net9" + dotnet: "net9.0" + image: "ADO-MMS22-SQL22" + operatingSystem: "Windows" + - displayName: "windows_net10" + dotnet: "net10.0" + image: "ADO-MMS22-SQL22" + operatingSystem: "Windows" + + - displayName: "linux_net8" + dotnet: "net8.0" + image: "ADO-UB22-SQL22" + operatingSystem: "Linux" + - displayName: "linux_net9" + dotnet: "net9.0" + image: "ADO-UB22-SQL22" + operatingSystem: "Linux" + - displayName: "linux_net10" + dotnet: "net10.0" + image: "ADO-UB22-SQL22" + operatingSystem: "Linux" + name: $(DayOfYear)$(Rev:rr) parameters: @@ -27,19 +61,19 @@ variables: stages: # Stage 1a: Build and pack all projects in the repository - - template: /eng/pipelines/pr/stages/pack-stage.yml + - template: /eng/pipelines/pr/stages/pack-stage.yml@self parameters: buildConfiguration: Debug buildSuffix: pr stageName: ${{ variables.stageNamePack }} # Stage 1b: Generate secrets - - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml + - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml@self parameters: stageName: ${{ variables.stageNameSecrets }} # Stage 2: Execute tests and collect code coverage - - template: /eng/pipelines/pr/stages/test-stages.yml + - template: /eng/pipelines/pr/stages/test-stages.yml@self parameters: buildConfiguration: Debug buildSuffix: pr @@ -49,36 +83,7 @@ stages: stageNameSecrets: ${{ variables.stageNameSecrets }} testResultsArtifactName: ${{ variables.testResultsArtifactName }} - platforms: - - displayName: "windows_net462" - dotnet: "net462" - image: "ADO-MMS22-SQL22" - operatingSystem: "Windows" - - displayName: "windows_net8" - dotnet: "net8.0" - image: "ADO-MMS22-SQL22" - operatingSystem: "Windows" - - displayName: "windows_net9" - dotnet: "net9.0" - image: "ADO-MMS22-SQL22" - operatingSystem: "Windows" - - displayName: "windows_net10" - dotnet: "net10.0" - image: "ADO-MMS22-SQL22" - operatingSystem: "Windows" - - - displayName: "linux_net8" - dotnet: "net8.0" - image: "ADO-UB22-SQL22" - operatingSystem: "Linux" - - displayName: "linux_net9" - dotnet: "net9.0" - image: "ADO-UB22-SQL22" - operatingSystem: "Linux" - - displayName: "linux_net10" - dotnet: "net10.0" - image: "ADO-UB22-SQL22" - operatingSystem: "Linux" + platforms: ${{ parameters.platforms }} manualTestAzureKeyVaultUrl: $(AzureKeyVaultUrl) manualTestAzureKeyVaultTenantId: $(AzureKeyVaultTenantId) @@ -92,4 +97,9 @@ stages: manualTestUserManagedIdentityClientId: $(UserManagedIdentityClientId) # Stage 3: Collect code coverage - # @TODO: + - template: /eng/pipelines/pr/stages/collect-coverage-stage.yml@self + parameters: + codeCovApiToken: $(CodeCovApiToken) + dependsOn: + - ${{ each platform in parameters.platforms }}: + - "test_${{ platform.displayName }}" diff --git a/eng/pipelines/pr/stages/collect-coverage-stage.yml b/eng/pipelines/pr/stages/collect-coverage-stage.yml index 15bd48662e..9c61a9c1d3 100644 --- a/eng/pipelines/pr/stages/collect-coverage-stage.yml +++ b/eng/pipelines/pr/stages/collect-coverage-stage.yml @@ -10,11 +10,15 @@ parameters: - name: codeCovApiToken type: string + # Names of the test stages that must complete before collecting coverage. + - name: dependsOn + type: object + default: [] + stages: - stage: collect_code_coverage displayName: "Collect code coverage" - dependsOn: - # @TODO: + dependsOn: ${{ parameters.dependsOn }} jobs: - job: collect_code_coverage From 5acf9e30c9e8bf500b892387d6a5afac82c25855 Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Tue, 9 Jun 2026 17:48:24 -0500 Subject: [PATCH 12/17] We only need *one* set of parameters --- eng/pipelines/pr/pr-pipeline.yml | 33 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/eng/pipelines/pr/pr-pipeline.yml b/eng/pipelines/pr/pr-pipeline.yml index 3cf180ca11..44c9b3c6ca 100644 --- a/eng/pipelines/pr/pr-pipeline.yml +++ b/eng/pipelines/pr/pr-pipeline.yml @@ -4,7 +4,23 @@ # See the LICENSE file in the project root for more information. # ################################################################################# +name: $(DayOfYear)$(Rev:rr) + parameters: + # General Parameters ===================================================== + + # Dotnet CLI verbosity level. + - name: dotnetVerbosity + displayName: dotnet CLI Verbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + - name: platforms type: object default: @@ -38,23 +54,6 @@ parameters: image: "ADO-UB22-SQL22" operatingSystem: "Linux" -name: $(DayOfYear)$(Rev:rr) - -parameters: - # General Parameters ===================================================== - - # Dotnet CLI verbosity level. - - name: dotnetVerbosity - displayName: dotnet CLI Verbosity - type: string - default: normal - values: - - quiet - - minimal - - normal - - detailed - - diagnostic - variables: - template: /eng/pipelines/common/variables/common-variables.yml@self - template: /eng/pipelines/pr/variables/pr-variables.yml@self From ba57adc66b45cfc3a2f4a086ea6d8c071f92437b Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Tue, 9 Jun 2026 17:49:29 -0500 Subject: [PATCH 13/17] We only need *one* pool name --- eng/pipelines/pr/stages/test-stages.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/pipelines/pr/stages/test-stages.yml b/eng/pipelines/pr/stages/test-stages.yml index 8766379131..5cc76ec78e 100644 --- a/eng/pipelines/pr/stages/test-stages.yml +++ b/eng/pipelines/pr/stages/test-stages.yml @@ -37,10 +37,6 @@ parameters: - name: poolName type: string - # Name of the pool to use for jobs that require customized VM images. - - name: poolName - type: string - # Name of the build/pack stage that this stage will depend on. - name: stageNamePack type: string From f6f4726ec30264b4b2320ca4c5d473276cf4bfbe Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Thu, 11 Jun 2026 14:55:22 -0500 Subject: [PATCH 14/17] Add triggers and fix code coverage verbosity thingy --- eng/pipelines/pr/pr-pipeline.yml | 48 +++++++++++++++++++ .../pr/stages/collect-coverage-stage.yml | 4 +- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/pr/pr-pipeline.yml b/eng/pipelines/pr/pr-pipeline.yml index 44c9b3c6ca..6d3e61c28d 100644 --- a/eng/pipelines/pr/pr-pipeline.yml +++ b/eng/pipelines/pr/pr-pipeline.yml @@ -4,8 +4,56 @@ # See the LICENSE file in the project root for more information. # ################################################################################# +# This pipeline builds and tests all products that we release. It is intended to run as validation +# for every PR. The set of tests it executes is not exhaustive, as many tests require special +# environments to provide complete coverage. Instead, this PR strikes a balance between complete +# confidence and speed of PR validation. +# +# It is triggered by pushes to PRs that target the main, dev/*, feat/*, and release/* branches in +# GitHub. The dev/* pattern includes dev/automation/* branches created by AI agents. +# +# It maps to the "sqlclient-pr" pipeline in the Public ADO project: +# https://sqlclientdrivers.visualstudio.com/public/_build?definitionId=2281 +# And the "pr-pipeline" pipeline in the internal ADO project: +# https://dev.azure.com/SqlClientDrivers/ADO.Net/_build?definitionId=2271 + +# Set the pipeline run name to the day-of-year and the daily run counter. name: $(DayOfYear)$(Rev:rr) +# Trigger PR validation runs for all pushes to PRs that target the specified branches. +# https://learn.microsoft.com/en-us/azure/devops/pipelines/repos/github?view=azure-devops&tabs=yaml#pr-triggers +pr: + branches: + include: + # GitHub repo branch targets that will trigger PR validation builds. + - dev/* + - feat/* + - main + - release/* + + paths: + include: + - .azuredevops/* + - .config/* + - doc/* + - eng/pipelines/* + - src/* + - tools/* + - azurepipelines-coverage.yml + - build.proj + - Directory.Packages.props + - dotnet-tools.json + - global.json + - NuGet.config + exclude: + - eng/pipelines/ci/* + - eng/pipelines/kerberos/* + - eng/pipelines/onebranch/* + - eng/pipelines/stress/* + +# Do not trigger commit or schedule runs for this pipeline. +trigger: none + parameters: # General Parameters ===================================================== diff --git a/eng/pipelines/pr/stages/collect-coverage-stage.yml b/eng/pipelines/pr/stages/collect-coverage-stage.yml index 9c61a9c1d3..c6ee438566 100644 --- a/eng/pipelines/pr/stages/collect-coverage-stage.yml +++ b/eng/pipelines/pr/stages/collect-coverage-stage.yml @@ -109,10 +109,12 @@ stages: # Upload the report # NOTE: pipeline name is used as "flag" (-F) to distinguish reports from other pipelines. + # NOTE: CodeCov's CLI requires that "options" go before the "command" + # https://docs.codecov.com/docs/the-codecov-cli - script: >- "${{ variables.workingDir }}/codecov" - upload-process --verbose + upload-process --fail-on-error -f "${{ variables.workingDir }}/merge/cobertura.xml" -t ${{ parameters.codeCovApiToken }} From 2f28f2401302c251520e34f2ed316c96dbcb0e67 Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Thu, 11 Jun 2026 16:13:00 -0500 Subject: [PATCH 15/17] :robot: fixes for codecov CLI --- eng/pipelines/pr/stages/collect-coverage-stage.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/pr/stages/collect-coverage-stage.yml b/eng/pipelines/pr/stages/collect-coverage-stage.yml index c6ee438566..a380203838 100644 --- a/eng/pipelines/pr/stages/collect-coverage-stage.yml +++ b/eng/pipelines/pr/stages/collect-coverage-stage.yml @@ -109,6 +109,8 @@ stages: # Upload the report # NOTE: pipeline name is used as "flag" (-F) to distinguish reports from other pipelines. + # NOTE: Override repository metadata because this ADO pipeline uploads coverage for the + # GitHub repository. CodeCov otherwise derives an invalid slug from the ADO metadata. # NOTE: CodeCov's CLI requires that "options" go before the "command" # https://docs.codecov.com/docs/the-codecov-cli - script: >- @@ -116,8 +118,11 @@ stages: --verbose upload-process --fail-on-error + --git-service github + --slug dotnet/SqlClient -f "${{ variables.workingDir }}/merge/cobertura.xml" - -t ${{ parameters.codeCovApiToken }} -F $(Build.DefinitionName) displayName: Upload coverage to CodeCov + env: + CODECOV_TOKEN: ${{ parameters.codeCovApiToken }} From 0172b62ef5333ebbacd9616a67fafd45fa4a541c Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Fri, 12 Jun 2026 13:26:55 -0500 Subject: [PATCH 16/17] :curly_haired_man: and :robot: comments --- eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml | 9 ++++++++- .../pr/{pr-pipeline.yml => sqlclient-pr-pipeline.yml} | 10 +++------- eng/pipelines/pr/stages/collect-coverage-stage.yml | 5 +---- eng/pipelines/pr/steps/publish-test-results-step.yml | 2 ++ 4 files changed, 14 insertions(+), 12 deletions(-) rename eng/pipelines/pr/{pr-pipeline.yml => sqlclient-pr-pipeline.yml} (95%) diff --git a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml index 313ce5fb44..e80e9995d9 100644 --- a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml +++ b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml @@ -132,6 +132,12 @@ jobs: - name: saPassword value: $[stageDependencies.${{ parameters.stageNameSecrets }}.secrets_job.outputs['SaPassword.Value']] + - name: testDisplayName + value: "sqlclient_manual_${{ parameters.configDisplayName}}_${{ parameters.testSet }}" + + - name: testResultsPath + value: "$(REPO_ROOT)/test_results/${{ parameters.platformDisplayName }}_$(testDisplayName)" + steps: # Install dotnet and the runtime that will run the tests (if it is not netframework) - template: /eng/pipelines/common/steps/install-dotnet.yml@self @@ -205,12 +211,13 @@ jobs: -p:BuildSuffix='${{ parameters.buildSuffix }}' -p:TestFramework=${{ parameters.platformDotnet }} -p:TestSet=${{ parameters.testSet }} + -p:TestResultsFolderPath=${{ variables.testResultsPath }} # Publish test results - template: /eng/pipelines/pr/steps/publish-test-results-step.yml@self parameters: buildConfiguration: ${{ parameters.buildConfiguration }} platformDisplayName: ${{ parameters.platformDisplayName }} - testDisplayName: "sqlclient_manual_${{ parameters.configDisplayName}}_${{ parameters.testSet }}" + testDisplayName: ${{ variables.testDisplayName }} testResultsArtifactName: ${{ parameters.testResultsArtifactName }} testResultsPath: ${{ variables.testResultsPath }} diff --git a/eng/pipelines/pr/pr-pipeline.yml b/eng/pipelines/pr/sqlclient-pr-pipeline.yml similarity index 95% rename from eng/pipelines/pr/pr-pipeline.yml rename to eng/pipelines/pr/sqlclient-pr-pipeline.yml index 6d3e61c28d..4c2d6fb01f 100644 --- a/eng/pipelines/pr/pr-pipeline.yml +++ b/eng/pipelines/pr/sqlclient-pr-pipeline.yml @@ -14,7 +14,7 @@ # # It maps to the "sqlclient-pr" pipeline in the Public ADO project: # https://sqlclientdrivers.visualstudio.com/public/_build?definitionId=2281 -# And the "pr-pipeline" pipeline in the internal ADO project: +# And the "sqlclient-pr" pipeline in the internal ADO project: # https://dev.azure.com/SqlClientDrivers/ADO.Net/_build?definitionId=2271 # Set the pipeline run name to the day-of-year and the daily run counter. @@ -36,7 +36,8 @@ pr: - .azuredevops/* - .config/* - doc/* - - eng/pipelines/* + - eng/pipelines/common/* + - eng/pipelines/pr/* - src/* - tools/* - azurepipelines-coverage.yml @@ -45,11 +46,6 @@ pr: - dotnet-tools.json - global.json - NuGet.config - exclude: - - eng/pipelines/ci/* - - eng/pipelines/kerberos/* - - eng/pipelines/onebranch/* - - eng/pipelines/stress/* # Do not trigger commit or schedule runs for this pipeline. trigger: none diff --git a/eng/pipelines/pr/stages/collect-coverage-stage.yml b/eng/pipelines/pr/stages/collect-coverage-stage.yml index a380203838..01aff672bb 100644 --- a/eng/pipelines/pr/stages/collect-coverage-stage.yml +++ b/eng/pipelines/pr/stages/collect-coverage-stage.yml @@ -51,11 +51,8 @@ stages: # However, we must respect the nuget.config to only download packages from governed # feeds. Thus, we copy the nuget.config from the root of the repo to a temp directory # then run dotnet tool installation from that folder. - - task: CopyFiles@2 + - script: cp "$(REPO_ROOT)/Nuget.config" "$(Agent.TempDirectory)" displayName: Copy nuget.config for tool installs - inputs: - contents: "$(REPO_ROOT)/nuget.config" - targetFolder: $(Agent.TempDirectory) - task: DotNetCoreCLI@2 displayName: Install dotnet-coverage diff --git a/eng/pipelines/pr/steps/publish-test-results-step.yml b/eng/pipelines/pr/steps/publish-test-results-step.yml index 222dea298b..83c066e785 100644 --- a/eng/pipelines/pr/steps/publish-test-results-step.yml +++ b/eng/pipelines/pr/steps/publish-test-results-step.yml @@ -41,9 +41,11 @@ steps: displayName: 'Publish Test Results' inputs: buildConfiguration: ${{ parameters.buildConfiguration }} + mergeTestResults: true testResultsFiles: | ${{ parameters.testResultsPath }}/*.trx ${{ parameters.testResultsPath }}/**/*.coverage + testResultsFormat: VSTest testRunTitle: "${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}" condition: succeededOrFailed() From aed442412c22a5af2ee9be93c8327e68bd201970 Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Fri, 12 Jun 2026 14:06:19 -0500 Subject: [PATCH 17/17] :robot: Use dotnet-tools.json --- .../pr/stages/collect-coverage-stage.yml | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/eng/pipelines/pr/stages/collect-coverage-stage.yml b/eng/pipelines/pr/stages/collect-coverage-stage.yml index 01aff672bb..3fff5810d1 100644 --- a/eng/pipelines/pr/stages/collect-coverage-stage.yml +++ b/eng/pipelines/pr/stages/collect-coverage-stage.yml @@ -41,26 +41,8 @@ stages: # Install .NET SDK - template: /eng/pipelines/common/steps/install-dotnet.yml@self - # Install additional dotnet tools - # - # We must work around a bug in the dotnet CLI that prevents tool installs when multiple - # project/solution files are present in the working directory. At various times, multiple - # project files have existed in the root of the repository. See SDK issue: - # https://github.com/dotnet/sdk/issues/9623 - # - # However, we must respect the nuget.config to only download packages from governed - # feeds. Thus, we copy the nuget.config from the root of the repo to a temp directory - # then run dotnet tool installation from that folder. - - script: cp "$(REPO_ROOT)/Nuget.config" "$(Agent.TempDirectory)" - displayName: Copy nuget.config for tool installs - - - task: DotNetCoreCLI@2 - displayName: Install dotnet-coverage - inputs: - command: custom - custom: tool - workingDirectory: $(Agent.TempDirectory) - arguments: install --global dotnet-coverage --version 18.3.2 + # Restore additional dotnet tools + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self # Download coverage reports from the test jobs. # NOTE: The artifact name is not specified, so *all* artifacts of the build will be @@ -73,12 +55,13 @@ stages: # Merge original files into a single Cobertura XML file - script: >- - dotnet-coverage merge "${{ variables.workingDir }}/originals/**/*.coverage" + dotnet tool run dotnet-coverage -- merge "${{ variables.workingDir }}/originals/**/*.coverage" --output "${{ variables.workingDir }}/merge/cobertura.xml" --output-format cobertura --log-file "${{ variables.workingDir }}/merge/merge.log" --log-level Verbose displayName: Merge coverage files + workingDirectory: $(REPO_ROOT) # Part 2) Publish merged results to the pipeline results/artifacts