Initial import #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Branch CI Monitor | ||
|
Check failure on line 1 in .github/workflows/nightly-branch-monitor.yml
|
||
| on: | ||
| schedule: | ||
| # 1 AM UTC daily (9 PM ET / 6 PM PT) | ||
| - cron: '0 1 * * *' | ||
| workflow_dispatch: | ||
| inputs: | ||
| branch: | ||
| description: 'Specific branch to test (optional, defaults to auto-detect)' | ||
| required: false | ||
| type: string | ||
| jobs: | ||
| detect-branches: | ||
| name: Detect Active Release Branches | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| branches: ${{ steps.get-branches.outputs.branches }} | ||
| count: ${{ steps.get-branches.outputs.count }} | ||
| steps: | ||
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | ||
| with: | ||
| persist-credentials: false | ||
| fetch-depth: 0 # Need full history for branch detection | ||
| - name: Set up Python | ||
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a20 # v5.3.0 | ||
| with: | ||
| python-version: '3.11' | ||
| - name: Install dependencies | ||
| run: pip install packaging | ||
| - name: Get active branches | ||
| id: get-branches | ||
| run: | | ||
| # Use manual branch if provided, otherwise auto-detect | ||
| if [ -n "${{ github.event.inputs.branch }}" ]; then | ||
| echo "branches=[\"${{ github.event.inputs.branch }}\"]" >> $GITHUB_OUTPUT | ||
| echo "count=1" >> $GITHUB_OUTPUT | ||
| else | ||
| python scripts/get_active_branches.py >> $GITHUB_OUTPUT | ||
| fi | ||
| - name: Display detected branches | ||
| run: | | ||
| echo "Monitoring branches: ${{ steps.get-branches.outputs.branches }}" | ||
| echo "Count: ${{ steps.get-branches.outputs.count }}" | ||
| validate-branch: | ||
| name: Validate Branch ${{ matrix.branch }} | ||
| needs: detect-branches | ||
| if: needs.detect-branches.outputs.count > 0 | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 240 # 4 hours max per branch | ||
| strategy: | ||
| matrix: | ||
| branch: ${{ fromJSON(needs.detect-branches.outputs.branches) }} | ||
| fail-fast: false # Continue testing all branches | ||
| max-parallel: 3 # Limit parallel jobs | ||
| outputs: | ||
| result-${{ matrix.branch }}: ${{ steps.set-result.outputs.result }} | ||
| steps: | ||
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | ||
| with: | ||
| ref: ${{ matrix.branch }} | ||
| persist-credentials: false | ||
| fetch-depth: 0 | ||
| - name: Branch validation info | ||
| run: | | ||
| echo "Validating branch: ${{ matrix.branch }}" | ||
| git log -1 --oneline | ||
| echo "Current commit: $(git rev-parse HEAD)" | ||
| - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1.12.0 | ||
| - name: Install latest stable toolchain | ||
| run: rustup update stable && rustup default stable && rustup component add rustfmt clippy | ||
| - name: Install hatch | ||
| uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # install | ||
| with: | ||
| version: "1.12.0" | ||
| - name: Set up Docker for system tests | ||
| run: | | ||
| # Start Docker service (already available on ubuntu-latest) | ||
| docker --version | ||
| # Run unit tests | ||
| - name: Run unit tests | ||
| id: unit-tests | ||
| continue-on-error: true | ||
| run: | | ||
| echo "Running unit tests on branch ${{ matrix.branch }}" | ||
| hatch run +py=3.12 multiple_os_tests:test | ||
| timeout-minutes: 30 | ||
| # Run framework tests (subset - most critical frameworks) | ||
| - name: Run framework tests | ||
| id: framework-tests | ||
| continue-on-error: true | ||
| run: | | ||
| echo "Running framework tests on branch ${{ matrix.branch }}" | ||
| # Run a subset of critical framework tests | ||
| hatch run +py=3.12 test.django || true | ||
| hatch run +py=3.12 test.flask || true | ||
| hatch run +py=3.12 test.fastapi || true | ||
| timeout-minutes: 90 | ||
| # Run build validation | ||
| - name: Run build validation | ||
| id: build-tests | ||
| continue-on-error: true | ||
| run: | | ||
| echo "Running build validation on branch ${{ matrix.branch }}" | ||
| # Test that the package can be built | ||
| hatch build | ||
| timeout-minutes: 40 | ||
| # Collect results | ||
| - name: Collect results | ||
| id: collect-results | ||
| if: always() | ||
| run: | | ||
| FAILED_JOBS="[]" | ||
| WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | ||
| if [ "${{ steps.unit-tests.outcome }}" == "failure" ]; then | ||
| FAILED_JOBS=$(echo "$FAILED_JOBS" | jq '. + [{"name":"unit-tests","url":"'"$WORKFLOW_URL"'"}]') | ||
| fi | ||
| if [ "${{ steps.framework-tests.outcome }}" == "failure" ]; then | ||
| FAILED_JOBS=$(echo "$FAILED_JOBS" | jq '. + [{"name":"framework-tests","url":"'"$WORKFLOW_URL"'"}]') | ||
| fi | ||
| if [ "${{ steps.build-tests.outcome }}" == "failure" ]; then | ||
| FAILED_JOBS=$(echo "$FAILED_JOBS" | jq '. + [{"name":"build-tests","url":"'"$WORKFLOW_URL"'"}]') | ||
| fi | ||
| echo "failed_jobs=$FAILED_JOBS" >> $GITHUB_OUTPUT | ||
| if [ "$FAILED_JOBS" != "[]" ]; then | ||
| echo "has_failures=true" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "has_failures=false" >> $GITHUB_OUTPUT | ||
| fi | ||
| - name: Upload results artifact | ||
| if: always() | ||
| uses: actions/upload-artifact@4.5.0 | ||
| with: | ||
| name: results-${{ matrix.branch }} | ||
| path: | | ||
| /tmp/nightly-results-${{ matrix.branch }}.json | ||
| retention-days: 7 | ||
| - name: Write results to file | ||
| if: always() | ||
| run: | | ||
| mkdir -p /tmp/nightly-results | ||
| cat > /tmp/nightly-results/${{ matrix.branch }}.json << EOF | ||
| { | ||
| "branch": "${{ matrix.branch }}", | ||
| "failed_jobs": ${{ steps.collect-results.outputs.failed_jobs }}, | ||
| "has_failures": "${{ steps.collect-results.outputs.has_failures }}" | ||
| } | ||
| EOF | ||
| - name: Upload results file | ||
| if: always() | ||
| uses: actions/upload-artifact@4.5.0 | ||
| with: | ||
| name: result-${{ matrix.branch }} | ||
| path: /tmp/nightly-results/${{ matrix.branch }}.json | ||
| retention-days: 7 | ||
| - name: Set job result output | ||
| id: set-result | ||
| if: always() | ||
| run: | | ||
| if [ "${{ steps.collect-results.outputs.has_failures }}" == "true" ]; then | ||
| echo "result=failure" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "result=success" >> $GITHUB_OUTPUT | ||
| fi | ||
| report-failures: | ||
| name: Report Failures and Alert | ||
| needs: [detect-branches, validate-branch] | ||
| if: always() && needs.detect-branches.outputs.count > 0 | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | ||
| with: | ||
| persist-credentials: false | ||
| - name: Set up Python | ||
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a20 # v5.3.0 | ||
| with: | ||
| python-version: '3.11' | ||
| - name: Download all results | ||
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | ||
| with: | ||
| path: results/ | ||
| - name: Process results and create incidents | ||
| id: process-results | ||
| env: | ||
| GH_TOKEN: ${{ github.token }} | ||
| run: | | ||
| INCIDENTS_CREATED="" | ||
| WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | ||
| # Make incident script executable | ||
| chmod +x scripts/create_ci_incident.py | ||
| # Process each branch result | ||
| for result_file in results/result-*/*.json; do | ||
| if [ ! -f "$result_file" ]; then | ||
| continue | ||
| fi | ||
| # Extract data from result file | ||
| BRANCH=$(jq -r '.branch' "$result_file") | ||
| FAILED_JOBS=$(jq -c '.failed_jobs' "$result_file") | ||
| HAS_FAILURES=$(jq -r '.has_failures' "$result_file") | ||
| echo "Processing branch: $BRANCH, has_failures: $HAS_FAILURES" | ||
| if [ "$HAS_FAILURES" == "true" ] && [ "$FAILED_JOBS" != "[]" ]; then | ||
| # Create or update incident | ||
| ISSUE_OUTPUT=$(python scripts/create_ci_incident.py \ | ||
| --branch "$BRANCH" \ | ||
| --failed-jobs "$FAILED_JOBS" \ | ||
| --workflow-url "$WORKFLOW_URL" 2>&1) | ||
| echo "$ISSUE_OUTPUT" | ||
| ISSUE_URL=$(echo "$ISSUE_OUTPUT" | grep "issue_url=" | cut -d'=' -f2) | ||
| if [ -n "$ISSUE_URL" ]; then | ||
| INCIDENTS_CREATED="${INCIDENTS_CREATED}\n- Branch \`${BRANCH}\`: ${ISSUE_URL}" | ||
| fi | ||
| else | ||
| # Branch is passing, close any existing incident | ||
| echo "Branch $BRANCH is passing, checking for existing incidents to close..." | ||
| python scripts/create_ci_incident.py \ | ||
| --branch "$BRANCH" \ | ||
| --resolved || true | ||
| fi | ||
| done | ||
| # Set output for Slack notification | ||
| if [ -n "$INCIDENTS_CREATED" ]; then | ||
| echo "should_notify=true" >> $GITHUB_OUTPUT | ||
| echo "incidents_created<<EOF" >> $GITHUB_OUTPUT | ||
| echo -e "$INCIDENTS_CREATED" >> $GITHUB_OUTPUT | ||
| echo "EOF" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "should_notify=false" >> $GITHUB_OUTPUT | ||
| echo "incidents_created=" >> $GITHUB_OUTPUT | ||
| fi | ||
| - name: Send Slack notification | ||
| if: steps.process-results.outputs.should_notify == 'true' | ||
| uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 | ||
| with: | ||
| channel-id: 'CHANNEL_ID_PLACEHOLDER' # Replace with actual #apm-python channel ID | ||
| payload: | | ||
| { | ||
| "text": ":warning: *Nightly CI Failures Detected*", | ||
| "blocks": [ | ||
| { | ||
| "type": "header", | ||
| "text": { | ||
| "type": "plain_text", | ||
| "text": ":warning: Nightly CI Failures Detected" | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "The following branches have failing nightly CI runs:\n${{ steps.process-results.outputs.incidents_created }}" | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "*Workflow Run:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\n\nAssigned to: @DataDog/apm-core-python" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| env: | ||
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | ||
| - name: Summary | ||
| if: always() | ||
| run: | | ||
| echo "## Nightly Branch Monitor Results" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| echo "**Monitored Branches:** ${{ needs.detect-branches.outputs.branches }}" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| if [ "${{ steps.process-results.outputs.should_notify }}" == "true" ]; then | ||
| echo "### :warning: Failures Detected" >> $GITHUB_STEP_SUMMARY | ||
| echo "${{ steps.process-results.outputs.incidents_created }}" >> $GITHUB_STEP_SUMMARY | ||
| else | ||
| echo "### :white_check_mark: All monitored branches passing" >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
| finished: | ||
| name: Nightly Monitor Complete | ||
| runs-on: ubuntu-latest | ||
| needs: [detect-branches, validate-branch, report-failures] | ||
| if: always() | ||
| steps: | ||
| - name: Check overall status | ||
| run: | | ||
| # This job always succeeds - failures are tracked via GitHub issues | ||
| echo "Nightly branch monitoring complete" | ||
| echo "Check GitHub issues with label 'ci-nightly-failure' for active incidents" | ||
| exit 0 | ||
