ingest.yml

name: Ingest SPY data

on:
  workflow_dispatch:
  schedule:
    - cron: "30 22 * * 1-5"

jobs:
  ingest:
    runs-on: ubuntu-latest
    timeout-minutes: 45

    strategy:
      fail-fast: false

    env:
      SPY_DB_PATH: data/spy_truth.db
      SYMBOL: SPY
      INTRADAY_TIMEFRAME: "15Min"
      INTRADAY_BARS_TABLE: spy_bars_15m

      ALPACA_API_KEY: ${{ secrets.ALPACA_API_KEY }}
      ALPACA_API_SECRET: ${{ secrets.ALPACA_API_SECRET }}
      ALPACA_TIMEOUT: 120
      ALPACA_RETRIES: 6
      ALPACA_FAIL_OPEN: 1
      FINRA_CLIENT_ID: ${{ secrets.FINRA_CLIENT_ID }}
      FINRA_CLIENT_SECRET: ${{ secrets.FINRA_CLIENT_SECRET }}

      FRED_API_KEY: ${{ secrets.FRED_API_KEY }}
      DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
         python-version: "3.11"

      - name: Install dependencies
        run: |
         python -m pip install --upgrade pip
         pip install pandas numpy matplotlib yfinance

      - name: Python quality gate
        run: |
          python scripts/utils/lint_python.py scripts tests

      - name: Unit tests
        run: |
          python -m unittest discover -s tests -v

      - name: Python industry-standard audit
        run: |
          python scripts/utils/audit_code_quality.py

      - name: Initialize pipeline health
        run: |
          mkdir -p outputs/health
          rm -f outputs/health/warnings.log
          echo "pipeline_start=$(date -u)" > outputs/health/run_state.txt

      - name: Ingest SPY daily bars
        run: |
          python scripts/ingest_spy_daily.py

      - name: Ingest SPY intraday (15m) from Alpaca
        continue-on-error: true
        run: |
          python scripts/ingest_spy_intraday_alpaca.py || echo "WARN: intraday ingest degraded" >> outputs/health/warnings.log

      - name: Aggregate options positioning metrics (0-1 DTE)
        continue-on-error: true
        run: |
          DTE_MIN=0 DTE_MAX=1 python scripts/aggregate_options_positioning_metrics.py || echo "WARN: 0-1 DTE positioning degraded" >> outputs/health/warnings.log

      - name: Aggregate options positioning metrics (2-3 DTE)
        continue-on-error: true
        run: |
          DTE_MIN=2 DTE_MAX=3 python scripts/aggregate_options_positioning_metrics.py || echo "WARN: 2-3 DTE positioning degraded" >> outputs/health/warnings.log

      - name: Build liquidity regime (daily)
        run: |
          python scripts/build_liquidity_regime_daily.py

      - name: Build open resolution regime (intraday)
        continue-on-error: true
        run: |
          python scripts/build_open_resolution_regime_intraday.py || echo "WARN: open resolution regime degraded" >> outputs/health/warnings.log

      - name: FINRA darkpool overlay
        continue-on-error: true
        run: |
          python scripts/ingest_finra_darkpool_overlay.py || echo "WARN: FINRA overlay degraded" >> outputs/health/warnings.log

      - name: FRED macro overlays
        continue-on-error: true
        run: |
          python scripts/ingest_fred_overlays.py || echo "WARN: FRED overlay degraded" >> outputs/health/warnings.log

      - name: Build regime (daily)
        run: |
          python scripts/build_regime_spy_daily.py

      - name: Build canonical auction expectancy dataset
        continue-on-error: true
        run: |
          python scripts/build_auction_expectancy_events.py || echo "WARN: auction expectancy build degraded" >> outputs/health/warnings.log

      - name: Classify auction fill paths
        continue-on-error: true
        run: |
          python scripts/classify_fill_paths.py || echo "WARN: fill path classification degraded" >> outputs/health/warnings.log

      - name: Measure gap excursions
        continue-on-error: true
        run: |
          python scripts/measure_gap_excursions.py || echo "WARN: gap excursion metrics degraded" >> outputs/health/warnings.log

      - name: Build conditional expectancy matrix
        continue-on-error: true
        run: |
          python scripts/build_conditional_expectancy_matrix.py || echo "WARN: conditional expectancy matrix degraded" >> outputs/health/warnings.log

      - name: Build SharpEdge 2.1 confidence matrix
        continue-on-error: true
        run: |
          python scripts/build_confidence_weights.py || echo "WARN: confidence weighting engine degraded" >> outputs/health/warnings.log

      - name: Build SharpEdge 2.1 risk decision layer
        continue-on-error: true
        run: |
          python scripts/build_risk_decision_layer.py || echo "WARN: risk decision layer degraded" >> outputs/health/warnings.log

      - name: Build regime×pressure×DTE expectancy
        continue-on-error: true
        run: python scripts/analysis/expectancy_engine.py

      - name: Publish SharpEdge 2.0 report
        continue-on-error: true
        run: |
          python scripts/publish_sharpedge_2_report.py || echo "WARN: SharpEdge 2.0 report degraded" >> outputs/health/warnings.log

      - name: Build Robinhood FVG monitor handoff
        continue-on-error: true
        run: |
          python scripts/build_robinhood_fvg_monitor.py || echo "WARN: Robinhood FVG monitor degraded" >> outputs/health/warnings.log

      - name: Run deterministic controller
        continue-on-error: true
        run: |
          python scripts/agents/controller_agent.py || echo "WARN: deterministic controller degraded" >> outputs/health/warnings.log

      - name: Build Agentic AI v1 decision contract
        continue-on-error: true
        run: |
          python scripts/agents/agent_v1_decision.py || echo "WARN: agentic AI v1 contract degraded" >> outputs/health/warnings.log

      - name: Build operator brief MVP
        continue-on-error: true
        run: |
          python scripts/agents/operator_brief.py || echo "WARN: operator brief degraded" >> outputs/health/warnings.log

      - name: Build operator session review
        continue-on-error: true
        run: |
          python scripts/agents/operator_session_review.py || echo "WARN: operator session review degraded" >> outputs/health/warnings.log

      - name: Build morning open dashboard
        continue-on-error: true
        run: |
          python scripts/agents/morning_open_dashboard.py || echo "WARN: morning open dashboard degraded" >> outputs/health/warnings.log

      - name: Build Robinhood beta execution handoff
        continue-on-error: true
        run: |
          python scripts/agents/robinhood_beta_execution.py || echo "WARN: Robinhood beta execution degraded" >> outputs/health/warnings.log

      - name: Commit updated DB + outputs
        continue-on-error: true
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add data/ outputs/ sql/ scripts/ || true
          git diff --staged --quiet && echo "No changes to commit" && exit 0
          git commit -m "Daily pipeline + SharpEdge 2.1 risk allocation engine" || true
          git push || true

      - name: Final pipeline state
        if: always()
        run: |
          echo "PIPELINE COMPLETE"
          if [ -f outputs/health/warnings.log ]; then
            echo "WARNINGS DETECTED:"
            cat outputs/health/warnings.log
          else
            echo "NO WARNINGS"
          fi