Ingest SPY data #366
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Ingest SPY data | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "30 22 * * 1-5" | |
| jobs: | |
| ingest: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| env: | |
| SPY_DB_PATH: data/spy_truth.db | |
| SYMBOL: SPY | |
| INTRADAY_TIMEFRAME: "15Min" | |
| INTRADAY_BARS_TABLE: spy_bars_15m | |
| ALPACA_API_KEY: ${{ secrets.ALPACA_API_KEY }} | |
| ALPACA_API_SECRET: ${{ secrets.ALPACA_API_SECRET }} | |
| ALPACA_TIMEOUT: 120 | |
| ALPACA_RETRIES: 6 | |
| ALPACA_FAIL_OPEN: 1 | |
| FINRA_CLIENT_ID: ${{ secrets.FINRA_CLIENT_ID }} | |
| FINRA_CLIENT_SECRET: ${{ secrets.FINRA_CLIENT_SECRET }} | |
| FRED_API_KEY: ${{ secrets.FRED_API_KEY }} | |
| DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pandas numpy matplotlib yfinance | |
| - name: Python quality gate | |
| run: | | |
| python scripts/utils/lint_python.py scripts tests | |
| - name: Unit tests | |
| run: | | |
| python -m unittest discover -s tests -v | |
| - name: Python industry-standard audit | |
| run: | | |
| python scripts/utils/audit_code_quality.py | |
| - name: Initialize pipeline health | |
| run: | | |
| mkdir -p outputs/health | |
| rm -f outputs/health/warnings.log | |
| echo "pipeline_start=$(date -u)" > outputs/health/run_state.txt | |
| - name: Ingest SPY daily bars | |
| run: | | |
| python scripts/ingest_spy_daily.py | |
| - name: Ingest SPY intraday (15m) from Alpaca | |
| continue-on-error: true | |
| run: | | |
| python scripts/ingest_spy_intraday_alpaca.py || echo "WARN: intraday ingest degraded" >> outputs/health/warnings.log | |
| - name: Aggregate options positioning metrics (0-1 DTE) | |
| continue-on-error: true | |
| run: | | |
| DTE_MIN=0 DTE_MAX=1 python scripts/aggregate_options_positioning_metrics.py || echo "WARN: 0-1 DTE positioning degraded" >> outputs/health/warnings.log | |
| - name: Aggregate options positioning metrics (2-3 DTE) | |
| continue-on-error: true | |
| run: | | |
| DTE_MIN=2 DTE_MAX=3 python scripts/aggregate_options_positioning_metrics.py || echo "WARN: 2-3 DTE positioning degraded" >> outputs/health/warnings.log | |
| - name: Build liquidity regime (daily) | |
| run: | | |
| python scripts/build_liquidity_regime_daily.py | |
| - name: Build open resolution regime (intraday) | |
| continue-on-error: true | |
| run: | | |
| python scripts/build_open_resolution_regime_intraday.py || echo "WARN: open resolution regime degraded" >> outputs/health/warnings.log | |
| - name: FINRA darkpool overlay | |
| continue-on-error: true | |
| run: | | |
| python scripts/ingest_finra_darkpool_overlay.py || echo "WARN: FINRA overlay degraded" >> outputs/health/warnings.log | |
| - name: FRED macro overlays | |
| continue-on-error: true | |
| run: | | |
| python scripts/ingest_fred_overlays.py || echo "WARN: FRED overlay degraded" >> outputs/health/warnings.log | |
| - name: Build regime (daily) | |
| run: | | |
| python scripts/build_regime_spy_daily.py | |
| - name: Build canonical auction expectancy dataset | |
| continue-on-error: true | |
| run: | | |
| python scripts/build_auction_expectancy_events.py || echo "WARN: auction expectancy build degraded" >> outputs/health/warnings.log | |
| - name: Classify auction fill paths | |
| continue-on-error: true | |
| run: | | |
| python scripts/classify_fill_paths.py || echo "WARN: fill path classification degraded" >> outputs/health/warnings.log | |
| - name: Measure gap excursions | |
| continue-on-error: true | |
| run: | | |
| python scripts/measure_gap_excursions.py || echo "WARN: gap excursion metrics degraded" >> outputs/health/warnings.log | |
| - name: Build conditional expectancy matrix | |
| continue-on-error: true | |
| run: | | |
| python scripts/build_conditional_expectancy_matrix.py || echo "WARN: conditional expectancy matrix degraded" >> outputs/health/warnings.log | |
| - name: Build SharpEdge 2.1 confidence matrix | |
| continue-on-error: true | |
| run: | | |
| python scripts/build_confidence_weights.py || echo "WARN: confidence weighting engine degraded" >> outputs/health/warnings.log | |
| - name: Build SharpEdge 2.1 risk decision layer | |
| continue-on-error: true | |
| run: | | |
| python scripts/build_risk_decision_layer.py || echo "WARN: risk decision layer degraded" >> outputs/health/warnings.log | |
| - name: Build regime×pressure×DTE expectancy | |
| continue-on-error: true | |
| run: python scripts/analysis/expectancy_engine.py | |
| - name: Publish SharpEdge 2.0 report | |
| continue-on-error: true | |
| run: | | |
| python scripts/publish_sharpedge_2_report.py || echo "WARN: SharpEdge 2.0 report degraded" >> outputs/health/warnings.log | |
| - name: Build Robinhood FVG monitor handoff | |
| continue-on-error: true | |
| run: | | |
| python scripts/build_robinhood_fvg_monitor.py || echo "WARN: Robinhood FVG monitor degraded" >> outputs/health/warnings.log | |
| - name: Run deterministic controller | |
| continue-on-error: true | |
| run: | | |
| python scripts/agents/controller_agent.py || echo "WARN: deterministic controller degraded" >> outputs/health/warnings.log | |
| - name: Build Agentic AI v1 decision contract | |
| continue-on-error: true | |
| run: | | |
| python scripts/agents/agent_v1_decision.py || echo "WARN: agentic AI v1 contract degraded" >> outputs/health/warnings.log | |
| - name: Build operator brief MVP | |
| continue-on-error: true | |
| run: | | |
| python scripts/agents/operator_brief.py || echo "WARN: operator brief degraded" >> outputs/health/warnings.log | |
| - name: Build operator session review | |
| continue-on-error: true | |
| run: | | |
| python scripts/agents/operator_session_review.py || echo "WARN: operator session review degraded" >> outputs/health/warnings.log | |
| - name: Build morning open dashboard | |
| continue-on-error: true | |
| run: | | |
| python scripts/agents/morning_open_dashboard.py || echo "WARN: morning open dashboard degraded" >> outputs/health/warnings.log | |
| - name: Build Robinhood beta execution handoff | |
| continue-on-error: true | |
| run: | | |
| python scripts/agents/robinhood_beta_execution.py || echo "WARN: Robinhood beta execution degraded" >> outputs/health/warnings.log | |
| - name: Commit updated DB + outputs | |
| continue-on-error: true | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add data/ outputs/ sql/ scripts/ || true | |
| git diff --staged --quiet && echo "No changes to commit" && exit 0 | |
| git commit -m "Daily pipeline + SharpEdge 2.1 risk allocation engine" || true | |
| git push || true | |
| - name: Final pipeline state | |
| if: always() | |
| run: | | |
| echo "PIPELINE COMPLETE" | |
| if [ -f outputs/health/warnings.log ]; then | |
| echo "WARNINGS DETECTED:" | |
| cat outputs/health/warnings.log | |
| else | |
| echo "NO WARNINGS" | |
| fi |