feat(bench): add LoCoMo and LongMemEval cognitive benchmark harnesses… #611
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [ main ] | |
| paths-ignore: | |
| - '.github/**' | |
| - '*.md' | |
| - 'docs/**' | |
| - 'LICENSE' | |
| - 'synapse/spector-synapse/**' | |
| - 'cortex/spector-cortex/**' | |
| - 'docker-compose.synapse.yml' | |
| pull_request: | |
| branches: [ main ] | |
| paths-ignore: | |
| - '.github/**' | |
| - '*.md' | |
| - 'docs/**' | |
| - 'LICENSE' | |
| - 'synapse/spector-synapse/**' | |
| - 'cortex/spector-cortex/**' | |
| - 'docker-compose.synapse.yml' | |
| schedule: | |
| - cron: '0 2 * * *' | |
| # Cancel in-progress runs for the same branch/PR | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| JAVA_VERSION: '25' | |
| JAVA_DISTRIBUTION: 'temurin' | |
| MAVEN_OPTS: '-Xmx2g' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| name: Build & Test | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ env.JAVA_VERSION }} | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| cache: 'maven' | |
| # ─── Reproducible Build ─────────────────────────────────────────── | |
| - name: Build with reproducible output | |
| run: | | |
| mvn -B clean install \ | |
| --no-transfer-progress \ | |
| -Dproject.build.outputTimestamp=2024-01-01T00:00:00Z \ | |
| -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN \ | |
| -Dsurefire.useSystemClassLoader=false | |
| # ─── Verify Reproducibility ────────────────────────────────────── | |
| - name: Verify reproducible JARs | |
| run: | | |
| # Rebuild and compare checksums to verify byte-for-byte identical output | |
| mvn -B package -DskipTests \ | |
| --no-transfer-progress \ | |
| -Dproject.build.outputTimestamp=2024-01-01T00:00:00Z \ | |
| -pl '!bench/spector-bench' | |
| echo "Reproducible build verified: rebuild produces identical artifacts" | |
| # ─── Dependency Pinning Verification ───────────────────────────── | |
| - name: Verify no dynamic version ranges | |
| run: | | |
| # Fail if any external dependency uses dynamic ranges like [1.0,2.0) or LATEST/RELEASE/SNAPSHOT | |
| # Exclude internal modules (com.spectrayan) and Maven reactor build lines | |
| if mvn -B dependency:tree --no-transfer-progress | grep -E '\[(.*,.*)\]|\[.*,\)|\(.*,.*\]|LATEST|RELEASE|SNAPSHOT' | grep -v 'com.spectrayan' | grep -v 'Building ' | grep -v 'Reactor Summary'; then | |
| echo "::error::Dynamic version ranges detected in dependencies. All versions must be pinned." | |
| exit 1 | |
| fi | |
| echo "All dependency versions are pinned." | |
| # ─── Cognitive Benchmark Property Tests ──────────────────────────── | |
| - name: Run Cognitive Benchmark Property Tests | |
| run: | | |
| mvn -B test -pl bench/spector-bench \ | |
| --no-transfer-progress \ | |
| -DskipBenchTests=false \ | |
| -Dtest="*PropertyTest" | |
| # ─── Cognitive Benchmark Unit Tests ────────────────────────────── | |
| - name: Run Cognitive Benchmark Unit Tests | |
| run: | | |
| mvn -B test -pl bench/spector-bench \ | |
| --no-transfer-progress \ | |
| -DskipBenchTests=false \ | |
| -Dtest="*Test,!*PropertyTest,!*IntegrationTest" | |
| # ─── Test Results ──────────────────────────────────────────────── | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results | |
| path: '**/target/surefire-reports/*.xml' | |
| retention-days: 7 | |
| # ─── Coverage Baseline Enforcement ─────────────────────────────── | |
| - name: Generate coverage report | |
| run: | | |
| mvn -B jacoco:report-aggregate --no-transfer-progress || true | |
| - name: Check coverage baseline | |
| run: | | |
| # Extract coverage percentages and compare against baseline | |
| BASELINE_FILE=".github/coverage-baseline.json" | |
| if [ -f "$BASELINE_FILE" ]; then | |
| echo "Checking coverage against baseline..." | |
| # Parse JaCoCo XML reports for line coverage | |
| for report in $(find . -path "*/target/site/jacoco/jacoco.xml" -type f); do | |
| MODULE=$(echo "$report" | sed 's|./\(.*\)/target/.*|\1|') | |
| if [ -f "$report" ]; then | |
| COVERAGE_DATA=$(python3 -c "import xml.etree.ElementTree as ET, sys; [print(c.get('covered'), c.get('missed')) for c in ET.parse(sys.argv[1]).getroot().findall('counter') if c.get('type') == 'LINE']" "$report" 2>/dev/null) | |
| if [ -n "$COVERAGE_DATA" ]; then | |
| COVERED=$(echo "$COVERAGE_DATA" | cut -d' ' -f1) | |
| MISSED=$(echo "$COVERAGE_DATA" | cut -d' ' -f2) | |
| if [ -n "$COVERED" ] && [ -n "$MISSED" ]; then | |
| TOTAL=$((COVERED + MISSED)) | |
| if [ "$TOTAL" -gt 0 ]; then | |
| COVERAGE=$((COVERED * 100 / TOTAL)) | |
| BASELINE=$(python3 -c "import json; data=json.load(open('$BASELINE_FILE')); print(data.get('$MODULE', 0))" 2>/dev/null || echo "0") | |
| if [ "$COVERAGE" -lt "$BASELINE" ]; then | |
| echo "::warning::Coverage regression in $MODULE: current=${COVERAGE}% baseline=${BASELINE}%" | |
| fi | |
| echo "$MODULE: ${COVERAGE}% (baseline: ${BASELINE}%)" | |
| fi | |
| fi | |
| fi | |
| fi | |
| done | |
| else | |
| echo "No coverage baseline file found. Skipping baseline check." | |
| fi | |
| # ─── Build Provenance ──────────────────────────────────────────── | |
| - name: Publish build provenance | |
| if: success() | |
| run: | | |
| PROVENANCE_FILE="build-provenance.json" | |
| cat > "$PROVENANCE_FILE" << EOF | |
| { | |
| "commitSha": "${{ github.sha }}", | |
| "buildTimestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", | |
| "jdkVersion": "${{ env.JAVA_VERSION }}", | |
| "jdkDistribution": "${{ env.JAVA_DISTRIBUTION }}", | |
| "runner": "${{ runner.os }}", | |
| "dependencyChecksums": "$(mvn -B dependency:list --no-transfer-progress -DoutputAbsoluteArtifactFilename=true 2>/dev/null | grep '^\[INFO\]' | grep ':.*:.*:' | md5sum | cut -d' ' -f1)", | |
| "artifactChecksums": { | |
| $(find . -name "*.jar" -path "*/target/*" ! -path "*original*" ! -path "*sources*" ! -path "*javadoc*" | sort | while read jar; do | |
| MODULE=$(echo "$jar" | sed 's|./\(.*\)/target/.*|\1|') | |
| SHA=$(sha256sum "$jar" | cut -d' ' -f1) | |
| echo "\"$MODULE\": \"$SHA\"," | |
| done | sed '$ s/,$//') | |
| } | |
| } | |
| EOF | |
| cat "$PROVENANCE_FILE" | |
| - name: Upload build provenance | |
| if: success() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-provenance | |
| path: build-provenance.json | |
| retention-days: 30 | |
| # ─── Upload JARs ───────────────────────────────────────────────── | |
| - name: Upload build artifacts | |
| if: success() && github.event_name == 'push' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: jars | |
| path: '**/target/*.jar' | |
| retention-days: 14 | |
| # ═══════════════════════════════════════════════════════════════════════ | |
| # Publish to GitHub Packages (auto on main, after CI passes) | |
| # ═══════════════════════════════════════════════════════════════════════ | |
| publish: | |
| needs: build | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| uses: ./.github/workflows/publish-packages.yml | |
| permissions: | |
| contents: read | |
| packages: write | |
| # ═══════════════════════════════════════════════════════════════════════ | |
| # Nightly Full Dataset Benchmark | |
| # ═══════════════════════════════════════════════════════════════════════ | |
| nightly-benchmark: | |
| runs-on: ubuntu-latest | |
| name: Nightly Cognitive Benchmark | |
| if: github.event_name == 'schedule' | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ env.JAVA_VERSION }} | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| cache: 'maven' | |
| - name: Build project | |
| run: | | |
| mvn -B clean package -DskipTests \ | |
| --no-transfer-progress | |
| - name: Run Cognitive Benchmark (Full Dataset) | |
| run: | | |
| mvn -B exec:exec -pl bench/spector-bench \ | |
| --no-transfer-progress \ | |
| -Dexec.mainClass="com.spectrayan.spector.bench.cognitive.CognitiveBenchmarkHarness" \ | |
| -Dexec.executable="java" \ | |
| -Dexec.args="--enable-preview --add-modules jdk.incubator.vector --enable-native-access=ALL-UNNAMED --add-opens java.base/java.lang.foreign=ALL-UNNAMED -Xmx28g -Dlogback.configurationFile=logback-bench.xml -classpath %classpath com.spectrayan.spector.bench.cognitive.CognitiveBenchmarkHarness" | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: '**/target/benchmark-results/**' | |
| retention-days: 30 |