Step3 Combine sketches #50
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Step3 Combine sketches | |
| on: | |
| workflow_run: | |
| workflows: ["Step2 Create Sketches"] | |
| types: [completed] | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: "Override RefSeq version (leave blank for auto-detect)" | |
| required: false | |
| default: '' | |
| run_id: | |
| description: "Step 2 Run ID (Required for manual runs)" | |
| required: false | |
| default: '' | |
| env: | |
| ZENODO_RECORD_ID: "19211013" | |
| jobs: | |
| pull_together: | |
| if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| actions: read | |
| steps: | |
| - name: Check Out Repository | |
| uses: actions/checkout@main | |
| with: | |
| # This ensures we have the ids.txt created in Step 1 | |
| ref: 'update-master-ids' | |
| lfs: true | |
| persist-credentials: false | |
| - name: Fetch Current RefSeq Version | |
| id: refseq | |
| run: | | |
| if [ -n "${{ github.event.inputs.version }}" ]; then | |
| RS_VERSION="${{ github.event.inputs.version }}" | |
| else | |
| RS_VERSION=$(curl -s https://ftp.ncbi.nlm.nih.gov/refseq/release/RELEASE_NUMBER) | |
| fi | |
| echo "VERSION=$RS_VERSION" >> $GITHUB_ENV | |
| - name: Download all Chunk Artifacts | |
| uses: actions/download-artifact@main | |
| with: | |
| pattern: msh-results-* | |
| path: all_sketches/ | |
| merge-multiple: true | |
| # Connects to the Step 2 Run ID | |
| run-id: ${{ inputs.run_id || github.event.workflow_run.id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install Mash v2.3 | |
| run: | | |
| wget -q https://github.com/marbl/Mash/releases/download/v2.3/mash-Linux64-v2.3.tar | |
| tar -xf mash-Linux64-v2.3.tar | |
| mv mash-Linux64-v2.3/mash . | |
| chmod +x mash | |
| echo "$(pwd)" >> $GITHUB_PATH | |
| - name: Paste and Compress | |
| run: | | |
| mkdir -p data/ | |
| rm -f data/RefSeqSketches*msh* | |
| VERSION_TAG="${{ env.VERSION }}" | |
| VERSIONED_NAME="RefSeqSketches_${VERSION_TAG}.msh" | |
| LATEST_NAME="RefSeqSketches_latest.msh" | |
| mash paste "data/${VERSIONED_NAME}" all_sketches/*.msh | |
| cp "data/${VERSIONED_NAME}" "${LATEST_NAME}" | |
| cp "data/${VERSIONED_NAME}" test.msh | |
| gzip -9 "data/${VERSIONED_NAME}" | |
| gzip -9 "${LATEST_NAME}" | |
| echo "VERSIONED_FILENAME=${VERSIONED_NAME}.gz" >> $GITHUB_ENV | |
| echo "LATEST_FILENAME=${LATEST_NAME}.gz" >> $GITHUB_ENV | |
| - name: Verify Accession Integrity | |
| run: | | |
| touch initial_accessions.txt final_accessions.txt missing_accessions.txt | |
| cut -f 1 data/ids.txt | sort | uniq > initial_accessions.txt || true | |
| mash info -t test.msh | cut -f 3 | grep G | rev | cut -f 1-2 -d _ | rev | sort | uniq > final_accessions.txt || true | |
| cat initial_accessions.txt | grep -vf final_accessions.txt > missing_accessions.txt || true | |
| ORIGINAL_COUNT=$(wc -l < initial_accessions.txt) | |
| MASH_COUNT=$(wc -l < final_accessions.txt) | |
| MISSING_COUNT=$(wc -l < missing_accessions.txt) | |
| echo "### Genomic Integrity Report (v${{ env.VERSION }})" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Total Accessions Expected:** $ORIGINAL_COUNT" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Total Accessions Sketched:** $MASH_COUNT" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Missing Accessions:** $MISSING_COUNT" >> $GITHUB_STEP_SUMMARY | |
| if [ "$MISSING_COUNT" -gt 0 ]; then | |
| echo "#### Missing Accession Samples (First 50)" >> $GITHUB_STEP_SUMMARY | |
| echo '```text' >> $GITHUB_STEP_SUMMARY | |
| head -n 50 missing_accessions.txt >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| # This determines if the Zenodo step should be triggered | |
| echo "SHOULD_PUBLISH=false" >> $GITHUB_ENV | |
| echo "Integrity check failed: $MISSING_COUNT accessions missing." | |
| else | |
| echo "SHOULD_PUBLISH=true" >> $GITHUB_ENV | |
| echo "Integrity check passed: All accessions accounted for." | |
| fi | |
| - name: Smoke Test | |
| run: | | |
| wget -q --no-check-certificate https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/019/048/245/GCF_019048245.1_ASM1904824v1/GCF_019048245.1_ASM1904824v1_genomic.fna.gz | |
| gunzip GCF_019048245.1_ASM1904824v1_genomic.fna.gz | |
| mash screen -p 4 test.msh GCF_019048245.1_ASM1904824v1_genomic.fna | sort -gr | head | |
| - name: Upload to Zenodo | |
| if: env.SHOULD_PUBLISH == 'true' | |
| env: | |
| ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} | |
| run: | | |
| # Start new version | |
| NEW_VER_JSON=$(curl -s -X POST "https://zenodo.org/api/deposit/depositions/${{ env.ZENODO_RECORD_ID }}/actions/newversion?access_token=$ZENODO_TOKEN") | |
| NEW_DRAFT_URL=$(echo $NEW_VER_JSON | jq -r '.links.latest_draft') | |
| NEW_DEP_ID=$(echo $NEW_DRAFT_URL | awk -F/ '{print $NF}') | |
| # Get draft details and bucket URL | |
| DRAFT_DETAILS=$(curl -s "$NEW_DRAFT_URL?access_token=$ZENODO_TOKEN") | |
| BUCKET_URL=$(echo $DRAFT_DETAILS | jq -r '.links.bucket') | |
| # Clean up the draft: Delete the old versioned file cloned from the previous release | |
| # Keep the 'latest' file for now as it will be overwritten | |
| echo "$DRAFT_DETAILS" | jq -r '.files[] | select(.filename | contains("latest") | not) | .links.self' | while read file_url; do | |
| curl -s -X DELETE "$file_url?access_token=$ZENODO_TOKEN" | |
| done | |
| # Upload BOTH the versioned file and the 'latest' pointer | |
| curl -S -o /dev/null -T "data/${{ env.VERSIONED_FILENAME }}" "$BUCKET_URL/${{ env.VERSIONED_FILENAME }}?access_token=$ZENODO_TOKEN" | |
| curl -S -o /dev/null -T "${{ env.LATEST_FILENAME }}" "$BUCKET_URL/RefSeqSketches_latest.msh.gz?access_token=$ZENODO_TOKEN" | |
| # Update Metadata | |
| curl -s -X PUT "$NEW_DRAFT_URL?access_token=$ZENODO_TOKEN" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"metadata\": { | |
| \"title\": \"RefSeq Prokaryotic Mash Reference (v${{ env.VERSION }})\", | |
| \"version\": \"${{ env.VERSION }}\", | |
| \"upload_type\": \"dataset\", | |
| \"description\": \"Automated Mash sketch of RefSeq prokaryotic representative genomes. Methods and source code at https://github.com/erinyoung/update_mash_dist. Version: ${{ env.VERSION }}\", | |
| \"creators\": [ | |
| { | |
| \"name\": \"Young, Erin\", | |
| \"orcid\": \"0000-0002-7535-006X\", | |
| \"affiliation\": \"Unified Public Health Laboratory, Department of Health and Human Services, State of Utah\" | |
| } | |
| ] | |
| } | |
| }" | |
| # Publish to ZENODO | |
| #curl -s -X POST "https://zenodo.org/api/deposit/depositions/$NEW_DEP_ID/actions/publish?access_token=$ZENODO_TOKEN" | |
| - name: Setup Git LFS | |
| run: | | |
| git lfs install | |
| # 1. Remove the old files from the Git Index only (--cached) | |
| # This stops LFS from looking for files that no longer exist on disk. | |
| git rm --cached data/*.msh.gz || true | |
| git rm --cached *.msh.gz || true | |
| # 2. Now track the patterns | |
| git lfs track "data/*.msh.gz" | |
| git lfs track "*.msh.gz" | |
| # 3. Add everything back | |
| # This stages the .gitattributes, the deletions, and the new files | |
| git add .gitattributes | |
| git add data/ | |
| git add *.msh.gz | |
| - name: Create Pull Request | |
| uses: peter-evans/create-pull-request@main | |
| with: | |
| commit-message: "Update mash reference to version ${{ env.VERSION }}" | |
| title: "Update Master Reference Sketch (v${{ env.VERSION }})" | |
| branch: "update-master-reference" | |
| base: main | |
| body: | | |
| Combined 100 genomic sketch chunks from Step 2. | |
| - RefSeq Version: ${{ env.VERSION }} | |
| - Zenodo Update: Completed (RefSeqSketches_latest.msh.gz) | |
| - Old version file removed from /data |