Step2 Create Sketches #277
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Step2 Create Sketches | |
| on: | |
| workflow_run: | |
| workflows: ["Step1 Create Reference"] | |
| types: [completed] | |
| workflow_dispatch: | |
| inputs: | |
| run_id: | |
| description: 'Step 1 Run ID (Find this in the Step 1 URL or Summary)' | |
| required: false | |
| default: '' | |
| jobs: | |
| download_and_sketch: | |
| if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} | |
| permissions: | |
| actions: read | |
| contents: read | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| chunk: ["000", "001", "002", "003", "004", "005", "006", "007", "008", "009", "010", "011", "012", "013", "014", "015", "016", "017", "018", "019", "020", "021", "022", "023", "024", "025", "026", "027", "028", "029", "030", "031", "032", "033", "034", "035", "036", "037", "038", "039", "040", "041", "042", "043", "044", "045", "046", "047", "048", "049", "050", "051", "052", "053", "054", "055", "056", "057", "058", "059", "060", "061", "062", "063", "064", "065", "066", "067", "068", "069", "070", "071", "072", "073", "074", "075", "076", "077", "078", "079", "080", "081", "082", "083", "084", "085", "086", "087", "088", "089", "090", "091", "092", "093", "094", "095", "096", "097", "098", "099"] | |
| # comment above and uncomment below for testing | |
| # chunk: ["000", "001", "099"] | |
| steps: | |
| - name: Check Out Repository | |
| uses: actions/checkout@main | |
| - name: Download ID Chunks | |
| uses: actions/download-artifact@main | |
| with: | |
| name: genome-chunks | |
| path: chunks/ | |
| # Prioritizes manual input 'run_id', falls back to automated trigger ID | |
| run-id: ${{ inputs.run_id || github.event.workflow_run.id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install Dependencies | |
| run: | | |
| wget -q https://ftp.ncbi.nlm.nih.gov/pub/datasets/command-line/v2/linux-amd64/datasets | |
| wget -q https://github.com/marbl/Mash/releases/download/v2.3/mash-Linux64-v2.3.tar | |
| tar -xf mash-Linux64-v2.3.tar | |
| mv mash-Linux64-v2.3/mash . | |
| chmod +x datasets mash | |
| echo "$(pwd)" >> $GITHUB_PATH | |
| - name: Download Batch | |
| run: | | |
| CHUNK_FILE="chunks/x${{ matrix.chunk }}.txt" | |
| cut -f 1 $CHUNK_FILE > ids.txt | |
| echo "Working on chunk ${{ matrix.chunk }}" | |
| datasets download genome accession --no-progressbar --inputfile ids.txt --dehydrated --filename genomes.zip | |
| unzip -n genomes.zip -d genomes | |
| # dehydrate and rehydrate is more stable | |
| datasets rehydrate --no-progressbar --directory genomes | |
| - name: Sketching Loop | |
| run: | | |
| ORIG_WORKSPACE=/home/runner/work/update_mash_dist/update_mash_dist | |
| DATASET_WORKSPACE=$(pwd)/tmp_datasets_workspace | |
| CHUNK_FILE="chunks/x${{ matrix.chunk }}.txt" | |
| cd $ORIG_WORKSPACE | |
| mkdir -p tmp_mash_workspace tmp_datasets_workspace | |
| # getting absolute paths for mash files | |
| NEW_MASH=$ORIG_WORKSPACE/new | |
| TEMP_MASH=$ORIG_WORKSPACE/temp | |
| FINAL_MASH=$ORIG_WORKSPACE/${{ matrix.chunk }} | |
| while read line | |
| do | |
| cd $ORIG_WORKSPACE | |
| rm -rf $DATASET_WORKSPACE/* | |
| id=$(echo "$line" | awk '{print $1}') | |
| ge=$(echo "$line" | awk '{print $2}') | |
| sp=$(echo "$line" | awk '{print $3}') | |
| # getting values for line | |
| echo "Looking for reference for" | |
| echo "Genome accession: $id" | |
| echo "Genus: $ge" | |
| echo "Species: $sp" | |
| # getting approximate line number | |
| wc -l $CHUNK_FILE | |
| grep -n $id $CHUNK_FILE | |
| GENOME_PATH="" | |
| # first looking to see if it downloaded in the batch (common) | |
| if [ -d "genomes/ncbi_dataset/data/${id}" ] | |
| then | |
| GENOME_PATH=$(find genomes/ncbi_dataset/data/${id} -name "*_genomic.fna" | head -n 1) | |
| fi | |
| # if didn't download as batch, try again as individual | |
| if [ ! -n "$GENOME_PATH" ] | |
| then | |
| cd $DATASET_WORKSPACE | |
| rm -rf $DATASET_WORKSPACE/* | |
| MAX_RETRIES=10 | |
| attempt=0 | |
| success=false | |
| echo "Accession $id was not found. Re-attempting download." | |
| until [ $attempt -ge $MAX_RETRIES ] | |
| do | |
| echo "Downloading attempt number $attempt" | |
| datasets download genome accession "${id}" --no-progressbar --no-progressbar --filename ncbi_dataset.zip < /dev/null | |
| if [ -f ncbi_dataset.zip ] | |
| then | |
| unzip ncbi_dataset.zip | |
| GENOME_PATH=$(find ncbi_dataset/data -name "*_genomic.fna" | head -n 1) | |
| fi | |
| if [ -n "$GENOME_PATH" ] | |
| then | |
| success=true | |
| break | |
| else | |
| rm -rf $DATASET_WORKSPACE/* | |
| attempt=$((attempt+1)) | |
| sleep $((attempt * 10))s | |
| fi | |
| done | |
| fi | |
| if [ -n "$GENOME_PATH" ] | |
| then | |
| if [ ! -f "$FINAL_MASH.msh" ]; then | |
| mash sketch "$GENOME_PATH" -o $FINAL_MASH -I "${ge}_${sp}_${id}" | |
| else | |
| mash sketch "$GENOME_PATH" -o $NEW_MASH -I "${ge}_${sp}_${id}" | |
| mash paste $TEMP_MASH $FINAL_MASH.msh $NEW_MASH.msh | |
| mv $TEMP_MASH.msh $FINAL_MASH.msh | |
| rm $NEW_MASH.msh | |
| fi | |
| else | |
| echo "Could not download the genome for $id" | |
| echo "FAILURE" | |
| exit 1 | |
| fi | |
| done < $CHUNK_FILE | |
| - name: Verify Accession Integrity | |
| run: | | |
| mash info -t ${{ matrix.chunk }}.msh | cut -f 3 | grep G | rev | cut -f 1-2 -d _ | rev | sort | uniq > final_accessions.txt || true | |
| cat ids.txt | grep -vf final_accessions.txt > missing_accessions.txt || true | |
| head *.txt || true | |
| ORIGINAL_COUNT=$(wc -l < ids.txt) | |
| MASH_COUNT=$(wc -l < final_accessions.txt) | |
| MISSING_COUNT=$(wc -l < missing_accessions.txt) | |
| echo "- **Total Accessions Expected:** $ORIGINAL_COUNT" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Total Accessions Sketched:** $MASH_COUNT" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Missing Accessions:** $MISSING_COUNT" >> $GITHUB_STEP_SUMMARY | |
| if [ "$MISSING_COUNT" -gt 0 ]; then | |
| echo "#### Missing Accession Samples (First 50)" >> $GITHUB_STEP_SUMMARY | |
| echo '```text' >> $GITHUB_STEP_SUMMARY | |
| head -n 50 missing_accessions.txt >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Smoke Test | |
| run: | | |
| wget -q --no-check-certificate https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/019/048/245/GCF_019048245.1_ASM1904824v1/GCF_019048245.1_ASM1904824v1_genomic.fna.gz | |
| gunzip GCF_019048245.1_ASM1904824v1_genomic.fna.gz | |
| mash screen -p 4 ${{ matrix.chunk }}.msh GCF_019048245.1_ASM1904824v1_genomic.fna | sort -gr | head | |
| - name: Upload Chunk Result | |
| uses: actions/upload-artifact@main | |
| with: | |
| name: msh-results-${{ matrix.chunk }} | |
| path: "${{ matrix.chunk }}.msh" | |
| retention-days: 7 |