Step1 Create Reference #53
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Step1 Create Reference | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| list_reference_genomes: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check Out Repository | |
| uses: actions/checkout@main | |
| - name: Download Datasets Tools | |
| run: | | |
| wget -q https://ftp.ncbi.nlm.nih.gov/pub/datasets/command-line/v2/linux-amd64/datasets | |
| wget -q https://ftp.ncbi.nlm.nih.gov/pub/datasets/command-line/v2/linux-amd64/dataformat | |
| chmod +x datasets dataformat | |
| - name: Generate Master List (ids.txt) | |
| run: | | |
| ./datasets summary genome taxon 2 --reference --as-json-lines | \ | |
| ./dataformat tsv genome --fields accession,organism-name --elide-header | \ | |
| sed 's/\[//g; s/\]//g; s/["'\'']//g; s/endosymbiont of /endosymbiont_of_/g' > ids.txt | |
| if [ -f data/include_anyway.txt ]; then | |
| awk '{print $1 "\t" $2 " " $3 }' data/include_anyway.txt >> ids.txt | |
| fi | |
| grep ^"G" ids.txt > ids_clean.txt | |
| mv ids_clean.txt ids.txt | |
| - name: Split into 100 Chunks | |
| run: | | |
| split -n l/100 ids.txt -d -a 3 --additional-suffix=.txt x | |
| echo "Created $(ls x*.txt | wc -l) chunk files." | |
| - name: Upload Master List | |
| uses: actions/upload-artifact@main | |
| with: | |
| name: master-id-list | |
| path: ids.txt | |
| retention-days: 1 | |
| - name: Upload Chunks | |
| uses: actions/upload-artifact@main | |
| with: | |
| name: genome-chunks | |
| path: x*.txt | |
| retention-days: 7 | |
| update_repo_master_list: | |
| needs: list_reference_genomes | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check Out Repository | |
| uses: actions/checkout@main | |
| - name: Download Master List Artifact | |
| uses: actions/download-artifact@main | |
| with: | |
| name: master-id-list | |
| - name: Move to Data Folder | |
| run: | | |
| mkdir -p data | |
| mv ids.txt data/ids.txt | |
| - name: Create Pull Request | |
| uses: peter-evans/create-pull-request@main | |
| with: | |
| commit-message: "Update master prokaryotic reference list" | |
| title: "Update Master ID List" | |
| branch: "update-master-ids" | |
| add-paths: | | |
| data/ids.txt | |
| body: | | |
| Updated the master reference list. | |
| The 100 split chunks have been stored as workflow artifacts. |