genome-nexus-annotation-pipeline/.circleci/config.yml at 5a7cbbaa3bd38d611abcc45f29c7543d57bdf2fc · genome-nexus/genome-nexus-annotation-pipeline · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
# Javascript Node CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-javascript/ for more details
#
version: 2
jobs:
  build:
    docker:
      # specify the version you desire here
      - image: cimg/openjdk:21.0.0-browsers

      # Specify service dependencies here if necessary
      # CircleCI maintains a library of pre-built images
      # documented at https://circleci.com/docs/2.0/circleci-images/
      # - image: circleci/mongo:3.4.4

    working_directory: ~/repo

    steps:
      - checkout

      # - run:
      #     name: Update npm
      #     command: 'sudo npm install -g npm@3 && npm update -g'

      # Download and cache dependencies
      - restore_cache:
          keys:
          - v1-dependencies-{{ checksum "pom.xml" }}
          # fallback to using the latest cache if no exact match is found
          - v1-dependencies-

      - run:
          name: "compile"
          command: "mvn clean install"

      - save_cache:
          paths:
          - ~/.m2
          key: v1-dependencies-{{ checksum "pom.xml" }}

      # run tests
      # EXAMPLE:
#      - run:
#          name: "check if maf file still the same when annotating with uniprot transcripts"
#          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/data_mutations_extended_100.in.txt  --output-filename test/data/data_mutations_extended_100.out.uniprot.txt --isoform-override uniprot && git diff -G "^[^#]" --exit-code test/data/data_mutations_extended_100.out.uniprot.txt || (echo MAF uniprot output changed test/data/data_mutations_extended_100.out.uniprot.txt && exit 1)'
      # run test
      - run:
          name: "check if maf file still the same when annotating with uniprot transcripts"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/data_mutations_extended_100.in.txt  --output-filename test/data/data_mutations_extended_100.out.uniprot.txt --isoform-override uniprot && diff <(tail -n +2 test/data/data_mutations_extended_100.out.uniprot.txt) <(git show HEAD:test/data/data_mutations_extended_100.out.uniprot.txt | tail -n +2) || (echo MAF uniprot output changed test/data/data_mutations_extended_100.out.uniprot.txt && exit 1)'

      - run:
          name: "check if maf file still the same when annotating with mskcc transcripts"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/data_mutations_extended_100.in.txt  --output-filename test/data/data_mutations_extended_100.out.mskcc.txt --isoform-override mskcc && diff <(tail -n +2 test/data/data_mutations_extended_100.out.mskcc.txt) <(git show HEAD:test/data/data_mutations_extended_100.out.mskcc.txt | tail -n +2) || (echo MAF mskcc output changed test/data/data_mutations_extended_100.out.mskcc.txt && exit 1)'

      # run test
      - run:
          name: "check if minimal example maf file still the same when annotating with mskcc transcripts"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/minimal_example.in.txt  --output-filename test/data/minimal_example.out.mskcc.txt --isoform-override mskcc && diff <(tail -n +2 test/data/minimal_example.out.mskcc.txt) <(git show HEAD:test/data/minimal_example.out.mskcc.txt | tail -n +2) || (echo simple MAF mskcc output changed test/data/minimal_example.out.mskcc.txt && exit 1)'

      - run:
          name: "check if columns prefixed by IGNORE_Genome_Nexus_Original are immutable"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/immutable_columns_test.in.txt  --output-filename test/data/immutable_columns_test.out.uniprot.txt --isoform-override uniprot && diff <(tail -n +2 test/data/immutable_columns_test.out.uniprot.txt) <(git show HEAD:test/data/immutable_columns_test.out.uniprot.txt | tail -n +2) || (echo simple MAF uniprot output changed test/data/immutable_columns_test.out.uniprot.txt && exit 1)'

      - run:
          name: "check if corner cases example maf file still the same when annotating with uniprot transcripts"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/corner_cases.two_tumor_seq_allele.in.txt  --output-filename test/data/corner_cases.two_tumor_seq_allele.out.uniprot.txt --isoform-override uniprot && diff <(tail -n +2 test/data/corner_cases.two_tumor_seq_allele.out.uniprot.txt) <(git show HEAD:test/data/corner_cases.two_tumor_seq_allele.out.uniprot.txt | tail -n +2) || (echo simple MAF uniprot output changed test/data/corner_cases.two_tumor_seq_allele.out.uniprot.txt && exit 1)'

      - run:
          name: "check if corner cases example maf file still the same when annotating with mskcc transcripts"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/corner_cases.two_tumor_seq_allele.in.txt  --output-filename test/data/corner_cases.two_tumor_seq_allele.out.mskcc.txt --isoform-override mskcc && diff <(tail -n +2 test/data/corner_cases.two_tumor_seq_allele.out.mskcc.txt) <(git show HEAD:test/data/corner_cases.two_tumor_seq_allele.out.mskcc.txt | tail -n +2) || (echo simple MAF mskcc output changed test/data/corner_cases.two_tumor_seq_allele.out.mskcc.txt && exit 1)'

      - run:
          name: "check if corner cases example maf file still the same when annotating with uniprot transcripts"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/corner_cases.in.txt  --output-filename test/data/corner_cases.out.uniprot.txt --isoform-override uniprot && diff <(tail -n +2 test/data/corner_cases.out.uniprot.txt) <(git show HEAD:test/data/corner_cases.out.uniprot.txt | tail -n +2) || (echo simple MAF uniprot output changed test/data/corner_cases.out.uniprot.txt && exit 1)'

      - run:
          name: "check if corner cases example maf file still the same when annotating with mskcc transcripts"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/corner_cases.in.txt  --output-filename test/data/corner_cases.out.mskcc.txt --isoform-override mskcc && diff <(tail -n +2 test/data/corner_cases.out.mskcc.txt) <(git show HEAD:test/data/corner_cases.out.mskcc.txt | tail -n +2) || (echo simple MAF mskcc output changed test/data/corner_cases.out.mskcc.txt && exit 1)'

      # - run:
      #     name: "Run vcf2maf test cases"
      #     command: 'sudo apt-get install make && ./test/scripts/vcf2maf_tests.sh'

      # run test
      - run:
          name: "check if maf file still the same when annotating with uniprot transcripts and POSTs"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/data_mutations_extended_100.in.txt  --output-filename test/data/data_mutations_extended_100.out.post.uniprot.txt --isoform-override uniprot && diff <(tail -n +2 test/data/data_mutations_extended_100.out.post.uniprot.txt) <(git show HEAD:test/data/data_mutations_extended_100.out.uniprot.txt | tail -n +2) || (echo MAF uniprot output changed test/data/data_mutations_extended_100.out.uniprot.txt)'

      - run:
          name: "check if maf file still the same when annotating with mskcc transcripts and POSTs"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/data_mutations_extended_100.in.txt  --output-filename test/data/data_mutations_extended_100.out.post.mskcc.txt --isoform-override mskcc && diff <(tail -n +2 test/data/data_mutations_extended_100.out.post.mskcc.txt) <(git show HEAD:test/data/data_mutations_extended_100.out.mskcc.txt | tail -n +2) || (echo MAF mskcc output changed test/data/data_mutations_extended_100.out.mskcc.txt)'

      - run:
          name: "check if minimal example maf file still the same when annotating with mskcc transcripts and POSTs"
          command: 'java -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/minimal_example.in.txt  --output-filename test/data/minimal_example.out.post.mskcc.txt --isoform-override mskcc && diff <(tail -n +2 test/data/minimal_example.out.post.mskcc.txt) <(git show HEAD:test/data/minimal_example.out.mskcc.txt | tail -n +2) || (echo simple MAF mskcc output changed test/data/minimal_example.out.mskcc.txt)'

      - run:
          name: "check if my_variant_info provides gnomad annotations"
          command: 'java -Dgenomenexus.enrichment_fields=annotation_summary,my_variant_info -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/my_variant_info_corner_cases.in.txt  --output-filename test/data/my_variant_info_corner_cases.out.uniprot.txt --isoform-override uniprot && diff <(tail -n +2 test/data/my_variant_info_corner_cases.out.uniprot.txt) <(git show HEAD:test/data/my_variant_info_corner_cases.out.uniprot.txt | tail -n +2) || (echo simple MAF uniprot output changed test/data/my_variant_info_corner_cases.out.uniprot.txt)'

      - run:
          name: "check if nucleotide_context provides Ref_Tri and Var_Tri columnsj"
          command: 'java -Dgenomenexus.enrichment_fields=annotation_summary,nucleotide_context -jar annotationPipeline/target/annotationPipeline-*.jar --filename test/data/data_mutations_extended_100.in.txt  --output-filename test/data/data_mutations_extended_100.out.uniprot.nucleotide_context.txt --isoform-override uniprot && diff <(tail -n +2 test/data/data_mutations_extended_100.out.uniprot.nucleotide_context.txt) <(git show HEAD:test/data/data_mutations_extended_100.out.uniprot.nucleotide_context.txt | tail -n +2) || (echo simple MAF uniprot output changed  test/data/data_mutations_extended_100.out.uniprot.nucleotide_context.txt)'

      - run:
          name: "performance benchmark: annotate CHOL TCGA GDC (~3764 variants) against grch38.genomenexus.org"
          command: |
            java -Dgenomenexus.base=https://grch38.genomenexus.org \
              -jar annotationPipeline/target/annotationPipeline-*.jar \
              --filename test/data/chol_tcga_gdc_data_mutations.txt \
              --output-filename test/data/chol_tcga_gdc_data_mutations.out.mskcc.txt \
              --isoform-override mskcc 2>&1 | tee test/data/benchmark.log

            # Parse stats into CSV
            LOG=test/data/benchmark.log
            # "records to annotate" is a log4j message that may not appear; fall back to counting input lines
            TOTAL_VARIANTS=$(grep -oP '\d+(?= records to annotate)' "$LOG" || echo "")
            if [ -z "$TOTAL_VARIANTS" ]; then
              TOTAL_VARIANTS=$(tail -n +2 test/data/chol_tcga_gdc_data_mutations.txt | grep -cv '^#' || echo "")
            fi
            RUNTIME=$(grep -oP '(?<=RUNTIME: )\d+' "$LOG" || echo "")
            ANNOTATION_TIME=$(grep -oP '(?<=ANNOTATION_TIME: )\d+' "$LOG" || echo "")
            AVG_RESPONSE=$(grep -oP '(?<=Average Response Time:  )[\d.]+' "$LOG" || echo "")
            TOTAL_RESPONSE=$(grep -oP '(?<=Total Response Time:  )\d+' "$LOG" || echo "")
            AMBIGUOUS=$(grep -oP '(?<=ambiguous SNP and INDEL allele changes:  )\d+' "$LOG" || echo "")
            FAILED=$(grep -oP '\d+(?= total failed annotations)' "$LOG" || echo "0")

            if [ -n "$TOTAL_VARIANTS" ] && [ -n "$ANNOTATION_TIME" ] && [ "$ANNOTATION_TIME" -gt 0 ]; then
              VARIANTS_PER_SEC=$(awk "BEGIN {printf \"%.1f\", $TOTAL_VARIANTS / $ANNOTATION_TIME}")
            else
              VARIANTS_PER_SEC=""
            fi

            CSV=test/data/benchmark_results.csv
            echo "dataset,server,total_variants,runtime_secs,annotation_time_secs,variants_per_sec,avg_response_secs,total_response_secs,ambiguous_records,failed_annotations" > "$CSV"
            echo "chol_tcga_gdc,grch38.genomenexus.org,$TOTAL_VARIANTS,$RUNTIME,$ANNOTATION_TIME,$VARIANTS_PER_SEC,$AVG_RESPONSE,$TOTAL_RESPONSE,$AMBIGUOUS,$FAILED" >> "$CSV"

            echo ""
            echo "=== Benchmark Results ==="
            cat "$CSV"

      - store_artifacts:
          path: test/data
          destination: /test-data-output

workflows:
  version: 2
  commit:
    jobs:
      - build
  nightly:
    triggers:
      - schedule:
          cron: "0 0 * * *"
          filters:
            branches:
              only:
                - master
    jobs:
      - build