馃 updated transfermarkt-scraper raw data
#750
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: | |
| branches: [master] | |
| jobs: | |
| build: | |
| if: github.event.pull_request.head.repo.fork == false | |
| concurrency: build_queue | |
| runs-on: ubuntu-latest | |
| container: | |
| image: dcaribou/transfermarkt-datasets:linux-amd64-master | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: Run tests | |
| run: | | |
| set -e | |
| just --set dbt_target prod dvc_pull test prepare_local | |
| - name: Export clean DuckDB file | |
| if: github.ref == 'refs/heads/master' && github.event_name == 'push' | |
| run: | | |
| python scripts/synching/export-duckdb.py --schema prod --commit $GITHUB_SHA | |
| - name: Upload prepared data to R2 | |
| if: github.ref == 'refs/heads/master' && github.event_name == 'push' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| run: | | |
| python scripts/synching/sync-r2.py | |
| build-image: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: Get changed files | |
| id: changed-files | |
| uses: tj-actions/changed-files@v40 | |
| with: | |
| files: | | |
| data/**/*.dvc | |
| - uses: extractions/setup-just@v2 | |
| - name: image build | |
| env: | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| # only run the build if the changed files are not only dvc files | |
| if: ${{ steps.changed-files.outputs.only_changed == 'false' }} | |
| run: | | |
| # set image tag accordingly | |
| REF=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }') | |
| if [ $REF = master ]; | |
| then | |
| TAG=master | |
| else | |
| TAG=dev | |
| fi | |
| # build and push | |
| just --set tag "$TAG" --set platform "linux/amd64" docker_build docker_push_dockerhub |