diff --git a/.github/workflows/velox_nightly.yml b/.github/workflows/velox_nightly.yml index 5dd0afeb3c..cb6648c122 100644 --- a/.github/workflows/velox_nightly.yml +++ b/.github/workflows/velox_nightly.yml @@ -26,6 +26,7 @@ on: env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + MVN_CMD: 'build/mvn -ntp' CCACHE_DIR: "${{ github.workspace }}/.ccache" concurrency: @@ -179,6 +180,103 @@ jobs: path: package/target/gluten-velox-bundle-*.jar retention-days: 7 + spark-test-spark35-smj: + if: ${{ startsWith(github.repository, 'apache/') }} + needs: build-native-lib-x86 + runs-on: ubuntu-22.04 + env: + SPARK_TESTING: true + container: apache/gluten:centos-8-jdk8 + steps: + - uses: actions/checkout@v4 + - name: Download All Artifacts + uses: actions/download-artifact@v4 + with: + name: velox-native-lib-${{github.sha}} + path: ./cpp/build/ + - name: Prepare + run: | + dnf module -y install python39 && \ + alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install setuptools==77.0.3 && \ + pip3 install pyspark==3.5.5 cython && \ + pip3 install pandas==2.2.3 pyarrow==20.0.0 + - name: Build and Run unit test for Spark 3.5.5 (other tests) + run: | + cd $GITHUB_WORKSPACE/ + export SPARK_SCALA_VERSION=2.12 + yum install -y java-17-openjdk-devel + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + java -version + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ -Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-report + path: '**/surefire-reports/TEST-*.xml' + - name: Upload unit tests log files + if: ${{ !success() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-test-log + path: | + **/target/*.log + **/gluten-ut/**/hs_err_*.log + **/gluten-ut/**/core.* + + spark-test-spark35-slow-smj: + if: ${{ startsWith(github.repository, 'apache/') }} + needs: build-native-lib-x86 + runs-on: ubuntu-22.04 + env: + SPARK_TESTING: true + LANG: C.UTF-8 #TODO remove after image update + LC_ALL: C.UTF-8 + container: apache/gluten:centos-8-jdk8 + steps: + - uses: actions/checkout@v4 + - name: Download All Artifacts + uses: actions/download-artifact@v4 + with: + name: velox-native-lib-${{github.sha}} + path: ./cpp/build/ + - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image update + run: | + rm -rf /opt/shims/spark35 + bash .github/workflows/util/install-spark-resources.sh 3.5 + - name: Build and Run unit test for Spark 3.5.5 (slow tests) + run: | + cd $GITHUB_WORKSPACE/ + yum install -y java-17-openjdk-devel + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + java -version + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ -Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ -Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \ + -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-report + path: '**/surefire-reports/TEST-*.xml' + - name: Upload unit tests log files + if: ${{ !success() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-test-log + path: | + **/target/*.log + **/gluten-ut/**/hs_err_*.log + **/gluten-ut/**/core.* + #build and package for arm64 build-native-lib-centos-8-arm64: if: ${{ startsWith(github.repository, 'apache/') }}