diff --git a/LICENSE b/LICENSE index b4f98a86e37..0a6c7cb3ec3 100644 --- a/LICENSE +++ b/LICENSE @@ -212,15 +212,6 @@ ./backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/SparkWriteFilesCommitProtocol.scala ./cpp-ch/local-engine/Parser/aggregate_function_parser/BloomFilterAggParser.cpp ./gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenExplainUtils.scala - ./shims/spark32/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala - ./shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala - ./shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala - ./shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/WriteFiles.scala - ./shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala - ./shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala - ./shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala.deprecated - ./shims/spark32/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala - ./shims/spark32/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala ./shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala ./shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/WriteFiles.scala ./shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala @@ -228,12 +219,6 @@ ./tools/gluten-it/common/src/main/scala/org/apache/spark/sql/TestUtils.scala Delta Lake(https://github.com/delta-io/delta) - ./backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/DeltaLog.scala - ./backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/Snapshot.scala - ./backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala - ./backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala - ./backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala - ./backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala ./backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/DeltaLog.scala ./backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/Snapshot.scala ./backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala diff --git a/dev/bloop-test.sh b/dev/bloop-test.sh index dc09b8340a7..5e97c1d313d 100755 --- a/dev/bloop-test.sh +++ b/dev/bloop-test.sh @@ -77,7 +77,6 @@ declare -A MODULE_MAP=( # Shims modules ["shims/common"]="spark-sql-columnar-shims-common" - ["shims/spark32"]="spark-sql-columnar-shims-spark32" ["shims/spark33"]="spark-sql-columnar-shims-spark33" ["shims/spark34"]="spark-sql-columnar-shims-spark34" ["shims/spark35"]="spark-sql-columnar-shims-spark35" @@ -87,7 +86,6 @@ declare -A MODULE_MAP=( # Unit test modules ["gluten-ut/common"]="gluten-ut-common" ["gluten-ut/test"]="gluten-ut-test" - ["gluten-ut/spark32"]="gluten-ut-spark32" ["gluten-ut/spark33"]="gluten-ut-spark33" ["gluten-ut/spark34"]="gluten-ut-spark34" ["gluten-ut/spark35"]="gluten-ut-spark35" diff --git a/dev/run-scala-test.sh b/dev/run-scala-test.sh index 3c03032d469..b73831931c9 100755 --- a/dev/run-scala-test.sh +++ b/dev/run-scala-test.sh @@ -164,7 +164,6 @@ declare -A MODULE_MAP=( # Shims modules (Java only, no scala subdirectory) ["spark-sql-columnar-shims-common"]="shims/common:java" - ["spark-sql-columnar-shims-spark32"]="shims/spark32:java" ["spark-sql-columnar-shims-spark33"]="shims/spark33:java" ["spark-sql-columnar-shims-spark34"]="shims/spark34:java" ["spark-sql-columnar-shims-spark35"]="shims/spark35:java" @@ -174,7 +173,6 @@ declare -A MODULE_MAP=( # Unit test modules ["gluten-ut-common"]="gluten-ut/common:scala" ["gluten-ut-test"]="gluten-ut/test:scala" - ["gluten-ut-spark32"]="gluten-ut/spark32:scala" ["gluten-ut-spark33"]="gluten-ut/spark33:scala" ["gluten-ut-spark34"]="gluten-ut/spark34:scala" ["gluten-ut-spark35"]="gluten-ut/spark35:scala" diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index 9fc761940da..27620395cf5 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -428,7 +428,7 @@ The following failure may occur in GitHub CI or local tests: ```log - TPC-H q5 *** FAILED *** Mismatch for query 5 - Actual Plan path: /tmp/tpch-approved-plan/v2-bhj/spark322/5.txt - Golden Plan path: /opt/gluten/backends-velox/target/scala-2.12/test-classes/tpch-approved-plan/v2-bhj/spark322/5.txt (VeloxTPCHSuite.scala:101) + Actual Plan path: /tmp/tpch-approved-plan/v1-bhj/spark35/5.txt + Golden Plan path: /opt/gluten/backends-velox/target/scala-2.12/test-classes/tpch-approved-plan/v1-bhj/spark35/5.txt (VeloxTPCHSuite.scala) ``` To update the golden files, find the actual Gluten plans in GitHub CI Artifacts or the local `/tmp/` directory, then update the corresponding golden files in the `tpch-approved-plan/` directory. diff --git a/ep/build-clickhouse/src/resources/bin/gluten.sh b/ep/build-clickhouse/src/resources/bin/gluten.sh index 4962d952c1e..3ae65b115c0 100755 --- a/ep/build-clickhouse/src/resources/bin/gluten.sh +++ b/ep/build-clickhouse/src/resources/bin/gluten.sh @@ -36,9 +36,7 @@ function start() { DRIVER_OPTIONS="${DRIVER_OPTIONS} $(cat ${GLUTEN_HOME}/conf/gluten.properties | grep "^spark.driver.extraJavaOptions" | cut -d "=" -f 2)" GLUTEN_JARS= - if [ "${SPARK_MAJOR_MINOR_VERSION}" == "3.2" ]; then - GLUTEN_JARS=${GLUTEN_HOME}/jars/spark32/* - elif [ "${SPARK_MAJOR_MINOR_VERSION}" == "3.3" ]; then + if [ "${SPARK_MAJOR_MINOR_VERSION}" == "3.3" ]; then GLUTEN_JARS=${GLUTEN_HOME}/jars/spark33/* else echo "Unsupported spark version: ${SPARK_MAJOR_MINOR_VERSION}" diff --git a/package/pom.xml b/package/pom.xml index 4ab4a01c37f..55318f85cf6 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -205,7 +205,6 @@ target - *spark3.2* *spark3.3* *spark3.4* *spark3.5*