From 3c11c8c4abaf10da3dff058271b434a5585dc643 Mon Sep 17 00:00:00 2001 From: zml1206 Date: Fri, 17 Jul 2026 15:00:44 +0800 Subject: [PATCH 1/3] [GLUTEN-11550][VL] Disable GlutenExplainSuite in Spark 4.x --- .../utils/velox/VeloxTestSettings.scala | 15 ++------- .../utils/velox/VeloxTestSettings.scala | 32 ++----------------- 2 files changed, 6 insertions(+), 41 deletions(-) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 39004877ff..a5ce9897e7 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -666,8 +666,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenFileMetadataStructSuite] enableSuite[GlutenParquetV1AggregatePushDownSuite] enableSuite[GlutenParquetV2AggregatePushDownSuite] - // TODO: Timestamp columns stats will lost if using int64 in parquet writer. - .exclude("aggregate push down - different data types") enableSuite[GlutenOrcV1AggregatePushDownSuite] .exclude("nested column: Count(nested sub-field) not push down") enableSuite[GlutenOrcV2AggregatePushDownSuite] @@ -895,7 +893,9 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDataFrameTransposeSuite] enableSuite[GlutenDefaultANSIValueSuite] enableSuite[GlutenDeprecatedDatasetAggregatorSuite] - // TODO: 4.x enableSuite[GlutenExplainSuite] // 1 failure + disableSuite[GlutenExplainSuite]( + "Validates Spark-specific physical plan and JVM codegen output that does not apply to " + + "Gluten's native execution plan") enableSuite[GlutenICUCollationsMapSuite] enableSuite[GlutenInlineTableParsingImprovementsSuite] enableSuite[GlutenJoinHintSuite] @@ -937,7 +937,6 @@ class VeloxTestSettings extends BackendTestSettings { // TODO: Velox parquet writer marks all struct fields as OPTIONAL (nullable), // but Spark's variant type requires REQUIRED fields. Needs Velox-side fix. .exclude("SPARK-47546: invalid variant binary") - .exclude("SPARK-47546: valid variant binary") enableSuite[GlutenVariantWriteShreddingSuite] enableSuite[GlutenXmlFunctionsSuite] enableSuite[GlutenApproxCountDistinctForIntervalsQuerySuite] @@ -1166,11 +1165,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenXPathFunctionsSuite] enableSuite[GlutenFallbackSuite] enableSuite[GlutenHashAggregationQuerySuite] - // TODO: fix on https://github.com/apache/gluten/issues/11919 - .exclude("udaf with all data types") enableSuite[GlutenHashAggregationQueryWithControlledFallbackSuite] - // TODO: fix on https://github.com/apache/gluten/issues/11919 - .exclude("udaf with all data types") enableSuite[GlutenHiveCommandSuite] enableSuite[GlutenHiveDDLSuite] enableSuite[GlutenHiveExplainSuite] @@ -1271,10 +1266,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("Dependent Batched Python UDFs and Scalar Pandas UDFs should not be combined") .exclude("Python UDF should not break column pruning/filter pushdown -- Parquet V2") enableSuite[GlutenStreamingQuerySuite] - // requires test resources that don't exist in Gluten repo - .exclude("detect escaped path and report the migration guide") - .exclude("ignore the escaped path check when the flag is off") - .excludeByPrefix("SPARK-51187") enableSuite[GlutenQueryExecutionSuite] // Rewritten to set root logger level to INFO so that logs can be parsed .exclude("Logging plan changes for execution") diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index f62a5dd0f4..5fe9594ab2 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -646,8 +646,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenFileMetadataStructSuite] enableSuite[GlutenParquetV1AggregatePushDownSuite] enableSuite[GlutenParquetV2AggregatePushDownSuite] - // TODO: Timestamp columns stats will lost if using int64 in parquet writer. - .exclude("aggregate push down - different data types") enableSuite[GlutenOrcV1AggregatePushDownSuite] .exclude("nested column: Count(nested sub-field) not push down") enableSuite[GlutenOrcV2AggregatePushDownSuite] @@ -875,7 +873,9 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDataFrameTableValuedFunctionsSuite] enableSuite[GlutenDataFrameTransposeSuite] enableSuite[GlutenDeprecatedDatasetAggregatorSuite] - // TODO: 4.x enableSuite[GlutenExplainSuite] // 1 failure + disableSuite[GlutenExplainSuite]( + "Validates Spark-specific physical plan and JVM codegen output that does not apply to " + + "Gluten's native execution plan") enableSuite[GlutenICUCollationsMapSuite] enableSuite[GlutenInlineTableParsingImprovementsSuite] enableSuite[GlutenJoinHintSuite] @@ -917,7 +917,6 @@ class VeloxTestSettings extends BackendTestSettings { // TODO: Velox parquet writer marks all struct fields as OPTIONAL (nullable), // but Spark's variant type requires REQUIRED fields. Needs Velox-side fix. .exclude("SPARK-47546: invalid variant binary") - .exclude("SPARK-47546: valid variant binary") enableSuite[GlutenVariantWriteShreddingSuite] enableSuite[GlutenXmlFunctionsSuite] enableSuite[GlutenApproxCountDistinctForIntervalsQuerySuite] @@ -1162,11 +1161,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenFallbackSuite] enableSuite[GlutenRowBasedChecksumSuite] enableSuite[GlutenHashAggregationQuerySuite] - // TODO: fix on https://github.com/apache/gluten/issues/11919 - .exclude("udaf with all data types") enableSuite[GlutenHashAggregationQueryWithControlledFallbackSuite] - // TODO: fix on https://github.com/apache/gluten/issues/11919 - .exclude("udaf with all data types") enableSuite[GlutenHiveCommandSuite] enableSuite[GlutenHiveDDLSuite] enableSuite[GlutenHiveExplainSuite] @@ -1270,30 +1265,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("Dependent Batched Python UDFs and Scalar Pandas UDFs should not be combined") .exclude("Python UDF should not break column pruning/filter pushdown -- Parquet V2") enableSuite[GlutenStreamingQuerySuite] - // requires test resources that don't exist in Gluten repo - .exclude("detect escaped path and report the migration guide") - .exclude("ignore the escaped path check when the flag is off") - .excludeByPrefix("SPARK-51187") - // TODO: fix on Spark-4.1 introduced by https://github.com/apache/spark/pull/52645 - .exclude("SPARK-53942: changing the number of stateless shuffle partitions via config") - .exclude("SPARK-53942: stateful shuffle partitions are retained from old checkpoint") enableSuite[GlutenStreamRealTimeModeAllowlistSuite] - // TODO: fix on Spark-4.1 see https://github.com/apache/spark/pull/52891 - .exclude("rtm operator allowlist") - .exclude("repartition not allowed") - .exclude("stateful queries not allowed") enableSuite[GlutenStreamRealTimeModeE2ESuite] - // TODO: fix on Spark-4.1 see https://github.com/apache/spark/pull/52870 - .exclude("foreach") - .exclude("union - foreach") - .exclude("mapPartitions") - .exclude("union - mapPartitions") - .exclude("stream static join") - .exclude("to_json and from_json round-trip") - .exclude("generateExec passthrough") enableSuite[GlutenStreamRealTimeModeSuite] - // TODO: fix on Spark-4.1 see https://github.com/apache/spark/pull/52473 - .exclude("processAllAvailable") enableSuite[GlutenQueryExecutionSuite] // Rewritten to set root logger level to INFO so that logs can be parsed .exclude("Logging plan changes for execution") From 5f97c9ed556498de97ed2959ce1ed3697b6f3ece Mon Sep 17 00:00:00 2001 From: zml1206 Date: Mon, 20 Jul 2026 10:56:52 +0800 Subject: [PATCH 2/3] fix --- .../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 1 + .../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 1 + 2 files changed, 2 insertions(+) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index a5ce9897e7..da89889199 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -937,6 +937,7 @@ class VeloxTestSettings extends BackendTestSettings { // TODO: Velox parquet writer marks all struct fields as OPTIONAL (nullable), // but Spark's variant type requires REQUIRED fields. Needs Velox-side fix. .exclude("SPARK-47546: invalid variant binary") + .exclude("SPARK-47546: valid variant binary") enableSuite[GlutenVariantWriteShreddingSuite] enableSuite[GlutenXmlFunctionsSuite] enableSuite[GlutenApproxCountDistinctForIntervalsQuerySuite] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 5fe9594ab2..1d61126565 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -917,6 +917,7 @@ class VeloxTestSettings extends BackendTestSettings { // TODO: Velox parquet writer marks all struct fields as OPTIONAL (nullable), // but Spark's variant type requires REQUIRED fields. Needs Velox-side fix. .exclude("SPARK-47546: invalid variant binary") + .exclude("SPARK-47546: valid variant binary") enableSuite[GlutenVariantWriteShreddingSuite] enableSuite[GlutenXmlFunctionsSuite] enableSuite[GlutenApproxCountDistinctForIntervalsQuerySuite] From dec338cb740d003475be31b9ddb34014f8a23a68 Mon Sep 17 00:00:00 2001 From: zml1206 Date: Mon, 20 Jul 2026 12:11:57 +0800 Subject: [PATCH 3/3] fix --- .../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 2 +- .../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index da89889199..6ad34180c2 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -894,7 +894,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDefaultANSIValueSuite] enableSuite[GlutenDeprecatedDatasetAggregatorSuite] disableSuite[GlutenExplainSuite]( - "Validates Spark-specific physical plan and JVM codegen output that does not apply to " + + "Validates Spark-specific physical plans and JVM codegen output, neither of which applies to " + "Gluten's native execution plan") enableSuite[GlutenICUCollationsMapSuite] enableSuite[GlutenInlineTableParsingImprovementsSuite] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 1d61126565..0b4a2041fa 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -874,7 +874,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDataFrameTransposeSuite] enableSuite[GlutenDeprecatedDatasetAggregatorSuite] disableSuite[GlutenExplainSuite]( - "Validates Spark-specific physical plan and JVM codegen output that does not apply to " + + "Validates Spark-specific physical plans and JVM codegen output, neither of which applies to " + "Gluten's native execution plan") enableSuite[GlutenICUCollationsMapSuite] enableSuite[GlutenInlineTableParsingImprovementsSuite]