From 7f00900332f53a36602a344f8e21d9adda7bf7be Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Mon, 22 Jun 2026 21:02:03 +0200 Subject: [PATCH 1/2] Added Iterable.toDataFrame() overload for DataRowSchema types, compiler plugin test in :samples, Issue #1880 --- .../kotlinx/dataframe/api/toDataFrame.kt | 8 +++ .../kotlinx/dataframe/tests/toDataFrame.kt | 55 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/tests/toDataFrame.kt diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toDataFrame.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toDataFrame.kt index f1d35d3b8f..fa129df0d1 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toDataFrame.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toDataFrame.kt @@ -6,6 +6,7 @@ import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.DataRow import org.jetbrains.kotlinx.dataframe.annotations.AccessApiOverload import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +import org.jetbrains.kotlinx.dataframe.annotations.DisableInterpretation import org.jetbrains.kotlinx.dataframe.annotations.Interpretable import org.jetbrains.kotlinx.dataframe.annotations.Refine import org.jetbrains.kotlinx.dataframe.columns.BaseColumn @@ -31,6 +32,13 @@ public inline fun Iterable.toDataFrame(): DataFrame = properties() } +@JvmName("dataRowSchemaIterableToDataFrame") +public inline fun Iterable.toDataFrame(): DataFrame = + @DisableInterpretation + toDataFrame { + properties() + } + @Refine @Interpretable("toDataFrameDsl") public inline fun Iterable.toDataFrame(noinline body: CreateDataFrameDsl.() -> Unit): DataFrame = diff --git a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/tests/toDataFrame.kt b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/tests/toDataFrame.kt new file mode 100644 index 0000000000..eb76f1c16d --- /dev/null +++ b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/tests/toDataFrame.kt @@ -0,0 +1,55 @@ +package org.jetbrains.kotlinx.dataframe.tests + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.annotations.ColumnName +import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +import org.jetbrains.kotlinx.dataframe.api.toDataFrame +import org.jetbrains.kotlinx.dataframe.api.toList +import org.jetbrains.kotlinx.dataframe.api.toListOf +import org.junit.Test + +class ToDataFrameTests { + + data class SimplePerson(val name: String, val age: Int) + + @DataSchema + data class DataSchemaPerson(val name: String, val age: Int) + + // Test case for #1880 + @Test + fun `simple Iterable to DataFrame`() { + val people = listOf( + SimplePerson("John", 25), + SimplePerson("Jane", 30), + ) + + // DataFrame + val df = people.toDataFrame() + df.name + df.age + + shouldThrow { df.toList() } + .message shouldContain "is not a data class. `toList` is supported only for data classes." + + df.toListOf() shouldBe people + } + + // Test case for #1880 + @Test + fun `DataSchema Iterable to DataFrame`() { + val people = listOf( + DataSchemaPerson("John", 25), + DataSchemaPerson("Jane", 30), + ) + + // DataFrame + val df: DataFrame = people.toDataFrame() + df.name + df.age + + df.toList() shouldBe people + } +} From 1d738644e72eca707f3289b2f1d232ca40f89132 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Tue, 23 Jun 2026 13:37:18 +0200 Subject: [PATCH 2/2] updated kotlin-spark dev example for #1880, taking #1908 into account --- .../dataframe/examples/kotlinSpark/typedDataset.kt | 14 +++++++++----- .../dataframe/examples/spark/typedDataset.kt | 4 +--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/projects/dev/kotlin-spark/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/kotlinSpark/typedDataset.kt b/examples/projects/dev/kotlin-spark/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/kotlinSpark/typedDataset.kt index 7d765549f9..a53c9fa5d9 100644 --- a/examples/projects/dev/kotlin-spark/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/kotlinSpark/typedDataset.kt +++ b/examples/projects/dev/kotlin-spark/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/kotlinSpark/typedDataset.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlinx.dataframe.examples.kotlinSpark import org.apache.spark.sql.Dataset import org.jetbrains.kotlinx.dataframe.annotations.DataSchema import org.jetbrains.kotlinx.dataframe.api.aggregate +import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.groupBy import org.jetbrains.kotlinx.dataframe.api.max import org.jetbrains.kotlinx.dataframe.api.mean @@ -13,7 +14,7 @@ import org.jetbrains.kotlinx.dataframe.api.print import org.jetbrains.kotlinx.dataframe.api.schema import org.jetbrains.kotlinx.dataframe.api.std import org.jetbrains.kotlinx.dataframe.api.toDataFrame -import org.jetbrains.kotlinx.dataframe.api.toListOf +import org.jetbrains.kotlinx.dataframe.api.toList import org.jetbrains.kotlinx.spark.api.withSpark /** @@ -45,6 +46,8 @@ fun main() = withSpark { // and convert it to DataFrame via a typed List val dataframe = dataset.collectAsList().toDataFrame() + // Due to #1908, we need to remove the platform type (`!`) by casting + .cast() dataframe.schema().print() dataframe.print(columnTypes = true, borders = true) @@ -59,10 +62,11 @@ fun main() = withSpark { ageStats.print(columnTypes = true, borders = true) - // and when we want to convert a DataFrame back to Spark, we can do the same trick via a typed List - // Using the compiler plugin, it's important to specify the target data class explicitly! - // The local compiler-plugin type is not a data class that can be instantiated. - val sparkDatasetAgain = dataframe.toListOf().toDS() + // When we want to convert a DataFrame back to Spark, we can do the same trick via a typed List. + // Using the compiler plugin, it's important to check the subtype of the dataframe is a + // data class that can be instantiated. In this case, `dataframe: DataFrame`, + // so `toList()` will work. + val sparkDatasetAgain = dataframe.toList().toDS() sparkDatasetAgain.printSchema() sparkDatasetAgain.show() } diff --git a/examples/projects/dev/kotlin-spark/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/spark/typedDataset.kt b/examples/projects/dev/kotlin-spark/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/spark/typedDataset.kt index cc82df8d1b..9f5e0f1938 100644 --- a/examples/projects/dev/kotlin-spark/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/spark/typedDataset.kt +++ b/examples/projects/dev/kotlin-spark/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/spark/typedDataset.kt @@ -90,13 +90,11 @@ fun main() { /** Creates a [bean encoder][Encoders.bean] for the given [T] instance. */ inline fun beanEncoderOf(): Encoder = Encoders.bean(T::class.java) -@DataSchema data class Name @JvmOverloads constructor(var firstName: String = "", var lastName: String = "") : Serializable -// The @DataSchema annotation is optional for this specific example, but is generally recommended -@DataSchema +// The @DataSchema annotation is optional data class Person @JvmOverloads constructor(