Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.DataRow
import org.jetbrains.kotlinx.dataframe.annotations.AccessApiOverload
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.annotations.DisableInterpretation
import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
import org.jetbrains.kotlinx.dataframe.annotations.Refine
import org.jetbrains.kotlinx.dataframe.columns.BaseColumn
Expand All @@ -31,6 +32,13 @@ public inline fun <reified T> Iterable<T>.toDataFrame(): DataFrame<T> =
properties()
}

@JvmName("dataRowSchemaIterableToDataFrame")
public inline fun <reified T : DataRowSchema> Iterable<T>.toDataFrame(): DataFrame<T> =
@DisableInterpretation
toDataFrame {
properties()
}

@Refine
@Interpretable("toDataFrameDsl")
public inline fun <reified T> Iterable<T>.toDataFrame(noinline body: CreateDataFrameDsl<T>.() -> Unit): DataFrame<T> =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package org.jetbrains.kotlinx.dataframe.examples.kotlinSpark
import org.apache.spark.sql.Dataset
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.api.aggregate
import org.jetbrains.kotlinx.dataframe.api.cast
import org.jetbrains.kotlinx.dataframe.api.groupBy
import org.jetbrains.kotlinx.dataframe.api.max
import org.jetbrains.kotlinx.dataframe.api.mean
Expand All @@ -13,7 +14,7 @@ import org.jetbrains.kotlinx.dataframe.api.print
import org.jetbrains.kotlinx.dataframe.api.schema
import org.jetbrains.kotlinx.dataframe.api.std
import org.jetbrains.kotlinx.dataframe.api.toDataFrame
import org.jetbrains.kotlinx.dataframe.api.toListOf
import org.jetbrains.kotlinx.dataframe.api.toList
import org.jetbrains.kotlinx.spark.api.withSpark

/**
Expand Down Expand Up @@ -45,6 +46,8 @@ fun main() = withSpark {

// and convert it to DataFrame via a typed List
val dataframe = dataset.collectAsList().toDataFrame()
// Due to #1908, we need to remove the platform type (`!`) by casting
.cast<Person>()
dataframe.schema().print()
dataframe.print(columnTypes = true, borders = true)

Expand All @@ -59,10 +62,11 @@ fun main() = withSpark {

ageStats.print(columnTypes = true, borders = true)

// and when we want to convert a DataFrame back to Spark, we can do the same trick via a typed List
// Using the compiler plugin, it's important to specify the target data class explicitly!
// The local compiler-plugin type is not a data class that can be instantiated.
val sparkDatasetAgain = dataframe.toListOf<Person>().toDS()
// When we want to convert a DataFrame back to Spark, we can do the same trick via a typed List.
// Using the compiler plugin, it's important to check the subtype of the dataframe is a
// data class that can be instantiated. In this case, `dataframe: DataFrame<Person>`,
// so `toList()` will work.
val sparkDatasetAgain = dataframe.toList().toDS()
sparkDatasetAgain.printSchema()
sparkDatasetAgain.show()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,11 @@ fun main() {
/** Creates a [bean encoder][Encoders.bean] for the given [T] instance. */
inline fun <reified T : Serializable> beanEncoderOf(): Encoder<T> = Encoders.bean(T::class.java)

@DataSchema
data class Name
@JvmOverloads
constructor(var firstName: String = "", var lastName: String = "") : Serializable

// The @DataSchema annotation is optional for this specific example, but is generally recommended
@DataSchema
// The @DataSchema annotation is optional
data class Person
@JvmOverloads
constructor(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package org.jetbrains.kotlinx.dataframe.tests

import io.kotest.assertions.throwables.shouldThrow
import io.kotest.matchers.shouldBe
import io.kotest.matchers.string.shouldContain
import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.annotations.ColumnName
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.api.toDataFrame
import org.jetbrains.kotlinx.dataframe.api.toList
import org.jetbrains.kotlinx.dataframe.api.toListOf
import org.junit.Test

class ToDataFrameTests {

data class SimplePerson(val name: String, val age: Int)

@DataSchema
data class DataSchemaPerson(val name: String, val age: Int)

// Test case for #1880
@Test
fun `simple Iterable to DataFrame`() {
val people = listOf(
SimplePerson("John", 25),
SimplePerson("Jane", 30),
)

// DataFrame<SimplePerson_XX>
val df = people.toDataFrame()
df.name
df.age

shouldThrow<IllegalArgumentException> { df.toList() }
.message shouldContain "is not a data class. `toList` is supported only for data classes."

df.toListOf<SimplePerson>() shouldBe people
}

// Test case for #1880
@Test
fun `DataSchema Iterable to DataFrame`() {
val people = listOf(
DataSchemaPerson("John", 25),
DataSchemaPerson("Jane", 30),
)

// DataFrame<DataSchemaPerson>
val df: DataFrame<DataSchemaPerson> = people.toDataFrame()
df.name
df.age

df.toList() shouldBe people
}
}
Loading