Skip to content

Commit aa2bd1b

Browse files
JolanrensenJolanrensen
committed
tests for parsing json columns to check behavior still matches. Added parseToDataFrameReadSource parser option.
1 parent 0033315 commit aa2bd1b

11 files changed

Lines changed: 383 additions & 103 deletions

File tree

‎core/api/core.api‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2475,13 +2475,15 @@ public abstract interface class org/jetbrains/kotlinx/dataframe/api/GlobalParser
24752475
public abstract fun getNulls ()Ljava/util/Set;
24762476
public abstract fun getParseExperimentalInstant ()Z
24772477
public abstract fun getParseExperimentalUuid ()Z
2478+
public abstract fun getParseToDataFrameReadSource ()Z
24782479
public abstract fun getSkipTypes ()Ljava/util/Set;
24792480
public abstract fun getUseFastDoubleParser ()Z
24802481
public abstract fun resetToDefault ()V
24812482
public abstract fun setDateTimeLibrary (Lorg/jetbrains/kotlinx/dataframe/api/ParseDateTimeLibrary;)V
24822483
public abstract fun setLocale (Ljava/util/Locale;)V
24832484
public abstract fun setParseExperimentalInstant (Z)V
24842485
public abstract fun setParseExperimentalUuid (Z)V
2486+
public abstract fun setParseToDataFrameReadSource (Z)V
24852487
public abstract fun setUseFastDoubleParser (Z)V
24862488
}
24872489

@@ -3622,18 +3624,19 @@ public final class org/jetbrains/kotlinx/dataframe/api/ParserOptions {
36223624
public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
36233625
public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;)V
36243626
public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
3625-
public fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)V
3626-
public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
3627-
public fun <init> (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)V
3628-
public synthetic fun <init> (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
3629-
public final fun copy (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/lang/Iterable;Ljava/lang/Iterable;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
3630-
public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/lang/Iterable;Ljava/lang/Iterable;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
3627+
public fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)V
3628+
public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
3629+
public fun <init> (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)V
3630+
public synthetic fun <init> (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
3631+
public final fun copy (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/lang/Iterable;Ljava/lang/Iterable;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
3632+
public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/lang/Iterable;Ljava/lang/Iterable;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
36313633
public fun equals (Ljava/lang/Object;)Z
36323634
public final fun getDateTime ()Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;
36333635
public final fun getLocale ()Ljava/util/Locale;
36343636
public final fun getNullStrings ()Ljava/util/Set;
36353637
public final fun getParseExperimentalInstant ()Ljava/lang/Boolean;
36363638
public final fun getParseExperimentalUuid ()Ljava/lang/Boolean;
3639+
public final fun getParseToDataFrameReadSource ()Ljava/lang/Boolean;
36373640
public final fun getSkipTypes ()Ljava/util/Set;
36383641
public final fun getUseFastDoubleParser ()Ljava/lang/Boolean;
36393642
public fun hashCode ()I

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/ParserOptions.kt‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,8 @@ public interface GlobalParserOptions {
291291
* @see [addJavaDateTimePattern]
292292
*/
293293
public var dateTimeLibrary: ParseDateTimeLibrary?
294+
295+
public var parseToDataFrameReadSource: Boolean
294296
}
295297

296298
/** @include [GlobalParserOptions] */
@@ -410,6 +412,7 @@ public class ParserOptions(
410412
public val useFastDoubleParser: Boolean? = null,
411413
public val parseExperimentalUuid: Boolean? = null,
412414
public val parseExperimentalInstant: Boolean? = null,
415+
public val parseToDataFrameReadSource: Boolean? = null,
413416
) {
414417
public fun copy(
415418
locale: Locale? = this.locale,
@@ -419,6 +422,7 @@ public class ParserOptions(
419422
useFastDoubleParser: Boolean? = this.useFastDoubleParser,
420423
parseExperimentalUuid: Boolean? = this.parseExperimentalUuid,
421424
parseExperimentalInstant: Boolean? = this.parseExperimentalInstant,
425+
parseToDataFrameReadSource: Boolean? = this.parseToDataFrameReadSource,
422426
): ParserOptions =
423427
ParserOptions(
424428
locale = locale,
@@ -428,6 +432,7 @@ public class ParserOptions(
428432
useFastDoubleParser = useFastDoubleParser,
429433
parseExperimentalUuid = parseExperimentalUuid,
430434
parseExperimentalInstant = parseExperimentalInstant,
435+
parseToDataFrameReadSource = parseToDataFrameReadSource,
431436
)
432437

433438
override fun equals(other: Any?): Boolean {
@@ -439,6 +444,7 @@ public class ParserOptions(
439444
if (useFastDoubleParser != other.useFastDoubleParser) return false
440445
if (parseExperimentalUuid != other.parseExperimentalUuid) return false
441446
if (parseExperimentalInstant != other.parseExperimentalInstant) return false
447+
if (parseToDataFrameReadSource != other.parseToDataFrameReadSource) return false
442448
if (locale != other.locale) return false
443449
if (dateTime != other.dateTime) return false
444450
if (nullStrings != other.nullStrings) return false
@@ -451,6 +457,8 @@ public class ParserOptions(
451457
var result = useFastDoubleParser?.hashCode() ?: 0
452458
result = 31 * result + (parseExperimentalUuid?.hashCode() ?: 0)
453459
result = 31 * result + (parseExperimentalInstant?.hashCode() ?: 0)
460+
result = 31 * result + (parseToDataFrameReadSource?.hashCode() ?: 0)
461+
result = 31 * result + (parseExperimentalInstant?.hashCode() ?: 0)
454462
result = 31 * result + (locale?.hashCode() ?: 0)
455463
result = 31 * result + (dateTime?.hashCode() ?: 0)
456464
result = 31 * result + (nullStrings?.hashCode() ?: 0)
@@ -459,7 +467,7 @@ public class ParserOptions(
459467
}
460468

461469
override fun toString(): String =
462-
"ParserOptions(locale=$locale, dateTimeParserOptions=$dateTime, nullStrings=$nullStrings, skipTypes=$skipTypes, useFastDoubleParser=$useFastDoubleParser, parseExperimentalUuid=$parseExperimentalUuid, parseExperimentalInstant=$parseExperimentalInstant)"
470+
"ParserOptions(locale=$locale, dateTimeParserOptions=$dateTime, nullStrings=$nullStrings, skipTypes=$skipTypes, useFastDoubleParser=$useFastDoubleParser, parseExperimentalUuid=$parseExperimentalUuid, parseExperimentalInstant=$parseExperimentalInstant, parseToDataFrameReadSource=$parseToDataFrameReadSource)"
463471

464472
// region deprecated constructors
465473

@@ -518,6 +526,7 @@ public class ParserOptions(
518526
useFastDoubleParser: Boolean? = null,
519527
parseExperimentalUuid: Boolean? = null,
520528
parseExperimentalInstant: Boolean? = null,
529+
parseToDataFrameReadSource: Boolean? = null,
521530
) : this(
522531
locale = locale,
523532
dateTime = 0.run {
@@ -535,6 +544,7 @@ public class ParserOptions(
535544
useFastDoubleParser = useFastDoubleParser,
536545
parseExperimentalUuid = parseExperimentalUuid,
537546
parseExperimentalInstant = parseExperimentalInstant,
547+
parseToDataFrameReadSource = parseToDataFrameReadSource,
538548
)
539549
// endregion
540550
}

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/convert.kt‎

Lines changed: 36 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -196,10 +196,6 @@ internal inline fun <T> convert(crossinline converter: (T) -> Any?): TypeConvert
196196

197197
private enum class DummyEnum
198198

199-
private val dataFrameReadSourceSupportedClasses by lazy {
200-
dataFrameReadSourceByType.keys.map { it.jvmErasure }.toSet()
201-
}
202-
203199
@Suppress("UNCHECKED_CAST")
204200
internal fun createConverter(from: KType, to: KType, options: ParserOptions? = null): TypeConverter? {
205201
if (from.isMarkedNullable) {
@@ -209,50 +205,46 @@ internal fun createConverter(from: KType, to: KType, options: ParserOptions? = n
209205
val fromClass = from.jvmErasure
210206
val toClass = to.jvmErasure
211207

212-
// readSource-backed conversions handle target types with type arguments (e.g. `DataFrame<*>`,
213-
// `DataRow<*>`), so they must run before the generic-arguments early-exit below.
214-
if (dataFrameReadSourceByType.any { from.isSubtypeOf(it.key) }) {
215-
val readSources = dataFrameReadSourceByType.entries
216-
.first { from.isSubtypeOf(it.key) }.value
217-
218-
when (toClass) {
219-
DataFrame::class ->
220-
return convert<Any> { source ->
221-
DataFrame.readSource(
222-
source = source,
223-
type = from,
224-
options = null,
225-
formats = readSources,
226-
)
227-
}
228-
229-
DataRow::class ->
230-
return convert<Any> { source ->
231-
DataRow.readSource(
232-
source = source,
233-
type = from,
234-
options = null,
235-
formats = readSources,
236-
)
237-
}
238-
239-
DataFrameSchema::class ->
240-
return convert<Any> { source ->
241-
DataFrameSchema.readSource(
242-
source = source,
243-
type = from,
244-
options = null,
245-
formats = readSources,
246-
)
247-
}
248-
}
208+
// early exit when we encounter types with generics (except DataFrame and DataRow), which we don't support
209+
if (from.arguments.isNotEmpty() ||
210+
(to.arguments.isNotEmpty() && toClass !in setOf(DataFrame::class, DataRow::class))
211+
) {
212+
return null
249213
}
250214

251-
return when {
252-
from.arguments.isNotEmpty() || to.arguments.isNotEmpty() -> null
215+
val fromTypeInDfReadSources =
216+
dataFrameReadSourceByType.keys.any { from.isSubtypeOf(it) } || from == typeOf<String>()
253217

218+
return when {
254219
fromClass == toClass -> TypeConverterIdentity
255220

221+
fromTypeInDfReadSources && toClass == DataFrame::class ->
222+
convert<Any> { source ->
223+
DataFrame.readSource(
224+
source = source,
225+
type = from,
226+
options = null,
227+
)
228+
}
229+
230+
fromTypeInDfReadSources && toClass == DataRow::class ->
231+
convert<Any> { source ->
232+
DataRow.readSource(
233+
source = source,
234+
type = from,
235+
options = null,
236+
)
237+
}
238+
239+
fromTypeInDfReadSources && toClass == DataFrameSchema::class ->
240+
convert<Any> { source ->
241+
DataFrameSchema.readSource(
242+
source = source,
243+
type = from,
244+
options = null,
245+
)
246+
}
247+
256248
// kotlin.time.Duration is a value class,
257249
// so it must be handled before the generic toClass.isValue / fromClass.isValue branches.
258250
toClass == Duration::class -> when (fromClass) {
@@ -277,7 +269,7 @@ internal fun createConverter(from: KType, to: KType, options: ParserOptions? = n
277269
val underlyingType = constructor.parameters.single().type
278270
val converter = getConverter(from, underlyingType)
279271
?: throw TypeConverterNotFoundException(from, underlyingType, null)
280-
return convert<Any> {
272+
convert<Any> {
281273
val converted = converter(it)
282274
if (converted == null && !underlyingType.isMarkedNullable) {
283275
throw TypeConversionException(it, from, underlyingType, null)

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/parse.kt‎

Lines changed: 70 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import org.jetbrains.kotlinx.dataframe.api.isFrameColumn
3939
import org.jetbrains.kotlinx.dataframe.api.isSubtypeOf
4040
import org.jetbrains.kotlinx.dataframe.api.map
4141
import org.jetbrains.kotlinx.dataframe.api.parser
42+
import org.jetbrains.kotlinx.dataframe.api.singleOrNull
4243
import org.jetbrains.kotlinx.dataframe.columns.TypeSuggestion
4344
import org.jetbrains.kotlinx.dataframe.columns.size
4445
import org.jetbrains.kotlinx.dataframe.exceptions.TypeConversionException
@@ -50,7 +51,10 @@ import org.jetbrains.kotlinx.dataframe.impl.createStarProjectedType
5051
import org.jetbrains.kotlinx.dataframe.impl.io.FastDoubleParser
5152
import org.jetbrains.kotlinx.dataframe.impl.javaDurationCanParse
5253
import org.jetbrains.kotlinx.dataframe.impl.lazyMapOf
54+
import org.jetbrains.kotlinx.dataframe.io.DataFrameReadSource
5355
import org.jetbrains.kotlinx.dataframe.io.isUrl
56+
import org.jetbrains.kotlinx.dataframe.io.newSupportedFormats
57+
import org.jetbrains.kotlinx.dataframe.io.readSourceImpl
5458
import org.jetbrains.kotlinx.dataframe.values
5559
import java.math.BigDecimal
5660
import java.math.BigInteger
@@ -334,6 +338,8 @@ internal object Parsers : GlobalParserOptions {
334338

335339
override var dateTimeLibrary: ParseDateTimeLibrary? = null
336340

341+
override var parseToDataFrameReadSource by Delegates.notNull<Boolean>()
342+
337343
override fun resetToDefault() {
338344
customGlobalJavaFormatters.values.forEach { it.clear() }
339345
nullStrings.clear()
@@ -343,6 +349,8 @@ internal object Parsers : GlobalParserOptions {
343349
useFastDoubleParser = true
344350
parseExperimentalUuid = false
345351
parseExperimentalInstant = true
352+
// disabled by default, because it can be very heavy
353+
parseToDataFrameReadSource = false
346354
_locale = null
347355
dateTimeLibrary = null
348356
nullStrings.addAll(listOf("null", "NULL", "NA", "N/A"))
@@ -941,40 +949,76 @@ internal object Parsers : GlobalParserOptions {
941949
stringParser<BigInteger> { it.toBigIntegerOrNull() },
942950
// BigDecimal
943951
stringParser<BigDecimal> { it.toBigDecimalOrNull() },
944-
// JSON array as DataFrame<*>
945-
stringParser<AnyFrame>(catch = true) {
946-
val trimmed = it.trim()
947-
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
948-
if (readJsonStrAnyFrame == null) {
949-
logger.warn {
950-
"parse() encountered a string that looks like a JSON array, but the dataframe-json dependency was not detected. Skipping for now."
951-
}
952-
null
953-
} else {
954-
readJsonStrAnyFrame!!(trimmed)
952+
// Char
953+
stringParser<Char> { it.singleOrNull() },
954+
stringParserWithOptions<AnyRow> { options, isConverter ->
955+
if (options?.parseToDataFrameReadSource ?: this.parseToDataFrameReadSource) {
956+
parseBy {
957+
readSourceImpl(
958+
source = it,
959+
sourceType = typeOf<String>(),
960+
options = null,
961+
formats = newSupportedFormats,
962+
resultKind = "DataRow",
963+
doStringToUrlConversion = isConverter,
964+
readOrNull = { source, sourceInfo, options ->
965+
readDataFrameOrNull(source, sourceInfo, options)?.singleOrNull()
966+
},
967+
).getOrNull()
955968
}
956969
} else {
957-
null
970+
SKIP_PARSER
958971
}
959972
},
960-
// JSON object as DataRow<*>
961-
stringParser<AnyRow>(catch = true) {
962-
val trimmed = it.trim()
963-
if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
964-
if (readJsonStrAnyRow == null) {
965-
logger.warn {
966-
"parse() encountered a string that looks like a JSON object, but the dataframe-json dependency was not detected. Skipping for now."
967-
}
968-
null
969-
} else {
970-
readJsonStrAnyRow!!(trimmed)
973+
stringParserWithOptions<AnyFrame> { options, isConverter ->
974+
if (options?.parseToDataFrameReadSource ?: this.parseToDataFrameReadSource) {
975+
parseBy {
976+
readSourceImpl(
977+
source = it,
978+
sourceType = typeOf<String>(),
979+
options = null,
980+
formats = newSupportedFormats,
981+
resultKind = "DataFrame",
982+
doStringToUrlConversion = isConverter,
983+
readOrNull = DataFrameReadSource::readDataFrameOrNull,
984+
).getOrNull()
971985
}
972986
} else {
973-
null
987+
SKIP_PARSER
974988
}
975989
},
976-
// Char
977-
stringParser<Char> { it.singleOrNull() },
990+
// // JSON array as DataFrame<*>
991+
// stringParser<AnyFrame>(catch = true) {
992+
// val trimmed = it.trim()
993+
// if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
994+
// if (readJsonStrAnyFrame == null) {
995+
// logger.warn {
996+
// "parse() encountered a string that looks like a JSON array, but the dataframe-json dependency was not detected. Skipping for now."
997+
// }
998+
// null
999+
// } else {
1000+
// readJsonStrAnyFrame!!(trimmed)
1001+
// }
1002+
// } else {
1003+
// null
1004+
// }
1005+
// },
1006+
// // JSON object as DataRow<*>
1007+
// stringParser<AnyRow>(catch = true) {
1008+
// val trimmed = it.trim()
1009+
// if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
1010+
// if (readJsonStrAnyRow == null) {
1011+
// logger.warn {
1012+
// "parse() encountered a string that looks like a JSON object, but the dataframe-json dependency was not detected. Skipping for now."
1013+
// }
1014+
// null
1015+
// } else {
1016+
// readJsonStrAnyRow!!(trimmed)
1017+
// }
1018+
// } else {
1019+
// null
1020+
// }
1021+
// },
9781022
// No parser found, return as String
9791023
// must be last in the list of parsers to return original unparsed string
9801024
stringParser<String> { it },

0 commit comments

Comments
 (0)