diff --git a/build.gradle.kts b/build.gradle.kts index 89f64525fc..bc330982b8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,9 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.jetbrains.kotlin.tooling.core.closure import org.jetbrains.kotlinx.publisher.apache2 import org.jetbrains.kotlinx.publisher.developer import org.jetbrains.kotlinx.publisher.githubRepo +import org.jmailen.gradle.kotlinter.KotlinterExtension @Suppress("DSL_SCOPE_VIOLATION", "UnstableApiUsage") plugins { @@ -51,6 +53,31 @@ allprojects { sourceCompatibility = JavaVersion.VERSION_1_8.toString() targetCompatibility = JavaVersion.VERSION_1_8.toString() } + + // Attempts to configure kotlinter for each sub-project that uses the plugin + afterEvaluate { + try { + kotlinter { + ignoreFailures = false + reporters = arrayOf("checkstyle", "plain") + experimentalRules = true + disabledRules = arrayOf( + "no-wildcard-imports", + "experimental:spacing-between-declarations-with-annotations", + "experimental:enum-entry-name-case", + "experimental:argument-list-wrapping", + "experimental:annotation", + "max-line-length", + "filename", + "comment-spacing", + "curly-spacing", + "experimental:annotation-spacing" + ) + } + } catch (_: UnknownDomainObjectException) { + logger.warn("Could not set kotlinter config on :${this.name}") + } + } } koverMerged { diff --git a/core/build.gradle.kts b/core/build.gradle.kts index c4910c9bc7..65fabeeebc 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -298,24 +298,6 @@ korro { } } -kotlinter { - ignoreFailures = false - reporters = arrayOf("checkstyle", "plain") - experimentalRules = true - disabledRules = arrayOf( - "no-wildcard-imports", - "experimental:spacing-between-declarations-with-annotations", - "experimental:enum-entry-name-case", - "experimental:argument-list-wrapping", - "experimental:annotation", - "max-line-length", - "filename", - "comment-spacing", - "curly-spacing", - "experimental:annotation-spacing" - ) -} - tasks.withType { dependsOn(tasks.generateKeywordsSrc) } diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt index 598b07a709..1486448cd1 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt @@ -50,9 +50,9 @@ internal inline fun JupyterHtmlRenderer.render( includeCss = true, ).plus( df.toHTML( - configuration = reifiedDisplayConfiguration, + // is added later to make sure it's put outside of potential iFrames + configuration = reifiedDisplayConfiguration.copy(enableFallbackStaticTables = false), cellRenderer = contextRenderer, - includeStatic = false, // is added later to make sure it's put outside of potential iFrames ) { footer } ).toJupyterHtmlData() diff --git a/dataframe-arrow/build.gradle.kts b/dataframe-arrow/build.gradle.kts index 38c903b725..68bb3c311f 100644 --- a/dataframe-arrow/build.gradle.kts +++ b/dataframe-arrow/build.gradle.kts @@ -2,6 +2,7 @@ plugins { kotlin("jvm") kotlin("libs.publisher") id("org.jetbrains.kotlinx.kover") + id("org.jmailen.kotlinter") } group = "org.jetbrains.kotlinx" diff --git a/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowWriterImpl.kt b/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowWriterImpl.kt index a37f2ea08f..2e87e1f109 100644 --- a/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowWriterImpl.kt +++ b/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowWriterImpl.kt @@ -54,7 +54,6 @@ import org.jetbrains.kotlinx.dataframe.exceptions.CellConversionException import org.jetbrains.kotlinx.dataframe.exceptions.TypeConverterNotFoundException import org.jetbrains.kotlinx.dataframe.name import org.jetbrains.kotlinx.dataframe.values -import java.nio.charset.Charset import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.typeOf @@ -85,15 +84,21 @@ internal class ArrowWriterImpl( private fun countTotalBytes(column: AnyCol): Long? { val columnType = column.type() return when { - columnType.isSubtypeOf(typeOf()) -> column.values.fold(0L) {totalBytes, value -> totalBytes + value.toString().length * 4} + columnType.isSubtypeOf(typeOf()) -> column.values.fold(0L) { totalBytes, value -> totalBytes + value.toString().length * 4 } else -> null } } private fun infillWithNulls(vector: FieldVector, size: Int) { when (vector) { - is BaseFixedWidthVector -> for (i in 0 until size) { vector.setNull(i) } - is BaseVariableWidthVector -> for (i in 0 until size) { vector.setNull(i) } + is BaseFixedWidthVector -> for (i in 0 until size) { + vector.setNull(i) + } + + is BaseVariableWidthVector -> for (i in 0 until size) { + vector.setNull(i) + } + else -> throw IllegalArgumentException("Can not infill ${vector.javaClass.canonicalName}") } vector.valueCount = size @@ -110,7 +115,8 @@ internal class ArrowWriterImpl( ArrowType.Int(32, true) -> column.convertToInt() ArrowType.Int(64, true) -> column.convertToLong() is ArrowType.Decimal -> column.convertToBigDecimal() - ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) -> column.convertToDouble().convertToFloat() // Use [convertToDouble] as locale logic step + ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) -> column.convertToDouble() + .convertToFloat() // Use [convertToDouble] as locale logic step ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) -> column.convertToDouble() ArrowType.Date(DateUnit.DAY) -> column.convertToLocalDate() ArrowType.Date(DateUnit.MILLISECOND) -> column.convertToLocalDateTime() @@ -123,25 +129,108 @@ internal class ArrowWriterImpl( private fun infillVector(vector: FieldVector, column: AnyCol) { when (vector) { - is VarCharVector -> column.convertToString().forEachIndexed { i, value -> value?.let { vector.set(i, Text(value)); value } ?: vector.setNull(i) } - is LargeVarCharVector -> column.convertToString().forEachIndexed { i, value -> value?.let { vector.set(i, Text(value)); value } ?: vector.setNull(i) } - is BitVector -> column.convertToBoolean().forEachIndexed { i, value -> value?.let { vector.set(i, value.compareTo(false)); value } ?: vector.setNull(i) } - is TinyIntVector -> column.convertToInt().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) } - is SmallIntVector -> column.convertToInt().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) } - is IntVector -> column.convertToInt().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) } - is BigIntVector -> column.convertToLong().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) } - is DecimalVector -> column.convertToBigDecimal().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) } - is Decimal256Vector -> column.convertToBigDecimal().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) } - is Float8Vector -> column.convertToDouble().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) } - is Float4Vector -> column.convertToFloat().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) } - - is DateDayVector -> column.convertToLocalDate().forEachIndexed { i, value -> value?.let { vector.set(i, (value.toJavaLocalDate().toEpochDay()).toInt()); value } ?: vector.setNull(i) } - is DateMilliVector -> column.convertToLocalDateTime().forEachIndexed { i, value -> value?.let { vector.set(i, value.toInstant( - TimeZone.UTC).toEpochMilliseconds()); value } ?: vector.setNull(i) } - is TimeNanoVector -> column.convertToLocalTime().forEachIndexed { i, value -> value?.let { vector.set(i, value.toNanoOfDay()); value } ?: vector.setNull(i) } - is TimeMicroVector -> column.convertToLocalTime().forEachIndexed { i, value -> value?.let { vector.set(i, value.toNanoOfDay() / 1000); value } ?: vector.setNull(i) } - is TimeMilliVector -> column.convertToLocalTime().forEachIndexed { i, value -> value?.let { vector.set(i, (value.toNanoOfDay() / 1000 / 1000).toInt()); value } ?: vector.setNull(i) } - is TimeSecVector -> column.convertToLocalTime().forEachIndexed { i, value -> value?.let { vector.set(i, (value.toNanoOfDay() / 1000 / 1000 / 1000).toInt()); value } ?: vector.setNull(i) } + is VarCharVector -> column.convertToString() + .forEachIndexed { i, value -> + value?.also { vector.set(i, Text(value)) } + ?: vector.setNull(i) + } + + is LargeVarCharVector -> column.convertToString() + .forEachIndexed { i, value -> + value?.also { vector.set(i, Text(value)) } + ?: vector.setNull(i) + } + + is BitVector -> column.convertToBoolean() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value.compareTo(false)) } + ?: vector.setNull(i) + } + + is TinyIntVector -> column.convertToInt() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value) } + ?: vector.setNull(i) + } + + is SmallIntVector -> column.convertToInt() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value) } + ?: vector.setNull(i) + } + + is IntVector -> column.convertToInt() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value) } + ?: vector.setNull(i) + } + + is BigIntVector -> column.convertToLong() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value) } + ?: vector.setNull(i) + } + + is DecimalVector -> column.convertToBigDecimal() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value) } + ?: vector.setNull(i) + } + + is Decimal256Vector -> column.convertToBigDecimal() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value) } + ?: vector.setNull(i) + } + + is Float8Vector -> column.convertToDouble() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value) } + ?: vector.setNull(i) + } + + is Float4Vector -> column.convertToFloat() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value) } + ?: vector.setNull(i) + } + + is DateDayVector -> column.convertToLocalDate() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value.toJavaLocalDate().toEpochDay().toInt()) } + ?: vector.setNull(i) + } + + is DateMilliVector -> column.convertToLocalDateTime() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value.toInstant(TimeZone.UTC).toEpochMilliseconds()) } + ?: vector.setNull(i) + } + + is TimeNanoVector -> column.convertToLocalTime() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value.toNanoOfDay()) } + ?: vector.setNull(i) + } + + is TimeMicroVector -> column.convertToLocalTime() + .forEachIndexed { i, value -> + value?.also { vector.set(i, value.toNanoOfDay() / 1000) } + ?: vector.setNull(i) + } + + is TimeMilliVector -> column.convertToLocalTime() + .forEachIndexed { i, value -> + value?.also { vector.set(i, (value.toNanoOfDay() / 1000 / 1000).toInt()) } + ?: vector.setNull(i) + } + + is TimeSecVector -> column.convertToLocalTime() + .forEachIndexed { i, value -> + value?.also { vector.set(i, (value.toNanoOfDay() / 1000 / 1000 / 1000).toInt()) } + ?: vector.setNull(i) + } + else -> { // TODO implement other vector types from [readField] (VarBinaryVector, UIntVector, DurationVector, StructVector) and may be others (ListVector, FixedSizeListVector etc) throw NotImplementedError("Saving to ${vector.javaClass.canonicalName} is currently not implemented") @@ -154,7 +243,12 @@ internal class ArrowWriterImpl( /** * Create Arrow FieldVector with [column] content cast to [field] type according to [strictType] and [strictNullable] settings. */ - private fun allocateVectorAndInfill(field: Field, column: AnyCol?, strictType: Boolean, strictNullable: Boolean): FieldVector { + private fun allocateVectorAndInfill( + field: Field, + column: AnyCol?, + strictType: Boolean, + strictNullable: Boolean, + ): FieldVector { val containNulls = (column == null || column.hasNulls()) // Convert the column to type specified in field. (If we already have target type, convertTo will do nothing) @@ -163,12 +257,19 @@ internal class ArrowWriterImpl( } catch (e: CellConversionException) { if (strictType) { // If conversion failed but strictType is enabled, throw the exception - val mismatch = ConvertingMismatch.TypeConversionFail.ConversionFailError(e.column?.name() ?: "", e.row, e) + val mismatch = + ConvertingMismatch.TypeConversionFail.ConversionFailError(e.column?.name() ?: "", e.row, e) mismatchSubscriber(mismatch) throw ConvertingException(mismatch) } else { // If strictType is not enabled, use original data with its type. Target nullable is saved at this step. - mismatchSubscriber(ConvertingMismatch.TypeConversionFail.ConversionFailIgnored(e.column?.name() ?: "", e.row, e)) + mismatchSubscriber( + ConvertingMismatch.TypeConversionFail.ConversionFailIgnored( + e.column?.name() ?: "", + e.row, + e + ) + ) column to column!!.toArrowField(mismatchSubscriber) } } catch (e: TypeConverterNotFoundException) { @@ -197,8 +298,17 @@ internal class ArrowWriterImpl( mismatchSubscriber(mismatch) throw ConvertingException(mismatch) } else { - mismatchSubscriber(ConvertingMismatch.NullableMismatch.NullValueIgnored(actualField.name, firstNullValue)) - Field(actualField.name, FieldType(true, actualField.fieldType.type, actualField.fieldType.dictionary), actualField.children).createVector(allocator)!! + mismatchSubscriber( + ConvertingMismatch.NullableMismatch.NullValueIgnored( + actualField.name, + firstNullValue + ) + ) + Field( + actualField.name, + FieldType(true, actualField.fieldType.type, actualField.fieldType.dictionary), + actualField.children + ).createVector(allocator)!! } } else { actualField.createVector(allocator)!! diff --git a/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/ConvertingMismatch.kt b/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/ConvertingMismatch.kt index b6dd76bbd2..c6fd988266 100644 --- a/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/ConvertingMismatch.kt +++ b/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/ConvertingMismatch.kt @@ -8,18 +8,19 @@ import org.jetbrains.kotlinx.dataframe.exceptions.TypeConverterNotFoundException * Can be sent to callback, written to log or encapsulated to exception */ public sealed class ConvertingMismatch( - /**Name of the column with mismatch*/ + /** Name of the column with mismatch */ public open val column: String, - /**Number of first row with mismatch (0-based) if defined*/ + /** Number of first row with mismatch (0-based) if defined */ public open val row: Int?, - /**Original exception if exist*/ - public open val cause: Exception? + /** Original exception if exist */ + public open val cause: Exception?, ) { public sealed class WideningMismatch(column: String) : ConvertingMismatch(column, null, null) { public data class AddedColumn(override val column: String) : WideningMismatch(column) { override fun toString(): String = "Added column \"$column\" not described in target schema" } + public data class RejectedColumn(override val column: String) : WideningMismatch(column) { override fun toString(): String = "Column \"$column\" is not described in target schema and was ignored" } @@ -27,55 +28,87 @@ public sealed class ConvertingMismatch( public sealed class NarrowingMismatch(column: String) : ConvertingMismatch(column, null, null) { public data class NotPresentedColumnIgnored(override val column: String) : NarrowingMismatch(column) { - override fun toString(): String = "Not nullable column \"$column\" is not presented in actual data, saving as is" + override fun toString(): String = + "Not nullable column \"$column\" is not presented in actual data, saving as is" } + public data class NotPresentedColumnError(override val column: String) : NarrowingMismatch(column) { - override fun toString(): String = "Not nullable column \"$column\" is not presented in actual data, can not save" + override fun toString(): String = + "Not nullable column \"$column\" is not presented in actual data, can not save" } } public sealed class TypeConversionNotFound( column: String, - cause: TypeConverterNotFoundException + cause: TypeConverterNotFoundException, ) : ConvertingMismatch(column, null, cause) { - public data class ConversionNotFoundIgnored(override val column: String, override val cause: TypeConverterNotFoundException) : TypeConversionNotFound(column, cause) { + + public data class ConversionNotFoundIgnored( + override val column: String, + override val cause: TypeConverterNotFoundException, + ) : TypeConversionNotFound(column, cause) { override fun toString(): String = "${cause.message} for column \"$column\", saving as is" } - public data class ConversionNotFoundError(override val column: String, val e: TypeConverterNotFoundException) : TypeConversionNotFound(column, e) { + + public data class ConversionNotFoundError( + override val column: String, + val e: TypeConverterNotFoundException, + ) : TypeConversionNotFound(column, e) { override fun toString(): String = "${e.message} for column \"$column\", can not save" } } public sealed class TypeConversionFail( - column: String, row: Int?, - public override val cause: CellConversionException + column: String, + row: Int?, + public override val cause: CellConversionException, ) : ConvertingMismatch(column, row, cause) { - public data class ConversionFailIgnored(override val column: String, override val row: Int?, override val cause: CellConversionException) : TypeConversionFail(column, row, cause) { + + public data class ConversionFailIgnored( + override val column: String, + override val row: Int?, + override val cause: CellConversionException, + ) : TypeConversionFail(column, row, cause) { override fun toString(): String = "${cause.message}, saving as is" } - public data class ConversionFailError(override val column: String, override val row: Int?, override val cause: CellConversionException) : TypeConversionFail(column, row, cause) { + + public data class ConversionFailError( + override val column: String, + override val row: Int?, + override val cause: CellConversionException, + ) : TypeConversionFail(column, row, cause) { override fun toString(): String = "${cause.message}, can not save" } } public data class SavedAsString( override val column: String, - val type: Class<*> + val type: Class<*>, ) : ConvertingMismatch(column, null, null) { override fun toString(): String = "Column \"$column\" has type ${type.canonicalName}, will be saved as String\"" } public sealed class NullableMismatch( column: String, - row: Int? + row: Int?, ) : ConvertingMismatch(column, row, null) { - public data class NullValueIgnored(override val column: String, override val row: Int?) : NullableMismatch(column, row) { - override fun toString(): String = "Column \"$column\" contains nulls in row $row but expected not nullable, saving as is" + public data class NullValueIgnored( + override val column: String, + override val row: Int?, + ) : NullableMismatch(column, row) { + override fun toString(): String = + "Column \"$column\" contains nulls in row $row but expected not nullable, saving as is" } - public data class NullValueError(override val column: String, override val row: Int?) : NullableMismatch(column, row) { - override fun toString(): String = "Column \"$column\" contains nulls in row $row but expected not nullable, can not save" + + public data class NullValueError( + override val column: String, + override val row: Int?, + ) : NullableMismatch(column, row) { + override fun toString(): String = + "Column \"$column\" contains nulls in row $row but expected not nullable, can not save" } } } -public class ConvertingException(public val mismatchCase: ConvertingMismatch) : IllegalArgumentException(mismatchCase.toString(), mismatchCase.cause) +public class ConvertingException(public val mismatchCase: ConvertingMismatch) : + IllegalArgumentException(mismatchCase.toString(), mismatchCase.cause) diff --git a/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/arrowReading.kt b/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/arrowReading.kt index 2e5e44324c..0ddcdf3fb9 100644 --- a/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/arrowReading.kt +++ b/dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/arrowReading.kt @@ -52,6 +52,7 @@ public fun DataFrame.Companion.readArrowIPC( allocator: RootAllocator = Allocator.ROOT, nullability: NullabilityOptions = NullabilityOptions.Infer, ): AnyFrame = readArrowIPCImpl(channel, allocator, nullability) + /** * Read [Arrow random access format](https://arrow.apache.org/docs/java/ipc.html#writing-and-reading-random-access-files) data from existing [channel] */ diff --git a/dataframe-arrow/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowKtTest.kt b/dataframe-arrow/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowKtTest.kt index 479070ae66..9aeaf5a7ce 100644 --- a/dataframe-arrow/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowKtTest.kt +++ b/dataframe-arrow/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowKtTest.kt @@ -35,6 +35,7 @@ internal class ArrowKtTest { fun testResource(resourcePath: String): URL = ArrowKtTest::class.java.classLoader.getResource(resourcePath)!! fun testArrowFeather(name: String) = testResource("$name.feather") + fun testArrowIPC(name: String) = testResource("$name.ipc") @Test @@ -47,7 +48,7 @@ internal class ArrowKtTest { mapOf( "c1" to Text("inner"), "c2" to 4.0, - "c3" to 50.0 + "c3" to 50.0, ) as Map ).toColumn() val d by columnOf("four") @@ -57,54 +58,204 @@ internal class ArrowKtTest { @Test fun testReadingAllTypesAsEstimated() { - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test.arrow"), NullabilityOptions.Infer), false, false) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test.arrow"), NullabilityOptions.Infer), false, false) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather(testArrowFeather("test.arrow"), NullabilityOptions.Infer), + expectedNullable = false, + hasNulls = false, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC(testArrowIPC("test.arrow"), NullabilityOptions.Infer), + expectedNullable = false, + hasNulls = false, + ) - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test.arrow"), NullabilityOptions.Checking), true, false) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test.arrow"), NullabilityOptions.Checking), true, false) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather(testArrowFeather("test.arrow"), NullabilityOptions.Checking), + expectedNullable = true, + hasNulls = false, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC(testArrowIPC("test.arrow"), NullabilityOptions.Checking), + expectedNullable = true, + hasNulls = false, + ) - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test.arrow"), NullabilityOptions.Widening), true, false) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test.arrow"), NullabilityOptions.Widening), true, false) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather(testArrowFeather("test.arrow"), NullabilityOptions.Widening), + expectedNullable = true, + hasNulls = false, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC(testArrowIPC("test.arrow"), NullabilityOptions.Widening), + expectedNullable = true, + hasNulls = false, + ) } @Test fun testReadingAllTypesAsEstimatedWithNulls() { - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test-with-nulls.arrow"), NullabilityOptions.Infer), true, true) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test-with-nulls.arrow"), NullabilityOptions.Infer), true, true) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather( + testArrowFeather("test-with-nulls.arrow"), + NullabilityOptions.Infer, + ), + expectedNullable = true, + hasNulls = true, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC( + testArrowIPC("test-with-nulls.arrow"), + NullabilityOptions.Infer, + ), + expectedNullable = true, + hasNulls = true, + ) - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test-with-nulls.arrow"), NullabilityOptions.Checking), true, true) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test-with-nulls.arrow"), NullabilityOptions.Checking), true, true) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather( + testArrowFeather("test-with-nulls.arrow"), + NullabilityOptions.Checking, + ), + expectedNullable = true, + hasNulls = true, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC( + testArrowIPC("test-with-nulls.arrow"), + NullabilityOptions.Checking, + ), + expectedNullable = true, + hasNulls = true, + ) - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test-with-nulls.arrow"), NullabilityOptions.Widening), true, true) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test-with-nulls.arrow"), NullabilityOptions.Widening), true, true) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather( + testArrowFeather("test-with-nulls.arrow"), + NullabilityOptions.Widening, + ), + expectedNullable = true, + hasNulls = true, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC( + testArrowIPC("test-with-nulls.arrow"), + NullabilityOptions.Widening, + ), + expectedNullable = true, + hasNulls = true, + ) } @Test fun testReadingAllTypesAsEstimatedNotNullable() { - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test-not-nullable.arrow"), NullabilityOptions.Infer), false, false) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test-not-nullable.arrow"), NullabilityOptions.Infer), false, false) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather( + testArrowFeather("test-not-nullable.arrow"), + NullabilityOptions.Infer, + ), + expectedNullable = false, + hasNulls = false, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC( + testArrowIPC("test-not-nullable.arrow"), + NullabilityOptions.Infer, + ), + expectedNullable = false, + hasNulls = false, + ) - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test-not-nullable.arrow"), NullabilityOptions.Checking), false, false) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test-not-nullable.arrow"), NullabilityOptions.Checking), false, false) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather( + testArrowFeather("test-not-nullable.arrow"), + NullabilityOptions.Checking, + ), + expectedNullable = false, + hasNulls = false, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC( + testArrowIPC("test-not-nullable.arrow"), + NullabilityOptions.Checking, + ), + expectedNullable = false, + hasNulls = false, + ) - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test-not-nullable.arrow"), NullabilityOptions.Widening), false, false) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test-not-nullable.arrow"), NullabilityOptions.Widening), false, false) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather( + testArrowFeather("test-not-nullable.arrow"), + NullabilityOptions.Widening, + ), + expectedNullable = false, + hasNulls = false, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC( + testArrowIPC("test-not-nullable.arrow"), + NullabilityOptions.Widening, + ), + expectedNullable = false, + hasNulls = false, + ) } @Test fun testReadingAllTypesAsEstimatedNotNullableWithNulls() { - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test-illegal.arrow"), NullabilityOptions.Infer), true, true) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test-illegal.arrow"), NullabilityOptions.Infer), true, true) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather( + testArrowFeather("test-illegal.arrow"), + NullabilityOptions.Infer, + ), + expectedNullable = true, + hasNulls = true, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC( + testArrowIPC("test-illegal.arrow"), + NullabilityOptions.Infer, + ), + expectedNullable = true, + hasNulls = true + ) shouldThrow { - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test-illegal.arrow"), NullabilityOptions.Checking), false, true) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather( + testArrowFeather("test-illegal.arrow"), + NullabilityOptions.Checking, + ), + expectedNullable = false, + hasNulls = true, + ) } shouldThrow { - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test-illegal.arrow"), NullabilityOptions.Checking), false, true) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC( + testArrowIPC("test-illegal.arrow"), + NullabilityOptions.Checking, + ), + expectedNullable = false, + hasNulls = true, + ) } - assertEstimations(DataFrame.readArrowFeather(testArrowFeather("test-illegal.arrow"), NullabilityOptions.Widening), true, true) - assertEstimations(DataFrame.readArrowIPC(testArrowIPC("test-illegal.arrow"), NullabilityOptions.Widening), true, true) + assertEstimations( + exampleFrame = DataFrame.readArrowFeather( + testArrowFeather("test-illegal.arrow"), + NullabilityOptions.Widening, + ), + expectedNullable = true, + hasNulls = true, + ) + assertEstimations( + exampleFrame = DataFrame.readArrowIPC( + testArrowIPC("test-illegal.arrow"), + NullabilityOptions.Widening, + ), + expectedNullable = true, + hasNulls = true, + ) } @Test @@ -118,7 +269,8 @@ internal class ArrowKtTest { citiesDeserialized["settled"].type() shouldBe typeOf() // cities["settled"].type() refers to FlexibleTypeImpl(LocalDate..LocalDate?) and does not match typeOf() citiesDeserialized["settled"].values() shouldBe citiesExampleFrame["settled"].values() citiesDeserialized["page_in_wiki"].type() shouldBe typeOf() // cities["page_in_wiki"].type() is URI, not supported by Arrow directly - citiesDeserialized["page_in_wiki"].values() shouldBe citiesExampleFrame["page_in_wiki"].values().map { it.toString() } + citiesDeserialized["page_in_wiki"].values() shouldBe + citiesExampleFrame["page_in_wiki"].values().map { it.toString() } } val testFile = File.createTempFile("cities", "arrow") @@ -138,8 +290,11 @@ internal class ArrowKtTest { citiesDeserialized["area"].type() shouldBe typeOf() citiesDeserialized["settled"].type() shouldBe typeOf() shouldThrow { citiesDeserialized["page_in_wiki"] } - citiesDeserialized["film_in_youtube"] shouldBe DataColumn.createValueColumn("film_in_youtube", arrayOfNulls( - citiesExampleFrame.rowsCount()).asList()) + citiesDeserialized["film_in_youtube"] shouldBe + DataColumn.createValueColumn( + name = "film_in_youtube", + values = arrayOfNulls(citiesExampleFrame.rowsCount()).asList(), + ) } @Test @@ -158,10 +313,13 @@ internal class ArrowKtTest { restrictWidening = false, restrictNarrowing = true, strictType = true, - strictNullable = true + strictNullable = true, ) ).use { it.saveArrowFeatherToByteArray() } - DataFrame.readArrowFeather(testAllowWidening)["page_in_wiki"].values() shouldBe citiesExampleFrame["page_in_wiki"].values().map { it.toString() } + DataFrame.readArrowFeather(testAllowWidening)["page_in_wiki"].values() shouldBe + citiesExampleFrame["page_in_wiki"] + .values() + .map { it.toString() } } @Test @@ -185,14 +343,19 @@ internal class ArrowKtTest { strictNullable = true ) ) { warning -> warnings.add(warning) }.use { it.saveArrowFeatherToByteArray() } - warnings.shouldContain( ConvertingMismatch.NarrowingMismatch.NotPresentedColumnIgnored("settled")) + warnings.shouldContain(ConvertingMismatch.NarrowingMismatch.NotPresentedColumnIgnored("settled")) shouldThrow { DataFrame.readArrowFeather(testAllowNarrowing)["settled"] } } @Test fun testStrictType() { val frameRenaming = citiesExampleFrame.copy().remove("settled") - val frameWithIncompatibleField = frameRenaming.add(frameRenaming["is_capital"].map { value -> value ?: false }.rename("settled").convertToBoolean()) + val frameWithIncompatibleField = + frameRenaming.add( + frameRenaming["is_capital"].map { value -> value ?: false } + .rename("settled") + .convertToBoolean() + ) frameWithIncompatibleField.arrowWriter( Schema.fromJSON(citiesExampleSchema), @@ -212,7 +375,14 @@ internal class ArrowKtTest { ) ) { warning -> warnings.add(warning) }.use { it.saveArrowFeatherToByteArray() } warnings.map { it.toString() }.shouldContain( - ConvertingMismatch.TypeConversionNotFound.ConversionNotFoundIgnored("settled", TypeConverterNotFoundException(typeOf(), typeOf(), pathOf("settled"))).toString() + ConvertingMismatch.TypeConversionNotFound.ConversionNotFoundIgnored( + "settled", + TypeConverterNotFoundException( + typeOf(), + typeOf(), + pathOf("settled") + ) + ).toString() ) DataFrame.readArrowFeather(testLoyalType)["settled"].type() shouldBe typeOf() } @@ -220,7 +390,12 @@ internal class ArrowKtTest { @Test fun testStrictNullable() { val frameRenaming = citiesExampleFrame.copy().remove("settled") - val frameWithNulls = frameRenaming.add(DataColumn.createValueColumn("settled", arrayOfNulls(frameRenaming.rowsCount()).asList())) + val frameWithNulls = frameRenaming.add( + DataColumn.createValueColumn( + "settled", + arrayOfNulls(frameRenaming.rowsCount()).asList(), + ) + ) frameWithNulls.arrowWriter( Schema.fromJSON(citiesExampleSchema), @@ -236,7 +411,7 @@ internal class ArrowKtTest { restrictWidening = true, restrictNarrowing = true, strictType = true, - strictNullable = false + strictNullable = false, ) ) { warning -> warnings.add(warning) }.use { it.saveArrowFeatherToByteArray() } warnings.shouldContain(ConvertingMismatch.NullableMismatch.NullValueIgnored("settled", 0)) @@ -252,16 +427,27 @@ internal class ArrowKtTest { val columnDoubleFraction = columnOf(12.345, 67.890) val columnDoubleRound = columnOf(12345.0, 67890.0) val targetType = FieldType.notNullable(ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)) - val targetSchema = Schema(listOf(Field("columnDot", targetType, emptyList()), Field("columnComma", targetType, emptyList()))) + val targetSchema = Schema( + listOf( + Field("columnDot", targetType, emptyList()), + Field("columnComma", targetType, emptyList()), + ) + ) val currentLocale = Locale.getDefault() try { Locale.setDefault(Locale.forLanguageTag("en-US")) val serializedAsUs = frameString.arrowWriter(targetSchema).saveArrowFeatherToByteArray() - DataFrame.readArrowFeather(serializedAsUs) shouldBe dataFrameOf("columnDot", "columnComma")(columnDoubleFraction, columnDoubleRound) + DataFrame.readArrowFeather(serializedAsUs) shouldBe dataFrameOf("columnDot", "columnComma")( + columnDoubleFraction, + columnDoubleRound + ) Locale.setDefault(Locale.forLanguageTag("ru-RU")) val serializedAsRu = frameString.arrowWriter(targetSchema).saveArrowFeatherToByteArray() - DataFrame.readArrowFeather(serializedAsRu) shouldBe dataFrameOf("columnDot", "columnComma")(columnDoubleFraction, columnDoubleFraction) + DataFrame.readArrowFeather(serializedAsRu) shouldBe dataFrameOf("columnDot", "columnComma")( + columnDoubleFraction, + columnDoubleFraction + ) } finally { Locale.setDefault(currentLocale) } diff --git a/dataframe-excel/build.gradle.kts b/dataframe-excel/build.gradle.kts index 959880f70d..78768409b0 100644 --- a/dataframe-excel/build.gradle.kts +++ b/dataframe-excel/build.gradle.kts @@ -2,6 +2,7 @@ plugins { kotlin("jvm") kotlin("libs.publisher") id("org.jetbrains.kotlinx.kover") + id("org.jmailen.kotlinter") } group = "org.jetbrains.kotlinx" diff --git a/dataframe-excel/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/XlsxTest.kt b/dataframe-excel/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/XlsxTest.kt index 13524f4710..976fd80e03 100644 --- a/dataframe-excel/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/XlsxTest.kt +++ b/dataframe-excel/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/XlsxTest.kt @@ -22,7 +22,8 @@ class XlsxTest { @Test fun `numerical columns`() { - DataFrame.readExcel(testResource("sample.xls"), "Sheet1") shouldBe dataFrameOf("col1", "col2")(1.0, 2.0) + DataFrame.readExcel(testResource("sample.xls"), "Sheet1") shouldBe + dataFrameOf("col1", "col2")(1.0, 2.0) } @Test @@ -53,7 +54,10 @@ class XlsxTest { @Test fun `first sheet is default sheet`() { - DataFrame.readExcel(testResource("sample.xls"), "Sheet1") shouldBe DataFrame.readExcel(testResource("sample.xls")) + DataFrame.readExcel( + testResource("sample.xls"), + "Sheet1" + ) shouldBe DataFrame.readExcel(testResource("sample.xls")) } @Test @@ -61,7 +65,7 @@ class XlsxTest { val temp = Files.createTempFile("excel", ".xlsx").toFile() val df = dataFrameOf("col1", "col2")( "string value", 3.2, - "string value 1", null + "string value 1", null, ) val extendedDf = List(10) { df }.concat() extendedDf.writeExcel(temp) @@ -93,8 +97,26 @@ class XlsxTest { @Test fun `consider skipRows when obtaining column indexes`() { val df = DataFrame.readExcel(testResource("header.xlsx"), skipRows = 6, rowsCount = 1) - df.columnNames() shouldBe listOf("Well", "Well Position", "Omit", "Sample Name", "Target Name", "Task", "Reporter", "Quencher") - df shouldBe dataFrameOf("Well", "Well Position", "Omit", "Sample Name", "Target Name", "Task", "Reporter", "Quencher")(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0,) + df.columnNames() shouldBe listOf( + "Well", + "Well Position", + "Omit", + "Sample Name", + "Target Name", + "Task", + "Reporter", + "Quencher", + ) + df shouldBe dataFrameOf( + "Well", + "Well Position", + "Omit", + "Sample Name", + "Target Name", + "Task", + "Reporter", + "Quencher", + )(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0) } @Test @@ -117,9 +139,24 @@ class XlsxTest { DataFrame.readExcel(testResource("iris_duplicated_column.xlsx")) } - val df = DataFrame.readExcel(testResource("iris_duplicated_column.xlsx"), nameRepairStrategy = NameRepairStrategy.MAKE_UNIQUE) - df.columnNames() shouldBe listOf("Sepal.Length", "Sepal.Width", "C", - "Petal.Length", "Petal.Width", "Species", "Other.Width", "Species1", "I", "Other.Width1", "Species2") + val df = DataFrame.readExcel( + testResource("iris_duplicated_column.xlsx"), + nameRepairStrategy = NameRepairStrategy.MAKE_UNIQUE, + ) + df.columnNames() shouldBe + listOf( + "Sepal.Length", + "Sepal.Width", + "C", + "Petal.Length", + "Petal.Width", + "Species", + "Other.Width", + "Species1", + "I", + "Other.Width1", + "Species2", + ) } @Test diff --git a/dataframe-jdbc/build.gradle.kts b/dataframe-jdbc/build.gradle.kts index 50652715e8..2c5505a46f 100644 --- a/dataframe-jdbc/build.gradle.kts +++ b/dataframe-jdbc/build.gradle.kts @@ -3,6 +3,7 @@ plugins { kotlin("libs.publisher") id("org.jetbrains.kotlinx.kover") kotlin("jupyter.api") + // TODO enable later: id("org.jmailen.kotlinter") } group = "org.jetbrains.kotlinx" diff --git a/dataframe-openapi/build.gradle.kts b/dataframe-openapi/build.gradle.kts index 2ac26b705f..d00075ad71 100644 --- a/dataframe-openapi/build.gradle.kts +++ b/dataframe-openapi/build.gradle.kts @@ -3,7 +3,7 @@ plugins { kotlin("libs.publisher") kotlin("plugin.serialization") id("org.jetbrains.kotlinx.kover") - + id("org.jmailen.kotlinter") kotlin("jupyter.api") } diff --git a/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/OpenApi.kt b/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/OpenApi.kt index 16af24d4d4..70a680826d 100644 --- a/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/OpenApi.kt +++ b/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/OpenApi.kt @@ -87,5 +87,3 @@ public class OpenApi : SupportedCodeGenerationFormat { override fun createDefaultReadMethod(pathRepresentation: String?): DefaultReadDfMethod = DefaultReadOpenApiMethod } - - diff --git a/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/resultTypes.kt b/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/resultTypes.kt index 8d063372be..4df1dc1026 100644 --- a/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/resultTypes.kt +++ b/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/resultTypes.kt @@ -108,4 +108,3 @@ internal sealed interface OpenApiTypeResult { /** Property is an enum with values [values]. */ data class Enum(val values: List, val nullable: Boolean) : OpenApiTypeResult } - diff --git a/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/utils.kt b/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/utils.kt index ab40094d37..cdb9526079 100644 --- a/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/utils.kt +++ b/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/utils.kt @@ -27,7 +27,6 @@ internal interface IsObject { val isObject: Boolean } - /** Helper function to create a [GeneratedField] without [GeneratedField.columnSchema]. */ internal fun generatedFieldOf( fieldName: ValidFieldName, diff --git a/dataframe-openapi/src/test/kotlin/OpenApiTests.kt b/dataframe-openapi/src/test/kotlin/OpenApiTests.kt index a1e7b6ca9b..fd8d7bc2cf 100644 --- a/dataframe-openapi/src/test/kotlin/OpenApiTests.kt +++ b/dataframe-openapi/src/test/kotlin/OpenApiTests.kt @@ -1064,15 +1064,14 @@ class OpenApiTests : JupyterReplTestCase() { @Test fun `Jupyter importDataSchema`() { + val filePath = apiGuruYaml.absolutePath.let { + if (separatorChar == '\\') it.replace("\\", "\\\\") + else it + } @Language("kts") val _1 = execRaw( """ - val ApiGuru = importDataSchema(File("${ - apiGuruYaml.absolutePath.let { - if (separatorChar == '\\') it.replace("\\", "\\\\") - else it - } - }")) + val ApiGuru = importDataSchema(File("$filePath")) """.trimIndent() ) diff --git a/plugins/dataframe-gradle-plugin/build.gradle.kts b/plugins/dataframe-gradle-plugin/build.gradle.kts index 30bc3d1d30..503f4c70b4 100644 --- a/plugins/dataframe-gradle-plugin/build.gradle.kts +++ b/plugins/dataframe-gradle-plugin/build.gradle.kts @@ -5,6 +5,7 @@ plugins { `java-gradle-plugin` `maven-publish` id("com.gradle.plugin-publish") version "0.15.0" + id("org.jmailen.kotlinter") } repositories { diff --git a/plugins/dataframe-gradle-plugin/src/integrationTest/kotlin/org/jetbrains/dataframe/gradle/ApiChangesDetectionTest.kt b/plugins/dataframe-gradle-plugin/src/integrationTest/kotlin/org/jetbrains/dataframe/gradle/ApiChangesDetectionTest.kt index b7dedddd02..bf20b25d07 100644 --- a/plugins/dataframe-gradle-plugin/src/integrationTest/kotlin/org/jetbrains/dataframe/gradle/ApiChangesDetectionTest.kt +++ b/plugins/dataframe-gradle-plugin/src/integrationTest/kotlin/org/jetbrains/dataframe/gradle/ApiChangesDetectionTest.kt @@ -8,9 +8,9 @@ import java.io.File class ApiChangesDetectionTest : AbstractDataFramePluginIntegrationTest() { - // GenerateDataSchemaTask::class, - // DefaultReadCsvMethod::class, - // DefaultReadJsonMethod::class + // GenerateDataSchemaTask::class, + // DefaultReadCsvMethod::class, + // DefaultReadJsonMethod::class @Test fun `cast api`() { compiles { @@ -43,6 +43,7 @@ class ApiChangesDetectionTest : AbstractDataFramePluginIntegrationTest() { """.trimIndent() } } + // GenerateDataSchemaTask::class, // DefaultReadCsvMethod::class, @Test diff --git a/plugins/dataframe-gradle-plugin/src/integrationTest/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPluginIntegrationTest.kt b/plugins/dataframe-gradle-plugin/src/integrationTest/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPluginIntegrationTest.kt index d6c1e7a299..381f682225 100644 --- a/plugins/dataframe-gradle-plugin/src/integrationTest/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPluginIntegrationTest.kt +++ b/plugins/dataframe-gradle-plugin/src/integrationTest/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPluginIntegrationTest.kt @@ -133,9 +133,11 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val kotlin = File(buildDir, "src/main/kotlin").also { it.mkdirs() } val main = File(kotlin, "Main.kt") - main.writeText(""" + main.writeText( + """ import org.example.Schema - """.trimIndent()) + """.trimIndent() + ) """ import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension @@ -176,42 +178,43 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val kotlin = File(buildDir, "src/jvmMain/kotlin").also { it.mkdirs() } val main = File(kotlin, "Main.kt") - main.writeText(""" + main.writeText( + """ import org.example.Schema - """.trimIndent()) + """.trimIndent() + ) """ - import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension - - plugins { - kotlin("multiplatform") version "$kotlinVersion" - id("org.jetbrains.kotlinx.dataframe") - } + import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension - repositories { - mavenCentral() - mavenLocal() - } + plugins { + kotlin("multiplatform") version "$kotlinVersion" + id("org.jetbrains.kotlinx.dataframe") + } + + repositories { + mavenCentral() + mavenLocal() + } + + kotlin { + jvm() - kotlin { - jvm() - - sourceSets { - val jvmMain by getting { - dependencies { - implementation(files("$dataframeJarPath")) - } + sourceSets { + val jvmMain by getting { + dependencies { + implementation(files("$dataframeJarPath")) } } } - - dataframes { - schema { - data = file("${TestData.csvName}") - name = "org.example.Schema" - } + } + + dataframes { + schema { + data = file("${TestData.csvName}") + name = "org.example.Schema" } + } """.trimIndent() - } result.task(":build")?.outcome shouldBe TaskOutcome.SUCCESS } @@ -236,7 +239,8 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val kotlin = File(buildDir, "src/main/kotlin").also { it.mkdirs() } val main = File(kotlin, "Main.kt") - main.writeText(""" + main.writeText( + """ import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.DataSchema import org.jetbrains.kotlinx.dataframe.io.read @@ -246,34 +250,35 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT fun main() { val df = DataFrame.read("${TestData.csvName}").cast() } - """.trimIndent()) + """.trimIndent() + ) """ - import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension - - plugins { - kotlin("jvm") version "$kotlinVersion" - id("org.jetbrains.kotlinx.dataframe") - } - - repositories { - mavenLocal() - mavenCentral() - } - - dependencies { - implementation(files("$dataframeJarPath")) - } + import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension - kotlin.sourceSets.getByName("main").kotlin.srcDir("build/generated/ksp/main/kotlin/") + plugins { + kotlin("jvm") version "$kotlinVersion" + id("org.jetbrains.kotlinx.dataframe") + } + + repositories { + mavenLocal() + mavenCentral() + } + + dependencies { + implementation(files("$dataframeJarPath")) + } + + kotlin.sourceSets.getByName("main").kotlin.srcDir("build/generated/ksp/main/kotlin/") - dataframes { - schema { - data = "${TestData.csvName}" - name = "Schema" - packageName = "" - } + dataframes { + schema { + data = "${TestData.csvName}" + name = "Schema" + packageName = "" } + } """.trimIndent() } result.task(":build")?.outcome shouldBe TaskOutcome.SUCCESS @@ -287,7 +292,8 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val kotlin = File(buildDir, "src/main/kotlin").also { it.mkdirs() } val main = File(kotlin, "Main.kt") - main.writeText(""" + main.writeText( + """ import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.DataSchema import org.jetbrains.kotlinx.dataframe.io.read @@ -303,7 +309,8 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val df = DataFrame.read("${TestData.csvName}").cast() val df1 = df.filter { age != null } } - """.trimIndent()) + """.trimIndent() + ) @Suppress("DuplicatedCode") """ @@ -337,7 +344,8 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val kotlin = File(buildDir, "src/main/kotlin").also { it.mkdirs() } val main = File(kotlin, "Main.kt") - main.writeText(""" + main.writeText( + """ @file:ImportDataSchema(name = "MySchema", path = "${TestData.csvName}") package test @@ -349,7 +357,8 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val df = MySchema.readCSV() val df1 = df.filter { age != null } } - """.trimIndent()) + """.trimIndent() + ) """ import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension @@ -377,19 +386,19 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT @Test @Ignore // TODO: test is broken - /* - e: file://test3901867314473689900/src/main/kotlin/Main.kt:12:43 Unresolved reference: readSqlTable - e: file://test3901867314473689900/src/main/kotlin/Main.kt:13:43 Unresolved reference: DatabaseConfiguration - e: file://test3901867314473689900/src/main/kotlin/Main.kt:19:28 Unresolved reference: readSqlTable - e: file://test3901867314473689900/src/main/kotlin/Main.kt:20:21 Unresolved reference: age - e: file://test3901867314473689900/src/main/kotlin/Main.kt:22:29 Unresolved reference: readSqlTable - e: file://test3901867314473689900/src/main/kotlin/Main.kt:23:22 Unresolved reference: age - e: file://test3901867314473689900/src/main/kotlin/Main.kt:25:24 Unresolved reference: DatabaseConfiguration - e: file://test3901867314473689900/src/main/kotlin/Main.kt:26:29 Unresolved reference: readSqlTable - e: file://test3901867314473689900/src/main/kotlin/Main.kt:27:22 Unresolved reference: age - e: file://test3901867314473689900/src/main/kotlin/Main.kt:29:29 Unresolved reference: readSqlTable - e: file://test3901867314473689900/src/main/kotlin/Main.kt:30:22 Unresolved reference: age - */ + /* + e: file://test3901867314473689900/src/main/kotlin/Main.kt:12:43 Unresolved reference: readSqlTable + e: file://test3901867314473689900/src/main/kotlin/Main.kt:13:43 Unresolved reference: DatabaseConfiguration + e: file://test3901867314473689900/src/main/kotlin/Main.kt:19:28 Unresolved reference: readSqlTable + e: file://test3901867314473689900/src/main/kotlin/Main.kt:20:21 Unresolved reference: age + e: file://test3901867314473689900/src/main/kotlin/Main.kt:22:29 Unresolved reference: readSqlTable + e: file://test3901867314473689900/src/main/kotlin/Main.kt:23:22 Unresolved reference: age + e: file://test3901867314473689900/src/main/kotlin/Main.kt:25:24 Unresolved reference: DatabaseConfiguration + e: file://test3901867314473689900/src/main/kotlin/Main.kt:26:29 Unresolved reference: readSqlTable + e: file://test3901867314473689900/src/main/kotlin/Main.kt:27:22 Unresolved reference: age + e: file://test3901867314473689900/src/main/kotlin/Main.kt:29:29 Unresolved reference: readSqlTable + e: file://test3901867314473689900/src/main/kotlin/Main.kt:30:22 Unresolved reference: age + */ fun `preprocessor imports schema from database`() { val connectionUrl = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=MySQL;DATABASE_TO_UPPER=false" DriverManager.getConnection(connectionUrl).use { @@ -400,41 +409,43 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val main = File(kotlin, "Main.kt") // this is a copy of the code snippet in the // DataFrameJdbcSymbolProcessorTest.`schema extracted via readFromDB method is resolved` - main.writeText(""" - @file:ImportDataSchema(name = "Customer", path = "$connectionUrl") - - package test - - import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema - import org.jetbrains.kotlinx.dataframe.api.filter - import org.jetbrains.kotlinx.dataframe.DataFrame - import org.jetbrains.kotlinx.dataframe.api.cast - import java.sql.Connection - import java.sql.DriverManager - import java.sql.SQLException - import org.jetbrains.kotlinx.dataframe.io.readSqlTable - import org.jetbrains.kotlinx.dataframe.io.DatabaseConfiguration - - fun main() { - Class.forName("org.h2.Driver") - val tableName = "Customer" - DriverManager.getConnection("$connectionUrl").use { connection -> - val df = DataFrame.readSqlTable(connection, tableName).cast() - df.filter { age > 30 } - - val df1 = DataFrame.readSqlTable(connection, tableName, 1).cast() - df1.filter { age > 30 } - - val dbConfig = DatabaseConfiguration(url = "$connectionUrl") - val df2 = DataFrame.readSqlTable(dbConfig, tableName).cast() - df2.filter { age > 30 } - - val df3 = DataFrame.readSqlTable(dbConfig, tableName, 1).cast() - df3.filter { age > 30 } - + main.writeText( + """ + @file:ImportDataSchema(name = "Customer", path = "$connectionUrl") + + package test + + import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema + import org.jetbrains.kotlinx.dataframe.api.filter + import org.jetbrains.kotlinx.dataframe.DataFrame + import org.jetbrains.kotlinx.dataframe.api.cast + import java.sql.Connection + import java.sql.DriverManager + import java.sql.SQLException + import org.jetbrains.kotlinx.dataframe.io.readSqlTable + import org.jetbrains.kotlinx.dataframe.io.DatabaseConfiguration + + fun main() { + Class.forName("org.h2.Driver") + val tableName = "Customer" + DriverManager.getConnection("$connectionUrl").use { connection -> + val df = DataFrame.readSqlTable(connection, tableName).cast() + df.filter { age > 30 } + + val df1 = DataFrame.readSqlTable(connection, tableName, 1).cast() + df1.filter { age > 30 } + + val dbConfig = DatabaseConfiguration(url = "$connectionUrl") + val df2 = DataFrame.readSqlTable(dbConfig, tableName).cast() + df2.filter { age > 30 } + + val df3 = DataFrame.readSqlTable(dbConfig, tableName, 1).cast() + df3.filter { age > 30 } + + } } - } - """.trimIndent()) + """.trimIndent() + ) """ import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension @@ -454,7 +465,7 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT } kotlin.sourceSets.getByName("main").kotlin.srcDir("build/generated/ksp/main/kotlin/") - """.trimIndent() + """.trimIndent() } result.task(":build")?.outcome shouldBe TaskOutcome.SUCCESS } @@ -464,23 +475,23 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT // Crate table Customer connection.createStatement().execute( """ - CREATE TABLE Customer ( - id INT PRIMARY KEY, - name VARCHAR(50), - age INT - ) - """.trimIndent() + CREATE TABLE Customer ( + id INT PRIMARY KEY, + name VARCHAR(50), + age INT + ) + """.trimIndent() ) // Create table Sale connection.createStatement().execute( """ - CREATE TABLE Sale ( - id INT PRIMARY KEY, - customerId INT, - amount DECIMAL(10, 2) - ) - """.trimIndent() + CREATE TABLE Sale ( + id INT PRIMARY KEY, + customerId INT, + amount DECIMAL(10, 2) + ) + """.trimIndent() ) // add data to the Customer table @@ -538,41 +549,43 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val dataFile = File(buildDir, TestData.csvName) val kotlin = File(buildDir, "src/jsMain/kotlin").also { it.mkdirs() } val main = File(kotlin, "Main.kt") - main.writeText(""" + main.writeText( + """ fun main() { console.log("Hello, Kotlin/JS!") } - """.trimIndent()) + """.trimIndent() + ) dataFile.writeText(TestData.csvSample) """ - import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension - - plugins { - kotlin("multiplatform") version "$kotlinVersion" - id("org.jetbrains.kotlinx.dataframe") - } - - repositories { - mavenCentral() - mavenLocal() - } + import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension - kotlin { - sourceSets { - js(IR) { - browser() - } + plugins { + kotlin("multiplatform") version "$kotlinVersion" + id("org.jetbrains.kotlinx.dataframe") + } + + repositories { + mavenCentral() + mavenLocal() + } + + kotlin { + sourceSets { + js(IR) { + browser() } } - - dataframes { - schema { - data = file("${TestData.csvName}") - name = "Schema" - packageName = "" - src = buildDir - } + } + + dataframes { + schema { + data = file("${TestData.csvName}") + name = "Schema" + packageName = "" + src = buildDir } + } """.trimIndent() } result.task(":build")?.outcome shouldBe TaskOutcome.SUCCESS @@ -584,39 +597,41 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val dataFile = File(buildDir, TestData.csvName) val kotlin = File(buildDir, "src/jvmMain/kotlin").also { it.mkdirs() } val main = File(kotlin, "Main.kt") - main.writeText(""" + main.writeText( + """ fun main() { println("Hello, Kotlin/JVM!") } - """.trimIndent()) + """.trimIndent() + ) dataFile.writeText(TestData.csvSample) """ - import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension - - plugins { - kotlin("multiplatform") version "$kotlinVersion" - id("org.jetbrains.kotlinx.dataframe") - } - - repositories { - mavenCentral() - mavenLocal() - } + import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension - kotlin { - sourceSets { - jvm() - } + plugins { + kotlin("multiplatform") version "$kotlinVersion" + id("org.jetbrains.kotlinx.dataframe") + } + + repositories { + mavenCentral() + mavenLocal() + } + + kotlin { + sourceSets { + jvm() } - - dataframes { - schema { - data = file("${TestData.csvName}") - name = "Schema" - packageName = "" - src = buildDir - } + } + + dataframes { + schema { + data = file("${TestData.csvName}") + name = "Schema" + packageName = "" + src = buildDir } + } """.trimIndent() } result.task(":build")?.outcome shouldBe TaskOutcome.SUCCESS @@ -637,28 +652,28 @@ class SchemaGeneratorPluginIntegrationTest : AbstractDataFramePluginIntegrationT val dataFile = File(buildDir, dataName) dataFile.writeText(dataSample) """ - import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension - - plugins { - kotlin("jvm") version "$kotlinVersion" - id("org.jetbrains.kotlinx.dataframe") - } - - repositories { - mavenCentral() - mavenLocal() - } - - dependencies { - implementation(files("$dataframeJarPath")) - } - - dataframes { - schema { - data = file("${dataName}") - name = "Schema" - } + import org.jetbrains.dataframe.gradle.SchemaGeneratorExtension + + plugins { + kotlin("jvm") version "$kotlinVersion" + id("org.jetbrains.kotlinx.dataframe") + } + + repositories { + mavenCentral() + mavenLocal() + } + + dependencies { + implementation(files("$dataframeJarPath")) + } + + dataframes { + schema { + data = file("$dataName") + name = "Schema" } + } """.trimIndent() } result.task(":build")?.outcome shouldBe TaskOutcome.SUCCESS diff --git a/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/GenerateDataSchemaTask.kt b/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/GenerateDataSchemaTask.kt index 2a954ddcf1..33c9d2b3d9 100644 --- a/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/GenerateDataSchemaTask.kt +++ b/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/GenerateDataSchemaTask.kt @@ -17,12 +17,6 @@ import org.jetbrains.kotlinx.dataframe.impl.codeGen.from import org.jetbrains.kotlinx.dataframe.impl.codeGen.toStandaloneSnippet import org.jetbrains.kotlinx.dataframe.impl.codeGen.urlCodeGenReader import org.jetbrains.kotlinx.dataframe.impl.codeGen.urlDfReader -import java.io.File -import java.lang.RuntimeException -import java.net.URL -import java.nio.file.Paths -import java.sql.Connection -import java.sql.DriverManager import org.jetbrains.kotlinx.dataframe.io.ArrowFeather import org.jetbrains.kotlinx.dataframe.io.CSV import org.jetbrains.kotlinx.dataframe.io.Excel @@ -33,6 +27,11 @@ import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.isURL import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema +import java.io.File +import java.net.URL +import java.nio.file.Paths +import java.sql.Connection +import java.sql.DriverManager abstract class GenerateDataSchemaTask : DefaultTask() { @@ -177,19 +176,23 @@ abstract class GenerateDataSchemaTask : DefaultTask() { private fun generateSchemaByJdbcOptions( jdbcOptions: JdbcOptionsDsl, - connection: Connection + connection: Connection, ): DataFrameSchema { logger.debug("Table name: ${jdbcOptions.tableName}") logger.debug("SQL query: ${jdbcOptions.sqlQuery}") - return if (jdbcOptions.tableName.isNotBlank()) + return if (jdbcOptions.tableName.isNotBlank()) { DataFrame.getSchemaForSqlTable(connection, jdbcOptions.tableName) - else if(jdbcOptions.sqlQuery.isNotBlank()) + } else if (jdbcOptions.sqlQuery.isNotBlank()) { DataFrame.getSchemaForSqlQuery(connection, jdbcOptions.sqlQuery) - else throw RuntimeException("Table name: ${jdbcOptions.tableName}, " + - "SQL query: ${jdbcOptions.sqlQuery} both are empty! " + - "Populate 'tableName' or 'sqlQuery' in jdbcOptions with value to generate schema " + - "for SQL table or result of SQL query!") + } else { + throw RuntimeException( + "Table name: ${jdbcOptions.tableName}, " + + "SQL query: ${jdbcOptions.sqlQuery} both are empty! " + + "Populate 'tableName' or 'sqlQuery' in jdbcOptions with value to generate schema " + + "for SQL table or result of SQL query!" + ) + } } private fun stringOf(data: Any): String = diff --git a/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPlugin.kt b/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPlugin.kt index 4aa4760cc6..a2e807fab7 100644 --- a/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPlugin.kt +++ b/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPlugin.kt @@ -18,6 +18,7 @@ import java.io.File import java.net.URL import java.nio.file.Path import java.nio.file.Paths +import java.util.* class SchemaGeneratorPlugin : Plugin { @@ -104,7 +105,7 @@ class SchemaGeneratorPlugin : Plugin { val defaultPath = schema.defaultPath ?: extension.defaultPath ?: true val delimiters = schema.withNormalizationBy ?: extension.withNormalizationBy ?: setOf('\t', ' ', '_') - return target.tasks.create("generateDataFrame${interfaceName}", GenerateDataSchemaTask::class.java) { + return target.tasks.create("generateDataFrame$interfaceName", GenerateDataSchemaTask::class.java) { (logging as? DefaultLoggingManager)?.setLevelInternal(LogLevel.QUIET) group = GROUP data.set(schema.data) @@ -124,7 +125,7 @@ class SchemaGeneratorPlugin : Plugin { val rawName = schema.name?.substringAfterLast('.') ?: fileName(schema.data) ?.toCamelCaseByDelimiters(delimiters) - ?.capitalize() + ?.replaceFirstChar { if (it.isLowerCase()) it.titlecase(Locale.getDefault()) else it.toString() } ?.removeSurrounding("`") ?: return null NameChecker.checkValidIdentifier(rawName) diff --git a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/DataFrameReadTest.kt b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/DataFrameReadTest.kt index a07602c592..2cf9c32cac 100644 --- a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/DataFrameReadTest.kt +++ b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/DataFrameReadTest.kt @@ -103,23 +103,23 @@ class DataFrameReadTest { // Create table Customer connection.createStatement().execute( """ - CREATE TABLE Customer ( - id INT PRIMARY KEY, - name VARCHAR(50), - age INT - ) - """.trimIndent() + CREATE TABLE Customer ( + id INT PRIMARY KEY, + name VARCHAR(50), + age INT + ) + """.trimIndent() ) // Create table Sale connection.createStatement().execute( """ - CREATE TABLE Sale ( - id INT PRIMARY KEY, - customerId INT, - amount DECIMAL(10, 2) - ) - """.trimIndent() + CREATE TABLE Sale ( + id INT PRIMARY KEY, + customerId INT, + amount DECIMAL(10, 2) + ) + """.trimIndent() ) // add data to the Customer table diff --git a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/FileTraversalTest.kt b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/FileTraversalTest.kt index 3c220e9008..94029d867f 100644 --- a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/FileTraversalTest.kt +++ b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/FileTraversalTest.kt @@ -37,7 +37,6 @@ internal class FileTraversalTest { temp.findDeepestCommonSubdirectory() shouldBe leaf } - @Test fun forkAtDepth0() { File(temp, "a/b/c").also { it.mkdirs() } @@ -53,10 +52,8 @@ internal class FileTraversalTest { temp.findDeepestCommonSubdirectory() shouldBe a } - @Test fun noSubdirectories() { temp.findDeepestCommonSubdirectory() shouldBe temp } - } diff --git a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPluginTest.kt b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPluginTest.kt index a36698c6fc..32fbc8a5c6 100644 --- a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPluginTest.kt +++ b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPluginTest.kt @@ -5,7 +5,6 @@ import io.kotest.inspectors.forOne import io.kotest.matchers.shouldBe import io.kotest.matchers.string.shouldContain import org.gradle.testkit.runner.TaskOutcome -import org.intellij.lang.annotations.Language import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension import org.junit.Test import java.io.File diff --git a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/TestData.kt b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/TestData.kt index 496f896c63..12d36fe2cf 100644 --- a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/TestData.kt +++ b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/TestData.kt @@ -1,10 +1,12 @@ package org.jetbrains.dataframe.gradle object TestData { - val csvSample = """ - name, age - Alice, 15 - Bob, + + val csvSample = + """ + name, age + Alice, 15 + Bob, """.trimIndent() val csvName = "data.csv" diff --git a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/makeProject.kt b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/makeProject.kt index a3cac428c0..4b19cb6c99 100644 --- a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/makeProject.kt +++ b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/makeProject.kt @@ -43,8 +43,5 @@ internal fun addBuildEventsListenerRegistryMock(project: Project) { } object BuildEventsListenerRegistryMock : BuildEventsListenerRegistry { - - override fun onTaskCompletion(listener: Provider?) { - } - + override fun onTaskCompletion(listener: Provider?) = Unit } diff --git a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/taskProperties/TaskDataSchemaPropertyTest.kt b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/taskProperties/TaskDataSchemaPropertyTest.kt index 34f7ed35cf..06a12feb3a 100644 --- a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/taskProperties/TaskDataSchemaPropertyTest.kt +++ b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/taskProperties/TaskDataSchemaPropertyTest.kt @@ -60,5 +60,4 @@ class TaskDataSchemaPropertyTest { (project.tasks.getByName("generateDataFrame321") as GenerateDataSchemaTask).dataSchema.get() .shouldBe(project.file("build/generated/dataframe/main1/kotlin/org/example/my/321.Generated.kt")) } - } diff --git a/plugins/expressions-converter/build.gradle.kts b/plugins/expressions-converter/build.gradle.kts index 931513c97d..8ef4946cf1 100644 --- a/plugins/expressions-converter/build.gradle.kts +++ b/plugins/expressions-converter/build.gradle.kts @@ -47,14 +47,6 @@ tasks.test { } } -kotlinter { - ignoreFailures = false - reporters = arrayOf("checkstyle", "plain") - disabledRules = arrayOf( - "max-line-length", - ) -} - sourceSets { main { java.setSrcDirs(listOf("src")) diff --git a/plugins/expressions-converter/tests/org/jetbrains/kotlinx/dataframe/classpathFromClassloader.kt b/plugins/expressions-converter/tests/org/jetbrains/kotlinx/dataframe/classpathFromClassloader.kt index 84e1e26d3e..5c1428c088 100644 --- a/plugins/expressions-converter/tests/org/jetbrains/kotlinx/dataframe/classpathFromClassloader.kt +++ b/plugins/expressions-converter/tests/org/jetbrains/kotlinx/dataframe/classpathFromClassloader.kt @@ -39,6 +39,7 @@ internal const val KOTLIN_COMPILER_CLASSPATH_PROPERTY = "kotlin.compiler.classpa internal const val KOTLIN_COMPILER_JAR_PROPERTY = "kotlin.compiler.jar" internal const val KOTLIN_STDLIB_JAR_PROPERTY = "kotlin.java.stdlib.jar" internal const val KOTLIN_REFLECT_JAR_PROPERTY = "kotlin.java.reflect.jar" + // obsolete name, but maybe still used in the wild // TODO: consider removing internal const val KOTLIN_RUNTIME_JAR_PROPERTY = "kotlin.java.runtime.jar" @@ -222,8 +223,9 @@ private fun allRelatedClassLoaders(clsLoader: ClassLoader, visited: MutableSet { val readDfMethod = codeGenResult.getReadDfMethod( pathRepresentation = importStatement @@ -268,19 +273,22 @@ class DataSchemaGenerator( private fun generateSchemaForImport( importStatement: ImportDataSchemaStatement, - connection: Connection + connection: Connection, ): DataFrameSchema { logger.info("Table name: ${importStatement.jdbcOptions.tableName}") logger.info("SQL query: ${importStatement.jdbcOptions.sqlQuery}") - return if (importStatement.jdbcOptions.tableName.isNotBlank()) + return if (importStatement.jdbcOptions.tableName.isNotBlank()) { DataFrame.getSchemaForSqlTable(connection, importStatement.jdbcOptions.tableName) - else if(importStatement.jdbcOptions.sqlQuery.isNotBlank()) + } else if (importStatement.jdbcOptions.sqlQuery.isNotBlank()) { DataFrame.getSchemaForSqlQuery(connection, importStatement.jdbcOptions.sqlQuery) - else throw RuntimeException("Table name: ${importStatement.jdbcOptions.tableName}, " + - "SQL query: ${importStatement.jdbcOptions.sqlQuery} both are empty! " + - "Populate 'tableName' or 'sqlQuery' in jdbcOptions with value to generate schema " + - "for SQL table or result of SQL query!") + } else { + throw RuntimeException( + "Table name: ${importStatement.jdbcOptions.tableName}, " + + "SQL query: ${importStatement.jdbcOptions.sqlQuery} both are empty! " + + "Populate 'tableName' or 'sqlQuery' in jdbcOptions with value to generate schema " + + "for SQL table or result of SQL query!" + ) + } } } - diff --git a/plugins/symbol-processor/src/main/kotlin/org/jetbrains/dataframe/ksp/ExtensionsGenerator.kt b/plugins/symbol-processor/src/main/kotlin/org/jetbrains/dataframe/ksp/ExtensionsGenerator.kt index 17740a726d..22ad6f0061 100644 --- a/plugins/symbol-processor/src/main/kotlin/org/jetbrains/dataframe/ksp/ExtensionsGenerator.kt +++ b/plugins/symbol-processor/src/main/kotlin/org/jetbrains/dataframe/ksp/ExtensionsGenerator.kt @@ -24,7 +24,7 @@ import java.io.OutputStreamWriter class ExtensionsGenerator( private val resolver: Resolver, private val codeGenerator: CodeGenerator, - private val logger: KSPLogger + private val logger: KSPLogger, ) { private companion object { val EXPECTED_VISIBILITIES = setOf(Visibility.PUBLIC, Visibility.INTERNAL) @@ -47,13 +47,13 @@ class ExtensionsGenerator( class DataSchemaDeclaration( val origin: KSClassDeclaration, - val properties: List + val properties: List, ) class KSAnnotatedWithType( private val declaration: KSAnnotated, val simpleName: KSName, - val type: KSTypeReference + val type: KSTypeReference, ) : KSAnnotated by declaration private fun KSClassDeclaration.toDataSchemaDeclarationOrNull(): DataSchemaDeclaration? { @@ -66,11 +66,13 @@ class ExtensionsGenerator( .toList(), ) } + else -> null } } - private fun KSClassDeclaration.isClassOrInterface() = classKind == ClassKind.INTERFACE || classKind == ClassKind.CLASS + private fun KSClassDeclaration.isClassOrInterface() = + classKind == ClassKind.INTERFACE || classKind == ClassKind.CLASS private fun KSClassDeclaration.effectivelyPublicOrInternal(): Boolean { return effectivelyPublicOrInternalOrNull(dataSchema = this) != null @@ -153,23 +155,35 @@ class ExtensionsGenerator( private fun getMarkerVisibility(dataSchema: KSClassDeclaration) = when (val visibility = dataSchema.getVisibility()) { - Visibility.PUBLIC -> if (dataSchema.modifiers.contains(Modifier.PUBLIC)) { - MarkerVisibility.EXPLICIT_PUBLIC - } else { - MarkerVisibility.IMPLICIT_PUBLIC - } - Visibility.INTERNAL -> MarkerVisibility.INTERNAL - Visibility.PRIVATE, Visibility.PROTECTED, Visibility.LOCAL, Visibility.JAVA_PACKAGE -> { - error("DataSchema declaration should have ${EXPECTED_VISIBILITIES}, but was $visibility") - } + Visibility.PUBLIC -> + if (dataSchema.modifiers.contains(Modifier.PUBLIC)) { + MarkerVisibility.EXPLICIT_PUBLIC + } else { + MarkerVisibility.IMPLICIT_PUBLIC + } + + Visibility.INTERNAL -> + MarkerVisibility.INTERNAL + + Visibility.PRIVATE, Visibility.PROTECTED, Visibility.LOCAL, Visibility.JAVA_PACKAGE -> + error("DataSchema declaration should have $EXPECTED_VISIBILITIES, but was $visibility") } private fun getColumnName(property: KSAnnotatedWithType): String { - val columnNameAnnotation = property.annotations.firstOrNull { annotation -> + val columnNameAnnotation = property.annotations.firstOrNull { annotation -> val annotationType = annotation.annotationType - (annotationType.element as? KSClassifierReference)?.referencedName() - .let { it == null || it == DataFrameNames.SHORT_COLUMN_NAME } - && annotationType.resolve().declaration.qualifiedName?.asString() == DataFrameNames.COLUMN_NAME + + val typeIsColumnNameOrNull = (annotationType.element as? KSClassifierReference) + ?.referencedName() + ?.let { it == DataFrameNames.SHORT_COLUMN_NAME } != false + + val declarationIsColumnName = annotationType + .resolve() + .declaration + .qualifiedName + ?.asString() == DataFrameNames.COLUMN_NAME + + typeIsColumnNameOrNull && declarationIsColumnName } return if (columnNameAnnotation != null) { when (val arg = columnNameAnnotation.arguments.singleOrNull()) { @@ -180,7 +194,7 @@ class ExtensionsGenerator( property.simpleName.asString() } } - + private fun typeMismatchError(property: KSAnnotatedWithType, arg: KSValueArgument): Nothing { error("Expected one argument of type String in annotation ColumnName on property ${property.simpleName}, but got ${arg.value}") } diff --git a/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameJdbcSymbolProcessorTest.kt b/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameJdbcSymbolProcessorTest.kt index 06ba64980f..9be7ec101c 100644 --- a/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameJdbcSymbolProcessorTest.kt +++ b/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameJdbcSymbolProcessorTest.kt @@ -40,7 +40,7 @@ class DataFrameJdbcSymbolProcessorTest { @JvmStatic @AfterClass - fun close(): Unit { + fun close() { try { connection.close() } catch (e: SQLException) { @@ -52,11 +52,11 @@ class DataFrameJdbcSymbolProcessorTest { // Crate table Customer connection.createStatement().execute( """ - CREATE TABLE Customer ( - id INT PRIMARY KEY, - name VARCHAR(50), - age INT - ) + CREATE TABLE Customer ( + id INT PRIMARY KEY, + name VARCHAR(50), + age INT + ) """.trimIndent() ) @@ -97,20 +97,20 @@ class DataFrameJdbcSymbolProcessorTest { SourceFile.kotlin( "MySources.kt", """ - @file:ImportDataSchema(name = "Customer", path = "123") - - package test - - import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema - import org.jetbrains.kotlinx.dataframe.annotations.JdbcOptions - import org.jetbrains.kotlinx.dataframe.api.filter - import org.jetbrains.kotlinx.dataframe.DataFrame - import org.jetbrains.kotlinx.dataframe.api.cast - import java.sql.Connection - import java.sql.DriverManager - import java.sql.SQLException - import org.jetbrains.kotlinx.dataframe.io.readSqlTable - import org.jetbrains.kotlinx.dataframe.io.DatabaseConfiguration + @file:ImportDataSchema(name = "Customer", path = "123") + + package test + + import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema + import org.jetbrains.kotlinx.dataframe.annotations.JdbcOptions + import org.jetbrains.kotlinx.dataframe.api.filter + import org.jetbrains.kotlinx.dataframe.DataFrame + import org.jetbrains.kotlinx.dataframe.api.cast + import java.sql.Connection + import java.sql.DriverManager + import java.sql.SQLException + import org.jetbrains.kotlinx.dataframe.io.readSqlTable + import org.jetbrains.kotlinx.dataframe.io.DatabaseConfiguration """.trimIndent() ) ) @@ -121,39 +121,39 @@ class DataFrameJdbcSymbolProcessorTest { @Test fun `schema is imported`() { - val result = KspCompilationTestRunner.compile( - TestCompilationParameters( - sources = listOf( - SourceFile.kotlin( - "MySources.kt", - """ - @file:ImportDataSchema( - "Customer", - "$CONNECTION_URL", - jdbcOptions = JdbcOptions("", "", tableName = "Customer") - ) - - package test - - import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema - import org.jetbrains.kotlinx.dataframe.annotations.JdbcOptions - import org.jetbrains.kotlinx.dataframe.api.filter - import org.jetbrains.kotlinx.dataframe.DataFrame - import org.jetbrains.kotlinx.dataframe.api.cast - import java.sql.Connection - import java.sql.DriverManager - import java.sql.SQLException - import org.jetbrains.kotlinx.dataframe.io.readSqlTable - import org.jetbrains.kotlinx.dataframe.io.DatabaseConfiguration - """.trimIndent() + val result = KspCompilationTestRunner.compile( + TestCompilationParameters( + sources = listOf( + SourceFile.kotlin( + "MySources.kt", + """ + @file:ImportDataSchema( + "Customer", + "$CONNECTION_URL", + jdbcOptions = JdbcOptions("", "", tableName = "Customer") ) + + package test + + import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema + import org.jetbrains.kotlinx.dataframe.annotations.JdbcOptions + import org.jetbrains.kotlinx.dataframe.api.filter + import org.jetbrains.kotlinx.dataframe.DataFrame + import org.jetbrains.kotlinx.dataframe.api.cast + import java.sql.Connection + import java.sql.DriverManager + import java.sql.SQLException + import org.jetbrains.kotlinx.dataframe.io.readSqlTable + import org.jetbrains.kotlinx.dataframe.io.DatabaseConfiguration + """.trimIndent() ) ) ) - println(result.kspGeneratedFiles) - result.inspectLines("Customer.Generated.kt") { - it.forAtLeastOne { it shouldContain "val name: String" } - } + ) + println(result.kspGeneratedFiles) + result.inspectLines("Customer.Generated.kt") { + it.forAtLeastOne { it shouldContain "val name: String" } + } } /** @@ -167,44 +167,44 @@ class DataFrameJdbcSymbolProcessorTest { SourceFile.kotlin( "MySources.kt", """ - @file:ImportDataSchema( - "Customer", - "$CONNECTION_URL", - jdbcOptions = JdbcOptions("", "", tableName = "Customer") - ) - - package test - - import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema - import org.jetbrains.kotlinx.dataframe.annotations.JdbcOptions - import org.jetbrains.kotlinx.dataframe.api.filter - import org.jetbrains.kotlinx.dataframe.DataFrame - import org.jetbrains.kotlinx.dataframe.api.cast - import java.sql.Connection - import java.sql.DriverManager - import java.sql.SQLException - import org.jetbrains.kotlinx.dataframe.io.readSqlTable - import org.jetbrains.kotlinx.dataframe.io.DatabaseConfiguration - - fun main() { - val tableName = "Customer" - DriverManager.getConnection("$CONNECTION_URL").use { connection -> - val df = DataFrame.readSqlTable(connection, tableName).cast() - df.filter { age > 30 } - - val df1 = DataFrame.readSqlTable(connection, tableName, 1).cast() - df1.filter { age > 30 } + @file:ImportDataSchema( + "Customer", + "$CONNECTION_URL", + jdbcOptions = JdbcOptions("", "", tableName = "Customer") + ) - val dbConfig = DatabaseConfiguration(url = "$CONNECTION_URL") - val df2 = DataFrame.readSqlTable(dbConfig, tableName).cast() - df2.filter { age > 30 } + package test - val df3 = DataFrame.readSqlTable(dbConfig, tableName, 1).cast() - df3.filter { age > 30 } - - } - } - """.trimIndent() + import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema + import org.jetbrains.kotlinx.dataframe.annotations.JdbcOptions + import org.jetbrains.kotlinx.dataframe.api.filter + import org.jetbrains.kotlinx.dataframe.DataFrame + import org.jetbrains.kotlinx.dataframe.api.cast + import java.sql.Connection + import java.sql.DriverManager + import java.sql.SQLException + import org.jetbrains.kotlinx.dataframe.io.readSqlTable + import org.jetbrains.kotlinx.dataframe.io.DatabaseConfiguration + + fun main() { + val tableName = "Customer" + DriverManager.getConnection("$CONNECTION_URL").use { connection -> + val df = DataFrame.readSqlTable(connection, tableName).cast() + df.filter { age > 30 } + + val df1 = DataFrame.readSqlTable(connection, tableName, 1).cast() + df1.filter { age > 30 } + + val dbConfig = DatabaseConfiguration(url = "$CONNECTION_URL") + val df2 = DataFrame.readSqlTable(dbConfig, tableName).cast() + df2.filter { age > 30 } + + val df3 = DataFrame.readSqlTable(dbConfig, tableName, 1).cast() + df3.filter { age > 30 } + + } + } + """.trimIndent() ) ) ) diff --git a/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameSymbolProcessorTest.kt b/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameSymbolProcessorTest.kt index ea8cd4c69a..cd5571d017 100644 --- a/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameSymbolProcessorTest.kt +++ b/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameSymbolProcessorTest.kt @@ -219,25 +219,25 @@ class DataFrameSymbolProcessorTest { SourceFile.kotlin( "MySources.kt", """ - @file:ImportDataSchema(name = "Repo", "$it") - - import org.jetbrains.kotlinx.dataframe.DataFrame - import org.jetbrains.kotlinx.dataframe.annotations.DataSchema - import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema - import org.jetbrains.kotlinx.dataframe.api.print - import org.jetbrains.kotlinx.dataframe.io.readJson - - @DataSchema - interface Repos { - val repositories: DataFrame - } - - fun main() { - val df: DataFrame = DataFrame.readJson("data/jetbrains_repositories.json") as DataFrame - df.repositories[0].print() - } - - """.trimIndent() + @file:ImportDataSchema(name = "Repo", "$it") + + import org.jetbrains.kotlinx.dataframe.DataFrame + import org.jetbrains.kotlinx.dataframe.annotations.DataSchema + import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema + import org.jetbrains.kotlinx.dataframe.api.print + import org.jetbrains.kotlinx.dataframe.io.readJson + + @DataSchema + interface Repos { + val repositories: DataFrame + } + + fun main() { + val df: DataFrame = DataFrame.readJson("data/jetbrains_repositories.json") as DataFrame + df.repositories[0].print() + } + + """.trimIndent() ) ) ) @@ -1271,7 +1271,6 @@ class DataFrameSymbolProcessorTest { } } - private val petstoreJson = File("../../dataframe-openapi/src/test/resources/petstore.json") @Test diff --git a/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/runner/KspCompilationTestRunner.kt b/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/runner/KspCompilationTestRunner.kt index 6dca3fe807..9b116959c9 100644 --- a/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/runner/KspCompilationTestRunner.kt +++ b/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/runner/KspCompilationTestRunner.kt @@ -40,7 +40,7 @@ internal object KspCompilationTestRunner { kspCompilation.kspArgs.putAll(params.options) kspCompilation.symbolProcessorProviders = listOf(DataFrameSymbolProcessorProvider()) kspCompilation.compile().also { - println(it.messages); + println(it.messages) if (it.exitCode == KotlinCompilation.ExitCode.COMPILATION_ERROR) { return KotlinCompileTestingCompilationResult( delegate = it, diff --git a/tests/build.gradle.kts b/tests/build.gradle.kts index d1227be119..0cf781c43a 100644 --- a/tests/build.gradle.kts +++ b/tests/build.gradle.kts @@ -95,21 +95,6 @@ tasks.withType { enabled = true } -kotlinter { - ignoreFailures = false - reporters = arrayOf("checkstyle", "plain") - experimentalRules = true - disabledRules = arrayOf( - "no-wildcard-imports", - "experimental:spacing-between-declarations-with-annotations", - "experimental:enum-entry-name-case", - "experimental:argument-list-wrapping", - "experimental:annotation", - "max-line-length", - "filename" - ) -} - tasks.test { jvmArgs = listOf("--add-opens", "java.base/java.nio=ALL-UNNAMED") }