Best-effort kotlin-reflect instantiation avoidance

Fixes #1819

core/api/kotlinx-serialization-core.api

@@ -117,7 +117,6 @@ public final class kotlinx/serialization/SerializersKt {
 	public static final fun serializer (Lkotlin/reflect/KType;)Lkotlinx/serialization/KSerializer;
 	public static final fun serializer (Lkotlinx/serialization/modules/SerializersModule;Ljava/lang/reflect/Type;)Lkotlinx/serialization/KSerializer;
 	public static final fun serializer (Lkotlinx/serialization/modules/SerializersModule;Lkotlin/reflect/KType;)Lkotlinx/serialization/KSerializer;
-	public static final fun serializerOrNull (Ljava/lang/Class;)Lkotlinx/serialization/KSerializer;
 	public static final fun serializerOrNull (Ljava/lang/reflect/Type;)Lkotlinx/serialization/KSerializer;
 	public static final fun serializerOrNull (Lkotlin/reflect/KClass;)Lkotlinx/serialization/KSerializer;
 	public static final fun serializerOrNull (Lkotlin/reflect/KType;)Lkotlinx/serialization/KSerializer;

core/commonMain/src/kotlinx/serialization/Serializers.kt

@@ -126,7 +126,6 @@ internal fun <T : Any> SerializersModule.reflectiveOrContextual(kClass: KClass<T
     return kClass.serializerOrNull() ?: getContextual(kClass, typeArgumentsSerializers)
 }
 
-
 /**
  * Retrieves a [KSerializer] for the given [KClass].
  * The given class must be annotated with [Serializable] or be one of the built-in types.

core/jvmMain/src/kotlinx/serialization/SerializersJvm.kt

@@ -56,7 +56,7 @@ public fun serializerOrNull(type: Type): KSerializer<Any>? = EmptySerializersMod
  */
 @ExperimentalSerializationApi
 public fun SerializersModule.serializer(type: Type): KSerializer<Any> =
-    serializerByJavaTypeImpl(type, failOnMissingTypeArgSerializer = true) ?: type.kclass().serializerNotRegistered()
+    serializerByJavaTypeImpl(type, failOnMissingTypeArgSerializer = true) ?: type.prettyClass().serializerNotRegistered()
 
 /**
  * Retrieves serializer for the given reflective Java [type] using
@@ -74,19 +74,6 @@ public fun SerializersModule.serializer(type: Type): KSerializer<Any> =
 public fun SerializersModule.serializerOrNull(type: Type): KSerializer<Any>? =
     serializerByJavaTypeImpl(type, failOnMissingTypeArgSerializer = false)
 
-/**
- * Retrieves a [KSerializer] for the given [java.lang.Class] instance of the Kotlin type or returns `null` if none is found.
- * The given class must be annotated with [Serializable] or be one of the built-in types.
- *
- * The API avoids instantiation of the corresponding [KClass][Class.kotlin] on the best-effort basis.
- *
- * This is a [Class] counterpart of [KClass.serializerOrNull] with all the restrictions the original function implies,
- * including the general recommendation to avoid uses of this API.
- */
-@InternalSerializationApi
-public fun <T : Any> Class<T>.serializerOrNull(): KSerializer<T>? =
-    constructSerializerForGivenTypeArgs() ?: kotlin.builtinSerializerOrNull()
-
 @OptIn(ExperimentalSerializationApi::class)
 private fun SerializersModule.serializerByJavaTypeImpl(type: Type, failOnMissingTypeArgSerializer: Boolean = true): KSerializer<Any>? =
     when (type) {
@@ -126,8 +113,7 @@ private fun SerializersModule.serializerByJavaTypeImpl(type: Type, failOnMissing
                     // probably we should deprecate this method because it can't differ nullable vs non-nullable types
                     // since it uses Java TypeToken, not Kotlin one
                     val varargs = argsSerializers.map { it as KSerializer<Any?> }
-                    (rootClass.kotlin.constructSerializerForGivenTypeArgs(*(varargs.toTypedArray())) as? KSerializer<Any>)
-                            ?: reflectiveOrContextual(rootClass.kotlin as KClass<Any>, varargs)
+                    reflectiveOrContextual(rootClass as Class<Any>, varargs)
                 }
             }
         }
@@ -143,10 +129,17 @@ private fun SerializersModule.typeSerializer(type: Class<*>, failOnMissingTypeAr
         val arraySerializer = ArraySerializer(eType.kotlin as KClass<Any>, s)
         arraySerializer as KSerializer<Any>
     } else {
-        reflectiveOrContextual(type.kotlin as KClass<Any>, emptyList())
+        reflectiveOrContextual(type as Class<Any>, emptyList())
     }
 }
 
+@OptIn(ExperimentalSerializationApi::class)
+private fun <T : Any> SerializersModule.reflectiveOrContextual(jClass: Class<T>, typeArgumentsSerializers: List<KSerializer<Any?>>): KSerializer<T>? {
+    jClass.constructSerializerForGivenTypeArgs(*typeArgumentsSerializers.toTypedArray())?.let { return it }
+    val kClass = jClass.kotlin
+    return kClass.builtinSerializerOrNull() ?: getContextual(kClass, typeArgumentsSerializers)
+}
+
 @OptIn(ExperimentalSerializationApi::class)
 private fun SerializersModule.genericArraySerializer(
     type: GenericArrayType,
@@ -167,11 +160,10 @@ private fun SerializersModule.genericArraySerializer(
     return ArraySerializer(kclass, serializer) as KSerializer<Any>
 }
 
-private fun Type.kclass(): KClass<*> = when (val it = this) {
-    is KClass<*> -> it
-    is Class<*> -> it.kotlin
-    is ParameterizedType -> it.rawType.kclass()
-    is WildcardType -> it.upperBounds.first().kclass()
-    is GenericArrayType -> it.genericComponentType.kclass()
+private fun Type.prettyClass(): Class<*> = when (val it = this) {
+    is Class<*> -> it
+    is ParameterizedType -> it.rawType.prettyClass()
+    is WildcardType -> it.upperBounds.first().prettyClass()
+    is GenericArrayType -> it.genericComponentType.prettyClass()
     else -> throw IllegalArgumentException("typeToken should be an instance of Class<?>, GenericArray, ParametrizedType or WildcardType, but actual type is $it ${it::class}")
 }

core/jvmMain/src/kotlinx/serialization/internal/Platform.kt

@@ -28,6 +28,13 @@ internal actual fun <T : Any, E : T?> ArrayList<E>.toNativeArrayImpl(eClass: KCl
 
 internal actual fun KClass<*>.platformSpecificSerializerNotRegistered(): Nothing = serializerNotRegistered()
 
+internal fun Class<*>.serializerNotRegistered(): Nothing {
+    throw SerializationException(
+        "Serializer for class '${simpleName}' is not found.\n" +
+                "Mark the class as @Serializable or provide the serializer explicitly."
+    )
+}
+
 internal actual fun <T : Any> KClass<T>.constructSerializerForGivenTypeArgs(vararg args: KSerializer<Any?>): KSerializer<T>? {
     return java.constructSerializerForGivenTypeArgs(*args)
 }

formats/json/jvmTest/src/kotlinx/serialization/features/SerializerByTypeTest.kt

@@ -56,12 +56,17 @@ class SerializerByTypeTest {
         assertSerializedWithType(IntBoxToken, """{"a":42}""", b)
     }
 
+    @Test
+    fun testNestedGenericParameter() {
+        val b = Box(Box(239))
+        assertSerializedWithType(typeTokenOf<Box<Box<Int>>>(), """{"a":{"a":239}}""", b)
+    }
+
     @Test
     fun testArray() {
         val myArr = arrayOf("a", "b", "c")
         val token = myArr::class.java
         assertSerializedWithType(token, """["a","b","c"]""", myArr)
-        myArr::class.assertKClassSerializer()
     }
 
     @Test
@@ -175,23 +180,20 @@ class SerializerByTypeTest {
     fun testNamedCompanion() {
         val namedCompanion = WithNamedCompanion(1)
         assertSerializedWithType(WithNamedCompanion::class.java, """{"a":1}""", namedCompanion)
-        WithNamedCompanion::class.assertKClassSerializer()
     }
 
     @Test
     fun testPrimitive() {
         val token = typeTokenOf<Int>()
         val serial = serializer(token)
         assertSame(Int.serializer() as KSerializer<*>, serial)
-        Int::class.assertKClassSerializer()
     }
 
     @Test
     fun testObject() {
         val token = typeTokenOf<SerializableObject>()
         val serial = serializer(token)
         assertEquals(SerializableObject.serializer().descriptor, serial.descriptor)
-        SerializableObject::class.assertKClassSerializer()
     }
 
     @Suppress("UNCHECKED_CAST")
@@ -276,9 +278,9 @@ class SerializerByTypeTest {
         assertFailsWithMessage<SerializationException>("for class 'NonSerializable'") {
             serializer(typeTokenOf<kotlinx.serialization.Box<NonSerializable>>())
         }
-    }
 
-    private fun <T: Any> KClass<T>.assertKClassSerializer() {
-        assertEquals(serializerOrNull(), java.serializerOrNull())
+        assertFailsWithMessage<SerializationException>("for class 'NonSerializable'") {
+            serializer(typeTokenOf<Array<NonSerializable>>())
+        }
     }
 }