[KxSerialization] Fixed initialization error for keep generated serializer feature with sealed classes

When the heir of a sealed class contains this sealed class as a property, cyclic initialization occurs due to the fact that the SealedClassSerializer in the constructor accesses all passed subclass serializers.

Initialization order:
Parent Sealed class serializer
\/
Create instance of SealedClassSerializer for Parent Sealed
\/
Access subclass serializer in constructor of SealedClassSerializer
\/
Call ChildClass.Compaion.generatedSerializer()
\/
Init ChildClass
\/
cache child serializers of ChildClass
\/
Create instance of SealedClassSerializer for Parent Sealed ---> /\

Fixes #KT-70516


Merge-request: KT-MR-17421
Merged-by: Sergei Shanshin <[email protected]>

Author: shanshin

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/BaseIrGenerator.kt

@@ -464,13 +464,15 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
         serializableProperties: List<IrSerializableProperty>
     ): List<IrExpression?> {
         return DeclarationIrBuilder(compilerContext, symbol).run {
-            serializableProperties.map { cacheableChildSerializerInstance(serializableClass, it) }
+            val hasKeepGeneratedSerializerAnnotation = serializableClass.hasKeepGeneratedSerializerAnnotation
+            serializableProperties.map { cacheableChildSerializerInstance(serializableClass, it, hasKeepGeneratedSerializerAnnotation) }
         }
     }
 
     private fun IrBuilderWithScope.cacheableChildSerializerInstance(
         serializableClass: IrClass,
-        property: IrSerializableProperty
+        property: IrSerializableProperty,
+        hasKeepGeneratedSerializerAnnotation: Boolean
     ): IrExpression? {
         // to avoid a cyclical dependency between the serializer cache and the cache of child serializers,
         // the class  should not cache its serializer as a child
@@ -486,14 +488,21 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
         val serializer = getIrSerialTypeInfo(property, compilerContext).serializer ?: return null
         if (serializer.owner.kind == ClassKind.OBJECT) return null
 
-        return serializerInstance(
+        // disable caching for sealed serializer because of initialization loop, see https://github.com/Kotlin/kotlinx.serialization/issues/2759
+        if (hasKeepGeneratedSerializerAnnotation && serializer.owner.classId == sealedSerializerId) {
+            return null
+        }
+
+        val serializerInstance = serializerInstance(
             serializer,
             compilerContext,
             property.type,
             null,
             serializableClass,
             null
         )
+
+        return serializerInstance
     }
 
     private fun IrSimpleType.checkTypeArgumentsHasSelf(itselfClass: IrClassSymbol): Boolean {

plugins/kotlinx-serialization/testData/boxIr/KeepGeneratedSerializer.kt

@@ -190,6 +190,19 @@ object ObjectSerializer: KSerializer<Object> {
     }
 }
 
+// == sealed loop ==
+@Serializable
+public sealed interface SealedInterface
+
+@Serializable(with = SealedChild.CustomSerializer::class)
+@SerialName("child")
+@KeepGeneratedSerializer
+data class SealedChild(
+    val child: SealedInterface,
+) : SealedInterface {
+    internal object CustomSerializer : KSerializer<SealedChild> by generatedSerializer()
+}
+
 fun box(): String = boxWrapper {
     val value = Value(42)
     val data = Data(42)
@@ -213,6 +226,8 @@ fun box(): String = boxWrapper {
 
     assertEquals("Object()", Object.generatedSerializer().descriptor.toString(), "Object.generatedSerializer() illegal")
     assertSame(Object.generatedSerializer(), Object.generatedSerializer(), "Object.generatedSerializer() instance differs")
+
+    assertEquals("SealedInterface", SealedInterface.serializer().descriptor.serialName, "SealedInterface.serializer() illegal")
 }
 
 inline fun <reified T : Any> test(