[FIR] Consider effective visibility of parent class during deserialization

For effective visibility calculation, it's essential to know the
effective visibility of the containing declaration, which was
completely missed in the implementation of the metadata deserializer

Also, make the effective visibility of deserialized declarations lazy,
as its computation requires subtyping, which is illegal to call during
deserialization.

^KT-74040

Author: demiurg906

compiler/fir/fir-deserialization/src/org/jetbrains/kotlin/fir/deserialization/ClassDeserialization.kt

@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.fir.*
 import org.jetbrains.kotlin.fir.declarations.*
 import org.jetbrains.kotlin.fir.declarations.builder.*
 import org.jetbrains.kotlin.fir.declarations.impl.FirResolvedDeclarationStatusImpl
+import org.jetbrains.kotlin.fir.declarations.impl.FirResolvedDeclarationStatusWithLazyEffectiveVisibility
 import org.jetbrains.kotlin.fir.declarations.utils.addDeclarations
 import org.jetbrains.kotlin.fir.declarations.utils.isCompanion
 import org.jetbrains.kotlin.fir.declarations.utils.moduleName
@@ -62,11 +63,12 @@ fun deserializeClassToSymbol(
     val kind = Flags.CLASS_KIND.get(flags)
     val modality = ProtoEnumFlags.modality(Flags.MODALITY.get(flags))
     val visibility = ProtoEnumFlags.visibility(Flags.VISIBILITY.get(flags))
-    val status = FirResolvedDeclarationStatusImpl(
-        visibility,
-        modality,
-        visibility.toEffectiveVisibility(parentContext?.outerClassSymbol, forClass = true)
-    ).apply {
+    val effectiveVisibility = visibility.toLazyEffectiveVisibility(
+        owner = parentContext?.outerClassSymbol,
+        session,
+        forClass = true
+    )
+    val status = FirResolvedDeclarationStatusWithLazyEffectiveVisibility(visibility, modality, effectiveVisibility).apply {
         isExpect = Flags.IS_EXPECT_CLASS.get(flags)
         isActual = false
         isCompanion = kind == ProtoBuf.Class.Kind.COMPANION_OBJECT
@@ -105,7 +107,8 @@ fun deserializeClassToSymbol(
             flexibleTypeFactory,
             constDeserializer,
             containerSource,
-            symbol
+            symbol,
+            status.effectiveVisibility
         )
     if (status.isCompanion) {
         parentContext?.let {

compiler/fir/fir-deserialization/src/org/jetbrains/kotlin/fir/deserialization/FirMemberDeserializer.kt

@@ -5,7 +5,9 @@
 
 package org.jetbrains.kotlin.fir.deserialization
 
+import org.jetbrains.kotlin.descriptors.EffectiveVisibility
 import org.jetbrains.kotlin.descriptors.Modality
+import org.jetbrains.kotlin.descriptors.Visibility
 import org.jetbrains.kotlin.fir.FirModuleData
 import org.jetbrains.kotlin.fir.FirSession
 import org.jetbrains.kotlin.fir.containingClassForStaticMemberAttr
@@ -14,7 +16,8 @@ import org.jetbrains.kotlin.fir.declarations.builder.*
 import org.jetbrains.kotlin.fir.declarations.impl.FirDefaultPropertyBackingField
 import org.jetbrains.kotlin.fir.declarations.impl.FirDefaultPropertyGetter
 import org.jetbrains.kotlin.fir.declarations.impl.FirDefaultPropertySetter
-import org.jetbrains.kotlin.fir.declarations.impl.FirResolvedDeclarationStatusImpl
+import org.jetbrains.kotlin.fir.declarations.impl.FirResolvedDeclarationStatusWithLazyEffectiveVisibility
+import org.jetbrains.kotlin.fir.declarations.utils.effectiveVisibility
 import org.jetbrains.kotlin.fir.declarations.utils.sourceElement
 import org.jetbrains.kotlin.fir.expressions.FirAnnotation
 import org.jetbrains.kotlin.fir.expressions.FirExpression
@@ -114,6 +117,7 @@ class FirDeserializationContext(
             typeParameterProtos = emptyList(),
             containerSource,
             outerClassSymbol = null,
+            outerClassEffectiveVisibility = EffectiveVisibility.Public,
             containingDeclarationSymbol = null
         )
 
@@ -126,7 +130,8 @@ class FirDeserializationContext(
             flexibleTypeFactory: FirTypeDeserializer.FlexibleTypeFactory,
             constDeserializer: FirConstDeserializer,
             containerSource: DeserializedContainerSource?,
-            outerClassSymbol: FirRegularClassSymbol
+            outerClassSymbol: FirRegularClassSymbol,
+            outerClassEffectiveVisibility: EffectiveVisibility,
         ): FirDeserializationContext = createRootContext(
             nameResolver,
             TypeTable(classProto.typeTable),
@@ -140,6 +145,7 @@ class FirDeserializationContext(
             classProto.typeParameterList,
             containerSource,
             outerClassSymbol,
+            outerClassEffectiveVisibility,
             outerClassSymbol
         )
 
@@ -156,6 +162,7 @@ class FirDeserializationContext(
             typeParameterProtos: List<ProtoBuf.TypeParameter>,
             containerSource: DeserializedContainerSource?,
             outerClassSymbol: FirRegularClassSymbol?,
+            outerClassEffectiveVisibility: EffectiveVisibility,
             containingDeclarationSymbol: FirBasedSymbol<*>?
         ): FirDeserializationContext {
             return FirDeserializationContext(
@@ -209,10 +216,10 @@ class FirMemberDeserializer(private val c: FirDeserializationContext) {
             origin = FirDeclarationOrigin.Library
             this.name = name
             val visibility = ProtoEnumFlags.visibility(Flags.VISIBILITY.get(flags))
-            status = FirResolvedDeclarationStatusImpl(
+            status = FirResolvedDeclarationStatusWithLazyEffectiveVisibility(
                 visibility,
                 Modality.FINAL,
-                visibility.toEffectiveVisibility(owner = null)
+                visibility.toLazyEffectiveVisibility(owner = null),
             ).apply {
                 isExpect = Flags.IS_EXPECT_CLASS.get(flags)
                 isActual = false
@@ -242,15 +249,15 @@ class FirMemberDeserializer(private val c: FirDeserializationContext) {
         val getterFlags = if (proto.hasGetterFlags()) proto.getterFlags else defaultAccessorFlags
         val visibility = ProtoEnumFlags.visibility(Flags.VISIBILITY.get(getterFlags))
         val accessorModality = ProtoEnumFlags.modality(Flags.MODALITY.get(getterFlags))
-        val effectiveVisibility = visibility.toEffectiveVisibility(classSymbol)
+        val effectiveVisibility = visibility.toLazyEffectiveVisibility(classSymbol)
         return if (Flags.IS_NOT_DEFAULT.get(getterFlags)) {
             buildPropertyAccessor {
                 moduleData = c.moduleData
                 origin = FirDeclarationOrigin.Library
                 this.returnTypeRef = returnTypeRef
                 resolvePhase = FirResolvePhase.ANALYZED_DEPENDENCIES
                 isGetter = true
-                status = FirResolvedDeclarationStatusImpl(visibility, accessorModality, effectiveVisibility).apply {
+                status = FirResolvedDeclarationStatusWithLazyEffectiveVisibility(visibility, accessorModality, effectiveVisibility).apply {
                     isInline = Flags.IS_INLINE_ACCESSOR.get(getterFlags)
                     isExternal = Flags.IS_EXTERNAL_ACCESSOR.get(getterFlags)
                 }
@@ -262,14 +269,12 @@ class FirMemberDeserializer(private val c: FirDeserializationContext) {
             }
         } else {
             FirDefaultPropertyGetter(
-                null,
+                source = null,
                 c.moduleData,
                 FirDeclarationOrigin.Library,
                 returnTypeRef,
-                visibility,
                 propertySymbol,
-                propertyModality,
-                effectiveVisibility,
+                status = FirResolvedDeclarationStatusWithLazyEffectiveVisibility(visibility, propertyModality, effectiveVisibility),
                 resolvePhase = FirResolvePhase.ANALYZED_DEPENDENCIES,
             )
         }.apply {
@@ -295,15 +300,15 @@ class FirMemberDeserializer(private val c: FirDeserializationContext) {
         val setterFlags = if (proto.hasSetterFlags()) proto.setterFlags else defaultAccessorFlags
         val visibility = ProtoEnumFlags.visibility(Flags.VISIBILITY.get(setterFlags))
         val accessorModality = ProtoEnumFlags.modality(Flags.MODALITY.get(setterFlags))
-        val effectiveVisibility = visibility.toEffectiveVisibility(classSymbol)
+        val effectiveVisibility = visibility.toLazyEffectiveVisibility(classSymbol)
         return if (Flags.IS_NOT_DEFAULT.get(setterFlags)) {
             buildPropertyAccessor {
                 moduleData = c.moduleData
                 origin = FirDeclarationOrigin.Library
                 this.returnTypeRef = FirImplicitUnitTypeRef(source)
                 resolvePhase = FirResolvePhase.ANALYZED_DEPENDENCIES
                 isGetter = false
-                status = FirResolvedDeclarationStatusImpl(visibility, accessorModality, effectiveVisibility).apply {
+                status = FirResolvedDeclarationStatusWithLazyEffectiveVisibility(visibility, accessorModality, effectiveVisibility).apply {
                     isInline = Flags.IS_INLINE_ACCESSOR.get(setterFlags)
                     isExternal = Flags.IS_EXTERNAL_ACCESSOR.get(setterFlags)
                 }
@@ -322,14 +327,12 @@ class FirMemberDeserializer(private val c: FirDeserializationContext) {
             }
         } else {
             FirDefaultPropertySetter(
-                null,
+                source = null,
                 c.moduleData,
                 FirDeclarationOrigin.Library,
                 returnTypeRef,
-                visibility,
                 propertySymbol,
-                propertyModality,
-                effectiveVisibility,
+                status = FirResolvedDeclarationStatusWithLazyEffectiveVisibility(visibility, propertyModality, effectiveVisibility),
                 resolvePhase = FirResolvePhase.ANALYZED_DEPENDENCIES,
             )
         }.apply {
@@ -394,7 +397,11 @@ class FirMemberDeserializer(private val c: FirDeserializationContext) {
             dispatchReceiverType = c.dispatchReceiver
             isLocal = false
             val visibility = ProtoEnumFlags.visibility(Flags.VISIBILITY.get(flags))
-            status = FirResolvedDeclarationStatusImpl(visibility, propertyModality, visibility.toEffectiveVisibility(classSymbol)).apply {
+            status = FirResolvedDeclarationStatusWithLazyEffectiveVisibility(
+                visibility,
+                propertyModality,
+                visibility.toLazyEffectiveVisibility(classSymbol)
+            ).apply {
                 isExpect = Flags.IS_EXPECT_PROPERTY.get(flags)
                 isActual = false
                 isOverride = false
@@ -546,10 +553,10 @@ class FirMemberDeserializer(private val c: FirDeserializationContext) {
 
             name = callableName
             val visibility = ProtoEnumFlags.visibility(Flags.VISIBILITY.get(flags))
-            status = FirResolvedDeclarationStatusImpl(
+            status = FirResolvedDeclarationStatusWithLazyEffectiveVisibility(
                 visibility,
                 ProtoEnumFlags.modality(Flags.MODALITY.get(flags)),
-                visibility.toEffectiveVisibility(classSymbol)
+                visibility.toLazyEffectiveVisibility(classSymbol)
             ).apply {
                 isExpect = Flags.IS_EXPECT_FUNCTION.get(flags)
                 isActual = false
@@ -625,10 +632,10 @@ class FirMemberDeserializer(private val c: FirDeserializationContext) {
             returnTypeRef = delegatedSelfType
             val visibility = ProtoEnumFlags.visibility(Flags.VISIBILITY.get(flags))
             val isInner = classBuilder.status.isInner
-            status = FirResolvedDeclarationStatusImpl(
+            status = FirResolvedDeclarationStatusWithLazyEffectiveVisibility(
                 visibility,
                 Modality.FINAL,
-                visibility.toEffectiveVisibility(classBuilder.symbol)
+                visibility.toLazyEffectiveVisibility(classBuilder.symbol)
             ).apply {
                 // We don't store information about expect modifier on constructors
                 // It is inherited from containing class
@@ -718,4 +725,40 @@ class FirMemberDeserializer(private val c: FirDeserializationContext) {
 
     private fun ProtoBuf.Type.toTypeRef(context: FirDeserializationContext): FirResolvedTypeRef =
         context.typeDeserializer.typeRef(this)
+
+    private fun Visibility.toLazyEffectiveVisibility(owner: FirClassLikeSymbol<*>?): Lazy<EffectiveVisibility> {
+        return this.toLazyEffectiveVisibility(owner, c.session, forClass = false)
+    }
+}
+
+fun Visibility.toLazyEffectiveVisibility(
+    owner: FirClassLikeSymbol<*>?,
+    session: FirSession,
+    forClass: Boolean
+): Lazy<EffectiveVisibility> {
+    /*
+     * `lowerBound` operation for `EffectiveVisibility.Protected` involves subtyping between container classes.
+     * In some cases, during deserialization, this subtyping might lead to the infinite recursion.
+     * Consider the following example:
+     *
+     * ```
+     * class Outer {
+     *     protected class Inner(protected val x: Any)
+     * }
+     * ```
+     *
+     * Here `Inner` class has effective visibility `protected (in Outer)` and `x` has `protected (in Inner)`.
+     * So to perform the `lowerBound` operation between these two visibilities, the compiler needs to check the
+     * subtyping between the `Outer` and `Inner`. BUT this happens during the deserialization in the following chain:
+     * `deserialize Outer -> deserialize Inner -> deserialize x`, and no class symbols are published yet (neither
+     * FIR element for them is created). So when subtyping tries to access supertypes of any of these classes, it triggers
+     * deserialization once again which leads to stack overflow eventually.
+     *
+     * Due to this situation, we cannot compute the effective visibility eagerly, so we postpone its computation
+     */
+    return lazy(LazyThreadSafetyMode.PUBLICATION) l@{
+        val selfEffectiveVisibility = this.toEffectiveVisibility(owner, forClass = forClass)
+        val parentEffectiveVisibility = owner?.effectiveVisibility ?: EffectiveVisibility.Public
+        parentEffectiveVisibility.lowerBound(selfEffectiveVisibility, session.typeContext)
+    }
 }

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/transformers/FirStatusResolver.kt

@@ -10,7 +10,6 @@ import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.fir.FirSession
 import org.jetbrains.kotlin.fir.declarations.*
 import org.jetbrains.kotlin.fir.declarations.impl.FirDeclarationStatusImpl
-import org.jetbrains.kotlin.fir.declarations.impl.FirResolvedDeclarationStatusImpl
 import org.jetbrains.kotlin.fir.declarations.utils.*
 import org.jetbrains.kotlin.fir.extensions.FirStatusTransformerExtension
 import org.jetbrains.kotlin.fir.extensions.extensionService
@@ -241,9 +240,6 @@ class FirStatusResolver(
     ): FirResolvedDeclarationStatus {
         if (status is FirResolvedDeclarationStatus) return status
         require(status is FirDeclarationStatusImpl)
-
-        @Suppress("UNCHECKED_CAST")
-        overriddenStatuses as List<FirResolvedDeclarationStatusImpl>
         val visibility = when (status.visibility) {
             Visibilities.Unknown -> when {
                 isLocal -> Visibilities.Local
@@ -262,7 +258,7 @@ class FirStatusResolver(
         if (overriddenStatuses.isNotEmpty()) {
             for (modifier in MODIFIERS_FROM_OVERRIDDEN) {
                 status[modifier] = status[modifier] || overriddenStatuses.fold(false) { acc, overriddenStatus ->
-                    acc || overriddenStatus[modifier]
+                    acc || (overriddenStatus as FirDeclarationStatusImpl)[modifier]
                 }
             }
             status[FirDeclarationStatusImpl.Modifier.OVERRIDE] = true
@@ -312,7 +308,7 @@ class FirStatusResolver(
         declaration: FirDeclaration,
         containingClass: FirClass?,
         containingProperty: FirProperty?,
-        overriddenStatuses: List<FirResolvedDeclarationStatusImpl>
+        overriddenStatuses: List<FirResolvedDeclarationStatus>
     ): Visibility {
         if (declaration is FirConstructor && containingClass?.hasPrivateConstructor() == true) return Visibilities.Private
 

compiler/fir/tree/src/org/jetbrains/kotlin/fir/EnumClassUtils.kt

@@ -22,6 +22,7 @@ import org.jetbrains.kotlin.fir.declarations.builder.buildValueParameter
 import org.jetbrains.kotlin.fir.declarations.impl.FirDeclarationStatusImpl
 import org.jetbrains.kotlin.fir.declarations.impl.FirDefaultPropertyGetter
 import org.jetbrains.kotlin.fir.declarations.impl.FirResolvedDeclarationStatusImpl
+import org.jetbrains.kotlin.fir.declarations.impl.FirResolvedDeclarationStatusWithLazyEffectiveVisibility
 import org.jetbrains.kotlin.fir.expressions.builder.buildEmptyExpressionBlock
 import org.jetbrains.kotlin.fir.symbols.impl.FirNamedFunctionSymbol
 import org.jetbrains.kotlin.fir.symbols.impl.FirPropertySymbol
@@ -257,11 +258,19 @@ fun generateEntriesGetter(
     }
 }
 
+@OptIn(FirImplementationDetail::class)
 private fun createStatus(parentStatus: FirDeclarationStatus): FirDeclarationStatusImpl {
-    val parentEffectiveVisibility = (parentStatus as? FirResolvedDeclarationStatusImpl)?.effectiveVisibility
-    return if (parentEffectiveVisibility != null) {
-        FirResolvedDeclarationStatusImpl(Visibilities.Public, Modality.FINAL, parentEffectiveVisibility)
-    } else {
-        FirDeclarationStatusImpl(Visibilities.Public, Modality.FINAL)
+    return when (parentStatus) {
+        is FirResolvedDeclarationStatusImpl -> FirResolvedDeclarationStatusImpl(
+            Visibilities.Public,
+            Modality.FINAL,
+            parentStatus.effectiveVisibility
+        )
+        is FirResolvedDeclarationStatusWithLazyEffectiveVisibility -> FirResolvedDeclarationStatusWithLazyEffectiveVisibility(
+            Visibilities.Public,
+            Modality.FINAL,
+            parentStatus.lazyEffectiveVisibility,
+        )
+        else -> FirDeclarationStatusImpl(Visibilities.Public, Modality.FINAL)
     }
 }