[release/9.0-staging] [STJ] Fix formatting of flag enums when values are overlapping.

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/EnumConverter.cs

@@ -562,6 +562,13 @@ private static EnumFieldInfo[] ResolveEnumFields(JsonNamingPolicy? namingPolicy)
                 enumFields[i] = new EnumFieldInfo(key, kind, originalName, jsonName);
             }
 
+            if (s_isFlagsEnum)
+            {
+                // Perform topological sort for flags enums to ensure values that are supersets of other values come first.
+                // This is important for flags enums to ensure proper parsing and formatting.
+                enumFields = TopologicalSortEnumFields(enumFields);
+            }
+
             return enumFields;
         }
 
@@ -668,6 +675,52 @@ static bool ConflictsWith(EnumFieldInfo current, EnumFieldInfo other)
             }
         }
 
+        /// <summary>
+        /// Performs a topological sort on enum fields to ensure values that are supersets of other values come first.
+        /// </summary>
+        private static EnumFieldInfo[] TopologicalSortEnumFields(EnumFieldInfo[] enumFields)
+        {
+            if (enumFields.Length <= 1)
+            {
+                return enumFields;
+            }
+
+            var indices = new (int negativePopCount, int index)[enumFields.Length];
+            for (int i = 0; i < enumFields.Length; i++)
+            {
+                // We want values with more bits set to come first so negate the pop count.
+                // Keep the index as a second comparand so that sorting stability is preserved.
+                indices[i] = (-PopCount(enumFields[i].Key), i);
+            }
+
+            Array.Sort(indices);
+
+            var sortedFields = new EnumFieldInfo[enumFields.Length];
+            for (int i = 0; i < indices.Length; i++)
+            {
+                // extract the index from the sorted tuple
+                int index = indices[i].index;
+                sortedFields[i] = enumFields[index];
+            }
+
+            return sortedFields;
+        }
+
+        private static int PopCount(ulong value)
+        {
+#if NET
+            return (int)ulong.PopCount(value);
+#else
+            int count = 0;
+            while (value != 0)
+            {
+                value &= value - 1;
+                count++;
+            }
+            return count;
+#endif
+        }
+
         private enum EnumFieldNameKind
         {
             Default = 0,

src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/EnumConverterTests.cs

@@ -1257,5 +1257,41 @@ public enum EnumWithVaryingNamingPolicies
             A,
             b,
         }
+
+        [Fact]
+        public static void EnumWithOverlappingBitsTests()
+        {
+            JsonSerializerOptions options = new() { Converters = { new JsonStringEnumConverter() } };
+
+            EnumWithOverlappingBits e1 = EnumWithOverlappingBits.BITS01 | EnumWithOverlappingBits.BIT3;
+            string json1 = JsonSerializer.Serialize(e1, options);
+            Assert.Equal("\"BITS01, BIT3\"", json1);
+
+            EnumWithOverlappingBits2 e2 = EnumWithOverlappingBits2.BITS01 | EnumWithOverlappingBits2.BIT3;
+            string json2 = JsonSerializer.Serialize(e2, options);
+            Assert.Equal("\"BITS01, BIT3\"", json2);
+        }
+
+        [Flags]
+        public enum EnumWithOverlappingBits
+        {
+            UNKNOWN = 0,
+            BIT0 = 1,
+            BIT1 = 2,
+            BIT2 = 4,
+            BIT3 = 8,
+            BITS01 = 3,
+        }
+
+        [Flags]
+        public enum EnumWithOverlappingBits2
+        {
+            UNKNOWN = 0,
+            BIT0 = 1,
+            // direct option for bit 1 missing
+            BIT2 = 4,
+            BIT3 = 8,
+            BITS01 = 3,
+        }
     }
 }