Merge pull request #80452 from vseanreesermsft/internal-merge-6.0-202…

…3-01-10-1108

Merging internal commits for release/6.0

src/libraries/System.Private.DataContractSerialization/src/System/Runtime/Serialization/DataContract.cs

@@ -18,6 +18,7 @@ namespace System.Runtime.Serialization
     using System.Collections.Concurrent;
     using System.Diagnostics.CodeAnalysis;
     using System.Diagnostics;
+    using System.Xml.Linq;
 
     internal abstract class DataContract
     {
@@ -1248,9 +1249,34 @@ private static void ValidatePreviousCollectionTypes(Type collectionType, Type it
             {
                 itemType = itemType.GetElementType()!;
             }
-            if (previousCollectionTypes.Contains(itemType))
+
+            // Do a breadth first traversal of the generic type tree to
+            // produce the closure of all generic argument types and
+            // check that none of these is in the previousCollectionTypes
+            List<Type> itemTypeClosure = new List<Type>();
+            Queue<Type> itemTypeQueue = new Queue<Type>();
+
+            itemTypeQueue.Enqueue(itemType);
+            itemTypeClosure.Add(itemType);
+
+            while (itemTypeQueue.Count > 0)
             {
-                throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(new InvalidDataContractException(SR.Format(SR.RecursiveCollectionType, GetClrTypeFullName(itemType))));
+                itemType = itemTypeQueue.Dequeue();
+                if (previousCollectionTypes.Contains(itemType))
+                {
+                    throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(new InvalidDataContractException(SR.Format(SR.RecursiveCollectionType, GetClrTypeFullName(itemType))));
+                }
+                if (itemType.IsGenericType)
+                {
+                    foreach (Type argType in itemType.GetGenericArguments())
+                    {
+                        if (!itemTypeClosure.Contains(argType))
+                        {
+                            itemTypeQueue.Enqueue(argType);
+                            itemTypeClosure.Add(argType);
+                        }
+                    }
+                }
             }
         }
 
@@ -1334,10 +1360,16 @@ internal static XmlQualifiedName GetStableName(Type type)
 
         [RequiresUnreferencedCode(DataContract.SerializerTrimmerWarning)]
         internal static XmlQualifiedName GetStableName(Type type, out bool hasDataContract)
+        {
+            return GetStableName(type, new HashSet<Type>(), out hasDataContract);
+        }
+
+        [RequiresUnreferencedCode(DataContract.SerializerTrimmerWarning)]
+        internal static XmlQualifiedName GetStableName(Type type, HashSet<Type> previousCollectionTypes, out bool hasDataContract)
         {
             type = UnwrapRedundantNullableType(type);
             XmlQualifiedName? stableName;
-            if (TryGetBuiltInXmlAndArrayTypeStableName(type, out stableName))
+            if (TryGetBuiltInXmlAndArrayTypeStableName(type, previousCollectionTypes, out stableName))
             {
                 hasDataContract = false;
             }
@@ -1351,7 +1383,7 @@ internal static XmlQualifiedName GetStableName(Type type, out bool hasDataContra
                 }
                 else
                 {
-                    stableName = GetNonDCTypeStableName(type);
+                    stableName = GetNonDCTypeStableName(type, previousCollectionTypes);
                     hasDataContract = false;
                 }
             }
@@ -1389,14 +1421,17 @@ private static XmlQualifiedName GetDCTypeStableName(Type type, DataContractAttri
         }
 
         [RequiresUnreferencedCode(DataContract.SerializerTrimmerWarning)]
-        private static XmlQualifiedName GetNonDCTypeStableName(Type type)
+        private static XmlQualifiedName GetNonDCTypeStableName(Type type, HashSet<Type> previousCollectionTypes)
         {
             string? name = null, ns = null;
 
             Type? itemType;
             CollectionDataContractAttribute? collectionContractAttribute;
             if (CollectionDataContract.IsCollection(type, out itemType))
-                return GetCollectionStableName(type, itemType, out collectionContractAttribute);
+            {
+                ValidatePreviousCollectionTypes(type, itemType, previousCollectionTypes);
+                return GetCollectionStableName(type, itemType, previousCollectionTypes, out collectionContractAttribute);
+            }
             name = GetDefaultStableLocalName(type);
 
             // ensures that ContractNamespaceAttribute is honored when used with non-attributed types
@@ -1412,7 +1447,7 @@ private static XmlQualifiedName GetNonDCTypeStableName(Type type)
         }
 
         [RequiresUnreferencedCode(DataContract.SerializerTrimmerWarning)]
-        private static bool TryGetBuiltInXmlAndArrayTypeStableName(Type type, [NotNullWhen(true)] out XmlQualifiedName? stableName)
+        private static bool TryGetBuiltInXmlAndArrayTypeStableName(Type type, HashSet<Type> previousCollectionTypes, [NotNullWhen(true)] out XmlQualifiedName? stableName)
         {
             stableName = null;
 
@@ -1431,8 +1466,9 @@ private static bool TryGetBuiltInXmlAndArrayTypeStableName(Type type, [NotNullWh
             }
             else if (type.IsArray)
             {
-                CollectionDataContractAttribute? collectionContractAttribute;
-                stableName = GetCollectionStableName(type, type.GetElementType()!, out collectionContractAttribute);
+                Type itemType = type.GetElementType()!;
+                ValidatePreviousCollectionTypes(type, itemType, previousCollectionTypes);
+                stableName = GetCollectionStableName(type, itemType, previousCollectionTypes, out _);
             }
             return stableName != null;
         }
@@ -1456,6 +1492,12 @@ internal static bool TryGetDCAttribute(Type type, [NotNullWhen(true)] out DataCo
 
         [RequiresUnreferencedCode(DataContract.SerializerTrimmerWarning)]
         internal static XmlQualifiedName GetCollectionStableName(Type type, Type itemType, out CollectionDataContractAttribute? collectionContractAttribute)
+        {
+            return GetCollectionStableName(type, itemType, new HashSet<Type>(), out collectionContractAttribute);
+        }
+
+        [RequiresUnreferencedCode(DataContract.SerializerTrimmerWarning)]
+        internal static XmlQualifiedName GetCollectionStableName(Type type, Type itemType, HashSet<Type> previousCollectionTypes, out CollectionDataContractAttribute? collectionContractAttribute)
         {
             string? name, ns;
             object[] collectionContractAttributes = type.GetCustomAttributes(Globals.TypeOfCollectionDataContractAttribute, false).ToArray();
@@ -1492,7 +1534,7 @@ internal static XmlQualifiedName GetCollectionStableName(Type type, Type itemTyp
             {
                 collectionContractAttribute = null;
                 string arrayOfPrefix = Globals.ArrayPrefix + GetArrayPrefix(ref itemType);
-                XmlQualifiedName elementStableName = GetStableName(itemType);
+                XmlQualifiedName elementStableName = GetStableName(itemType, previousCollectionTypes, out _);
                 name = arrayOfPrefix + elementStableName.Name;
                 ns = GetCollectionNamespace(elementStableName.Namespace);
             }

src/libraries/System.Runtime.Serialization.Xml/tests/DataContractSerializer.cs

@@ -805,7 +805,7 @@ public static void DCS_EnumerableMembers()
         Assert.True(y.RO1.Count == 1);
         Assert.True((char)y.RO1[0] == 'x');
     }
-	
+
     [Fact]
     public static void DCS_EnumerableMemberConcreteTypeWithoutDefaultContructor()
     {
@@ -2151,7 +2151,21 @@ public static void DCS_RecursiveCollection()
     {
         Assert.Throws<InvalidDataContractException>(() =>
         {
-            (new DataContractSerializer(typeof (RecursiveCollection))).WriteObject(new MemoryStream(), new RecursiveCollection());
+            (new DataContractSerializer(typeof(RecursiveCollection))).WriteObject(new MemoryStream(), new RecursiveCollection());
+        });
+
+        Assert.Throws<InvalidDataContractException>(() =>
+        {
+            (new DataContractSerializer(typeof(RecursiveEnumerable))).WriteObject(new MemoryStream(), new RecursiveEnumerable());
+        });
+
+        // When constructing an element name, we seem to ignore recursion for types marked with [CollectionDataContract].
+        var name = new XsdDataContractExporter().GetRootElementName(typeof(RecursiveCollection));
+
+        // But 'RecursiveEnumerable' is not marked with the attribute. So it takes a code path that looks for this.
+        Assert.Throws<InvalidDataContractException>(() =>
+        {
+            new XsdDataContractExporter().GetRootElementName(typeof(RecursiveEnumerable));
         });
     }
 

src/libraries/System.Runtime.Serialization.Xml/tests/SerializationTypes.RuntimeOnly.cs

@@ -2580,6 +2580,23 @@ public TypeWithEnumerableInterfaceGetOnlyCollection(List<string> items)
     }
 }
 
+// RecursiveEnumerable
+// =============================
+// IEnumerable<>
+// Not-explicit [CollectionDataContract]
+// Direct recursion
+//
+// RecursiveCollection
+// =============================
+// List<>
+// Explicit [CollectionDataContract]
+// Indirect recursion
+public class RecursiveEnumerable : IEnumerable<RecursiveEnumerable>
+{
+    public IEnumerator<RecursiveEnumerable> GetEnumerator() => null;
+    IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+}
+
 [CollectionDataContract]
 public class RecursiveCollection : List<RecursiveCollection2>
 {