POC: Test failover envchange token

src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlInternalConnectionTds.cs

@@ -1946,10 +1946,10 @@ TimeoutTimer timeout
                         throw;  // Caller will call LoginFailure()
                     }
 
-                    if (!ADP.IsAzureSqlServerEndpoint(connectionOptions.DataSource) && IsConnectionDoomed)
+                    /*if (!ADP.IsAzureSqlServerEndpoint(connectionOptions.DataSource) && IsConnectionDoomed)
                     {
                         throw;
-                    }
+                    }*/
 
                     if (1 == attemptNumber % 2)
                     {

src/Microsoft.Data.SqlClient/tests/FunctionalTests/Microsoft.Data.SqlClient.FunctionalTests.csproj

@@ -94,6 +94,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="Microsoft.SqlServer.Types" />
     <PackageReference Include="Newtonsoft.Json" />
+    <PackageReference Include="System.Configuration.ConfigurationManager" />
     <PackageReference Include="System.Security.Cryptography.Pkcs" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetGroup)' == 'netcoreapp'">

src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConnectionBasicTests.cs

@@ -149,6 +149,39 @@ public void TransientFaultDisabledTest(uint errorCode)
             Assert.Equal(ConnectionState.Closed, connection.State);
         }
 
+        [Fact]
+        public void ConnectionPoolInvalidatedOnFault()
+        {
+            AppContext.SetSwitch("Switch.Microsoft.Data.SqlClient.UseManagedNetworkingOnWindows", true);
+
+            // Arrange: Create a few pooled connections to a server that will raise a transient fault.
+            using TransientFaultTDSServer failoverServer = TransientFaultTDSServer.StartTestServer(false, false, 40613, "localhost,1234");
+            var failoverDataSource = $"localhost,{failoverServer.Port}";
+            using TransientFaultTDSServer server = TransientFaultTDSServer.StartTestServer(false, false, 40613, failoverDataSource);
+            SqlConnectionStringBuilder builder = new()
+            {
+                DataSource = "localhost," + server.Port,
+                IntegratedSecurity = true,
+                ConnectRetryCount = 0,
+                Encrypt = SqlConnectionEncryptOption.Optional,
+                FailoverPartner = failoverDataSource,
+                InitialCatalog = "test"
+            };
+
+            using SqlConnection connection = new(builder.ConnectionString);
+            connection.Open();
+            {
+                using SqlConnection connection2 = new(builder.ConnectionString);
+                connection2.Open();
+            }
+            using SqlConnection connection3 = new(builder.ConnectionString);
+            connection3.Open();
+
+            server.SetErrorBehavior(true, 40613, "Transient fault occurred.");
+            using SqlConnection failedConnection = new(builder.ConnectionString);
+            failedConnection.Open();
+        }
+
         [Fact]
         public void SqlConnectionDbProviderFactoryTest()
         {

src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS.Servers/TransientFaultTDSServer.cs

@@ -8,7 +8,9 @@
 using System.Threading;
 using Microsoft.SqlServer.TDS.Done;
 using Microsoft.SqlServer.TDS.EndPoint;
+using Microsoft.SqlServer.TDS.EnvChange;
 using Microsoft.SqlServer.TDS.Error;
+using Microsoft.SqlServer.TDS.FeatureExtAck;
 using Microsoft.SqlServer.TDS.Login7;
 
 namespace Microsoft.SqlServer.TDS.Servers
@@ -22,6 +24,18 @@ public class TransientFaultTDSServer : GenericTDSServer, IDisposable
 
         public int Port { get; set; }
 
+        public string ConnectionString { get; private set; }
+
+        public void SetErrorBehavior(bool isEnabledTransientFault, uint errorNumber, string message)
+        {
+            if (Arguments is TransientFaultTDSServerArguments ServerArguments)
+            {
+                ServerArguments.IsEnabledTransientError = isEnabledTransientFault;
+                ServerArguments.Number = errorNumber;
+                ServerArguments.Message = message;
+            }
+        }
+
         /// <summary>
         /// Constructor
         /// </summary>
@@ -114,17 +128,67 @@ public override TDSMessageCollection OnLogin7Request(ITDSServerSession session,
             return base.OnLogin7Request(session, request);
         }
 
-        public static TransientFaultTDSServer StartTestServer(bool isEnabledTransientFault, bool enableLog, uint errorNumber, [CallerMemberName] string methodName = "")
-         => StartServerWithQueryEngine(null, isEnabledTransientFault, enableLog, errorNumber, methodName);
+        /// <summary>
+        /// Complete login sequence
+        /// </summary>
+        protected override TDSMessageCollection OnAuthenticationCompleted(ITDSServerSession session)
+        {
+            // Delegate to the base class
+            TDSMessageCollection responseMessageCollection = base.OnAuthenticationCompleted(session);
+
+            // Check if arguments are of routing server
+            if (Arguments is TransientFaultTDSServerArguments)
+            {
+                // Cast to transient fault TDS server arguments
+                TransientFaultTDSServerArguments serverArguments = Arguments as TransientFaultTDSServerArguments;
+
+                if (serverArguments.FailoverPartner == "")
+                {
+                    return responseMessageCollection;
+                } 
+
+                var envChangeToken = new TDSEnvChangeToken(TDSEnvChangeTokenType.RealTimeLogShipping, serverArguments.FailoverPartner);
+
+                // Log response
+                TDSUtilities.Log(Arguments.Log, "Response", envChangeToken);
+
+                // Get the first message
+                TDSMessage targetMessage = responseMessageCollection[0];
+
+                // Index at which to insert the routing token
+                int insertIndex = targetMessage.Count - 1;
+
+                // VSTS# 1021027 - Read-Only Routing yields TDS protocol error
+                // Resolution: Send TDS FeatureExtAct token before TDS ENVCHANGE token with routing information
+                TDSPacketToken featureExtAckToken = targetMessage.Find(t => t is TDSFeatureExtAckToken);
+
+                // Check if found
+                if (featureExtAckToken != null)
+                {
+                    // Find token position
+                    insertIndex = targetMessage.IndexOf(featureExtAckToken);
+                }
+
+                // Insert right before the done token
+                targetMessage.Insert(insertIndex, envChangeToken);
+
+            }
+
+            return responseMessageCollection;
+        }
+
+        public static TransientFaultTDSServer StartTestServer(bool isEnabledTransientFault, bool enableLog, uint errorNumber, string failoverPartner = "", [CallerMemberName] string methodName = "")
+         => StartServerWithQueryEngine(null, isEnabledTransientFault, enableLog, errorNumber, failoverPartner, methodName);
 
-        public static TransientFaultTDSServer StartServerWithQueryEngine(QueryEngine engine, bool isEnabledTransientFault, bool enableLog, uint errorNumber, [CallerMemberName] string methodName = "")
+        public static TransientFaultTDSServer StartServerWithQueryEngine(QueryEngine engine, bool isEnabledTransientFault, bool enableLog, uint errorNumber, string failoverPartner = "", [CallerMemberName] string methodName = "")
         {
             TransientFaultTDSServerArguments args = new TransientFaultTDSServerArguments()
             {
                 Log = enableLog ? Console.Out : null,
                 IsEnabledTransientError = isEnabledTransientFault,
                 Number = errorNumber,
-                Message = GetErrorMessage(errorNumber)
+                Message = GetErrorMessage(errorNumber),
+                FailoverPartner = failoverPartner
             };
 
             TransientFaultTDSServer server = engine == null ? new TransientFaultTDSServer(args) : new TransientFaultTDSServer(engine, args);

src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS.Servers/TransientFaultTDSServerArguments.cs

@@ -21,6 +21,11 @@ public class TransientFaultTDSServerArguments : TDSServerArguments
         /// </summary>
         public bool IsEnabledTransientError { get; set; }
 
+        /// <summary>
+        /// Routing destination protocol
+        /// </summary>
+        public string FailoverPartner { get; set; }
+
         /// <summary>
         /// Constructor to initialize
         /// </summary>
@@ -29,6 +34,7 @@ public TransientFaultTDSServerArguments()
             Number = 0;
             Message = string.Empty;
             IsEnabledTransientError = false;
+            FailoverPartner = string.Empty;
         }
     }
 }

src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS/TDS.csproj

@@ -106,4 +106,7 @@
     <Compile Include="TDSVersion.cs" />
     <Compile Include="TransactionIsolationLevelType.cs" />
   </ItemGroup>
+  <ItemGroup>
+    <Folder Include="microsoft.data.sqlclient\tests\tools\tds\tds\envchange\" />
+  </ItemGroup>
 </Project>