Test improvements

src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs

@@ -11,6 +11,7 @@
 using System.Linq;
 using System.Reflection;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
 using Xunit;
@@ -445,14 +446,31 @@ public static string GenerateRandomCharacters(string prefix)
             return prefix + path;
         }
 
-        public static void RunNonQuery(string connectionString, string sql)
+        public static void RunNonQuery(string connectionString, string sql, int numberOfRetries = 0)
         {
-            using (SqlConnection connection = new SqlConnection(connectionString))
-            using (SqlCommand command = connection.CreateCommand())
+            int retries = 0;
+            while (true)
             {
-                connection.Open();
-                command.CommandText = sql;
-                command.ExecuteNonQuery();
+                try
+                {
+                    using (SqlConnection connection = new SqlConnection(connectionString))
+                    using (SqlCommand command = connection.CreateCommand())
+                    {
+                        connection.Open();
+                        command.CommandText = sql;
+                        command.ExecuteNonQuery();
+                        break;
+                    }
+                }
+                catch (Exception)
+                {
+                    if (retries >= numberOfRetries)
+                    {
+                        throw;
+                    }
+                    retries++;
+                    Thread.Sleep(1000);
+                }
             }
         }
 

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/Common/InternalConnectionWrapper.cs

@@ -23,7 +23,7 @@ public class InternalConnectionWrapper
         /// </summary>
         /// <param name="connection">Live outer connection to grab the inner connection from</param>
         /// <param name="supportKillByTSql">If true then we will query the server for this connection's SPID details (to be used in the KillConnectionByTSql method)</param>
-        public InternalConnectionWrapper(SqlConnection connection, bool supportKillByTSql = false)
+        public InternalConnectionWrapper(SqlConnection connection, bool supportKillByTSql = false, string originalConnectionString = "")
         {
             if (connection == null)
                 throw new ArgumentNullException(nameof(connection));
@@ -33,6 +33,17 @@ public InternalConnectionWrapper(SqlConnection connection, bool supportKillByTSq
 
             if (supportKillByTSql)
             {
+                SqlConnectionStringBuilder csb = new SqlConnectionStringBuilder(ConnectionString);
+                if (!csb.IntegratedSecurity &&
+                    string.IsNullOrWhiteSpace(originalConnectionString))
+                {
+                    throw new ArgumentException("Must provide originalConnectionString if using supportKillByTSql and not using Integrated Security.");
+                }
+                else if (!string.IsNullOrWhiteSpace(originalConnectionString))
+                {
+                    ConnectionString = originalConnectionString;
+                }
+
                 // Save the SPID for later use
                 using (SqlCommand command = new SqlCommand("SELECT @@SPID", connection))
                 {

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectivityTests/ConnectivityTest.cs

@@ -207,9 +207,9 @@ public static void ConnectionKilledTest()
                 DataTestUtility.RunNonQuery(s_dbConnectionString, s_createTableCmd);
 
                 // Kill all the connections and set Database to SINGLE_USER Mode.
-                DataTestUtility.RunNonQuery(s_connectionString, s_alterDatabaseSingleCmd);
+                DataTestUtility.RunNonQuery(s_connectionString, s_alterDatabaseSingleCmd, 4);
                 // Set Database back to MULTI_USER Mode
-                DataTestUtility.RunNonQuery(s_connectionString, s_alterDatabaseMultiCmd);
+                DataTestUtility.RunNonQuery(s_connectionString, s_alterDatabaseMultiCmd, 4);
 
                 // Execute SELECT statement.
                 DataTestUtility.RunNonQuery(s_dbConnectionString, s_selectTableCmd);
@@ -222,8 +222,58 @@ public static void ConnectionKilledTest()
             finally
             {
                 // Kill all the connections, set Database to SINGLE_USER Mode and drop Database
-                DataTestUtility.RunNonQuery(s_connectionString, s_alterDatabaseSingleCmd);
-                DataTestUtility.RunNonQuery(s_connectionString, s_dropDatabaseCmd);
+                DataTestUtility.RunNonQuery(s_connectionString, s_alterDatabaseSingleCmd, 4);
+                DataTestUtility.RunNonQuery(s_connectionString, s_dropDatabaseCmd, 4);
+            }
+        }
+
+        [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))]
+        public static void ConnectionResiliencyTest()
+        {
+            SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString);
+            builder.ConnectRetryCount = 0;
+            builder.ConnectRetryInterval = 5;
+
+            // No connection resiliency
+            using (SqlConnection conn = new SqlConnection(builder.ConnectionString))
+            {
+                conn.Open();
+                InternalConnectionWrapper wrapper = new InternalConnectionWrapper(conn, true, builder.ConnectionString);
+                using (SqlCommand cmd = conn.CreateCommand())
+                {
+                    cmd.CommandText = "SELECT TOP 1 * FROM dbo.Employees";
+                    wrapper.KillConnectionByTSql();
+                    bool cmdSuccess = false;
+                    try
+                    {
+                        cmd.ExecuteScalar();
+                        cmdSuccess = true;
+                    }
+                    // Windows always throws SqlException. Unix sometimes throws AggregateException against Azure SQL DB.
+                    catch (Exception ex) when (ex is SqlException || ex is AggregateException) { }
+                    Assert.False(cmdSuccess);
+                }
+            }
+
+            builder.ConnectRetryCount = 2;
+            using (SqlConnection conn = new SqlConnection(builder.ConnectionString))
+            {
+                conn.Open();
+                InternalConnectionWrapper wrapper = new InternalConnectionWrapper(conn, true, builder.ConnectionString);
+                using (SqlCommand cmd = conn.CreateCommand())
+                {
+                    cmd.CommandText = "SELECT TOP 1 * FROM dbo.Employees";
+                    using (SqlDataReader reader = cmd.ExecuteReader())
+                        while (reader.Read())
+                        { }
+
+                    wrapper.KillConnectionByTSql();
+
+                    // Connection resiliency should reconnect transparently
+                    using (SqlDataReader reader = cmd.ExecuteReader())
+                        while (reader.Read())
+                        { }
+                }
             }
         }
     }

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlCredentialTest/SqlCredentialTest.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Linq;
 using System.Security;
+using System.Threading;
 using Xunit;
 
 namespace Microsoft.Data.SqlClient.ManualTesting.Tests
@@ -152,7 +153,24 @@ public static void OldCredentialsShouldFail()
                     SecureString newPassword = new SecureString();
                     "newPassword".ToCharArray().ToList().ForEach(x => newPassword.AppendChar(x));
                     newPassword.MakeReadOnly();
-                    SqlConnection.ChangePassword(sqlConnectionStringBuilder.ConnectionString, credential, newPassword);
+                    int numberOfRetries = 0;
+                    while (true) // ChangePassword produces an intermittent server error during test runs so retry on failure
+                    {
+                        try
+                        {
+                            SqlConnection.ChangePassword(sqlConnectionStringBuilder.ConnectionString, credential, newPassword);
+                            break;
+                        }
+                        catch (SqlException)
+                        {
+                            if (numberOfRetries >= 4)
+                            {
+                                throw;
+                            }
+                            numberOfRetries++;
+                            Thread.Sleep(1000);
+                        }
+                    }
                     using (SqlConnection conn5 = new SqlConnection(sqlConnectionStringBuilder.ConnectionString, new SqlCredential(user, password)))
                     {
                         Assert.Throws<SqlException>(() => conn5.Open());