[Apple] Handle iOS 18+ test results via file copy instead of TCP tunnel

Directory.Packages.props

@@ -13,7 +13,7 @@
     <PackageVersion Include="Mono.Options" Version="6.12.0.148" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="Selenium.WebDriver" Version="4.0.0-alpha05" />
-    <PackageVersion Include="Microsoft.Tools.Mlaunch" Version="1.0.256" />
+    <PackageVersion Include="Microsoft.Tools.Mlaunch" Version="1.1.14" />
     <PackageVersion Include="NUnit" Version="3.13.0" />
     <PackageVersion Include="NUnit.Engine" Version="3.13.0" />
     <PackageVersion Include="xunit.extensibility.execution" Version="$(XUnitVersion)" />

src/Microsoft.DotNet.XHarness.Apple/AppOperations/AppTester.cs

@@ -204,6 +204,7 @@ public AppTester(
                 timeout,
                 null,
                 (level, message) => _mainLog.WriteLine(message));
+            IResultFileHandler resultFileHandler = new ResultFileHandler(_processManager, _mainLog);
 
             deviceListener.ConnectedTask
                 .TimeoutAfter(testLaunchTimeout)
@@ -252,10 +253,13 @@ await RunSimulatorTests(
                     mlaunchArguments,
                     crashReporter,
                     testReporter,
+                    resultFileHandler,
+                    deviceListener,
                     (ISimulatorDevice)device,
                     companionDevice as ISimulatorDevice,
                     timeout,
-                    cancellationToken);
+                    cancellationToken,
+                    runMode);
             }
             else
             {
@@ -283,15 +287,18 @@ await RunSimulatorTests(
                         appEndTag);
 
                     await RunDeviceTests(
+                        appInformation,
                         mlaunchArguments,
                         crashReporter,
                         testReporter,
+                        resultFileHandler,
                         deviceListener,
                         device,
                         appOutputLog,
                         timeout,
                         extraEnvVariables,
-                        cancellationToken);
+                        cancellationToken,
+                        runMode);
                 }
             }
 
@@ -305,10 +312,13 @@ private async Task RunSimulatorTests(
         MlaunchArguments mlaunchArguments,
         ICrashSnapshotReporter crashReporter,
         ITestReporter testReporter,
+        IResultFileHandler resultFileHandler,
+        ISimpleListener deviceListener,
         ISimulatorDevice simulator,
         ISimulatorDevice? companionSimulator,
         TimeSpan timeout,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken,
+        RunMode runMode)
     {
         var result = await RunSimulatorApp(
             appInformation,
@@ -321,18 +331,36 @@ private async Task RunSimulatorTests(
             cancellationToken);
 
         await testReporter.CollectSimulatorResult(result);
+
+        // On iOS 18 and later, transferring results over a TCP tunnel isn’t supported.
+        // Instead, copy the results file from the device to the host machine.
+        if (deviceListener.TestLog != null &&
+            !await resultFileHandler.CopyResultsAsync(
+                runMode,
+                true,
+                simulator.OSVersion,
+                simulator.UDID,
+                appInformation.BundleIdentifier,
+                deviceListener.TestLog.FullPath,
+                cancellationToken))
+        {
+            throw new InvalidOperationException("Failed to copy test results from simulator to host.");
+        }
     }
 
     private async Task RunDeviceTests(
+        AppBundleInformation appInformation,
         MlaunchArguments mlaunchArguments,
         ICrashSnapshotReporter crashReporter,
         ITestReporter testReporter,
+        IResultFileHandler resultFileHandler,
         ISimpleListener deviceListener,
         IDevice device,
         ILog appOutputLog,
         TimeSpan timeout,
         IEnumerable<(string, string)> extraEnvVariables,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken,
+        RunMode runMode)
     {
         var deviceSystemLog = _logs.Create($"device-{device.Name}-{_helpers.Timestamp}.log", LogType.SystemLog.ToString());
         deviceSystemLog.Timestamp = false;
@@ -391,6 +419,21 @@ private async Task RunDeviceTests(
         {
             _mainLog.WriteLine("Device log captured in {0}", deviceSystemLog.FullPath);
         }
+
+        // On iOS 18 and later, transferring results over a TCP tunnel isn’t supported.
+        // Instead, copy the results file from the device to the host machine.
+        if (deviceListener.TestLog != null &&
+            !await resultFileHandler.CopyResultsAsync(
+                runMode,
+                false,
+                device.OSVersion,
+                device.UDID,
+                appInformation.BundleIdentifier,
+                deviceListener.TestLog.FullPath,
+                cancellationToken))
+        {
+            throw new InvalidOperationException("Failed to copy test results from device to host.");
+        }
     }
 
     /// <summary>

src/Microsoft.DotNet.XHarness.Apple/AppOperations/AppTesterFactory.cs

@@ -14,7 +14,7 @@ namespace Microsoft.DotNet.XHarness.Apple;
 
 public interface IAppTesterFactory
 {
-    IAppTester Create(CommunicationChannel communicationChannel, bool isSimulator, IFileBackedLog log, ILogs logs, Action<string>? logCallback);
+    IAppTester Create(CommunicationChannel communicationChannel, bool isSimulator, IFileBackedLog log, ILogs logs, Action<string>? logCallback, Version? osVersion);
 }
 
 public class AppTesterFactory : IAppTesterFactory
@@ -50,9 +50,11 @@ public IAppTester Create(
         bool isSimulator,
         IFileBackedLog log,
         ILogs logs,
-        Action<string>? logCallback)
+        Action<string>? logCallback,
+        Version? osVersion)
     {
-        var tunnelBore = (communicationChannel == CommunicationChannel.UsbTunnel && !isSimulator)
+        // On iOS 18 and later, transferring results over a TCP tunnel isn’t supported.
+        var tunnelBore = (communicationChannel == CommunicationChannel.UsbTunnel && !isSimulator && osVersion !=null && osVersion.Major < 18)
             ? new TunnelBore(_processManager)
             : null;
 

src/Microsoft.DotNet.XHarness.Apple/ExitCodeDetector.cs

@@ -85,7 +85,11 @@ public class iOSExitCodeDetector : ExitCodeDetector, IiOSExitCodeDetector
 {
     // Example line coming from the mlaunch log
     // [07:02:21.6637600] Application 'net.dot.iOS.Simulator.PInvoke.Test' terminated (with exit code '42' and/or crashing signal ').
-    private Regex DeviceExitCodeRegex { get; } = new Regex(@"terminated \(with exit code '(?<exitCode>-?[0-9]+)' and/or crashing signal", RegexOptions.Compiled);
+    private Regex[] DeviceExitCodeRegexes { get; } = new Regex[]
+    {
+        new Regex(@"terminated \(with exit code '(?<exitCode>-?[0-9]+)' and/or crashing signal", RegexOptions.Compiled),
+        new Regex(@"Failed to execute 'devicectl':.*returned the exit code (?<exitCode>\d+)\.", RegexOptions.Compiled)
+    };
 
     protected override Match? IsSignalLine(AppBundleInformation appBundleInfo, string logLine)
     {
@@ -96,7 +100,14 @@ public class iOSExitCodeDetector : ExitCodeDetector, IiOSExitCodeDetector
 
         if (logLine.Contains(appBundleInfo.BundleIdentifier))
         {
-            return DeviceExitCodeRegex.Match(logLine);
+            foreach (var regex in DeviceExitCodeRegexes)
+            {
+                var m = regex.Match(logLine);
+                if (m.Success)
+                {
+                    return m;
+                }
+            }
         }
 
         return null;

src/Microsoft.DotNet.XHarness.Apple/Orchestration/RunOrchestrator.cs

@@ -313,7 +313,9 @@ private ExitCode ParseResult(
             return ExitCode.TIMED_OUT;
         }
 
-        if (!result.Succeeded)
+        // On iOS 18 and later, mlaunch returns exit code 1 with the following error message:
+        // "Failed to execute 'devicectl': returned the exit code <exit code>."
+        if (!result.Succeeded && result.ExitCode != 1)
         {
             _logger.LogError($"App run has failed. mlaunch exited with {result.ExitCode}");
             return ExitCode.APP_LAUNCH_FAILURE;

src/Microsoft.DotNet.XHarness.Apple/Orchestration/TestOrchestrator.cs

@@ -214,9 +214,10 @@ private async Task<ExitCode> ExecuteApp(
         bool signalAppEnd,
         CancellationToken cancellationToken)
     {
+        bool versionParsed = Version.TryParse(device.OSVersion, out var version);
         // iOS 14+ devices do not allow local network access and won't work unless the user confirms a dialog on the screen
         // https://developer.apple.com/forums/thread/663858
-        if (Version.TryParse(device.OSVersion, out var version) && version.Major >= 14 && target.Platform.ToRunMode() == RunMode.iOS && communicationChannel == CommunicationChannel.Network)
+        if (versionParsed && version!.Major >= 14 && target.Platform.ToRunMode() == RunMode.iOS && communicationChannel == CommunicationChannel.Network)
         {
             _logger.LogWarning(
                 "Applications need user permission for communication over local network on iOS 14 and newer." + Environment.NewLine +
@@ -231,7 +232,7 @@ private async Task<ExitCode> ExecuteApp(
 
         _logger.LogInformation("Starting test run for " + appBundleInfo.BundleIdentifier + "..");
 
-        var appTester = GetAppTester(communicationChannel, target.Platform.IsSimulator());
+        var appTester = GetAppTester(communicationChannel, target.Platform.IsSimulator(), version);
 
         (TestExecutingResult testResult, string resultMessage) = await appTester.TestApp(
             appBundleInfo,
@@ -255,7 +256,7 @@ private async Task<ExitCode> ExecuteApp(
             // Copy system and application logs to the main log for better failure investigation.
             CopyLogsToMainLog();
         }
-        
+
         return exitCode;
     }
 
@@ -272,7 +273,7 @@ private async Task<ExitCode> ExecuteMacCatalystApp(
         bool signalAppEnd,
         CancellationToken cancellationToken)
     {
-        var appTester = GetAppTester(communicationChannel, TestTarget.MacCatalyst.IsSimulator());
+        var appTester = GetAppTester(communicationChannel, TestTarget.MacCatalyst.IsSimulator(), null);
 
         (TestExecutingResult testResult, string resultMessage) = await appTester.TestMacCatalystApp(
             appBundleInfo,
@@ -294,12 +295,12 @@ private async Task<ExitCode> ExecuteMacCatalystApp(
         return exitCode;
     }
 
-    private IAppTester GetAppTester(CommunicationChannel communicationChannel, bool isSimulator)
+    private IAppTester GetAppTester(CommunicationChannel communicationChannel, bool isSimulator, Version? osVersion)
     {
         // Only add the extra callback if we do know that the feature was indeed enabled
         Action<string>? logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(_logger, l) : null;
 
-        return _appTesterFactory.Create(communicationChannel, isSimulator, _mainLog, _logs, logCallback);
+        return _appTesterFactory.Create(communicationChannel, isSimulator, _mainLog, _logs, logCallback, osVersion);
     }
 
     private ExitCode ParseResult(TestExecutingResult testResult, string resultMessage, bool listenerConnected)
@@ -389,7 +390,7 @@ private void CopyLogsToMainLog()
         {
             _mainLog.WriteLine($"==================== {log.Description} ====================");
             _mainLog.WriteLine($"Log file: {log.FullPath}");
-            
+
             try
             {
                 // Read and append log content to the main log
@@ -406,7 +407,7 @@ private void CopyLogsToMainLog()
             {
                 _mainLog.WriteLine($"Failed to read {log.Description}: {ex.Message}");
             }
-            
+
             _mainLog.WriteLine($"==================== End of {log.Description} ====================");
             _mainLog.WriteLine(string.Empty);
         }

src/Microsoft.DotNet.XHarness.TestRunners.Common/iOSApplicationEntryPointBase.cs

@@ -3,35 +3,58 @@
 // See the LICENSE file in the project root for more information.
 using System;
 using System.Threading.Tasks;
+using System.IO;
 
 #nullable enable
 namespace Microsoft.DotNet.XHarness.TestRunners.Common;
 
 public abstract class iOSApplicationEntryPointBase : ApplicationEntryPoint
 {
+    /// <summary>
+    /// Logger used for outputting logs. Defaults to Console.Out.
+    /// </summary>
+    public TextWriter? Logger = Console.Out;
+
+    /// <summary>
+    /// The final path where test results in XML format will be saved.
+    /// </summary>
+    public string TestsResultsFinalPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), "test-results.xml");
+
     public override async Task RunAsync()
     {
         var options = ApplicationOptions.Current;
-        TcpTextWriter? writer;
 
-        try
+        // On iOS 18 and later, transferring results over a TCP tunnel isn’t supported.
+        // Instead, copy the results file from the device to the host machine.
+        if (!OperatingSystem.IsMacCatalyst() && Environment.OSVersion.Version.Major >= 18)
         {
-            writer = options.UseTunnel
-                ? TcpTextWriter.InitializeWithTunnelConnection(options.HostPort)
-                : TcpTextWriter.InitializeWithDirectConnection(options.HostName, options.HostPort);
+            using TextWriter? resultsFileMaybe = options.EnableXml ? System.IO.File.CreateText(TestsResultsFinalPath) : null;
+            await InternalRunAsync(options, Logger, resultsFileMaybe);
+            Console.WriteLine($"Test results saved to: {TestsResultsFinalPath}");
         }
-        catch (Exception ex)
+        else
         {
-            Console.WriteLine("Failed to initialize TCP writer. Continuing on console." + Environment.NewLine + ex);
-            writer = null; // null means we will fall back to console output
-        }
+            TcpTextWriter? writer;
 
-        using (writer)
-        {
-            var logger = (writer == null || options.EnableXml) ? new LogWriter(Device) : new LogWriter(Device, writer);
-            logger.MinimumLogLevel = MinimumLogLevel.Info;
+            try
+            {
+                writer = options.UseTunnel
+                    ? TcpTextWriter.InitializeWithTunnelConnection(options.HostPort)
+                    : TcpTextWriter.InitializeWithDirectConnection(options.HostName, options.HostPort);
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Failed to initialize TCP writer. Continuing on console." + Environment.NewLine + ex);
+                writer = null; // null means we will fall back to console output
+            }
+
+            using (writer)
+            {
+                var logger = (writer == null || options.EnableXml) ? new LogWriter(Device) : new LogWriter(Device, writer);
+                logger.MinimumLogLevel = MinimumLogLevel.Info;
 
-            await InternalRunAsync(options, writer, writer);
+                await InternalRunAsync(options, writer, writer);
+            }
         }
     }
 }

src/Microsoft.DotNet.XHarness.iOS.Shared/IResultFileHandler.cs

@@ -0,0 +1,24 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Threading;
+using System.Threading.Tasks;
+
+#nullable enable
+namespace Microsoft.DotNet.XHarness.iOS.Shared;
+
+public interface IResultFileHandler
+{
+    /// <summary>
+    /// Copy the XML results file from the app container (simulator or device) to the host path.
+    /// </summary>
+    Task<bool> CopyResultsAsync(
+        RunMode runMode,
+        bool isSimulator,
+        string osVersion,
+        string udid,
+        string bundleIdentifier,
+        string hostDestinationPath,
+        CancellationToken token);
+}