Add the WASI runtime test target

README.md

@@ -4,7 +4,7 @@ This repo contains the code to build the **XHarness dotnet tool** and a **TestRu
 
 ## What is XHarness
 
-XHarness is primarily a command line tool that enables running xUnit like tests on Android, Apple iOS / tvOS / WatchOS / Mac Catalyst and desktop browsers (WASM).
+XHarness is primarily a command line tool that enables running xUnit like tests on Android, Apple iOS / tvOS / WatchOS / Mac Catalyst, WASI and desktop browsers (WASM).
 It can
 - locate devices/emulators
 - install a given application, run it and collect results uninstalling it after,

src/Microsoft.DotNet.XHarness.CLI/CommandArguments/WASI/Arguments/WasmEngineArgument.cs

@@ -0,0 +1,32 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+
+namespace Microsoft.DotNet.XHarness.CLI.CommandArguments.Wasi;
+
+internal class WasmEngineArgument : Argument<WasmEngine?>
+{
+    public WasmEngineArgument()
+        : base("engine=|e=", "Specifies the Wasm engine to be used", null)
+    {
+    }
+
+    public override void Action(string argumentValue) =>
+        Value = ParseArgument<WasmEngine>("engine", argumentValue);
+
+    // Set WasmTime as default engine
+    public override void Validate() => Value ??= WasmEngine.WasmTime;
+}
+
+/// <summary>
+/// Specifies a name of a wasm engine used to run WASI application.
+/// </summary>
+internal enum WasmEngine
+{
+    /// <summary>
+    /// WasmTime
+    /// </summary>
+    WasmTime,
+}

src/Microsoft.DotNet.XHarness.CLI/CommandArguments/WASI/Arguments/WasmEngineArguments.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace Microsoft.DotNet.XHarness.CLI.CommandArguments.Wasi;
+
+internal class WasmEngineArguments : RepeatableArgument
+{
+    public WasmEngineArguments()
+        : base("engine-arg=", "Argument to pass to the wasm engine")
+    {
+    }
+}

src/Microsoft.DotNet.XHarness.CLI/CommandArguments/WASI/Arguments/WasmEngineLocationArgument.cs

@@ -0,0 +1,12 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace Microsoft.DotNet.XHarness.CLI.CommandArguments.Wasi;
+
+internal class WasmEngineLocationArgument : StringArgument
+{
+    public WasmEngineLocationArgument() : base("wasm-engine-path=", "Path to the wasm engine to be used. This must correspond to the engine specified with -e")
+    {
+    }
+}

src/Microsoft.DotNet.XHarness.CLI/CommandArguments/WASI/WasiTestCommandArguments.cs

@@ -0,0 +1,28 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.DotNet.XHarness.CLI.CommandArguments.Wasi;
+
+internal class WasiTestCommandArguments : XHarnessCommandArguments
+{
+    public WasmEngineArgument Engine { get; } = new();
+    public WasmEngineLocationArgument EnginePath { get; } = new();
+    public WasmEngineArguments EngineArgs { get; } = new();
+    public ExpectedExitCodeArgument ExpectedExitCode { get; } = new((int)Common.CLI.ExitCode.SUCCESS);
+    public OutputDirectoryArgument OutputDirectory { get; } = new();
+    public TimeoutArgument Timeout { get; } = new(TimeSpan.FromMinutes(15));
+
+    protected override IEnumerable<Argument> GetArguments() => new Argument[]
+    {
+            Engine,
+            EnginePath,
+            EngineArgs,
+            OutputDirectory,
+            Timeout,
+            ExpectedExitCode,
+    };
+}

src/Microsoft.DotNet.XHarness.CLI/Commands/WASI/Engine/WasiTestCommand.cs

@@ -0,0 +1,153 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.DotNet.XHarness.CLI.CommandArguments.Wasi;
+using Microsoft.DotNet.XHarness.CLI.Commands.Wasm;
+using Microsoft.DotNet.XHarness.Common;
+using Microsoft.DotNet.XHarness.Common.CLI;
+using Microsoft.DotNet.XHarness.Common.Execution;
+using Microsoft.DotNet.XHarness.Common.Logging;
+using Microsoft.DotNet.XHarness.Common.Utilities;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.DotNet.XHarness.CLI.Commands.Wasi;
+internal class WasiTestCommand : XHarnessCommand<WasiTestCommandArguments>
+{
+    private const string CommandHelp = "Executes tests on WASI using a selected engine";
+
+    protected override WasiTestCommandArguments Arguments { get; } = new();
+    protected override string CommandUsage { get; } = "wasi test [OPTIONS] -- [ENGINE OPTIONS]";
+    protected override string CommandDescription { get; } = CommandHelp;
+
+    public WasiTestCommand() : base(TargetPlatform.WASI, "test", true, new ServiceCollection(), CommandHelp)
+    {
+    }
+
+    protected override async Task<ExitCode> InvokeInternal(ILogger logger)
+    {
+        var processManager = ProcessManagerFactory.CreateProcessManager();
+
+        string engineBinary = Arguments.Engine.Value switch
+        {
+            WasmEngine.WasmTime => "wasmtime",
+            _ => throw new ArgumentException("Engine not set")
+        };
+
+        if (!string.IsNullOrEmpty(Arguments.EnginePath.Value))
+        {
+            engineBinary = Arguments.EnginePath.Value;
+            if (Path.IsPathRooted(engineBinary) && !File.Exists(engineBinary))
+                throw new ArgumentException($"Could not find js engine at the specified path - {engineBinary}");
+        }
+        else
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {                
+                engineBinary = FileUtils.FindFileInPath(engineBinary + ".cmd");
+            }
+        }
+
+        var cts = new CancellationTokenSource();
+        try
+        {
+            logger.LogInformation($"Using wasm engine {Arguments.Engine.Value} from path {engineBinary}");
+            await PrintVersionAsync(Arguments.Engine.Value.Value, engineBinary);
+
+            var engineArgs = new List<string>();
+            engineArgs.AddRange(Arguments.EngineArgs.Value);
+            engineArgs.AddRange(PassThroughArguments);
+
+            var xmlResultsFilePath = Path.Combine(Arguments.OutputDirectory, "testResults.xml");
+            File.Delete(xmlResultsFilePath);
+
+            var stdoutFilePath = Path.Combine(Arguments.OutputDirectory, "wasi-console.log");
+            File.Delete(stdoutFilePath);
+
+            var logProcessor = new WasmTestMessagesProcessor(xmlResultsFilePath,
+                                                             stdoutFilePath,
+                                                             logger);
+            var logProcessorTask = Task.Run(() => logProcessor.RunAsync(cts.Token));
+
+            var processTask = processManager.ExecuteCommandAsync(
+                engineBinary,
+                engineArgs,
+                log: new CallbackLog(m => logger.LogInformation(m.Trim())),
+                stdoutLog: new CallbackLog(msg => logProcessor.Invoke(msg)),
+                stderrLog: new CallbackLog(logProcessor.ProcessErrorMessage),
+                Arguments.Timeout);
+
+            var tasks = new Task[]
+            {
+                logProcessorTask,
+                processTask,
+                Task.Delay(Arguments.Timeout)
+            };
+
+            var task = await Task.WhenAny(tasks).ConfigureAwait(false);
+            if (task == tasks[^1] || cts.IsCancellationRequested || task.IsCanceled)
+            {
+                logger.LogError($"Tests timed out after {((TimeSpan)Arguments.Timeout).TotalSeconds}secs");
+                if (!cts.IsCancellationRequested)
+                    cts.Cancel();
+
+                return ExitCode.TIMED_OUT;
+            }
+
+            if (task.IsFaulted)
+            {
+                logger.LogError($"task faulted {task.Exception}");
+                throw task.Exception!;
+            }
+
+            // if the log processor completed without errors, then the
+            // process should be done too, or about to be done!
+            var result = await processTask;
+            ExitCode logProcessorExitCode = await logProcessor.CompleteAndFlushAsync();
+
+            if (result.ExitCode != Arguments.ExpectedExitCode)
+            {
+                logger.LogError($"Application has finished with exit code {result.ExitCode} but {Arguments.ExpectedExitCode} was expected");
+                return ExitCode.GENERAL_FAILURE;
+            }
+            else
+            {
+                logger.LogInformation("Application has finished with exit code: " + result.ExitCode);
+                // return SUCCESS if logProcess also returned SUCCESS
+                return logProcessorExitCode;
+            }
+        }
+        catch (Win32Exception e) when (e.NativeErrorCode == 2)
+        {
+            logger.LogCritical($"The engine binary `{engineBinary}` was not found");
+            return ExitCode.APP_LAUNCH_FAILURE;
+        }
+        finally
+        {
+            if (!cts.IsCancellationRequested)
+            {
+                cts.Cancel();
+            }
+        }
+
+        Task PrintVersionAsync(WasmEngine engine, string engineBinary)
+        {
+            return processManager.ExecuteCommandAsync(
+                        engineBinary,
+                        new[] { "--version" },
+                        log: new CallbackLog(m => logger.LogDebug(m.Trim())),
+                        stdoutLog: new CallbackLog(msg => logger.LogInformation(msg.Trim())),
+                        stderrLog: new CallbackLog(msg => logger.LogError(msg.Trim())),
+                        TimeSpan.FromSeconds(10));
+        }
+    }
+}

src/Microsoft.DotNet.XHarness.CLI/Commands/WASI/WasiCommandSet.cs

@@ -0,0 +1,18 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Mono.Options;
+
+namespace Microsoft.DotNet.XHarness.CLI.Commands.Wasi;
+
+// Main WASI command set that contains the platform specific commands. This allows the command line to
+// support different options in different platforms.
+// Whenever the behavior does match, the goal is to have the same arguments for all platforms
+public class WasiCommandSet : CommandSet
+{
+    public WasiCommandSet() : base("wasi")
+    {
+        Add(new WasiTestCommand());
+    }
+}

src/Microsoft.DotNet.XHarness.CLI/Commands/WASM/JS/WasmTestCommand.cs

@@ -15,6 +15,7 @@
 using Microsoft.DotNet.XHarness.Common.CLI;
 using Microsoft.DotNet.XHarness.Common.Execution;
 using Microsoft.DotNet.XHarness.Common.Logging;
+using Microsoft.DotNet.XHarness.Common.Utilities;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 
@@ -30,27 +31,7 @@ internal class WasmTestCommand : XHarnessCommand<WasmTestCommandArguments>
 
     public WasmTestCommand() : base(TargetPlatform.WASM, "test", true, new ServiceCollection(), CommandHelp)
     {
-    }
-
-    private static string FindEngineInPath(string engineBinary)
-    {
-        if (File.Exists(engineBinary) || Path.IsPathRooted(engineBinary))
-            return engineBinary;
-
-        var path = Environment.GetEnvironmentVariable("PATH");
-
-        if (path == null)
-            return engineBinary;
-
-        foreach (var folder in path.Split(Path.PathSeparator))
-        {
-            var fullPath = Path.Combine(folder, engineBinary);
-            if (File.Exists(fullPath))
-                return fullPath;
-        }
-
-        return engineBinary;
-    }
+    }    
 
     protected override async Task<ExitCode> InvokeInternal(ILogger logger)
     {
@@ -76,18 +57,18 @@ protected override async Task<ExitCode> InvokeInternal(ILogger logger)
             if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
             {
                 if (engineBinary.Equals("node"))
-                    engineBinary = FindEngineInPath(engineBinary + ".exe"); // NodeJS ships as .exe rather than .cmd
+                    engineBinary = FileUtils.FindFileInPath(engineBinary + ".exe"); // NodeJS ships as .exe rather than .cmd
                 else
-                    engineBinary = FindEngineInPath(engineBinary + ".cmd");
+                    engineBinary = FileUtils.FindFileInPath(engineBinary + ".cmd");
             }
         }
 
-        logger.LogInformation($"Using js engine {Arguments.Engine.Value} from path {engineBinary}");
-        await PrintVersionAsync(Arguments.Engine.Value.Value, engineBinary);
-
         var cts = new CancellationTokenSource();
         try
         {
+            logger.LogInformation($"Using js engine {Arguments.Engine.Value} from path {engineBinary}");
+            await PrintVersionAsync(Arguments.Engine.Value.Value, engineBinary);
+
             ServerURLs? serverURLs = null;
             if (Arguments.WebServerMiddlewarePathsAndTypes.Value.Count > 0)
             {

src/Microsoft.DotNet.XHarness.CLI/Commands/WASM/WasmTestMessagesProcessor.cs

@@ -38,7 +38,7 @@ public class WasmTestMessagesProcessor
     // Set once `WASM EXIT` message is received
     public TaskCompletionSource WasmExitReceivedTcs { get; } = new ();
 
-    public WasmTestMessagesProcessor(string xmlResultsFilePath, string stdoutFilePath, ILogger logger, string? errorPatternsFile, WasmSymbolicatorBase? symbolicator)
+    public WasmTestMessagesProcessor(string xmlResultsFilePath, string stdoutFilePath, ILogger logger, string? errorPatternsFile = null, WasmSymbolicatorBase? symbolicator = null)
     {
         _xmlResultsFilePath = xmlResultsFilePath;
         _stdoutFileWriter = File.CreateText(stdoutFilePath);

src/Microsoft.DotNet.XHarness.CLI/Commands/XHarnessHelpCommand.cs

@@ -10,6 +10,7 @@
 using Microsoft.DotNet.XHarness.CLI.Commands.Apple;
 using Microsoft.DotNet.XHarness.CLI.Commands.Apple.Simulators;
 using Microsoft.DotNet.XHarness.CLI.Commands.Wasm;
+using Microsoft.DotNet.XHarness.CLI.Commands.Wasi;
 using Microsoft.DotNet.XHarness.Common.CLI;
 using Mono.Options;
 
@@ -67,8 +68,11 @@ public override int Invoke(IEnumerable<string> arguments)
             case "wasm":
                 PrintCommandHelp(new WasmCommandSet(), subCommand);
                 break;
+            case "wasi":
+                PrintCommandHelp(new WasiCommandSet(), subCommand);
+                break;
             default:
-                Console.WriteLine($"No help available for command '{command}'. Allowed commands are 'apple', 'wasm' and 'android'");
+                Console.WriteLine($"No help available for command '{command}'. Allowed commands are 'apple', 'wasm', 'wasi' and 'android'");
                 break;
         }
 

src/Microsoft.DotNet.XHarness.CLI/Program.cs

@@ -10,6 +10,7 @@
 using Microsoft.DotNet.XHarness.CLI.Commands;
 using Microsoft.DotNet.XHarness.CLI.Commands.Apple;
 using Microsoft.DotNet.XHarness.CLI.Commands.Wasm;
+using Microsoft.DotNet.XHarness.CLI.Commands.Wasi;
 using Microsoft.DotNet.XHarness.Common.CLI;
 using Mono.Options;
 
@@ -84,6 +85,7 @@ public static CommandSet GetXHarnessCommandSet()
         commandSet.Add(new AndroidCommandSet());
         commandSet.Add(new AndroidHeadlessCommandSet());
         commandSet.Add(new WasmCommandSet());
+        commandSet.Add(new WasiCommandSet());
         commandSet.Add(new XHarnessHelpCommand());
         commandSet.Add(new XHarnessVersionCommand());
 

src/Microsoft.DotNet.XHarness.Common/CommandDiagnostics.cs

@@ -75,6 +75,7 @@ public CommandDiagnostics(ILogger logger, TargetPlatform platform, string comman
             TargetPlatform.Android => "android",
             TargetPlatform.Apple => "apple",
             TargetPlatform.WASM => "wasm",
+            TargetPlatform.WASI => "wasi",
             _ => throw new ArgumentOutOfRangeException(nameof(platform)),
         };
     }

src/Microsoft.DotNet.XHarness.Common/TargetPlatform.cs

@@ -9,4 +9,5 @@ public enum TargetPlatform
     Android,
     Apple,
     WASM,
+    WASI,
 }