Use correct python executable inside venv for perf tests on Windows (#4339)

* Use correct python executable inside venv

* Put "echo on" on windows, not linux

Author: caaavik-msft

scripts/run_performance_job.py

@@ -122,11 +122,6 @@ def get_pre_commands(args: RunPerformanceJobArgs, v8_version: str):
     if args.os_group != "windows":
         helix_pre_commands += ["export CRYPTOGRAPHY_ALLOW_OPENSSL_102=true"]
 
-    if args.os_group == "windows":
-        python = "py -3"
-    else:
-        python = "python3"
-
     # Create separate list of commands to handle the next part. 
     # On non-Windows, these commands are chained together with && so they will stop if any fail
     install_prerequisites: list[str] = []
@@ -144,6 +139,7 @@ def get_pre_commands(args: RunPerformanceJobArgs, v8_version: str):
             install_prerequisites += [
                 "py -3 -m venv %HELIX_WORKITEM_ROOT%\\.venv",
                 "call %HELIX_WORKITEM_ROOT%\\.venv\\Scripts\\activate.bat",
+                "echo on" # venv activate script turns echo off, so turn it back on
             ]
         else:
             if args.os_group != "osx" and args.os_sub_group != "_musl":
@@ -162,13 +158,13 @@ def get_pre_commands(args: RunPerformanceJobArgs, v8_version: str):
 
         # Install python pacakges needed to upload results to azure storage
         install_prerequisites += [
-            f"{python} -m pip install -U pip --force-reinstall",
-            f"{python} -m pip install azure.storage.blob==12.13.0 --force-reinstall",
-            f"{python} -m pip install azure.storage.queue==12.4.0 --force-reinstall",
-            f"{python} -m pip install azure.identity==1.16.1 --force-reinstall",
-            f"{python} -m pip install urllib3==1.26.19 --force-reinstall",
-            f"{python} -m pip install opentelemetry-api==1.23.0 --force-reinstall",
-            f"{python} -m pip install opentelemetry-sdk==1.23.0 --force-reinstall",
+            f"python -m pip install -U pip --force-reinstall",
+            f"python -m pip install azure.storage.blob==12.13.0 --force-reinstall",
+            f"python -m pip install azure.storage.queue==12.4.0 --force-reinstall",
+            f"python -m pip install azure.identity==1.16.1 --force-reinstall",
+            f"python -m pip install urllib3==1.26.19 --force-reinstall",
+            f"python -m pip install opentelemetry-api==1.23.0 --force-reinstall",
+            f"python -m pip install opentelemetry-sdk==1.23.0 --force-reinstall",
         ]
 
         # Install prereqs for NodeJS https://github.com/dotnet/runtime/pull/40667 
@@ -619,11 +615,11 @@ def run_performance_job(args: RunPerformanceJobArgs):
 
     # ensure work item directory is not empty
     shutil.copytree(os.path.join(args.performance_repo_dir, "docs"), work_item_dir)
-    
+
     if args.os_group == "windows":
-        python = "py -3"
+        agent_python = "py -3"
     else:
-        python = "python3"
+        agent_python = "python3"
 
     helix_pre_commands = get_pre_commands(args, v8_version)
     helix_post_commands = get_post_commands(args)
@@ -747,15 +743,15 @@ def run_performance_job(args: RunPerformanceJobArgs):
         os.environ["Architecture"] = args.architecture
         os.environ["TargetsWindows"] = "true" if args.os_group == "windows" else "false"
         os.environ["HelixTargetQueues"] = args.queue
-        os.environ["Python"] = python
+        os.environ["Python"] = agent_python
         os.environ["RuntimeFlavor"] = args.runtime_flavor or ''
         os.environ["HybridGlobalization"] = str(args.hybrid_globalization)
 
         # TODO: See if these commands are needed for linux as they were being called before but were failing.
         if args.os_group == "windows" or args.os_group == "osx":
-            RunCommand([*(python.split(" ")), "-m", "pip", "install", "--user", "--upgrade", "pip"]).run()
-            RunCommand([*(python.split(" ")), "-m", "pip", "install", "--user", "urllib3==1.26.19"]).run()
-            RunCommand([*(python.split(" ")), "-m", "pip", "install", "--user", "requests"]).run()
+            RunCommand([*(agent_python.split(" ")), "-m", "pip", "install", "--user", "--upgrade", "pip"]).run()
+            RunCommand([*(agent_python.split(" ")), "-m", "pip", "install", "--user", "urllib3==1.26.19"]).run()
+            RunCommand([*(agent_python.split(" ")), "-m", "pip", "install", "--user", "requests"]).run()
 
         scenarios_path = os.path.join(args.performance_repo_dir, "src", "scenarios")
         script_path = os.path.join(args.performance_repo_dir, "scripts")
@@ -843,12 +839,12 @@ def run_performance_job(args: RunPerformanceJobArgs):
 
     if args.os_group == "windows":
         work_item_command = [
-            python,
+            "python",
             "%HELIX_WORKITEM_ROOT%\\performance\\scripts\\benchmarks_ci.py", 
             "--csproj", f"%HELIX_WORKITEM_ROOT%\\performance\\{args.target_csproj}"]
     else:
         work_item_command = [
-            python,
+            "python",
             "$HELIX_WORKITEM_ROOT/performance/scripts/benchmarks_ci.py", 
             "--csproj", f"$HELIX_WORKITEM_ROOT/performance/{args.target_csproj}"]
 
@@ -895,7 +891,7 @@ def run_performance_job(args: RunPerformanceJobArgs):
         download_files_from_helix=True,
         targets_windows=args.os_group == "windows",
         helix_results_destination_dir=helix_results_destination_dir,
-        python=python,
+        python="python",
         affinity=args.affinity)
 
     if args.send_to_helix:

src/scenarios/shared/runner.py

@@ -245,16 +245,20 @@ def run(self):
         Runs the specified scenario
         '''
         self.parseargs()
+
+        python_command = pythoncommand().split(' ')
+        python_exe = python_command[0]
+        python_args = " ".join(python_command[1:])
         if self.testtype == const.INNERLOOP:
             startup = StartupWrapper()
             self.traits.add_traits(scenarioname=self.scenarioname,
             scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP],
             apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext),
-            innerloopcommand=pythoncommand(),
-            iterationsetup=pythoncommand(),
-            setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
-            iterationcleanup=pythoncommand(),
-            cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
+            innerloopcommand=python_exe,
+            iterationsetup=python_exe,
+            setupargs='%s %s setup_build' % (python_args, const.ITERATION_SETUP_FILE),
+            iterationcleanup=python_exe,
+            cleanupargs='%s %s cleanup' % (python_args, const.ITERATION_SETUP_FILE),
             affinity=self.affinity)
             startup.runtests(self.traits)
 
@@ -263,11 +267,11 @@ def run(self):
             self.traits.add_traits(scenarioname=self.scenarioname,
             scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD],
             apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext),
-            innerloopcommand=pythoncommand(),
-            iterationsetup=pythoncommand(),
-            setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
-            iterationcleanup=pythoncommand(),
-            cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
+            innerloopcommand=python_exe,
+            iterationsetup=python_exe,
+            setupargs='%s %s setup_build' % (python_args, const.ITERATION_SETUP_FILE),
+            iterationcleanup=python_exe,
+            cleanupargs='%s %s cleanup' % (python_args, const.ITERATION_SETUP_FILE),
             affinity=self.affinity)
             startup.runtests(self.traits)
 
@@ -276,11 +280,11 @@ def run(self):
             self.traits.add_traits(scenarioname=self.scenarioname,
             scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH],
             apptorun='dotnet', appargs='watch -v',
-            innerloopcommand=pythoncommand(),
-            iterationsetup=pythoncommand(),
-            setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
-            iterationcleanup=pythoncommand(),
-            cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
+            innerloopcommand=python_exe,
+            iterationsetup=python_exe,
+            setupargs='%s %s setup_build' % (python_args, const.ITERATION_SETUP_FILE),
+            iterationcleanup=python_exe,
+            cleanupargs='%s %s cleanup' % (python_args, const.ITERATION_SETUP_FILE),
             affinity=self.affinity)
             self.traits.add_traits(workingdir = const.APPDIR)
             startup.runtests(self.traits)
@@ -312,10 +316,10 @@ def run(self):
                     scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD),
                     apptorun=const.DOTNET,
                     appargs='build',
-                    iterationsetup=pythoncommand(),
-                    setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
-                    iterationcleanup=pythoncommand(),
-                    cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
+                    iterationsetup=python_exe,
+                    setupargs='%s %s setup_build' % (python_args, const.ITERATION_SETUP_FILE),
+                    iterationcleanup=python_exe,
+                    cleanupargs='%s %s cleanup' % (python_args, const.ITERATION_SETUP_FILE),
                     workingdir=const.APPDIR,
                     environmentvariables=envlistcleanbuild,
                 )
@@ -344,10 +348,10 @@ def run(self):
                     apptorun=const.DOTNET,
                     scenarioname=self.scenarioname,
                     scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE),
-                    iterationsetup=pythoncommand(),
-                    setupargs='%s %s setup_new' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
-                    iterationcleanup=pythoncommand(),
-                    cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
+                    iterationsetup=python_exe,
+                    setupargs='%s %s setup_new' % (python_args, const.ITERATION_SETUP_FILE),
+                    iterationcleanup=python_exe,
+                    cleanupargs='%s %s cleanup' % (python_args, const.ITERATION_SETUP_FILE),
                     workingdir=const.APPDIR
                 )
                 self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME)

src/scenarios/shared/util.py

@@ -70,9 +70,16 @@ def getruntimeidentifier():
 
     return rid
 
+# https://stackoverflow.com/a/42580137
+def is_venv(): 
+    return (hasattr(sys, 'real_prefix') or
+            (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix))
+
 def pythoncommand():
-    if iswin():
-        return 'py'
+    if is_venv():
+        return 'python'
+    elif iswin():
+        return 'py -3'
     else:
         return 'python3'