From d30eb39e91cf2ae6addb0f6f14723588f04cd32a Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Wed, 15 Jan 2025 15:34:36 -0700
Subject: [PATCH 01/11] Fix typo

---
 .ci/tests/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/tests/build.sh b/.ci/tests/build.sh
index a0598fd499..135a751ca1 100755
--- a/.ci/tests/build.sh
+++ b/.ci/tests/build.sh
@@ -3,7 +3,7 @@ help()
 {
   echo "./build.sh as_host workingdir [options] [-- <hostenv.sh options>]"
   echo "  as_host                   First argument must be the host configuration to use for environment loading"
-  echo "  workingdir                First argument must be the working dir to immediate cd to"
+  echo "  workingdir                Second argument must be the working dir to immediate cd to"
   echo "  -c                        Configuration build type, piped directly into configure"
   echo "  -n                        Configuration nesting type, piped directly into configure"
   echo "  -o                        Configuration optstring passed into configure"

From 1f387f72ef47301535233aa6803b871ebcd6ecfe Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Wed, 15 Jan 2025 15:35:08 -0700
Subject: [PATCH 02/11] Add script to run namelists

This script is based on the very well-named script.csh out of the also
very well-named repo :
https://github.com/wrf-model/SCRIPTS

While it runs the rd_l2_norm.py, if not present it does not error out.
Instead diffwrf is used as that is more precise in its comparisons.
---
 .ci/tests/runNamelists.sh | 392 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 392 insertions(+)
 create mode 100755 .ci/tests/runNamelists.sh

diff --git a/.ci/tests/runNamelists.sh b/.ci/tests/runNamelists.sh
new file mode 100755
index 0000000000..797e6542dd
--- /dev/null
+++ b/.ci/tests/runNamelists.sh
@@ -0,0 +1,392 @@
+#!/bin/sh
+help()
+{
+  echo "./runNamelists.sh [workingdir] [options] [-- <hostenv.sh options>]"
+  echo "  as_host                   First argument must be the host configuration to use for environment loading"
+  echo "  workingdir                Second argument must be the working dir to immediate cd to"
+  echo "  -c <folder>               Directory for specific core built"
+  echo "  -r <folder>               Directory to run in using mkdir and symlinking -c <folder> for each namelist"
+  echo "  -b <exec>                 Binary executable for WRF front-end (ideal/real/etc.)"
+  echo "  -f <folder>               Namelist folder to look for namelists under"
+  echo "  -n <fileA,fileB,...>      Comma-delimited list of namelists to test, each of which will in serial be run by replacing namelist.input"
+  echo "  -d <folder>               Data directory to link into run directory"
+  echo "  -p <mpirun cmd>           Parallel launch command (MPI), e.g. mpirun, mpiexec_mpt, mpiexec -np 8 --oversubscribe"
+  echo "  -k <diffwrf cmd>          diffwrf command binary path, if not set will use external/io_netcdf/diffwrf"
+  echo "  -s <folder>               Save result data to prefix location, full path for run constructed as <work>/<thisfolder>/<namelist>/ "
+  echo "  -i <folder>               Folder for bitwise-identical results, full path for run constructed as <work>/<thisfolder>/<namelist>/ "
+  echo "  -e <varA=val,varB,...>    environment variables in comma-delimited list, e.g. var=1,foo,bar=0"
+  echo "  -- <hostenv.sh options>   Directly pass options to hostenv.sh, equivalent to hostenv.sh <options>"
+  echo "  -h                  Print this message"
+  echo ""
+  echo "If you wish to use an env var in your arg such as '-c \$SERIAL -e SERIAL=32', you must"
+  echo "you will need to do '-c \\\$SERIAL -e SERIAL=32' to delay shell expansion when input from shell/CLI"
+  echo ""
+  echo "If -i <folder> is provided, bitwise-identical checks are performed as part of checks"
+}
+
+echo "Input arguments:"
+echo "$*"
+
+AS_HOST=$1
+shift
+if [ $AS_HOST = "-h" ]; then
+  help
+  exit 0
+fi
+
+workingDirectory=$1
+shift
+
+cd $workingDirectory
+
+# Get some helper functions
+. .ci/env/helpers.sh
+
+while getopts c:r:b:f:n:d:p:k:s:i:e:h opt; do
+  case $opt in
+    c)
+      coreDir="$OPTARG"
+    ;;
+    r)
+      runDir="$OPTARG"
+    ;;
+    b)
+      binFirst="$OPTARG"
+    ;;
+    f)
+      namelistFolder="$OPTARG"
+    ;;
+    n)
+      namelists="$OPTARG"
+    ;;
+    d)
+      data="$OPTARG"
+    ;;
+    p)
+      parallelExec="$OPTARG"
+    ;;
+    k)
+      diffExec="$OPTARG"
+    ;;
+    s)
+      moveFolder="$OPTARG"
+    ;;
+    i)
+      identicalFolder="$OPTARG"
+    ;;
+    e)
+      envVars="$envVars,$OPTARG"
+    ;;
+    h)  help; exit 0 ;;
+    *)  help; exit 1 ;;
+    :)  help; exit 1 ;;
+    \?) help; exit 1 ;;
+  esac
+done
+
+shift "$((OPTIND - 1))"
+
+# Everything else goes to our env setup
+. .ci/env/hostenv.sh $*
+
+# Now evaluate env vars in case it pulls from hostenv.sh
+if [ ! -z $envVars ]; then
+  setenvStr "$envVars"
+fi
+
+# Re-evaluate input values for delayed expansion
+eval "coreDir=\$( realpath \"$coreDir\" )"
+eval "namelistFolder=\$( realpath \"$namelistFolder\" )"
+eval "namelists=\"$namelists\""
+eval "data=\$( realpath \"$data\" )"
+eval "parallelExec=\"$parallelExec\""
+eval "moveFolder=\"$moveFolder\""
+eval "identicalFolder=\"$identicalFolder\""
+
+eval "runDir=\"$runDir\""
+mkdir -p $runDir
+# Now set to realpath since it exists 
+runDir=$( realpath $runDir )
+
+# And prep so other values can be derived
+ln -sf $coreDir/* $runDir/
+
+eval "binFirst=\$( realpath \"$runDir/$binFirst\" )"
+
+
+wrf=$( realpath $( find $runDir -type f -name wrf -o -name wrf.exe | head -n 1 ) )
+rd_12_norm=$( realpath .ci/tests/SCRIPTS/rd_l2_norm.py )
+
+# Check our paths
+if [ ! -x "${wrf}" ]; then
+  echo "No wrf executable found"
+  exit 1
+fi
+
+if [ ! -x "${binFirst}" ]; then
+  echo "No domain preparation executable found"
+  exit 1
+fi
+
+if [ ! -d "${data}" ]; then
+  echo "No valid data path provided"
+  exit 1
+fi
+
+if [ ! -d "${namelistFolder}" ]; then
+  echo "No valid namelist folder provided"
+  exit 1
+fi
+
+if [ -z "${diffExec}" ]; then
+  diffExec=$( realpath $workingDirectory/external/io_netcdf/diffwrf )
+else
+  eval "diffExec=\$( realpath \"$diffExec\" )"
+fi
+
+
+
+
+################################################################################
+#
+# Things done only once
+# Go to core dir to make sure it exists
+cd $coreDir || exit $?
+# Clean up previous runs
+rm wrfinput_d* wrfbdy_d* wrfout_d* wrfchemi_d* wrf_chem_input_d* rsl* real.print.out* wrf.print.out* wrf_d0*_runstats.out qr_acr_qg_V4.dat fort.98 fort.88 -rf
+
+
+# Go to run location now - We only operate here from now on
+cd $runDir || exit $?
+# Clean up previous runs
+rm wrfinput_d* wrfbdy_d* wrfout_d* wrfchemi_d* wrf_chem_input_d* rsl* real.print.out* wrf.print.out* wrf_d0*_runstats.out qr_acr_qg_V4.dat fort.98 fort.88 -rf
+
+# Link in data in here
+ln -sf $data/* .
+
+#
+################################################################################
+
+
+################################################################################
+#
+# Loop testing namelists
+tmpIFS=$IFS
+IFS=","
+
+# Since we might fail or succeed on certain namelists, make a running set of failures
+errorMsg=""
+
+for namelist in $namelists; do
+  banner 42 "START $namelist"
+
+  parallelExecToUse="$parallelExec"
+  #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  # SPECIFIC TO NESTED DOMAINS - WE MUST RUN AN ODD NUMBER OF MPI TASKS
+  # THIS IS NOTED BY THE NAMELIST HAVING 'NE' OR 'VN' AT THE END
+  #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  if [ -n "$parallelExec" ] && { [ $( echo $namelist | grep -Ec "NE$" ) -eq 1 ] || [ $( echo $namelist | grep -Ec "VN$" ) -eq 1 ]; } then
+    # Check if parallel exec np is even and if so, reduce by 1
+    parallelExecSplit=$( echo "$parallelExec" | sed -e "s/\(.*\?-np[ ]*\)\([0-9]\+\)\(.*\?\)/\1;\2;\3/g" )
+    partFront=$( echo "$parallelExecSplit" | awk -F';' '{print $1}' )
+    partEnd=$( echo "$parallelExecSplit" | awk -F';' '{print $3}' )
+    numProcs=$( echo "$parallelExecSplit" | awk -F';' '{print $2}' )
+    if [ $(( $numProcs % 2 )) -eq 0 ]; then
+      echo "MPI runs with nested namelist domains require odd-number tasks, reducing by one"
+      numProcs=$(( $numProcs - 1 ))
+      parallelExecToUse="$partFront$numProcs$partEnd"
+      parallelExecToUseBinFirst="$partFront$numProcs$partEnd"
+      echo "New command will be '$parallelExecToUse'"
+    fi
+
+    # Check if we are runnng ideal
+    if [ $( contains $binFirst "ideal" ) -eq 0 ]; then
+      echo "Ideal test case initial conditions must be generated with one MPI rank at most"
+      parallelExecToUseBinFirst="$partFront"1"$partEnd"
+    fi
+  fi
+
+
+  # Clean up output of any of these runs
+  rm wrfinput_d* wrfbdy_d* wrfout_d* rsl* real.print.out* wrf.print.out* wrf_d0*_runstats.out qr_acr_qg_V4.dat fort.98 fort.88 -rf
+
+  # Copy namelist
+  echo "Setting $namelistFolder/$namelist as namelist.input"
+  # remove old namelist.input which may be a symlink in which case this would have failed
+  rm namelist.input
+  cp $namelistFolder/$namelist namelist.input || exit $?
+
+  # Run setup
+
+  echo "Running $parallelExecToUseBinFirst $binFirst"
+
+  eval "$parallelExecToUseBinFirst $binFirst | tee setup.print.out"
+  result=$?
+  if [ -n "$parallelExecToUseBinFirst" ]; then
+    if [ $( ls ./rsl.out.* 2>/dev/null | wc -l ) -ne 0 ]; then
+      # Output the rsl. output
+      cat $( ls ./rsl.out.* | sort | head -n 1 )
+    fi
+  fi
+
+  if [ $result -ne 0 ]; then
+    currentErrorMsg="[$namelist] $parallelExecToUseBinFirst $binFirst failed"
+    echo "$currentErrorMsg"
+    errorMsg="$errorMsg\n$currentErrorMsg"
+    continue
+  fi
+
+  # run wrf
+  echo "Running $parallelExecToUse $wrf"
+
+  eval "$parallelExecToUse $wrf | tee wrf.print.out"
+  result=$?
+  if [ -n "$parallelExecToUse" ]; then
+    if [ $( ls ./rsl.out.* 2>/dev/null | wc -l ) -ne 0 ]; then
+      # Output the rsl. output
+      cat $( ls ./rsl.out.* | sort | head -n 1 )
+    fi
+  fi
+
+  
+
+  if [ $result -ne 0 ]; then
+    currentErrorMsg="[$namelist] $parallelExecToUse $wrf failed"
+    echo "$currentErrorMsg"
+    errorMsg="$errorMsg\n$currentErrorMsg"
+    continue
+  fi
+
+  # Check output per domain
+  maxDom=$( grep max_dom namelist.input |  awk '{print $3}' | tr -d ',' )
+  currentDom=0
+  while [ $currentDom -lt $maxDom ]; do
+    currentDom=$(( $currentDom + 1 ))
+
+    domFiles=$( ls -1 | grep wrfout_d0${currentDom} | wc -l | awk '{print $1}' )
+
+    if [ $domFiles -eq 0 ]; then
+      currentErrorMsg="[$namelist] No output files generated for domain $currentDom"
+      echo "$currentErrorMsg"
+      errorMsg="$errorMsg\n$currentErrorMsg"
+      continue
+    fi
+
+    ncdump wrfout_d0${currentDom}_* | grep -i nan | grep -vi dominant
+    okNan=$?
+    timeSteps=$( ncdump -h wrfout_d0${currentDom}_* | grep "Time = UNLIMITED" | cut -d"(" -f2 | awk '{print $1}' )
+
+    if [ $okNan -eq 1 ] && [ $timeSteps -eq 2 ]; then
+      # Super ok, store output in file for comparison later
+      python3 $rd_12_norm wrfout_d0${currentDom}_* > wrf_d0${currentDom}_runstats.out
+    else
+      # Super NOT ok
+      currentErrorMsg="[$namelist] Checks on output failed. okNan=$okNan, timeSteps=$timeSteps when expected okNan=1 && timeSteps=2"
+      echo "$currentErrorMsg"
+      errorMsg="$errorMsg\n$currentErrorMsg"
+      continue
+    fi
+  done
+
+  # try comp
+  if [ ! -z $identicalFolder ]; then
+    if [ -d $workingDirectory/$identicalFolder/$namelist/ ]; then
+      echo "Comparing current $namelist output to output stored in $workingDirectory/$identicalFolder/$namelist/"
+
+      for io_type in input out; do
+        for dom in d01 d02 d03; do
+          if [ -e $workingDirectory/$identicalFolder/$namelist/wrf${io_type}_${dom}* ]; then
+            if [ ! -e ./wrf${io_type}_${dom}* ]; then
+              # Domain does not exist in our current run but in the provided folder, that should not happen -FAIL!
+              currentErrorMsg="[$namelist] Domain $( ls wrf${io_type}_${dom}* ) file $dom exists in $workingDirectory/$identicalFolder/$namelist but not in this run, cannot compare results"
+              echo "$currentErrorMsg"
+              errorMsg="$errorMsg\n$currentErrorMsg"
+              continue
+            fi
+
+            # We have a domain to check - should exist in both (we are treating the identical folder as truth)
+            if [ "${io_type}" = "out" ]; then
+              diff $workingDirectory/$identicalFolder/$namelist/wrf_${dom}_runstats.out $(pwd)/wrf_${dom}_runstats.out
+              statDiff=$?
+            fi
+
+            $diffExec $workingDirectory/$identicalFolder/$namelist/wrf${io_type}_${dom}* wrf${io_type}_${dom}*
+            result=$?
+
+            if [ $result -ne 0 ]; then
+              currentErrorMsg="[$namelist] $diffExec failed"
+              echo "$currentErrorMsg"
+              errorMsg="$errorMsg\n$currentErrorMsg"
+              continue
+            fi
+
+            if [ -e fort.98 ] || [ -e fort.88 ]; then
+              currentErrorMsg="[$namelist] $( ls $workingDirectory/$identicalFolder/$namelist/wrf${io_type}_${dom}* ) and current wrf${io_type}_${dom}* differ"
+              if [ -n "$statDiff" ] && [ $statDiff -eq 0 ]; then
+                currentErrorMsg="$currentErrorMsg - but are statistically equivalent based on checksum"
+              fi
+              echo "$currentErrorMsg"
+              cat fort.98 fort.88 2>/dev/null
+              errorMsg="$errorMsg\n$currentErrorMsg"
+              continue
+            fi
+          elif [ -e ./wrf${io_type}_${dom}* ]; then
+            echo "Domain file $( ls wrf${io_type}_${dom}* ) exists in $( pwd ) but not in $workingDirectory/$identicalFolder/$namelist/, is this an error?"
+          else 
+            # neither has it, skip
+            echo "Domain file wrf${io_type}_${dom}* not generated for namelist $namelist, skipping check"
+          fi
+        done
+      done
+    else
+      echo "No output to check in $workingDirectory/$identicalFolder for $namelist"
+    fi
+  fi
+
+  # Now if we have a move folder, do that
+  if [ ! -z $moveFolder ]; then
+    mkdir -p $workingDirectory/$moveFolder/$namelist/
+
+    # we are in core dir, find our output
+    echo "Moving output files to $workingDirectory/$moveFolder/$namelist/"
+    find . -type f -name "wrfinput_d*"       \
+                -o -name "wrfbdy_d*"         \
+                -o -name "wrfout_d*"         \
+                -o -name "wrfchemi_d*"       \
+                -o -name "wrf_chem_input_d*" \
+                -o -name "rsl*"              \
+                -o -name "setup.print.out*"  \
+                -o -name "wrf.print.out*"    \
+                -o -name "wrf_d0*_runstats.out"
+    # Now move
+    find . -type f \(  -name "wrfinput_d*"       \
+                    -o -name "wrfbdy_d*"         \
+                    -o -name "wrfout_d*"         \
+                    -o -name "wrfchemi_d*"       \
+                    -o -name "wrf_chem_input_d*" \
+                    -o -name "rsl*"              \
+                    -o -name "setup.print.out*"  \
+                    -o -name "wrf.print.out*"    \
+                    -o -name "wrf_d0*_runstats.out" \) \
+                -exec mv {} $workingDirectory/$moveFolder/$namelist/ \;
+  fi
+
+  banner 42 "STOP $namelist"
+done
+IFS=$tmpIFS
+
+# If we passed, clean up after ourselves
+if [ -z "$errorMsg" ]; then
+  # Unlink everything we linked in
+  ls $data/ | xargs -I{} rm {}
+
+  # Clean up once more since we passed
+  rm -rf wrfinput_d* wrfbdy_d* wrfout_d* wrfchemi_d* wrf_chem_input_d* rsl* setup.print.out* wrf.print.out* qr_acr_qg_V4.dat fort.98 fort.88
+
+  # We passed!
+  echo "TEST $(basename $0) PASS"
+else
+  printf "%b" "$errorMsg"
+  exit 1
+fi
+

From 18b7a78f65dd7638c2c301070ee212026a940337 Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Wed, 15 Jan 2025 15:48:16 -0700
Subject: [PATCH 03/11] Add workflow file for tests

These tests are based on :
https://github.com/kkeene44/wrf-coop/blob/version16_update/build.csh
---
 .ci/wrf_em_real_tests.json | 844 +++++++++++++++++++++++++++++++++++++
 1 file changed, 844 insertions(+)
 create mode 100644 .ci/wrf_em_real_tests.json

diff --git a/.ci/wrf_em_real_tests.json b/.ci/wrf_em_real_tests.json
new file mode 100644
index 0000000000..18e1fbf215
--- /dev/null
+++ b/.ci/wrf_em_real_tests.json
@@ -0,0 +1,844 @@
+{
+  "submit_options" :
+  {
+    "timelimit" : "00:10:00",
+    "working_directory" : "..",
+    "arguments" : 
+    {
+      ".*build.*::base_env_numprocs"          : [ "-e", "NUM_PROCS=4" ],
+      ".*testcase.*::base_env_numprocs"       : [ "-e", "NUM_PROCS=4" ],
+
+      ".*build.*::args_nesting"    : [ "-n", "1" ],
+      ".*build.*::args_configopt"  : [ "-o", "-d" ],
+      ".*build.*::args_build_tgt"  : [ "-b", "em_real -j $NUM_PROCS" ],
+
+      ".*build-serial::args_config": [ "-c", "32" ],
+      ".*build-omp::args_config"   : [ "-c", "33" ],
+      ".*build-mpi::args_config"   : [ "-c", "34" ],
+
+      ".*testcase.*::global_env"   : [ "-e", "DATA=/path/to/data" ],
+      ".*testcase.*::args_always"  : [ "-c", "test/em_real", "-b", "real.exe", "-d", "$DATA/em_real" ],
+      ".*testcase-mpi.*::exec_mpi" : [ "-p", "mpirun -np $NUM_PROCS" ],
+      ".*testcase-omp.*::env"      : [ "-e", "OMP_NUM_THREADS=$NUM_PROCS" ],
+
+      ".*testcase.*::args_namelist_dir" : [ "-r", "runNamelists/$NAMELIST_DIR" ],
+
+      ".*testcase-serial.*::dir"   : [ "-f", ".ci/tests/SCRIPTS/Namelists/weekly/$NAMELIST_DIR/SERIAL/", "-s", "serial_results/$NAMELIST_DIR" ],
+      ".*testcase-omp.*::dir"      : [ "-f", ".ci/tests/SCRIPTS/Namelists/weekly/$NAMELIST_DIR/OPENMP/", "-i", "serial_results/$NAMELIST_DIR" ],
+      ".*testcase-mpi.*::dir"      : [ "-f", ".ci/tests/SCRIPTS/Namelists/weekly/$NAMELIST_DIR/MPI/",    "-i", "serial_results/$NAMELIST_DIR" ]
+    },
+    "mmm-mystic"   :
+    {
+      "submission" : "LOCAL",
+      "arguments"  :
+      {
+        ".*build.*::base_env_numprocs"          : [ "-e", "NUM_PROCS=10" ],
+        ".*testcase.*::base_env_numprocs"       : [ "-e", "NUM_PROCS=5" ],
+        ".*::first_global_env" : [ "-e", "NETCDF=$HOME/dependencies/netcdf_for_wrf" ],
+        ".*testcase.*::global_env"     :
+        [ 
+          "-e", "DATA=$HOME/wrf-model/DATA/namelists/Data/",
+          "-e", "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NETCDF/lib"
+        ],
+        ".*mpi.*::path" : [ "-e", "PATH=$PATH:$HOME/dependencies/mpich/bin" ]
+      }
+    },
+    "hsn.de.hpc"  :
+    {
+      "submission" : "PBS",
+      "queue"      : "main",
+      "hpc_arguments"  : 
+      {
+        ".*testcase-serial.*::node_select" : 
+        { 
+          "-l " : { "select"       : 1, "ncpus" : 1  }
+        },
+        "^((?!testcase-serial).)*$::node_select" : 
+        {
+          "-l " : { "select"       : 1, "ncpus" : 8,
+                    ".*testcase-mpi.*::mpiprocs" : 8,
+                    ".*testcase-omp.*::ompthreads" : 8 } 
+        },
+        "priority"    : { "-l " : { "job_priority" : "economy"       } }
+      },
+      "arguments"  : 
+      {
+        ".*build.*::base_env_numprocs"          : [ "-e", "NUM_PROCS=12" ],
+        ".*testcase.*::base_env_numprocs"       : [ "-e", "NUM_PROCS=8" ],
+        ".*testcase.*::global_env"     :
+        [ 
+          "-e", "DATA=/glade/work/aislas/github/data/wrf/regtest/Data/",
+          "-e", "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NETCDF/lib"
+        ],
+        "global_modules" : [ "gcc", "netcdf" ],
+        ".*mpi.*::mpi_module" : [ "cray-mpich" ]
+      }
+    }
+  },
+  "em_real" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist_dir" : [ ],
+        ".*testcase.*0.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.3dtke",
+          "-e", "NAMELIST_DIR=em_real",
+          "-r", "runNamelists/$NAMELIST_DIR/0"
+        ],
+        ".*testcase.*1.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.conus",
+          "-e", "NAMELIST_DIR=em_real",
+          "-r", "runNamelists/$NAMELIST_DIR/1"
+        ],
+        ".*testcase.*2.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.rap",
+          "-e", "NAMELIST_DIR=em_real",
+          "-r", "runNamelists/$NAMELIST_DIR/2"
+        ],
+        ".*testcase.*3.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.tropical",
+          "-e", "NAMELIST_DIR=em_real",
+          "-r", "runNamelists/$NAMELIST_DIR/3"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial0" :
+      {
+        "submit_options" : { "timelimit" : "00:40:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "testcase-serial1" :
+      {
+        "submit_options" : { "timelimit" : "00:40:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "testcase-serial2" :
+      {
+        "submit_options" : { "timelimit" : "00:40:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "testcase-serial3" :
+      {
+        "submit_options" : { "timelimit" : "00:40:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : 
+        {
+          "testcase-serial0" : "afterok",
+          "testcase-serial1" : "afterok",
+          "testcase-serial2" : "afterok",
+          "testcase-serial3" : "afterok"
+        }
+      },
+      "testcase-omp0" :
+      {
+        "submit_options" : { "timelimit" : "00:25:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "testcase-omp1" :
+      {
+        "submit_options" : { "timelimit" : "00:25:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "testcase-omp0" : "afterok" }
+      },
+      "testcase-omp2" :
+      {
+        "submit_options" : { "timelimit" : "00:25:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "testcase-omp1" : "afterok" }
+      },
+      "testcase-omp3" :
+      {
+        "submit_options" : { "timelimit" : "00:25:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "testcase-omp2" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp3" : "afterok" }
+      },
+      "testcase-mpi0" :
+      {
+        "submit_options" : { "timelimit" : "00:20:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      },
+      "testcase-mpi1" :
+      {
+        "submit_options" : { "timelimit" : "00:20:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "testcase-mpi0" : "afterok" }
+      },
+      "testcase-mpi2" :
+      {
+        "submit_options" : { "timelimit" : "00:20:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "testcase-mpi1" : "afterok" }
+      },
+      "testcase-mpi3" :
+      {
+        "submit_options" : { "timelimit" : "00:20:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "testcase-mpi2" : "afterok" }
+      }
+    }
+  },
+  "em_real8" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*build.*::args_configopt"   : [ "-o", "-d -r8" ],
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.14,namelist.input.17AD",
+          "-e", "NAMELIST_DIR=em_real8"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realA" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.03,namelist.input.03DF",
+          "-e", "NAMELIST_DIR=em_realA"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realB" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.10,namelist.input.11,namelist.input.14",
+          "-e", "NAMELIST_DIR=em_realB"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realC" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.17,namelist.input.20",
+          "-e", "NAMELIST_DIR=em_realC"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realD" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.38,namelist.input.48,namelist.input.49",
+          "-e", "NAMELIST_DIR=em_realD"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realE" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.52DF",
+          "-e", "NAMELIST_DIR=em_realE"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realF" :
+  {
+    "submit_options" :
+    {
+      "timelimit" : "00:40:00",
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.65DF",
+          "-e", "NAMELIST_DIR=em_realF"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realG" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.kiaps1NE,namelist.input.kiaps2",
+          "-e", "NAMELIST_DIR=em_realG"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realH" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.52,namelist.input.cmt,namelist.input.solaraNE,namelist.input.urb3bNE,namelist.input.03ST",
+          "-e", "NAMELIST_DIR=em_realH"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realI" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.03FD,namelist.input.06",
+          "-e", "NAMELIST_DIR=em_realI"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realJ" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.50,namelist.input.51",
+          "-e", "NAMELIST_DIR=em_realJ"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realK" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.52FD,namelist.input.60NE",
+          "-e", "NAMELIST_DIR=em_realK"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial" : "afterok" }
+      },
+      "testcase-omp" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp" : "afterok" }
+      },
+      "testcase-mpi" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      }
+    }
+  },
+  "em_realL" :
+  {
+    "submit_options" :
+    {
+      "arguments" :
+      {
+        ".*testcase.*::args_namelist_dir" : [ ],
+        ".*testcase.*0.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.66FD",
+          "-e", "NAMELIST_DIR=em_realL",
+          "-r", "runNamelists/$NAMELIST_DIR/0"
+        ],
+        ".*testcase.*1.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.71",
+          "-e", "NAMELIST_DIR=em_realL",
+          "-r", "runNamelists/$NAMELIST_DIR/1"
+        ],
+        ".*testcase.*2.*::args_namelist" : 
+        [ 
+          "-n", "namelist.input.78,namelist.input.79",
+          "-e", "NAMELIST_DIR=em_realL",
+          "-r", "runNamelists/$NAMELIST_DIR/1"
+        ]
+      }
+    },
+    "steps" :
+    {
+      "build-serial" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      },
+      "testcase-serial0" :
+      {
+        "submit_options" : { "timelimit" : "00:30:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "testcase-serial1" :
+      {
+        "submit_options" : { "timelimit" : "00:30:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "testcase-serial2" :
+      {
+        "submit_options" : { "timelimit" : "00:30:00" },
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-serial" : "afterok" }
+      },
+      "build-omp" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-serial0" : "afterok", "testcase-serial1" : "afterok", "testcase-serial2" : "afterok" }
+      },
+      "testcase-omp0" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-omp" : "afterok" }
+      },
+      "testcase-omp1" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "testcase-omp0" : "afterok" }
+      },
+      "testcase-omp2" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "testcase-omp1" : "afterok" }
+      },
+      "build-mpi" :
+      {
+        "command"      : ".ci/tests/build.sh",
+        "dependencies" : { "testcase-omp2" : "afterok" }
+      },
+      "testcase-mpi0" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "build-mpi" : "afterok" }
+      },
+      "testcase-mpi1" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "testcase-mpi0" : "afterok" }
+      },
+      "testcase-mpi2" :
+      {
+        "command"      : ".ci/tests/runNamelists.sh",
+        "dependencies" : { "testcase-mpi1" : "afterok" }
+      }
+    }
+  }
+}

From 8d3f60d46a6812917cb62e0275eecda9dfdd3399 Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Wed, 15 Jan 2025 16:13:06 -0700
Subject: [PATCH 04/11] Add the new em_real tests to the ci/cd regression suite

---
 .ci/hpc-workflows                   |  2 +-
 .github/workflows/ci.yml            | 72 ++++++++++++++++++++++++++++-
 .github/workflows/entry_point.yml   |  7 ++-
 .github/workflows/test_workflow.yml |  6 +++
 4 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/.ci/hpc-workflows b/.ci/hpc-workflows
index 043a4b6711..e5d2d0d5a8 160000
--- a/.ci/hpc-workflows
+++ b/.ci/hpc-workflows
@@ -1 +1 @@
-Subproject commit 043a4b6711e7f7d6dd41bd932e6e7336334e9a53
+Subproject commit e5d2d0d5a8f1298cb13ab78e04195fbf83515599
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 79a88db082..42ee642d05 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -32,6 +32,7 @@ on:
         options:
           - all-tests
           - compile-tests
+          - em_real-tests
 
 
 # https://docs.github.com/en/actions/sharing-automations/reusing-workflows#supported-keywords-for-jobs-that-call-a-reusable-workflow
@@ -67,7 +68,7 @@ jobs:
             name : "Make Compilation Tests"
             id   : make-tests
             fileroot : wrf_compilation_tests-make
-            args : -j='{"node_select":{"-l ":{"select":1}}}'
+            args : -ff hsn.de.hpc.ucar.edu -j='{"node_select":{"-l ":{"select":1}}}'
             pool  : 8
             tpool : 1
             mkdirs : true
@@ -123,6 +124,75 @@ jobs:
       pull-requests: write
       statuses: write
     name : Test ${{ matrix.testSet.name }} on ${{ matrix.testSet.host }}
+  
+  em_real:
+    if : ${{ contains( fromJson('["em_real-tests","all-tests"]'), inputs.test ) || inputs.event_name == 'push' }}
+    strategy:
+      max-parallel: 4
+      fail-fast: false
+      matrix:
+      
+        testSet  :
+          - host : derecho
+            hpc-workflows_path : .ci/hpc-workflows
+            archive : /glade/work/aislas/github/runners/wrf/derecho/logs/
+            account : NMMM0012
+            name : "em_real Tests"
+            id   : em_real-tests
+            fileroot : wrf_em_real_tests
+            args : -ff hsn.de.hpc.ucar.edu -j='{"node_select":{"-l ":{"select":1}}}'
+            pool  : 14
+            tpool : 4 # This really only applies to em_real and em_realL
+            mkdirs : true
+            tests :
+              - em_real
+              - em_real8
+              - em_realA
+              - em_realB
+              - em_realC
+              - em_realD
+              - em_realE
+              - em_realF
+              - em_realG
+              - em_realH
+              - em_realI
+              - em_realJ
+              - em_realK
+              - em_realL
+
+    uses : ./.github/workflows/test_workflow.yml
+    with :
+      # This should be the only hard-coded value, we don't use ${{ inputs.test }}
+      # to avoid 'all-tests' to be used in this workflow
+      label    : em_real-tests
+
+      # Everything below this should remain the same and comes from the testSet matrix
+      hpc-workflows_path : ${{ matrix.testSet.hpc-workflows_path }}
+      archive  : ${{ matrix.testSet.archive }}
+      name     : ${{ matrix.testSet.name }}
+      id       : ${{ matrix.testSet.id }}
+      host     : ${{ matrix.testSet.host }}
+      fileroot : ${{ matrix.testSet.fileroot }}
+      account  : ${{ matrix.testSet.account }}
+      tests    : ${{ toJson( matrix.testSet.tests ) }}
+      mkdirs   : ${{ matrix.testSet.mkdirs }}
+      args     : ${{ matrix.testSet.args }}
+      pool     : ${{ matrix.testSet.pool }}
+      tpool    : ${{ matrix.testSet.tpool }}
+
+      # required to emulate event trigger
+      event_name   : ${{ inputs.event_name }}
+      event_number : ${{ inputs.event_number }}
+      event_label  : ${{ inputs.test }}
+      ref          : ${{ inputs.ref }}
+      sha          : ${{ inputs.sha }}
+
+    # I am leaving this here for posterity if this is to be replicated in private repositories for testing
+    permissions:
+      contents: read
+      pull-requests: write
+      statuses: write
+    name : Test ${{ matrix.testSet.name }} on ${{ matrix.testSet.host }}
 
 
   # In the event that 'all-tests' is used, this final job will be the one to remove
diff --git a/.github/workflows/entry_point.yml b/.github/workflows/entry_point.yml
index ed4ce200cd..04cad8fcdc 100644
--- a/.github/workflows/entry_point.yml
+++ b/.github/workflows/entry_point.yml
@@ -22,7 +22,12 @@ on:
 # https://stackoverflow.com/a/68940067
 jobs:
   queue_tests:
-    if : ${{ contains( fromJson('["compile-tests","all-tests"]'), github.event.label.name ) || github.event_name == 'push' }}
+    # https://stackoverflow.com/a/67532120
+    if : >
+      ${{ contains(
+            fromJson('["compile-tests","em_real-tests","all-tests"]'), 
+            github.event.label.name ) || 
+          github.event_name == 'push' }}
     name: Queue Test (${{ github.event_name == 'push' && github.ref_name || github.event.label.name }})
     runs-on: ubuntu-latest
     permissions:
diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml
index a4060dcb6c..52ee2ea0a7 100644
--- a/.github/workflows/test_workflow.yml
+++ b/.github/workflows/test_workflow.yml
@@ -99,6 +99,12 @@ jobs:
         path      : main
         submodules: true
         ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }}
+    # More checkout
+    - name: Checkout extra via manage_externals
+      id  : manage_externals
+      run : |
+        cd main
+        ./tools/manage_externals/checkout_externals -e arch/Externals.cfg 
 
     
     # Immediately copy out to # of tests to do

From 35001ba88e74a70c68fbe5a6caafe4e75fa8a217 Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Wed, 15 Jan 2025 16:23:33 -0700
Subject: [PATCH 05/11] Add the test namelists to the repo externals

---
 .github/workflows/test_workflow.yml |  8 +++++++-
 arch/OptionalExternals_testing.cfg  | 10 ++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)
 create mode 100644 arch/OptionalExternals_testing.cfg

diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml
index 52ee2ea0a7..a709d51804 100644
--- a/.github/workflows/test_workflow.yml
+++ b/.github/workflows/test_workflow.yml
@@ -99,12 +99,18 @@ jobs:
         path      : main
         submodules: true
         ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }}
-    # More checkout
+
+    ############################################################################
+    # WRF Specific
+    # More checkout, just do everything so we have it all at the ready. This unfortunately will limit testing 
+    # of on-the-fly checkouts
     - name: Checkout extra via manage_externals
       id  : manage_externals
       run : |
         cd main
         ./tools/manage_externals/checkout_externals -e arch/Externals.cfg 
+        ./tools/manage_externals/checkout_externals -e arch/OptionalExternals_testing.cfg
+    ############################################################################
 
     
     # Immediately copy out to # of tests to do
diff --git a/arch/OptionalExternals_testing.cfg b/arch/OptionalExternals_testing.cfg
new file mode 100644
index 0000000000..b25e87d510
--- /dev/null
+++ b/arch/OptionalExternals_testing.cfg
@@ -0,0 +1,10 @@
+[testing_namelists]
+local_path = .ci/tests/SCRIPTS
+protocol = git
+repo_url = https://github.com/wrf-model/SCRIPTS.git
+hash     = 416918d
+
+required = True
+
+[externals_description]
+schema_version = 1.0.0

From 9a484f4053cd605415fa9a5e7cd4d284e18338e4 Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Thu, 23 Jan 2025 17:47:39 -0700
Subject: [PATCH 06/11] Update to latest to allow comments in json

---
 .ci/hpc-workflows | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/hpc-workflows b/.ci/hpc-workflows
index e5d2d0d5a8..97ca807b0e 160000
--- a/.ci/hpc-workflows
+++ b/.ci/hpc-workflows
@@ -1 +1 @@
-Subproject commit e5d2d0d5a8f1298cb13ab78e04195fbf83515599
+Subproject commit 97ca807b0e1b7d67a35ab10e938b29774242e48e

From 21a4ef1ad6b71574e146a646f5e3fa6854665d5a Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Thu, 23 Jan 2025 18:03:41 -0700
Subject: [PATCH 07/11] Move tests to jsonc file for comments

---
 ...eal_tests.json => wrf_em_real_tests.jsonc} | 23 ++++++++++++-----
 .github/workflows/ci.yml                      |  8 +++---
 .github/workflows/test_workflow.yml           | 25 ++++++++++++-------
 3 files changed, 37 insertions(+), 19 deletions(-)
 rename .ci/{wrf_em_real_tests.json => wrf_em_real_tests.jsonc} (96%)

diff --git a/.ci/wrf_em_real_tests.json b/.ci/wrf_em_real_tests.jsonc
similarity index 96%
rename from .ci/wrf_em_real_tests.json
rename to .ci/wrf_em_real_tests.jsonc
index 18e1fbf215..412162dac7 100644
--- a/.ci/wrf_em_real_tests.json
+++ b/.ci/wrf_em_real_tests.jsonc
@@ -1,3 +1,7 @@
+// Tests emulating the code that is generated and run from
+// https://github.com/kkeene44/wrf-coop/blob/version16_update/build.csh
+// and
+// https://github.com/wrf-model/SCRIPTS/blob/master/script.csh
 {
   "submit_options" :
   {
@@ -27,13 +31,14 @@
       ".*testcase-omp.*::dir"      : [ "-f", ".ci/tests/SCRIPTS/Namelists/weekly/$NAMELIST_DIR/OPENMP/", "-i", "serial_results/$NAMELIST_DIR" ],
       ".*testcase-mpi.*::dir"      : [ "-f", ".ci/tests/SCRIPTS/Namelists/weekly/$NAMELIST_DIR/MPI/",    "-i", "serial_results/$NAMELIST_DIR" ]
     },
+    // This is an example of how one might set up the environment to run locally
     "mmm-mystic"   :
     {
       "submission" : "LOCAL",
       "arguments"  :
       {
         ".*build.*::base_env_numprocs"          : [ "-e", "NUM_PROCS=10" ],
-        ".*testcase.*::base_env_numprocs"       : [ "-e", "NUM_PROCS=5" ],
+        ".*testcase.*::base_env_numprocs"       : [ "-e", "NUM_PROCS=10" ],
         ".*::first_global_env" : [ "-e", "NETCDF=$HOME/dependencies/netcdf_for_wrf" ],
         ".*testcase.*::global_env"     :
         [ 
@@ -53,18 +58,22 @@
         { 
           "-l " : { "select"       : 1, "ncpus" : 1  }
         },
+        // for any test case that isn't serial apply the following
         "^((?!testcase-serial).)*$::node_select" : 
         {
-          "-l " : { "select"       : 1, "ncpus" : 8,
-                    ".*testcase-mpi.*::mpiprocs" : 8,
-                    ".*testcase-omp.*::ompthreads" : 8 } 
+          "-l " : { "select" : 1,          "ncpus" : 10,
+                    // Only apply this to MPI tests
+                    ".*testcase-mpi.*::mpiprocs"   : 10,
+                    // only for OMP tests
+                    ".*testcase-omp.*::ompthreads" : 10 } 
         },
         "priority"    : { "-l " : { "job_priority" : "economy"       } }
       },
       "arguments"  : 
       {
-        ".*build.*::base_env_numprocs"          : [ "-e", "NUM_PROCS=12" ],
-        ".*testcase.*::base_env_numprocs"       : [ "-e", "NUM_PROCS=8" ],
+        // These need to be representative of the resources we've requested
+        ".*build.*::base_env_numprocs"          : [ "-e", "NUM_PROCS=10" ],
+        ".*testcase.*::base_env_numprocs"       : [ "-e", "NUM_PROCS=10" ],
         ".*testcase.*::global_env"     :
         [ 
           "-e", "DATA=/glade/work/aislas/github/data/wrf/regtest/Data/",
@@ -81,6 +90,7 @@
     {
       "arguments" :
       {
+        // Remove auto-filled dir and supply sub-testcase specific ones
         ".*testcase.*::args_namelist_dir" : [ ],
         ".*testcase.*0.*::args_namelist" : 
         [ 
@@ -143,6 +153,7 @@
         "command"      : ".ci/tests/build.sh",
         "dependencies" : 
         {
+          // We parallelized the serial tests, now sync
           "testcase-serial0" : "afterok",
           "testcase-serial1" : "afterok",
           "testcase-serial2" : "afterok",
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 42ee642d05..79f4c5497c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -67,7 +67,7 @@ jobs:
             account : NMMM0012
             name : "Make Compilation Tests"
             id   : make-tests
-            fileroot : wrf_compilation_tests-make
+            config : .ci/wrf_compilation_tests-make.json
             args : -ff hsn.de.hpc.ucar.edu -j='{"node_select":{"-l ":{"select":1}}}'
             pool  : 8
             tpool : 1
@@ -103,7 +103,7 @@ jobs:
       name     : ${{ matrix.testSet.name }}
       id       : ${{ matrix.testSet.id }}
       host     : ${{ matrix.testSet.host }}
-      fileroot : ${{ matrix.testSet.fileroot }}
+      config   : ${{ matrix.testSet.config }}
       account  : ${{ matrix.testSet.account }}
       tests    : ${{ toJson( matrix.testSet.tests ) }}
       mkdirs   : ${{ matrix.testSet.mkdirs }}
@@ -139,7 +139,7 @@ jobs:
             account : NMMM0012
             name : "em_real Tests"
             id   : em_real-tests
-            fileroot : wrf_em_real_tests
+            config : .ci/wrf_em_real_tests.jsonc
             args : -ff hsn.de.hpc.ucar.edu -j='{"node_select":{"-l ":{"select":1}}}'
             pool  : 14
             tpool : 4 # This really only applies to em_real and em_realL
@@ -172,7 +172,7 @@ jobs:
       name     : ${{ matrix.testSet.name }}
       id       : ${{ matrix.testSet.id }}
       host     : ${{ matrix.testSet.host }}
-      fileroot : ${{ matrix.testSet.fileroot }}
+      config   : ${{ matrix.testSet.config }}
       account  : ${{ matrix.testSet.account }}
       tests    : ${{ toJson( matrix.testSet.tests ) }}
       mkdirs   : ${{ matrix.testSet.mkdirs }}
diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml
index a709d51804..4ff22d7ba0 100644
--- a/.github/workflows/test_workflow.yml
+++ b/.github/workflows/test_workflow.yml
@@ -22,7 +22,7 @@ on :
       host     :
         required : true
         type     : string
-      fileroot :
+      config :
         required : true
         type     : string
       account  :
@@ -94,6 +94,13 @@ jobs:
             state:      'pending'
           })
 
+    - name: Get fileroot of test config
+      id: get_fileroot
+      run: |
+        filename=$( basename $config )
+        echo "rel_dir=$( dirname $config )" >> "$GITHUB_ENV"
+        echo "fileroot=${filename%.*}" >> "$GITHU_ENV"
+    
     - uses: actions/checkout@v4
       with:
         path      : main
@@ -132,10 +139,10 @@ jobs:
       id  : runTest
       run: |
         if [ "${{ inputs.mkdirs }}" = "true" ]; then
-          ALT_DIRS="-alt ../${{ join( fromJson( inputs.tests ), '/.ci ../' ) }}/.ci"
+          ALT_DIRS="-alt ../${{ join( fromJson( inputs.tests ), format( '/{0} ../', env.rel_dir ) ) }}/$rel_dir"
         fi
         ./main/${{ inputs.hpc-workflows_path }}/.ci/runner.py                   \
-          ./main/.ci/${{ inputs.fileroot }}.json             \
+          ./main/${{ inputs.config }}                   \
           -t   ${{ join( fromJson( inputs.tests ), ' ' ) }}  \
           -a "${{ inputs.account }}"                         \
           -p ${{ inputs.pool}} -tp ${{ inputs.tpool }}       \
@@ -147,14 +154,14 @@ jobs:
       if : ${{ failure() }}
       run : |
         # move log files to safe location
-        ./main/${{ inputs.hpc-workflows_path }}/.ci/relocator.py ./main/.ci/${{ inputs.fileroot }}.log ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }}
+        ./main/${{ inputs.hpc-workflows_path }}/.ci/relocator.py ./main/$rel_dir/$fileroot.log ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }}
 
         # report on them - alt dirs need extra help
         if [ "${{ inputs.mkdirs }}" = "true" ]; then
-          masterlogLoc=main/.ci
+          masterlogLoc=main/$rel_dir
         fi
-        ./main/${{ inputs.hpc-workflows_path }}/.ci/reporter.py ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }}/$masterlogLoc/${{ inputs.fileroot }}.log \
-                                             -e ./${{ inputs.hpc-workflows_path }}/.ci/runner.py -p ./.ci \
+        ./main/${{ inputs.hpc-workflows_path }}/.ci/reporter.py ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }}/$masterlogLoc/$fileroot.log \
+                                             -e ./${{ inputs.hpc-workflows_path }}/.ci/runner.py -p ./$rel_dir \
                                              -o GITHUB -m -n # only mark fail steps with gh syntax
 
         echo "Relaying results to summary..."
@@ -162,8 +169,8 @@ jobs:
         # report on them
         echo "# Summary for ${{ inputs.name }}" >> $GITHUB_STEP_SUMMARY
         echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-        ./main/${{ inputs.hpc-workflows_path }}/.ci/reporter.py ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }}/$masterlogLoc/${{ inputs.fileroot }}.log \
-                                             -e ./${{ inputs.hpc-workflows_path }}/.ci/runner.py -p ./.ci \
+        ./main/${{ inputs.hpc-workflows_path }}/.ci/reporter.py ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }}/$masterlogLoc/$fileroot.log \
+                                             -e ./${{ inputs.hpc-workflows_path }}/.ci/runner.py -p ./$rel_dir \
                                              -s -n >> $GITHUB_STEP_SUMMARY
         echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
         # We know this is a failure

From 33feef0a37416d072047a9baf9ea128ed5e7a14c Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Fri, 24 Jan 2025 17:16:49 -0700
Subject: [PATCH 08/11] Disable unreliable w.r.t. reproducibility tests

---
 .ci/wrf_em_real_tests.jsonc | 141 ++++++++++++++++++------------------
 .github/workflows/ci.yml    |  14 ++--
 2 files changed, 79 insertions(+), 76 deletions(-)

diff --git a/.ci/wrf_em_real_tests.jsonc b/.ci/wrf_em_real_tests.jsonc
index 412162dac7..494130940c 100644
--- a/.ci/wrf_em_real_tests.jsonc
+++ b/.ci/wrf_em_real_tests.jsonc
@@ -104,12 +104,13 @@
           "-e", "NAMELIST_DIR=em_real",
           "-r", "runNamelists/$NAMELIST_DIR/1"
         ],
-        ".*testcase.*2.*::args_namelist" : 
-        [ 
-          "-n", "namelist.input.rap",
-          "-e", "NAMELIST_DIR=em_real",
-          "-r", "runNamelists/$NAMELIST_DIR/2"
-        ],
+        // This is producing inconsistent bitwise identical results
+        // ".*testcase.*2.*::args_namelist" : 
+        // [ 
+        //   "-n", "namelist.input.rap",
+        //   "-e", "NAMELIST_DIR=em_real",
+        //   "-r", "runNamelists/$NAMELIST_DIR/2"
+        // ],
         ".*testcase.*3.*::args_namelist" : 
         [ 
           "-n", "namelist.input.tropical",
@@ -136,12 +137,12 @@
         "command"      : ".ci/tests/runNamelists.sh",
         "dependencies" : { "build-serial" : "afterok" }
       },
-      "testcase-serial2" :
-      {
-        "submit_options" : { "timelimit" : "00:40:00" },
-        "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "build-serial" : "afterok" }
-      },
+      // "testcase-serial2" :
+      // {
+      //   "submit_options" : { "timelimit" : "00:40:00" },
+      //   "command"      : ".ci/tests/runNamelists.sh",
+      //   "dependencies" : { "build-serial" : "afterok" }
+      // },
       "testcase-serial3" :
       {
         "submit_options" : { "timelimit" : "00:40:00" },
@@ -172,12 +173,12 @@
         "command"      : ".ci/tests/runNamelists.sh",
         "dependencies" : { "testcase-omp0" : "afterok" }
       },
-      "testcase-omp2" :
-      {
-        "submit_options" : { "timelimit" : "00:25:00" },
-        "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "testcase-omp1" : "afterok" }
-      },
+      // "testcase-omp2" :
+      // {
+      //   "submit_options" : { "timelimit" : "00:25:00" },
+      //   "command"      : ".ci/tests/runNamelists.sh",
+      //   "dependencies" : { "testcase-omp1" : "afterok" }
+      // },
       "testcase-omp3" :
       {
         "submit_options" : { "timelimit" : "00:25:00" },
@@ -201,12 +202,12 @@
         "command"      : ".ci/tests/runNamelists.sh",
         "dependencies" : { "testcase-mpi0" : "afterok" }
       },
-      "testcase-mpi2" :
-      {
-        "submit_options" : { "timelimit" : "00:20:00" },
-        "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "testcase-mpi1" : "afterok" }
-      },
+      // "testcase-mpi2" :
+      // {
+      //   "submit_options" : { "timelimit" : "00:20:00" },
+      //   "command"      : ".ci/tests/runNamelists.sh",
+      //   "dependencies" : { "testcase-mpi1" : "afterok" }
+      // },
       "testcase-mpi3" :
       {
         "submit_options" : { "timelimit" : "00:20:00" },
@@ -446,52 +447,52 @@
       }
     }
   },
-  "em_realE" :
-  {
-    "submit_options" :
-    {
-      "arguments" :
-      {
-        ".*testcase.*::args_namelist" : 
-        [ 
-          "-n", "namelist.input.52DF",
-          "-e", "NAMELIST_DIR=em_realE"
-        ]
-      }
-    },
-    "steps" :
-    {
-      "build-serial" :
-      {
-        "command"      : ".ci/tests/build.sh"
-      },
-      "testcase-serial" :
-      {
-        "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "build-serial" : "afterok" }
-      },
-      "build-omp" :
-      {
-        "command"      : ".ci/tests/build.sh",
-        "dependencies" : { "testcase-serial" : "afterok" }
-      },
-      "testcase-omp" :
-      {
-        "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "build-omp" : "afterok" }
-      },
-      "build-mpi" :
-      {
-        "command"      : ".ci/tests/build.sh",
-        "dependencies" : { "testcase-omp" : "afterok" }
-      },
-      "testcase-mpi" :
-      {
-        "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "build-mpi" : "afterok" }
-      }
-    }
-  },
+  // "em_realE" :
+  // {
+  //   "submit_options" :
+  //   {
+  //     "arguments" :
+  //     {
+  //       ".*testcase.*::args_namelist" : 
+  //       [ 
+  //         "-n", "namelist.input.52DF",
+  //         "-e", "NAMELIST_DIR=em_realE"
+  //       ]
+  //     }
+  //   },
+  //   "steps" :
+  //   {
+  //     "build-serial" :
+  //     {
+  //       "command"      : ".ci/tests/build.sh"
+  //     },
+  //     "testcase-serial" :
+  //     {
+  //       "command"      : ".ci/tests/runNamelists.sh",
+  //       "dependencies" : { "build-serial" : "afterok" }
+  //     },
+  //     "build-omp" :
+  //     {
+  //       "command"      : ".ci/tests/build.sh",
+  //       "dependencies" : { "testcase-serial" : "afterok" }
+  //     },
+  //     "testcase-omp" :
+  //     {
+  //       "command"      : ".ci/tests/runNamelists.sh",
+  //       "dependencies" : { "build-omp" : "afterok" }
+  //     },
+  //     "build-mpi" :
+  //     {
+  //       "command"      : ".ci/tests/build.sh",
+  //       "dependencies" : { "testcase-omp" : "afterok" }
+  //     },
+  //     "testcase-mpi" :
+  //     {
+  //       "command"      : ".ci/tests/runNamelists.sh",
+  //       "dependencies" : { "build-mpi" : "afterok" }
+  //     }
+  //   }
+  // },
   "em_realF" :
   {
     "submit_options" :
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 79f4c5497c..b3be8bbbfc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -151,14 +151,16 @@ jobs:
               - em_realB
               - em_realC
               - em_realD
-              - em_realE
+              # This test intermittently fails
+              # - em_realE
               - em_realF
               - em_realG
-              - em_realH
-              - em_realI
-              - em_realJ
-              - em_realK
-              - em_realL
+              # Unstable and cannot get bitwise identical results
+              # - em_realH
+              # - em_realI
+              # - em_realJ
+              # - em_realK
+              # - em_realL
 
     uses : ./.github/workflows/test_workflow.yml
     with :

From 50b163fc08af7a1016eb663370a38fcfc1abe1c8 Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Thu, 24 Apr 2025 13:40:37 -0700
Subject: [PATCH 09/11] Derecho now has netcdf and netcdff installed in two
 different locations

---
 .ci/wrf_em_real_tests.jsonc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/wrf_em_real_tests.jsonc b/.ci/wrf_em_real_tests.jsonc
index 494130940c..ce8621efaf 100644
--- a/.ci/wrf_em_real_tests.jsonc
+++ b/.ci/wrf_em_real_tests.jsonc
@@ -77,7 +77,7 @@
         ".*testcase.*::global_env"     :
         [ 
           "-e", "DATA=/glade/work/aislas/github/data/wrf/regtest/Data/",
-          "-e", "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NETCDF/lib"
+          "-e", "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NETCDF/lib:$NETCDF/lib64"
         ],
         "global_modules" : [ "gcc", "netcdf" ],
         ".*mpi.*::mpi_module" : [ "cray-mpich" ]

From 8b5f6a14c94f5b90a100d1573630c314a1f0fe50 Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Thu, 24 Apr 2025 15:59:11 -0700
Subject: [PATCH 10/11] Disable tests that require BUILD_RRTMG_FAST and
 BUILD_RRTMK

---
 .ci/wrf_em_real_tests.jsonc | 107 +++++++++++++++++++-----------------
 .github/workflows/ci.yml    |   7 ++-
 2 files changed, 60 insertions(+), 54 deletions(-)

diff --git a/.ci/wrf_em_real_tests.jsonc b/.ci/wrf_em_real_tests.jsonc
index ce8621efaf..88dc9d81f7 100644
--- a/.ci/wrf_em_real_tests.jsonc
+++ b/.ci/wrf_em_real_tests.jsonc
@@ -157,7 +157,7 @@
           // We parallelized the serial tests, now sync
           "testcase-serial0" : "afterok",
           "testcase-serial1" : "afterok",
-          "testcase-serial2" : "afterok",
+          // "testcase-serial2" : "afterok",
           "testcase-serial3" : "afterok"
         }
       },
@@ -183,7 +183,7 @@
       {
         "submit_options" : { "timelimit" : "00:25:00" },
         "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "testcase-omp2" : "afterok" }
+        "dependencies" : { "testcase-omp1" : "afterok" }
       },
       "build-mpi" :
       {
@@ -212,7 +212,7 @@
       {
         "submit_options" : { "timelimit" : "00:20:00" },
         "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "testcase-mpi2" : "afterok" }
+        "dependencies" : { "testcase-mpi1" : "afterok" }
       }
     }
   },
@@ -271,7 +271,9 @@
       {
         ".*testcase.*::args_namelist" : 
         [ 
-          "-n", "namelist.input.03,namelist.input.03DF",
+          // The following namelists require BUILD_RRTMG_FAST and BUILD_RRTMK
+          // namelist.input.03
+          "-n", "namelist.input.03DF",
           "-e", "NAMELIST_DIR=em_realA"
         ]
       }
@@ -409,7 +411,9 @@
       {
         ".*testcase.*::args_namelist" : 
         [ 
-          "-n", "namelist.input.38,namelist.input.48,namelist.input.49",
+          "-n", "namelist.input.38",
+          // The following namelists require BUILD_RRTMG_FAST and BUILD_RRTMK
+          // ,namelist.input.48,namelist.input.49",
           "-e", "NAMELIST_DIR=em_realD"
         ]
       }
@@ -540,52 +544,53 @@
       }
     }
   },
-  "em_realG" :
-  {
-    "submit_options" :
-    {
-      "arguments" :
-      {
-        ".*testcase.*::args_namelist" : 
-        [ 
-          "-n", "namelist.input.kiaps1NE,namelist.input.kiaps2",
-          "-e", "NAMELIST_DIR=em_realG"
-        ]
-      }
-    },
-    "steps" :
-    {
-      "build-serial" :
-      {
-        "command"      : ".ci/tests/build.sh"
-      },
-      "testcase-serial" :
-      {
-        "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "build-serial" : "afterok" }
-      },
-      "build-omp" :
-      {
-        "command"      : ".ci/tests/build.sh",
-        "dependencies" : { "testcase-serial" : "afterok" }
-      },
-      "testcase-omp" :
-      {
-        "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "build-omp" : "afterok" }
-      },
-      "build-mpi" :
-      {
-        "command"      : ".ci/tests/build.sh",
-        "dependencies" : { "testcase-omp" : "afterok" }
-      },
-      "testcase-mpi" :
-      {
-        "command"      : ".ci/tests/runNamelists.sh",
-        "dependencies" : { "build-mpi" : "afterok" }
-      }
-    }
-  },
+  // This requires BUILD_RRTMG_FAST and BUILD_RRTMK
+  // "em_realG" :
+  // {
+  //   "submit_options" :
+  //   {
+  //     "arguments" :
+  //     {
+  //       ".*testcase.*::args_namelist" : 
+  //       [ 
+  //         "-n", "namelist.input.kiaps1NE,namelist.input.kiaps2",
+  //         "-e", "NAMELIST_DIR=em_realG"
+  //       ]
+  //     }
+  //   },
+  //   "steps" :
+  //   {
+  //     "build-serial" :
+  //     {
+  //       "command"      : ".ci/tests/build.sh"
+  //     },
+  //     "testcase-serial" :
+  //     {
+  //       "command"      : ".ci/tests/runNamelists.sh",
+  //       "dependencies" : { "build-serial" : "afterok" }
+  //     },
+  //     "build-omp" :
+  //     {
+  //       "command"      : ".ci/tests/build.sh",
+  //       "dependencies" : { "testcase-serial" : "afterok" }
+  //     },
+  //     "testcase-omp" :
+  //     {
+  //       "command"      : ".ci/tests/runNamelists.sh",
+  //       "dependencies" : { "build-omp" : "afterok" }
+  //     },
+  //     "build-mpi" :
+  //     {
+  //       "command"      : ".ci/tests/build.sh",
+  //       "dependencies" : { "testcase-omp" : "afterok" }
+  //     },
+  //     "testcase-mpi" :
+  //     {
+  //       "command"      : ".ci/tests/runNamelists.sh",
+  //       "dependencies" : { "build-mpi" : "afterok" }
+  //     }
+  //   }
+  // },
   "em_realH" :
   {
     "submit_options" :
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b3be8bbbfc..81d66e4e71 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -147,14 +147,15 @@ jobs:
             tests :
               - em_real
               - em_real8
-              - em_realA
+              - em_realA # partially implemented
               - em_realB
               - em_realC
-              - em_realD
+              - em_realD # partially implemented
               # This test intermittently fails
               # - em_realE
               - em_realF
-              - em_realG
+              # This test requires BUILD_RRTMG_FAST and BUILD_RRTMK
+              # - em_realG
               # Unstable and cannot get bitwise identical results
               # - em_realH
               # - em_realI

From ab9efe4ce5450270f7c176c7325c4c1c170ffec2 Mon Sep 17 00:00:00 2001
From: Anthony Islas <aislas@ucar.edu>
Date: Tue, 29 Apr 2025 16:28:24 -0700
Subject: [PATCH 11/11] Fix typo fixes that were lost in rebase

---
 .github/workflows/test_workflow.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml
index 4ff22d7ba0..8468956a9a 100644
--- a/.github/workflows/test_workflow.yml
+++ b/.github/workflows/test_workflow.yml
@@ -97,9 +97,9 @@ jobs:
     - name: Get fileroot of test config
       id: get_fileroot
       run: |
-        filename=$( basename $config )
-        echo "rel_dir=$( dirname $config )" >> "$GITHUB_ENV"
-        echo "fileroot=${filename%.*}" >> "$GITHU_ENV"
+        filename=$( basename ${{ inputs.config }} )
+        echo "rel_dir=$( dirname ${{ inputs.config }} )" >> "$GITHUB_ENV"
+        echo "fileroot=${filename%.*}" >> "$GITHUB_ENV"
     
     - uses: actions/checkout@v4
       with: