[WIP] Decomp init for testing work

.gitmodules

@@ -27,8 +27,11 @@
 #
 [submodule "fates"]
 path = src/fates
-url = https://github.com/NGEET/fates
-fxtag = sci.1.87.2_api.41.0.0
+#url = https://github.com/NGEET/fates
+url = https://github.com/ekluzek/fates
+#fxtag = sci.1.87.2_api.41.0.0
+#fxtag = add_file_to_cmake_unittest_list
+fxtag = fa4016d1b59ffdb509a7833da113a88872a8a56c
 fxrequired = AlwaysRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/NGEET/fates

bld/CLMBuildNamelist.pm

@@ -5315,6 +5315,7 @@ sub write_output_files {
   push @groups, "clm_canopy_inparm";
   push @groups, "prigentroughness";
   push @groups, "zendersoilerod";
+  push @groups, "for_testing_options";
   if (remove_leading_and_trailing_quotes($nl->get_value('snow_cover_fraction_method')) eq 'SwensonLawrence2012') {
      push @groups, "scf_swenson_lawrence_2012_inparm";
   }

bld/namelist_files/namelist_definition_ctsm.xml

@@ -1259,12 +1259,38 @@ Whether to use subgrid fluxes for snow
 Whether snow on the vegetation canopy affects the radiation/albedo calculations
 </entry>
 
+<!-- ========================================================================================  -->
+<!-- Namelist items for doing specific things for testing                                      -->
+<!-- for_testing section:                                                                      -->
+<!-- ========================================================================================  -->
+
+<entry id="for_testing_bypass_init" type="logical" category="default_settings"
+       group="for_testing_options" >
+For testing whether to bypass the rest of the initialization after the self test driver is run
+</entry>
+
+<entry id="for_testing_bypass_run" type="logical" category="default_settings"
+       group="for_testing_options" >
+For testing whether to bypass most of the run phase other than the clock advance
+</entry>
+
+<entry id="for_testing_exit_after_self_tests" type="logical" category="default_settings"
+       group="for_testing_options" >
+Whether to exit early after the initialization self tests are run. This is typically only used in automated tests.
+</entry>
+
 <entry id="for_testing_run_ncdiopio_tests" type="logical" category="default_settings"
-       group="clm_inparm" >
+       group="for_testing_options" >
 Whether to run some tests of ncdio_pio as part of the model run. This is
 typically only used in automated tests.
 </entry>
 
+<entry id="for_testing_run_decomp_init_tests" type="logical" category="default_settings"
+       group="for_testing_options" >
+Whether to run some tests of decompInit (to get the gridcell to MPI task decomposition) as part of the model run. This is
+typically only used in automated tests.
+</entry>
+
 <entry id="for_testing_use_second_grain_pool" type="logical" category="default_settings"
        group="clm_inparm" >
 If true, allocate memory for and use a second crop grain pool. This is

cime_config/buildlib

@@ -135,6 +135,7 @@ def _main_func():
                 os.path.join(lnd_root, "src", "dyn_subgrid"),
                 os.path.join(lnd_root, "src", "init_interp"),
                 os.path.join(lnd_root, "src", "self_tests"),
+                os.path.join(lnd_root, "src", "unit_test_shr"),
                 os.path.join(lnd_root, "src", "fates"),
                 os.path.join(lnd_root, "src", "fates", "main"),
                 os.path.join(lnd_root, "src", "fates", "biogeophys"),

cime_config/testdefs/ExpectedTestFails.xml

@@ -381,13 +381,6 @@
 
   <!-- decomp_init test list-->
 
-  <test name="SMS_Ln1_PL.mpasa15_mpasa15.I2000Clm45Sp.derecho_intel.clm-run_self_tests">
-    <phase name="RUN">
-      <status>FAIL</status>
-      <issue>#3316</issue>
-    </phase>
-  </test>
-
   <!-- interim_restart (and aux_clm) test list-->
 
   <test name="ERR_Ld7.f10_f10_mg37.I2000Clm50BgcCropRtm.derecho_gnu.clm-default">

cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/user_nl_clm

@@ -1,3 +1,6 @@
+! Exit early and bypass the run phase
+for_testing_exit_after_self_tests = .true.
+
 ! Turn off history, restarts, and output
 hist_empty_htapes = .true.
 use_noio = .true.

cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands

@@ -0,0 +1,9 @@
+#!/bin/bash
+./xmlchange CLM_FORCE_COLDSTART="on"
+
+# We use this testmod in a _Ln1 test; this requires forcing the ROF coupling frequency to every time step
+./xmlchange ROF_NCPL=48
+
+# Restarts aren't allowed for these tests, and turn off CPL history
+./xmlchange REST_OPTION="never"
+./xmlchange HIST_OPTION="never"

cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm

@@ -1,5 +1,15 @@
+! Bypass as much of the init phase as can be done
+! Bypassing the run phase already was inherited from the for_testing_fastsetup_bypassrun testmod
+for_testing_bypass_init = .true.
+
+! Turn on some of the self tests
 for_testing_run_ncdiopio_tests = .true.
+for_testing_run_decomp_init_tests = .true.
 
 ! Turn off history, restarts, and output
 hist_empty_htapes = .true.
 use_noio = .true.
+for_testing_run_decomp_init_tests = .true.
+
+! Exit initialization phase after the self tests
+for_testing_bypass_init = .true.

src/CMakeLists.txt

@@ -59,41 +59,29 @@ add_subdirectory(${CLM_ROOT}/src/main clm_main)
 add_subdirectory(${CLM_ROOT}/src/init_interp clm_init_interp)
 add_subdirectory(${CLM_ROOT}/src/self_tests clm_self_tests)
 
+# Add FATES source directories
+add_subdirectory(${CLM_ROOT}/src/fates/main fates_main)
+add_subdirectory(${CLM_ROOT}/src/fates/biogeochem fates_biogeochem)
+add_subdirectory(${CLM_ROOT}/src/fates/biogeophys fates_biogeophys)
+add_subdirectory(${CLM_ROOT}/src/fates/parteh fates_parteh)
+add_subdirectory(${CLM_ROOT}/src/fates/fire fates_fire)
+add_subdirectory(${CLM_ROOT}/src/fates/radiation fates_radiation)
+
 # Add general unit test directories (stubbed out files, etc.)
 add_subdirectory(unit_test_stubs)
 add_subdirectory(unit_test_shr)
 
-# Remove some things from share_sources
-#
-# TODO: this should be moved into a general-purpose function in Sourcelist_utils.
-# Then each removal could be replaced with a single call, like:
-# remove_source_file(${share_sources} "shr_mpi_mod.F90")
-foreach (sourcefile ${share_sources})
-  # Remove shr_mpi_mod from share_sources.
-  # This is needed because we want to use the mock shr_mpi_mod in place of the real one
-  string(REGEX MATCH "shr_mpi_mod.F90" match_found ${sourcefile})
-  if(match_found)
-    list(REMOVE_ITEM share_sources ${sourcefile})
-  endif()
-
-  # Remove shr_pio_mod from share_sources. This is needed to avoid an explicit dependency
-  # on PIO. This removal is needed on some systems but not on others: the unit test build
-  # works without this removal on a Mac with a pre-built PIO library, but failed (with
-  # error message, "Cannot open module file 'pio.mod'") on a Mac without a pre-built PIO
-  # (where ESMF was built with its internal PIO).
-  string(REGEX MATCH "shr_pio_mod.F90" match_found ${sourcefile})
-  if(match_found)
-    list(REMOVE_ITEM share_sources ${sourcefile})
-  endif()
-endforeach()
-
 # Build libraries containing stuff needed for the unit tests.
 # Eventually, these add_library calls should probably be distributed into the correct location, rather than being in this top-level CMakeLists.txt file.
 add_library(csm_share ${share_sources} ${drv_sources_needed})
 declare_generated_dependencies(csm_share "${share_genf90_sources}")
 add_library(clm ${clm_sources})
+add_library(fates ${fates_sources})
 declare_generated_dependencies(clm "${clm_genf90_sources}")
-add_dependencies(clm csm_share esmf)
+# Add explicit dependencies to lower level libraries, for each library
+add_dependencies(csm_share esmf)
+add_dependencies(clm fates csm_share esmf)
+add_dependencies(fates csm_share esmf)
 
 # We need to look for header files here, in order to pick up shr_assert.h
 include_directories(${CLM_ROOT}/share/include)

src/cpl/nuopc/lnd_comp_nuopc.F90

@@ -49,6 +49,7 @@ module lnd_comp_nuopc
   use lnd_import_export      , only : advertise_fields, realize_fields, import_fields, export_fields
   use lnd_comp_shr           , only : mesh, model_meshfile, model_clock
   use perf_mod               , only : t_startf, t_stopf, t_barrierf
+  use SelfTestDriver         , only : for_testing_exit_after_self_tests
 
   implicit none
   private ! except
@@ -351,6 +352,8 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
     use lnd_set_decomp_and_domain , only : lnd_set_decomp_and_domain_from_readmesh
     use lnd_set_decomp_and_domain , only : lnd_set_mesh_for_single_column
     use lnd_set_decomp_and_domain , only : lnd_set_decomp_and_domain_for_single_column
+    use SelfTestDriver            , only : for_testing_bypass_init_after_self_tests, &
+                                           for_testing_exit_after_self_tests
 
     ! input/output variables
     type(ESMF_GridComp)  :: gcomp
@@ -500,6 +503,12 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
     else
        single_column = .false.
     end if
+    !if ( for_testing_exit_after_self_tests) then
+       ! *******************
+       ! *** RETURN HERE ***
+       ! *******************
+       !RETURN
+    !end if
 
     !----------------------------------------------------------------------------
     ! Reset shr logging to my log file
@@ -676,14 +685,19 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
     call t_startf('clm_init2')
     call initialize2(ni, nj, currtime)
     call t_stopf('clm_init2')
+    if (for_testing_exit_after_self_tests) then
+       RETURN
+    end if
 
     !--------------------------------
     ! Create land export state
     !--------------------------------
+    if ( .not. for_testing_bypass_init_after_self_tests() ) then
     call get_proc_bounds(bounds)
     call export_fields(gcomp, bounds, glc_present, rof_prognostic, &
          water_inst%waterlnd2atmbulk_inst, lnd2atm_inst, lnd2glc_inst, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    end if
 
     ! Set scalars in export state
     call State_SetScalar(dble(ldomain%ni), flds_scalar_index_nx, exportState, &
@@ -731,6 +745,7 @@ subroutine ModelAdvance(gcomp, rc)
     use clm_instMod , only : water_inst, atm2lnd_inst, glc2lnd_inst, lnd2atm_inst, lnd2glc_inst
     use decompMod   , only : bounds_type, get_proc_bounds
     use clm_driver  , only : clm_drv
+    use SelfTestDriver, only : for_testing_bypass_init_after_self_tests
 
     ! input/output variables
     type(ESMF_GridComp)  :: gcomp
@@ -786,6 +801,9 @@ subroutine ModelAdvance(gcomp, rc)
     if (single_column .and. .not. scol_valid) then
        RETURN
     end if
+    !if (for_testing_exit_after_self_tests) then
+      ! RETURN
+    !end if
 
     !$  call omp_set_num_threads(nthrds)
 
@@ -818,16 +836,20 @@ subroutine ModelAdvance(gcomp, rc)
          flds_scalar_index_nextsw_cday, nextsw_cday, &
          flds_scalar_name, flds_scalar_num, rc)
 
-    ! Get proc bounds
-    call get_proc_bounds(bounds)
-
     !--------------------------------
     ! Unpack import state
     !--------------------------------
 
+    if ( .not. for_testing_bypass_init_after_self_tests() ) then
+    ! Get proc bounds for both import and export
+    call get_proc_bounds(bounds)
+
+    call t_startf ('lc_lnd_import')
     call import_fields( gcomp, bounds, glc_present, rof_prognostic, &
          atm2lnd_inst, glc2lnd_inst, water_inst%wateratm2lndbulk_inst, rc )
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    call t_stopf ('lc_lnd_import')
+    end if
 
     !--------------------------------
     ! Run model
@@ -917,9 +939,13 @@ subroutine ModelAdvance(gcomp, rc)
     ! Pack export state
     !--------------------------------
 
+    if ( .not. for_testing_bypass_init_after_self_tests() ) then
+    call t_startf ('lc_lnd_export')
     call export_fields(gcomp, bounds, glc_present, rof_prognostic, &
          water_inst%waterlnd2atmbulk_inst, lnd2atm_inst, lnd2glc_inst, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    call t_stopf ('lc_lnd_export')
+    end if
 
     !--------------------------------
     ! Advance ctsm time step
@@ -1009,6 +1035,7 @@ subroutine ModelSetRunClock(gcomp, rc)
     rc = ESMF_SUCCESS
     call ESMF_LogWrite(subname//' called', ESMF_LOGMSG_INFO)
     if (.not. scol_valid) return
+    !if (for_testing_exit_after_self_tests) return
 
     ! query the Component for its clocks
     call NUOPC_ModelGet(gcomp, driverClock=dclock, modelClock=mclock, rc=rc)
@@ -1320,6 +1347,9 @@ subroutine CheckImport(gcomp, rc)
     if (single_column .and. .not. scol_valid) then
        RETURN
     end if
+    !if (for_testing_exit_after_self_tests) then
+       !RETURN
+    !end if
     ! The remander of this should be equivalent to the NUOPC internal routine
     ! from NUOPC_ModeBase.F90
 

src/main/CMakeLists.txt

@@ -19,6 +19,7 @@ list(APPEND clm_sources
   clm_varsur.F90 
   column_varcon.F90
   decompMod.F90
+  decompInitMod.F90
   filterColMod.F90
   FireMethodType.F90
   glc2lndMod.F90
@@ -30,6 +31,7 @@ list(APPEND clm_sources
   ncdio_utils.F90
   organicFileMod.F90
   paramUtilMod.F90
+  subgridMod.F90
   subgridAveMod.F90
   subgridWeightsMod.F90
   surfrdUtilsMod.F90

src/main/clm_driver.F90

@@ -85,6 +85,7 @@ module clm_driver
   use clm_instMod
   use SoilMoistureStreamMod  , only : PrescribedSoilMoistureInterp, PrescribedSoilMoistureAdvance
   use SoilBiogeochemDecompCascadeConType , only : no_soil_decomp, decomp_method
+  use SelfTestDriver         , only : for_testing_bypass_run_except_clock_advance
   !
   ! !PUBLIC TYPES:
   implicit none
@@ -165,6 +166,7 @@ subroutine clm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate, ro
     ! CalcIrrigationNeeded. Simply declaring this variable makes the ICE go away.
     real(r8), allocatable :: dummy1_to_make_pgi_happy(:)
     !-----------------------------------------------------------------------
+    if ( for_testing_bypass_run_except_clock_advance() ) return
 
     ! Determine processor bounds and clumps for this processor
 
@@ -1576,6 +1578,8 @@ subroutine clm_drv_init(bounds, &
     integer :: fp, fc                  ! filter indices
     !-----------------------------------------------------------------------
 
+    if ( for_testing_bypass_run_except_clock_advance() ) return
+
     associate(                                                             &
          snl                => col%snl                                   , & ! Input:  [integer  (:)   ]  number of snow layers
 
@@ -1657,6 +1661,7 @@ subroutine clm_drv_patch2col (bounds, &
     ! !LOCAL VARIABLES:
     integer :: c,fc              ! indices
     ! -----------------------------------------------------------------
+    if ( for_testing_bypass_run_except_clock_advance() ) return
 
     ! Note: lake points are excluded from many of the following
     ! averages. For some fields, this is because the field doesn't
@@ -1752,6 +1757,8 @@ subroutine write_diagnostic (bounds, nstep, lnd2atm_inst)
     integer :: status(MPI_STATUS_SIZE) ! mpi status
     !------------------------------------------------------------------------
 
+    if ( for_testing_bypass_run_except_clock_advance() ) return
+
     call get_proc_global(ng=numg)
 
     if (masterproc) then