diff --git a/MG5aMC/mg5amcnlo b/MG5aMC/mg5amcnlo index cb4771efc5..3597d123f6 160000 --- a/MG5aMC/mg5amcnlo +++ b/MG5aMC/mg5amcnlo @@ -1 +1 @@ -Subproject commit cb4771efc507953c5ae03320ef5ee5496fabbff3 +Subproject commit 3597d123f6c954bdba4a49387b8e89a5f2b06dcc diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/MG5aMC_patches/PROD/patch.common b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/MG5aMC_patches/PROD/patch.common index 2e272b419b..8f96dab5ef 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/MG5aMC_patches/PROD/patch.common +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/MG5aMC_patches/PROD/patch.common @@ -25,7 +25,7 @@ index 617f10b93..dbe08b846 100644 $(RM) *.o $(LIBRARIES) $(BINARIES) cd PDF; make clean; cd .. cd PDF/gammaUPC; make clean; cd ../../ -@@ -132,4 +132,9 @@ clean: +@@ -132,4 +132,11 @@ clean: cd BIAS/ptj_bias; make clean; cd ../.. if [ -d $(CUTTOOLSDIR) ]; then cd $(CUTTOOLSDIR); make clean; cd ..; fi if [ -d $(IREGIDIR) ]; then cd $(IREGIDIR); make clean; cd ..; fi @@ -33,8 +33,11 @@ index 617f10b93..dbe08b846 100644 +clean: cleanSource for i in `ls -d ../SubProcesses/P*`; do cd $$i; make clean; cd -; done; + -+cleanall: cleanSource ++cleanavx: + for i in `ls -d ../SubProcesses/P*`; do cd $$i; make cleanavxs; cd -; done; ++cleanall: cleanSource # THIS IS THE ONE ++ for i in `ls -d ../SubProcesses/P*`; do cd $$i; make cleanavxs; cd -; done; ++ diff --git b/epochX/cudacpp/gg_tt.mad/SubProcesses/makefile a/epochX/cudacpp/gg_tt.mad/SubProcesses/makefile index 348c283be..74db44d84 100644 --- b/epochX/cudacpp/gg_tt.mad/SubProcesses/makefile @@ -78,12 +81,12 @@ index 348c283be..74db44d84 100644 +CUDACPP_MAKEENV:=$(shell echo '$(.VARIABLES)' | tr " " "\n" | egrep "(USEBUILDDIR|AVX|FPTYPE|HELINL|HRDCOD)") +###$(info CUDACPP_MAKEENV=$(CUDACPP_MAKEENV)) +###$(info $(foreach v,$(CUDACPP_MAKEENV),$(v)="$($(v))")) -+CUDACPP_BUILDDIR:=$(shell $(MAKE) $(foreach v,$(CUDACPP_MAKEENV),$(v)="$($(v))") -f $(CUDACPP_MAKEFILE) -pn |& awk '/Building/{print $$3}' | sed s/BUILDDIR=//) -+ifeq ($(CUDACPP_BUILDDIR),) -+$(error CUDACPP_BUILDDIR='$(CUDACPP_BUILDDIR)' should not be empty!) -+else ++CUDACPP_BUILDDIR:=$(shell $(MAKE) $(foreach v,$(CUDACPP_MAKEENV),$(v)="$($(v))") -f $(CUDACPP_MAKEFILE) -pn 2>/dev/null | awk '/Building/{print $$3}' | sed s/BUILDDIR=//) ++#ifeq ($(CUDACPP_BUILDDIR),) ++#$(error CUDACPP_BUILDDIR='$(CUDACPP_BUILDDIR)' should not be empty!) ++#else +$(info CUDACPP_BUILDDIR='$(CUDACPP_BUILDDIR)') -+endif ++#endif +CUDACPP_COMMONLIB=mg5amc_common +CUDACPP_CXXLIB=mg5amc_$(processid_short)_cpp +CUDACPP_CULIB=mg5amc_$(processid_short)_cuda @@ -110,7 +113,7 @@ index 348c283be..74db44d84 100644 -$(PROG): $(PROCESS) auto_dsig.o $(LIBS) $(MATRIX) - $(FC) -o $(PROG) $(PROCESS) $(MATRIX) $(LINKLIBS) $(LDFLAGS) $(BIASDEPENDENCIES) -fopenmp -+LDFLAGS+=-Wl,--no-relax # avoid 'failed to convert GOTPCREL relocation' error #458 ++#LDFLAGS+=-Wl,--no-relax # avoid 'failed to convert GOTPCREL relocation' error #458 (flag not universal -> skip?) -$(PROG)_forhel: $(PROCESS) auto_dsig.o $(LIBS) $(MATRIX_HEL) - $(FC) -o $(PROG)_forhel $(PROCESS) $(MATRIX_HEL) $(LINKLIBS) $(LDFLAGS) $(BIASDEPENDENCIES) -fopenmp @@ -283,44 +286,6 @@ index 348c283be..74db44d84 100644 + +distclean: cleanall # Clean all fortran and cudacpp builds as well as the googletest installation + $(MAKE) -f $(CUDACPP_MAKEFILE) distclean -diff --git b/epochX/cudacpp/gg_tt.mad/Source/make_opts a/epochX/cudacpp/gg_tt.mad/Source/make_opts -index 57f5f7bb9..bd3c24228 100644 ---- b/epochX/cudacpp/gg_tt.mad/Source/make_opts -+++ a/epochX/cudacpp/gg_tt.mad/Source/make_opts -@@ -1,12 +1,17 @@ -+pdlabel1= -+pdlabel2= -+lhapdf= -+PYTHIA8_PATH=NotInstalled -+MG5AMC_VERSION=3.5.0_lo_vect - GLOBAL_FLAG=-O3 -ffast-math -fbounds-check -+ALOHA_FLAG= -+MATRIX_FLAG= - DEFAULT_CPP_COMPILER=g++ - MACFLAG= - STDLIB=-lstdc++ - STDLIB_FLAG= - DEFAULT_F_COMPILER=gfortran - DEFAULT_F2PY_COMPILER=f2py3 --MG5AMC_VERSION=SpecifiedByMG5aMCAtRunTime --PYTHIA8_PATH=NotInstalled - #end_of_make_opts_variables - - BIASLIBDIR=../../../lib/ -diff --git b/epochX/cudacpp/gg_tt.mad/bin/internal/banner.py a/epochX/cudacpp/gg_tt.mad/bin/internal/banner.py -index 8f8df219d..7624b9f55 100755 ---- b/epochX/cudacpp/gg_tt.mad/bin/internal/banner.py -+++ a/epochX/cudacpp/gg_tt.mad/bin/internal/banner.py -@@ -4187,7 +4187,8 @@ class RunCardLO(RunCard): - self.add_param('mxxmin4pdg',[-1.], system=True) - self.add_param('mxxpart_antipart', [False], system=True) - -- -+ # CUDACPP parameters -+ self.add_param('cudacpp_backend', 'CPP', include=False, hidden=False) - - def check_validity(self): - """ """ diff --git b/epochX/cudacpp/gg_tt.mad/bin/internal/gen_ximprove.py a/epochX/cudacpp/gg_tt.mad/bin/internal/gen_ximprove.py index 4dd71db86..3b8ec3121 100755 --- b/epochX/cudacpp/gg_tt.mad/bin/internal/gen_ximprove.py diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/__init__.py b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/__init__.py index f302f9c746..a0cd9dbfb3 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/__init__.py +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/__init__.py @@ -32,7 +32,12 @@ # allows the command "output myformat PATH" in madgraph. # MYCLASS should inherit from class madgraph.iolibs.export_v4.VirtualExporter import PLUGIN.CUDACPP_OUTPUT.output as output - new_output = { 'standalone_cudacpp' : output.PLUGIN_ProcessExporter } + new_output = { 'madevent_simd' : output.SIMD_ProcessExporter, + 'madevent_gpu' : output.GPU_ProcessExporter, + 'standalone_cudacpp' : output.PLUGIN_ProcessExporter, + 'standalone_simd' : output.SIMD_ProcessExporter, + 'standalone_cuda' : output.GPU_ProcessExporter, + } # 2. Define new way to handle the cluster. # Example: new_cluster = {'mycluster': MYCLUSTERCLASS} diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/launch_plugin.py b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/launch_plugin.py new file mode 100644 index 0000000000..23271b846e --- /dev/null +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/launch_plugin.py @@ -0,0 +1,100 @@ + +import logging +import os +import subprocess +pjoin = os.path.join +logger = logging.getLogger('cmdprint') # for stdout + +try: + import madgraph +except ImportError: + import internal.madevent_interface as madevent_interface + import internal.misc as misc + import internal.extended_cmd as extended_cmd + import internal.banner as banner_mod +else: + import madgraph.interface.madevent_interface as madevent_interface + import madgraph.various.misc as misc + import madgraph.interface.extended_cmd as extended_cmd + import madgraph.various.banner as banner_mod + +class CPPMEInterface(madevent_interface.MadEventCmdShell): + + def compile(self, *args, **opts): + """ """ + + import multiprocessing + if not self.options['nb_core'] or self.options['nb_core'] == 'None': + self.options['nb_core'] = multiprocessing.cpu_count() + + if args and args[0][0] == 'madevent' and hasattr(self, 'run_card'): + import pathlib + import os + pjoin = os.path.join + + + + + cudacpp_backend = self.run_card['cudacpp_backend'].upper() # the default value is defined in banner.py + logger.info("Building madevent in madevent_interface.py with '%s' matrix elements"%cudacpp_backend) + if cudacpp_backend == 'FORTRAN': + args[0][0] = 'madevent_fortran_link' + elif cudacpp_backend == 'CPP': + args[0][0] = 'madevent_cpp_link' + elif cudacpp_backend == 'CUDA': + args[0][0] = 'madevent_cuda_link' + else: + raise Exception("Invalid cudacpp_backend='%s': only 'FORTRAN', 'CPP', 'CUDA' are supported") + return misc.compile(nb_core=self.options['nb_core'], *args, **opts) + else: + return misc.compile(nb_core=self.options['nb_core'], *args, **opts) + +class CPPRunCard(banner_mod.RunCardLO): + + def reset_simd(self, old_value, new_value, name): + if not hasattr(self, 'path'): + raise Exception + + if name == "vector_size" and new_value <= int(old_value): + # code can handle the new size -> do not recompile + return + + Sourcedir = pjoin(os.path.dirname(os.path.dirname(self.path)), 'Source') + subprocess.call(['make', 'cleanavx'], cwd=Sourcedir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + def plugin_input(self, finput): + return + + def default_setup(self): + super().default_setup() + self.add_param('cudacpp_backend', 'CPP', include=False, hidden=False) + + + def write_one_include_file(self, output_dir, incname, output_file=None): + """write one include file at the time""" + + if incname == "vector.inc" and 'vector_size' not in self.user_set: + return + super().write_one_include_file(output_dir, incname, output_file) + + + def check_validity(self): + """ensure that PLUGIN information are consistent""" + + super().check_validity() + + if self['SDE_strategy'] != 1: + logger.warning('SDE_strategy different of 1 is not supported with SMD/GPU mode') + self['sde_strategy'] = 1 + + if self['hel_recycling']: + self['hel_recycling'] = False + +class GPURunCard(CPPRunCard): + + def default_setup(self): + super(CPPRunCard, self).default_setup() + self.add_param('cudacpp_backend', 'CUDA', include=False, hidden=False) + +MEINTERFACE = CPPMEInterface +RunCard = CPPRunCard diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/MatrixElementKernels.cc b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/MatrixElementKernels.cc index 30257195b6..cfed7d2af5 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/MatrixElementKernels.cc +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/MatrixElementKernels.cc @@ -112,10 +112,17 @@ namespace mg5amcCpu // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu bool ok = true; // this is just an assumption! const std::string tag = "arm neon (128bit as in SSE4.2)"; -#else +#elif defined(__x86_64__) || defined(__i386__) bool known = true; bool ok = __builtin_cpu_supports( "sse4.2" ); const std::string tag = "nehalem (SSE4.2)"; +#else + bool known = false; // __builtin_cpu_supports is not supported + // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html + // See https://stackoverflow.com/q/62783908 + // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu + bool ok = true; // this is just an assumption! + const std::string tag = "arm neon (128bit as in SSE4.2)"; #endif #else bool known = true; diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/cudacpp.mk b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/cudacpp.mk index 222d75f846..49a5856085 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/cudacpp.mk +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/cudacpp.mk @@ -27,6 +27,7 @@ UNAME_S := $(shell uname -s) UNAME_P := $(shell uname -p) ###$(info UNAME_P='$(UNAME_P)') +include ../../Source/make_opts #------------------------------------------------------------------------------- #=== Configure common compiler flags for C++ and CUDA @@ -220,7 +221,7 @@ else ifneq ($(shell $(CXX) --version | egrep '^(clang)'),) override OMPFLAGS = -fopenmp ###override OMPFLAGS = # disable OpenMP MT on clang (was not ok without or with nvcc before #578) else ifneq ($(shell $(CXX) --version | egrep '^(Apple clang)'),) -override OMPFLAGS = # disable OpenMP MT on Apple clang (builds fail in the CI #578) +override OMPFLAGS = -fopenmp # disable OpenMP MT on Apple clang (builds fail in the CI #578) else override OMPFLAGS = -fopenmp ###override OMPFLAGS = # disable OpenMP MT (default before #575) @@ -554,7 +555,7 @@ endif $(LIBDIR)/lib$(MG5AMC_CXXLIB).so: $(BUILDDIR)/fbridge.o $(LIBDIR)/lib$(MG5AMC_CXXLIB).so: cxx_objects_lib += $(BUILDDIR)/fbridge.o $(LIBDIR)/lib$(MG5AMC_CXXLIB).so: $(LIBDIR)/lib$(MG5AMC_COMMONLIB).so $(cxx_objects_lib) - $(CXX) -shared -o $@ $(cxx_objects_lib) $(CXXLIBFLAGSRPATH2) -L$(LIBDIR) -l$(MG5AMC_COMMONLIB) + $(CXX) -shared -o $@ $(cxx_objects_lib) $(CXXLIBFLAGSRPATH2) -L$(LIBDIR) -l$(MG5AMC_COMMONLIB) $(LIBFLAGS) -fopenmp ifneq ($(NVCC),) $(LIBDIR)/lib$(MG5AMC_CULIB).so: $(BUILDDIR)/fbridge_cu.o diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_function_definitions.inc b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_function_definitions.inc index c3c0812b94..a3ce5ad2dc 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_function_definitions.inc +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_function_definitions.inc @@ -405,8 +405,7 @@ namespace mg5amcCpu { // nprocesses>1 was last observed for "mirror processes" in uux_ttx in the 270 branch (see issue #343 and PRs #360 and #396) constexpr int nprocesses = %(nproc)i; - static_assert( nprocesses == 1, "Assume nprocesses == 1" ); - // process_id corresponds to the index of DSIG1 Fortran functions (must be 1 because cudacpp is unable to handle DSIG2) + static_assert( nprocesses == 1 || nprocesses == 2, "Assume nprocesses == 1 or 2" ); constexpr int process_id = %(proc_id)i; // code generation source: %(proc_id_source)s static_assert( process_id == 1, "Assume process_id == 1" ); } diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/model_handling.py b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/model_handling.py index a73c41d7c3..0e634c4fe6 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/model_handling.py +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/model_handling.py @@ -196,6 +196,7 @@ def get_header_txt(self, name=None, couplings=None,mode=''): list_arg = '[]' # AV from cxtype_sv to fptype array (running alphas #373) point = self.type2def['pointer_coup'] args.append('%s %s%s%s'% (type, point, argname, list_arg)) + args.append('double Ccoeff%s'% argname[7:]) else: args.append('%s %s%s'% (type, argname, list_arg)) if not self.offshell: @@ -534,6 +535,15 @@ def write_MultContainer(self, obj, prefactor=True): text = '%(factors)s' return text % data + def change_var_format(self, obj): + """ """ + if obj.startswith('COUP'): + out = super().change_var_format(obj) + postfix = out[4:] + return "Ccoeff%s*%s" % (postfix, out) + else: + return super().change_var_format(obj) + # AV - new method (based on implementation of write_obj and write_MultVariable) def objIsSimpleVariable(self, obj) : ###print ( obj.vartype, obj.prefactor, len( obj ), obj ) # AV - FOR DEBUGGING @@ -726,7 +736,6 @@ def write_hardcoded_parameters(self, params): for par in pars.split(','): ###print(len(pardef_lines), par) # for debugging pardef_lines[par] = ( 'constexpr ' + type + ' ' + par ) - misc.sprint( 'pardef_lines size =', len(pardef_lines), ', keys size =', len(pardef_lines.keys()) ) ###print( pardef_lines ) # for debugging ###for line in pardef_lines: misc.sprint(line) # for debugging parset_pars = [] @@ -1043,14 +1052,14 @@ class PLUGIN_OneProcessExporter(PLUGIN_export_cpp.OneProcessExporterGPU): # AV - overload export_cpp.OneProcessExporterGPU constructor (rename gCPPProcess to CPPProcess, set include_multi_channel) def __init__(self, *args, **kwargs): - misc.sprint('Entering PLUGIN_OneProcessExporter.__init__') +# misc.sprint('Entering PLUGIN_OneProcessExporter.__init__') for kwarg in kwargs: misc.sprint( 'kwargs[%s] = %s' %( kwarg, kwargs[kwarg] ) ) super().__init__(*args, **kwargs) self.process_class = 'CPPProcess' ###if self.in_madevent_mode: proc_id = kwargs['prefix']+1 # madevent+cudacpp (NB: HERE SELF.IN_MADEVENT_MODE DOES NOT WORK!) if 'prefix' in kwargs: proc_id = kwargs['prefix']+1 # madevent+cudacpp (ime+1 from ProcessExporterFortranMEGroup.generate_subprocess_directory) else: proc_id = 0 # standalone_cudacpp - misc.sprint(proc_id) +# misc.sprint(proc_id) self.proc_id = proc_id # AV - overload export_cpp.OneProcessExporterGPU method (indent comments in process_lines) @@ -1148,12 +1157,18 @@ def get_process_function_definitions(self, write=True): # AV - modify export_cpp.OneProcessExporterGPU method (add debug printouts for multichannel #342) def get_sigmaKin_lines(self, color_amplitudes, write=True): - misc.sprint('Entering PLUGIN_OneProcessExporter.get_sigmaKin_lines') - misc.sprint(self.include_multi_channel) - misc.sprint(self.support_multichannel) +# misc.sprint('Entering PLUGIN_OneProcessExporter.get_sigmaKin_lines') +# misc.sprint(self.include_multi_channel) +# misc.sprint(self.support_multichannel) replace_dict = super().get_sigmaKin_lines(color_amplitudes, write=False) replace_dict['proc_id'] = self.proc_id if self.proc_id>0 else 1 replace_dict['proc_id_source'] = 'madevent + cudacpp exporter' if self.proc_id>0 else 'standalone_cudacpp' # FIXME? use self.in_madevent_mode instead? + + # Extract denominator (avoid to extend size for mirroring) + den_factors = [str(me.get_denominator_factor()) for me in \ + self.matrix_elements] + replace_dict['den_factors'] = ",".join(den_factors) + if write: file = self.read_template_file(self.process_sigmaKin_function_template) % replace_dict file = '\n'.join( file.split('\n')[8:] ) # skip first 8 lines in process_sigmaKin_function.inc (copyright) @@ -1167,14 +1182,14 @@ def get_all_sigmaKin_lines(self, color_amplitudes, class_name): ret_lines = [] if self.single_helicities: ###assert self.include_multi_channel # remove this assert: must handle both cases and produce two different code bases (#473) - misc.sprint(type(self.helas_call_writer)) - misc.sprint(self.support_multichannel, self.include_multi_channel) +# misc.sprint(type(self.helas_call_writer)) +# misc.sprint(self.support_multichannel, self.include_multi_channel) multi_channel = None if self.include_multi_channel: if not self.support_multichannel: raise Exception("link with madevent not supported") multi_channel = self.get_multi_channel_dictionary(self.matrix_elements[0].get('diagrams'), self.include_multi_channel) - misc.sprint(multi_channel) +# misc.sprint(multi_channel) ###misc.sprint( 'before get_matrix_element_calls', self.matrix_elements[0].get_number_of_wavefunctions() ) # WRONG value of nwf, eg 7 for gg_tt helas_calls = self.helas_call_writer.get_matrix_element_calls(\ self.matrix_elements[0], @@ -1302,12 +1317,9 @@ def get_process_info_lines(self, matrix_element): # AV - replace the export_cpp.OneProcessExporterGPU method (invert .cc/.cu, add debug printouts) def generate_process_files(self): """Generate mgOnGpuConfig.h, CPPProcess.cc, CPPProcess.h, check_sa.cc, gXXX.cu links""" - misc.sprint('Entering PLUGIN_OneProcessExporter.generate_process_files') - if self.include_multi_channel: - misc.sprint('self.include_multi_channel is already defined: this is madevent+second_exporter mode') # FIXME? use self.in_madevent_mode instead? - else: - misc.sprint('self.include_multi_channel is not yet defined: this is standalone_cudacpp mode') # see issue #473 - # I move those line to standalone_cudacpp mode (but do we need those at all???) +# misc.sprint('Entering PLUGIN_OneProcessExporter.generate_process_files') + if not self.include_multi_channel: + #this condition is likely wrong and need to be removed if self.matrix_elements[0].get('has_mirror_process'): self.matrix_elements[0].set('has_mirror_process', False) self.nprocesses/=2 @@ -1341,15 +1353,15 @@ def generate_process_files(self): ###template_ref = 'dump_CPUTest.'+self.process_name+'.txt' template_ref = self.template_path + '/../../../test/ref/' + 'dump_CPUTest.' + self.process_name + '.txt' if os.path.exists( template_ref ): - misc.sprint( 'Copying test reference file: ', template_ref ) +# misc.sprint( 'Copying test reference file: ', template_ref ) PLUGIN_export_cpp.cp( template_ref, self.path + '/../../test/ref' ) - else: - misc.sprint( 'Test reference file does not exist and will not be copied: ', template_ref ) +# else: +# misc.sprint( 'Test reference file does not exist and will not be copied: ', template_ref ) # SR - generate CMakeLists.txt file inside the P* directory def edit_CMakeLists(self): """Generate CMakeLists.txt""" - misc.sprint('Entering PLUGIN_OneProcessExporter.edit_CMakeLists') +# misc.sprint('Entering PLUGIN_OneProcessExporter.edit_CMakeLists') template = open(pjoin(self.template_path,'CMake/SubProcesses/CMakeLists_P.txt'),'r').read() ff = open(pjoin(self.path, 'CMakeLists.txt'),'w') ff.write(template) @@ -1358,7 +1370,7 @@ def edit_CMakeLists(self): # AV - replace the export_cpp.OneProcessExporterGPU method (invert .cc/.cu, add debug printouts) def edit_check_sa(self): """Generate check_sa.cc and fcheck_sa.f""" - misc.sprint('Entering PLUGIN_OneProcessExporter.edit_check_sa') +# misc.sprint('Entering PLUGIN_OneProcessExporter.edit_check_sa') ff = open(pjoin(self.path, 'check_sa.cc'),'w') template = open(pjoin(self.template_path,'gpu','check_sa.cc'),'r').read() ff.write(template) # nothing to replace in check_sa.cc @@ -1375,7 +1387,7 @@ def edit_check_sa(self): # AV - replace the export_cpp.OneProcessExporterGPU method (add debug printouts and multichannel handling #473) def edit_mgonGPU(self): """Generate mgOnGpuConfig.h""" - misc.sprint('Entering PLUGIN_OneProcessExporter.edit_mgonGPU') +# misc.sprint('Entering PLUGIN_OneProcessExporter.edit_mgonGPU') template = open(pjoin(self.template_path,'gpu','mgOnGpuConfig.h'),'r').read() replace_dict = {} nexternal, nincoming = self.matrix_elements[0].get_nexternal_ninitial() @@ -1395,7 +1407,7 @@ def edit_mgonGPU(self): # AV - new method def edit_processidfile(self): """Generate epoch_process_id.h""" - misc.sprint('Entering PLUGIN_OneProcessExporter.edit_processidfile') +# misc.sprint('Entering PLUGIN_OneProcessExporter.edit_processidfile') template = open(pjoin(self.template_path,'gpu','epoch_process_id.h'),'r').read() replace_dict = {} replace_dict['processid'] = self.get_process_name() @@ -1407,7 +1419,7 @@ def edit_processidfile(self): # AV - new method def edit_coloramps(self): """Generate coloramps.h""" - misc.sprint('Entering PLUGIN_OneProcessExporter.edit_coloramps') +# misc.sprint('Entering PLUGIN_OneProcessExporter.edit_coloramps') template = open(pjoin(self.template_path,'gpu','coloramps.h'),'r').read() ff = open(pjoin(self.path, 'coloramps.h'),'w') # The following five lines from OneProcessExporterCPP.get_sigmaKin_lines (using OneProcessExporterCPP.get_icolamp_lines) @@ -1425,7 +1437,7 @@ def edit_coloramps(self): # AV - new method def edit_testxxx(self): """Generate testxxx.cc""" - misc.sprint('Entering PLUGIN_OneProcessExporter.edit_testxxx') +# misc.sprint('Entering PLUGIN_OneProcessExporter.edit_testxxx') template = open(pjoin(self.template_path,'gpu','testxxx.cc'),'r').read() replace_dict = {} replace_dict['model_name'] = self.model_name @@ -1436,7 +1448,7 @@ def edit_testxxx(self): # AV - new method def edit_memorybuffers(self): """Generate MemoryBuffers.h""" - misc.sprint('Entering PLUGIN_OneProcessExporter.edit_memorybuffers') +# misc.sprint('Entering PLUGIN_OneProcessExporter.edit_memorybuffers') template = open(pjoin(self.template_path,'gpu','MemoryBuffers.h'),'r').read() replace_dict = {} replace_dict['model_name'] = self.model_name @@ -1447,7 +1459,7 @@ def edit_memorybuffers(self): # AV - new method def edit_memoryaccesscouplings(self): """Generate MemoryAccessCouplings.h""" - misc.sprint('Entering PLUGIN_OneProcessExporter.edit_memoryaccesscouplings') +# misc.sprint('Entering PLUGIN_OneProcessExporter.edit_memoryaccesscouplings') template = open(pjoin(self.template_path,'gpu','MemoryAccessCouplings.h'),'r').read() replace_dict = {} replace_dict['model_name'] = self.model_name @@ -1459,7 +1471,7 @@ def edit_memoryaccesscouplings(self): # [*NB export_cpp.UFOModelConverterGPU.write_process_h_file is not called!*] def write_process_h_file(self, writer): """Generate final gCPPProcess.h""" - misc.sprint('Entering PLUGIN_OneProcessExporter.write_process_h_file') +# misc.sprint('Entering PLUGIN_OneProcessExporter.write_process_h_file') out = super().write_process_h_file(writer) writer.seek(-1, os.SEEK_CUR) writer.truncate() @@ -1481,7 +1493,7 @@ def super_write_process_cc_file(self, writer): # AV - overload the export_cpp.OneProcessExporterGPU method (add debug printout and truncate last \n) def write_process_cc_file(self, writer): """Generate CPPProcess.cc""" - misc.sprint('Entering PLUGIN_OneProcessExporter.write_process_cc_file') +# misc.sprint('Entering PLUGIN_OneProcessExporter.write_process_cc_file') ###out = super().write_process_cc_file(writer) out = self.super_write_process_cc_file(writer) writer.seek(-1, os.SEEK_CUR) @@ -1625,13 +1637,14 @@ def format_coupling(self, call): alias[coup] = len(alias) if name == 'cIPD': call = call.replace('m_pars->%s%s' % (sign, coup), - '%s%s[%s]' % (sign, name, alias[coup])) + '%s%s[%s]' % (sign, name, alias[coup])) else: ###call = call.replace('m_pars->%s%s' % (sign, coup), ### '%scxmake( cIPC[%s], cIPC[%s] )' % ### (sign, 2*alias[coup],2*alias[coup]+1)) + # AV from cIPCs to COUP array (running alphas #373) call = call.replace('m_pars->%s%s' % (sign, coup), - '%sCOUPs[%s]' % (sign, alias[coup])) # AV from cIPCs to COUP array (running alphas #373) + 'COUPs[%s], %s' % (alias[coup], '1.0' if not sign else '-1.0')) return call # AV - new method for formatting wavefunction/amplitude calls @@ -1660,7 +1673,7 @@ def super_get_matrix_element_calls(self, matrix_element, color_amplitudes, multi color[namp][njamp] = coeff me = matrix_element.get('diagrams') matrix_element.reuse_outdated_wavefunctions(me) - misc.sprint(multi_channel_map) +# misc.sprint(multi_channel_map) res = [] ###res.append('for(int i=0;i<%s;i++){jamp[i] = cxtype(0.,0.);}' % len(color_amplitudes)) res.append("""constexpr size_t nxcoup = ndcoup + nicoup; // both dependent and independent couplings @@ -1715,7 +1728,7 @@ def super_get_matrix_element_calls(self, matrix_element, color_amplitudes, multi sum([diagrams[idiag].get('amplitudes') for \ idiag in multi_channel_map[config]], [])] diag_to_config[amp[0]] = config - misc.sprint(diag_to_config) +# misc.sprint(diag_to_config) id_amp = 0 for diagram in matrix_element.get('diagrams'): ###print('DIAGRAM %3d: #wavefunctions=%3d, #diagrams=%3d' % @@ -1803,71 +1816,51 @@ def get_external(self, wf, argument): def get_external_line(self, wf, argument): call = '' call = call + helas_call_writers.HelasCallWriter.mother_dict[\ - argument.get_spin_state_number()].lower() - if wf.get('mass').lower() != 'zero' or argument.get('spin') != 2: - # Fill out with X up to 6 positions - call = call + 'x' * (6 - len(call)) - # Specify namespace for Helas calls - call = call + '( momenta,' - if argument.get('spin') != 1: - # For non-scalars, need mass and helicity - call = call + 'm_pars->%s, cHel[ihel][%d],' - else: - # AV This seems to be for scalars (spin==1???), pass neither mass nor helicity (#351) - ###call = call + 'm_pars->%s,' - call = call - call = call + '%+d, w_sv[%d], %d );' - if argument.get('spin') == 1: - # AV This seems to be for scalars (spin==1???), pass neither mass nor helicity (#351) - return call % \ - ( - ###wf.get('mass'), - # For boson, need initial/final here - (-1) ** (wf.get('state') == 'initial'), - wf.get('me_id')-1, - wf.get('number_external')-1) - elif argument.is_boson(): - misc.sprint(call) - misc.sprint( (wf.get('mass'), - wf.get('number_external')-1, - # For boson, need initial/final here - (-1) ** (wf.get('state') == 'initial'), - wf.get('me_id')-1, - wf.get('number_external')-1)) - return self.format_coupling(call % \ - (wf.get('mass'), - wf.get('number_external')-1, - # For boson, need initial/final here - (-1) ** (wf.get('state') == 'initial'), - wf.get('me_id')-1, - wf.get('number_external')-1)) - else: - return self.format_coupling(call % \ - (wf.get('mass'), - wf.get('number_external')-1, - # For fermions, need particle/antiparticle - - (-1) ** wf.get_with_flow('is_part'), - wf.get('me_id')-1, - wf.get('number_external')-1)) + argument.get_spin_state_number()].lower() + # Fill out with X up to 6 positions + call = call + 'x' * (6 - len(call)) + # Specify namespace for Helas calls + call = call + '( momenta,' + if argument.get('spin') != 1: + # For non-scalars, need mass and helicity + call = call + 'm_pars->%s, cHel[ihel][%d],' + else: + # AV This seems to be for scalars (spin==1???), pass neither mass nor helicity (#351) + ###call = call + 'm_pars->%s,' + call = call + call = call + '%+d, w_sv[%d], %d );' + if argument.get('spin') == 1: + # AV This seems to be for scalars (spin==1???), pass neither mass nor helicity (#351) + return call % \ + ( + ###wf.get('mass'), + # For boson, need initial/final here + (-1) ** (wf.get('state') == 'initial'), + wf.get('me_id')-1, + wf.get('number_external')-1) + elif argument.is_boson(): +# misc.sprint(call) +# misc.sprint( (wf.get('mass'), +# wf.get('number_external')-1, +# # For boson, need initial/final here +# (-1) ** (wf.get('state') == 'initial'), +# wf.get('me_id')-1, +# wf.get('number_external')-1)) + return self.format_coupling(call % \ + (wf.get('mass'), + wf.get('number_external')-1, + # For boson, need initial/final here + (-1) ** (wf.get('state') == 'initial'), + wf.get('me_id')-1, + wf.get('number_external')-1)) else: - if wf.get('number_external') == 1: - call += 'pz' - elif wf.get('number_external') == 2: - call += 'mz' - else: - call += 'xz' - comment = '' # AV - call = call + 'x' * (6 - len(call)) - if wf.get('number_external') == 1 or wf.get('number_external') == 2: # AV - comment = ' // NB: ' + call + ' only uses pz' # AV skip '(not E,px,py)' to avoid interference with comma parsing in get_external - # Specify namespace for Helas calls - call = call + '( momenta, cHel[ihel][%d], %+d, w_sv[%d], %d );' + comment # AV vectorize and add comment return self.format_coupling(call % \ - (wf.get('number_external')-1, - # For fermions, need particle/antiparticle - - (-1) ** wf.get_with_flow('is_part'), - wf.get('me_id')-1, - wf.get('number_external')-1)) + (wf.get('mass'), + wf.get('number_external')-1, + # For fermions, need particle/antiparticle + - (-1) ** wf.get_with_flow('is_part'), + wf.get('me_id')-1, + wf.get('number_external')-1)) # AV - replace helas_call_writers.GPUFOHelasCallWriter method (vectorize w_sv and amp_sv) def generate_helas_call(self, argument): @@ -1931,7 +1924,8 @@ def generate_helas_call(self, argument): # AV FOR PR #434: determine if this call needs aS-dependent or aS-independent parameters usesdepcoupl = None for coup in argument.get('coupling'): - if coup.startswith('-'): coup = coup[1:] + if coup.startswith('-'): + coup = coup[1:] # Use the same implementation as in UFOModelConverterCPP.prepare_couplings (assume self.model is the same) for key, coup_list in self.get('model')['couplings'].items(): if coup in coup_list: diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/output.py b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/output.py index 14522ebe14..53cbf7f621 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/output.py +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/output.py @@ -4,6 +4,7 @@ # Further modified by: O. Mattelaer, S. Roiser, A. Valassi, Z. Wettersten (2021-2023) for the MG5aMC CUDACPP plugin. import os +import subprocess # AV - load an independent 2nd copy of the export_cpp module (as PLUGIN_export_cpp) and use that within the plugin (workaround for #341) # See https://stackoverflow.com/a/11285504 @@ -34,6 +35,7 @@ from os.path import join as pjoin import madgraph.various.misc as misc +import madgraph.iolibs.files as files # AV - define the plugin's process exporter # (NB: this is the plugin's main class, enabled in the new_output dictionary in __init__.py) @@ -201,13 +203,24 @@ def finalize(self, matrix_element, cmdhistory, MG5options, outputflag): cmdhistory is the list of command used so far. MG5options are all the options of the main interface outputflags is a list of options provided when doing the output command""" - misc.sprint('Entering PLUGIN_ProcessExporter.finalize', self.in_madevent_mode) + misc.sprint('Entering PLUGIN_ProcessExporter.finalize', self.in_madevent_mode, type(self)) if self.in_madevent_mode: self.add_input_for_banner() if 'CUDACPP_CODEGEN_PATCHLEVEL' in os.environ: patchlevel = os.environ['CUDACPP_CODEGEN_PATCHLEVEL'] else: patchlevel = '' - if os.system(PLUGINDIR + os.sep + 'patchMad.sh ' + self.dir_path + ' PROD ' + patchlevel) != 0: + plugin_path = os.path.dirname(os.path.realpath( __file__ )) +# path = os.path.realpath(os.curdir + os.sep + 'PLUGIN' + os.sep + 'CUDACPP_OUTPUT') +# misc.sprint(path) + p = subprocess.Popen([pjoin(plugin_path, 'patchMad.sh'), self.dir_path , 'PROD', str(patchlevel)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.returncode != 0: + logger.debug("####### \n stdout is \n %s", stdout) + logger.info("####### \n stderr is \n %s", stderr) + logger.info("return code is %s\n", p.returncode) raise Exception('ERROR! the O/S call to patchMad.sh failed') + + self.add_madevent_plugin_fct() return super().finalize(matrix_element, cmdhistory, MG5options, outputflag) # AV (default from OM's tutorial) - overload settings and add a debug printout @@ -228,4 +241,55 @@ def add_input_for_banner(self): for entry in new_parameters: finput.write(entry) + # OM adding a new way to "patch" python file such that the launch command of MG5aMC is working + # this consist in a file plugin_interface.py + # which contains a series of functions and one dictionary variable TO_OVERWRITE + # that will be used to have temporary overwrite of all the key variable passed as string by their value. + # all variable that are file related should be called as madgraph.dir.file.variable + def add_madevent_plugin_fct(self): + """this consist in a file plugin_interface.py + which contains a series of functions and one dictionary variable TO_OVERWRITE + that will be used to have temporary overwrite of all the key variable passed as string by their value. + all variable that are file related should be called as madgraph.dir.file.variable + """ + + plugin_path = os.path.dirname(os.path.realpath( __file__ )) + files.cp(pjoin(plugin_path, 'plugin_interface.py'), pjoin(self.dir_path, 'bin', 'internal')) + files.cp(pjoin(plugin_path, 'launch_plugin.py'), pjoin(self.dir_path, 'bin', 'internal')) + files.ln( pjoin(self.dir_path, 'lib'), pjoin(self.dir_path, 'SubProcesses')) + + #------------------------------------------------------------------------------------ +class SIMD_ProcessExporter(PLUGIN_ProcessExporter): + + def change_output_args(args, cmd): + """ """ + cmd._export_format = "madevent" + args.append('--hel_recycling=False') + args.append('--me_exporter=standalone_simd') + if 'vector_size' not in ''.join(args): + args.append('--vector_size=16') + return args + + + +class GPU_ProcessExporter(PLUGIN_ProcessExporter): + + def change_output_args(args, cmd): + """ """ + cmd._export_format = "madevent" + args.append('--hel_recycling=False') + args.append('--me_exporter=standalone_cuda') + if 'vector_size' not in ''.join(args): + args.append('--vector_size=16384') + return args + + def finalize(self, matrix_element, cmdhistory, MG5options, outputflag): + + misc.sprint("enter dedicated function") + out = super().finalize(matrix_element, cmdhistory, MG5options, outputflag) + #change RunCard class to have default for GPU + text = open(pjoin(self.dir_path, 'bin', 'internal', 'launch_plugin.py'), 'r').read() + text = text.replace('RunCard = CPPRunCard', 'RunCard = GPURunCard') + open(pjoin(self.dir_path, 'bin', 'internal', 'launch_plugin.py'), 'w').write(text) + return out diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/patchMad.sh b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/patchMad.sh index 9a9bfd55cc..548a7fdcc8 100755 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/patchMad.sh +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/patchMad.sh @@ -41,53 +41,43 @@ dir_patches=$2 if [ ! -e ${dir} ]; then echo "ERROR! Directory $dir does not exist"; exit 1; fi # These two steps are part of "cd Source; make" but they actually are code-generating steps -${dir}/bin/madevent treatcards run -${dir}/bin/madevent treatcards param +#${dir}/bin/madevent treatcards run +#${dir}/bin/madevent treatcards param # Cleanup -\rm -f ${dir}/crossx.html -\rm -f ${dir}/index.html -\rm -f ${dir}/madevent.tar.gz -\rm -f ${dir}/Cards/delphes_trigger.dat -\rm -f ${dir}/Cards/plot_card.dat -\rm -f ${dir}/bin/internal/run_plot* -\rm -f ${dir}/HTML/* -\rm -rf ${dir}/bin/internal/__pycache__ -\rm -rf ${dir}/bin/internal/ufomodel/__pycache__ -touch ${dir}/HTML/.keep # new file +#\rm -f ${dir}/crossx.html +#\rm -f ${dir}/index.html +#\rm -f ${dir}/madevent.tar.gz +#\rm -f ${dir}/Cards/delphes_trigger.dat +#\rm -f ${dir}/Cards/plot_card.dat +#\rm -f ${dir}/bin/internal/run_plot* +#\rm -f ${dir}/HTML/* +#\rm -rf ${dir}/bin/internal/__pycache__ +#\rm -rf ${dir}/bin/internal/ufomodel/__pycache__ +#touch ${dir}/HTML/.keep # new file # Exit here for patchlevel 0 (--upstream) if [ "${patchlevel}" == "0" ]; then exit $status; fi # Add global flag '-O3 -ffast-math -fbounds-check' as in previous gridpacks -echo "GLOBAL_FLAG=-O3 -ffast-math -fbounds-check" > ${dir}/Source/make_opts.new -cat ${dir}/Source/make_opts >> ${dir}/Source/make_opts.new -\mv ${dir}/Source/make_opts.new ${dir}/Source/make_opts +#echo "GLOBAL_FLAG=-O3 -ffast-math -fbounds-check" > ${dir}/Source/make_opts.new +#cat ${dir}/Source/make_opts >> ${dir}/Source/make_opts.new +#\mv ${dir}/Source/make_opts.new ${dir}/Source/make_opts # Patch the default Fortran code to provide the integration with the cudacpp plugin # (1) Process-independent patches touch ${dir}/Events/.keep # this file should already be present (mg5amcnlo copies it from Template/LO/Events/.keep) -\cp -dpr ${scrdir}/MG5aMC_patches/${dir_patches}/fbridge_common.inc ${dir}/SubProcesses # new file -\cp -dpr ${scrdir}/MG5aMC_patches/${dir_patches}/{counters.cc,ompnumthreads.cc} ${dir}/SubProcesses/ -sed -i 's/2 = sde_strategy/1 = sde_strategy/' ${dir}/Cards/run_card.dat # use strategy SDE=1 in multichannel mode (see #419) -sed -i 's/SDE_STRAT = 2/SDE_STRAT = 1/' ${dir}/Source/run_card.inc # use strategy SDE=1 in multichannel mode (see #419) +\cp -pr ${scrdir}/MG5aMC_patches/${dir_patches}/fbridge_common.inc ${dir}/SubProcesses # new file +\cp -pr ${scrdir}/MG5aMC_patches/${dir_patches}/{counters.cc,ompnumthreads.cc} ${dir}/SubProcesses/ +#sed -i 's/2 = sde_strategy/1 = sde_strategy/' ${dir}/Cards/run_card.dat # use strategy SDE=1 in multichannel mode (see #419) +#sed -i 's/SDE_STRAT = 2/SDE_STRAT = 1/' ${dir}/Source/run_card.inc # use strategy SDE=1 in multichannel mode (see #419) if [ "${patchlevel}" == "2" ]; then cd ${dir} - sed -i 's/DEFAULT_F2PY_COMPILER=f2py3.*/DEFAULT_F2PY_COMPILER=f2py3/' Source/make_opts + #sed -i 's/DEFAULT_F2PY_COMPILER=f2py3.*/DEFAULT_F2PY_COMPILER=f2py3/' Source/make_opts echo "DEBUG: cd ${PWD}; patch -p4 -i ${scrdir}/MG5aMC_patches/${dir_patches}/patch.common" if ! patch -p4 -i ${scrdir}/MG5aMC_patches/${dir_patches}/patch.common; then status=1; fi \rm -f Source/*.orig \rm -f bin/internal/*.orig - echo " -#********************************************************************* -# Options for the cudacpp plugin -#********************************************************************* - -# Set cudacpp-specific values of non-cudacpp-specific options --O3 -ffast-math -fbounds-check = global_flag ! build flags for Fortran code (for a fair comparison to cudacpp) - -# New cudacpp-specific options (default values are defined in banner.py) -CPP = cudacpp_backend ! valid backends are FORTRAN, CPP, CUDA" >> Cards/run_card.dat cd - > /dev/null fi for p1dir in ${dir}/SubProcesses/P*; do diff --git a/epochX/kokkos/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum/Makefile b/epochX/kokkos/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum/Makefile deleted file mode 120000 index d0b0e8e008..0000000000 --- a/epochX/kokkos/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum/Makefile +++ /dev/null @@ -1 +0,0 @@ -../Makefile \ No newline at end of file diff --git a/epochX/kokkos/gg_tt.sa/SubProcesses/P1_Sigma_sm_gg_ttx/Makefile b/epochX/kokkos/gg_tt.sa/SubProcesses/P1_Sigma_sm_gg_ttx/Makefile deleted file mode 120000 index d0b0e8e008..0000000000 --- a/epochX/kokkos/gg_tt.sa/SubProcesses/P1_Sigma_sm_gg_ttx/Makefile +++ /dev/null @@ -1 +0,0 @@ -../Makefile \ No newline at end of file diff --git a/epochX/kokkos/gg_ttg.sa/SubProcesses/P1_Sigma_sm_gg_ttxg/Makefile b/epochX/kokkos/gg_ttg.sa/SubProcesses/P1_Sigma_sm_gg_ttxg/Makefile deleted file mode 120000 index d0b0e8e008..0000000000 --- a/epochX/kokkos/gg_ttg.sa/SubProcesses/P1_Sigma_sm_gg_ttxg/Makefile +++ /dev/null @@ -1 +0,0 @@ -../Makefile \ No newline at end of file diff --git a/epochX/kokkos/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg/Makefile b/epochX/kokkos/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg/Makefile deleted file mode 120000 index d0b0e8e008..0000000000 --- a/epochX/kokkos/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg/Makefile +++ /dev/null @@ -1 +0,0 @@ -../Makefile \ No newline at end of file diff --git a/epochX/kokkos/gg_ttggg.sa/SubProcesses/P1_Sigma_sm_gg_ttxggg/Makefile b/epochX/kokkos/gg_ttggg.sa/SubProcesses/P1_Sigma_sm_gg_ttxggg/Makefile deleted file mode 120000 index d0b0e8e008..0000000000 --- a/epochX/kokkos/gg_ttggg.sa/SubProcesses/P1_Sigma_sm_gg_ttxggg/Makefile +++ /dev/null @@ -1 +0,0 @@ -../Makefile \ No newline at end of file