Merge remote-tracking branch 'origin/master' into 2228_unknown_kernel…

…_symbol

.github/workflows/lfric_test.yml

@@ -0,0 +1,113 @@
+# -----------------------------------------------------------------------------
+# BSD 3-Clause License
+#
+# Copyright (c) 2023, Science and Technology Facilities Council.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+# Author S. Siso, STFC Daresbury Lab
+
+# This workflow will use a self-hosted runner to perform the more expensive
+# integrations tests that are not run on GHA systems.
+
+name: LFRic Integration Tests
+
+on:
+  push
+
+jobs:
+  run_if_on_mirror:
+    if: ${{ github.repository == 'stfc/PSyclone-mirror' }}
+    runs-on: self-hosted
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+        # This is required to get the commit history for merge commits for
+        # the ci-skip check below.
+        fetch-depth: '0'
+    - name: Check for [skip ci] in commit message
+      uses: mstachniuk/ci-skip@v1
+      with:
+        # This setting causes the tests to 'fail' if [skip ci] is specified
+        fail-fast: true
+        commit-filter: '[skip ci]'
+    - name: Install dependencies
+      run: |
+        python -m venv .runner_venv
+        . .runner_venv/bin/activate
+        python -m pip install --upgrade pip
+        # If you wish to install the version of fparser pointed to by the
+        # submodule instead of the released version (from PyPI) then
+        # uncomment the following line:
+        pip install external/fparser
+        pip install .[test]
+        pip install jinja2
+
+    # PSyclone, compile and run MetOffice LFRic with 6 MPI ranks
+    - name: LFRic passthrough (with DistributedMemory)
+      run: |
+        . .runner_venv/bin/activate
+        export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
+        export MINIAPP_DIR=${HOME}/LFRic/miniapps/gungho_model
+        cd ${MINIAPP_DIR}
+        # Compile
+        module load lfric_env
+        export PSYCLONE_CONFIG=${HOME}/lfric_psyclone_config.cfg
+        make clean
+        make -j 6 build
+        # Run
+        cd example
+        cp ${HOME}/lfric_gunho_configuration.nml configuration.nml
+        mpirun -n 6 ../bin/gungho_model configuration.nml
+        tail PET0.gungho_model.Log
+        cat timer.txt
+
+    # PSyclone, compile and run MetOffice LFRic with all optimisations and 6 OpenMP threads
+    - name: LFRic with all transformations
+      run: |
+        . .runner_venv/bin/activate
+        export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
+        export MINIAPP_DIR=${HOME}/LFRic/miniapps/gungho_model
+        cd ${MINIAPP_DIR}
+        # Prepare script (LFRics expects a folder with a global.py)
+        mkdir -p psyclone-test
+        cp ${PSYCLONE_LFRIC_DIR}/everything_everywhere_all_at_once.py psyclone-test/global.py
+        # Compile
+        module load lfric_env
+        export PSYCLONE_CONFIG=${HOME}/lfric_psyclone_config.cfg
+        make clean
+        make OPTIMISATION_PATH=${MINIAPP_DIR}/psyclone-test -j 6 build
+        # Run
+        cd example
+        cp ${HOME}/lfric_gunho_configuration.nml configuration.nml
+        export OMP_NUM_THREADS=6
+        mpirun -n 1 ../bin/gungho_model configuration.nml
+        cat timer.txt

changelog

@@ -523,6 +523,10 @@
 	177) PR #2158 for #81. Ensures all example/test Fortran code for
 	LFRic follows the LFRic naming convention.
 
+	178) PR #2218 towards #2132. Adds LFRic integration tests and
+	new, associated optimisation scripts. Includes bug fixes for
+	matmul inlining transformation.
+
 release 2.3.1 17th of June 2022
 
 	1) PR #1747 for #1720. Adds support for If blocks to PSyAD.

examples/lfric/scripts/async_halo_exchanges.py

@@ -0,0 +1,77 @@
+# -----------------------------------------------------------------------------
+# BSD 3-Clause License
+#
+# Copyright (c) 2018-2022, Science and Technology Facilities Council
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+# Authors: R. Ford, A. R. Porter and S. Siso, STFC Daresbury Laboratory
+
+'''File containing a PSyclone transformation script for the Dynamo0p3
+API to make asynchronous halo exchanges and overlap their
+communication with computation. This can be applied via the -s option
+in the generator.py script.
+
+'''
+
+from psyclone.dynamo0p3 import DynHaloExchange, DynHaloExchangeStart
+from psyclone.transformations import Dynamo0p3AsyncHaloExchangeTrans, \
+    MoveTrans, TransformationError
+
+
+def trans(psy):
+    '''A transformation script to use asynchronous halo exchanges with
+    overlapping compute and communication for the LFRic model. '''
+
+    for invoke in psy.invokes.invoke_list:
+        schedule = invoke.schedule
+
+        # This transformation splits the three synchronous halo exchanges
+        ahex_trans = Dynamo0p3AsyncHaloExchangeTrans()
+        for h_ex in schedule.walk(DynHaloExchange):
+            ahex_trans.apply(h_ex)
+
+        # This transformation moves the start of the halo exchanges as far
+        # as possible offering the potential for overlap between communication
+        # and computation.
+        mtrans = MoveTrans()
+        location_cursor = 0
+        for ahex in schedule.walk(DynHaloExchangeStart):
+            if ahex.position <= location_cursor:
+                continue
+            try:
+                mtrans.apply(ahex, schedule.children[location_cursor])
+                location_cursor += 1
+            except TransformationError:
+                pass
+
+        print(f"{location_cursor} AsyncHaloExchanges have been rearranged"
+              f" in {invoke.name}")
+
+    return psy

examples/lfric/scripts/everything_everywhere_all_at_once.py

@@ -0,0 +1,143 @@
+# -----------------------------------------------------------------------------
+# BSD 3-Clause License
+#
+# Copyright (c) 2023, Science and Technology Facilities Council
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+# Authors: R. W. Ford, A. R. Porter and S. Siso, STFC Daresbury Lab
+#          I. Kavcic, Met Office
+#          J. Henrichs, Bureau of Meteorology
+
+'''PSyclone transformation script for the LFRic API to apply all the
+DistibutedMemory, OpenMP coloring and serial transformations possible.
+
+'''
+from psyclone.domain.common.transformations import KernelModuleInlineTrans
+from psyclone.domain.lfric import LFRicConstants
+from psyclone.dynamo0p3 import DynHaloExchange, DynHaloExchangeStart
+from psyclone.psyir.transformations import Matmul2CodeTrans
+from psyclone.psyir.nodes import BinaryOperation, Container, KernelSchedule
+from psyclone.transformations import Dynamo0p3ColourTrans, \
+                                     Dynamo0p3OMPLoopTrans, \
+                                     OMPParallelTrans, \
+                                     Dynamo0p3RedundantComputationTrans, \
+                                     Dynamo0p3AsyncHaloExchangeTrans, \
+                                     MoveTrans, \
+                                     TransformationError
+
+ENABLE_REDUNDANT_COMPUTATION = True
+ENABLE_ASYNC_HALOS = True
+ENABLE_OMP_COLOURING = True
+ENABLE_INTRINSIC_INLINING = True
+# LFRicLoopFuseTrans and DynKernelConstTrans could also be included but there
+# are some issues to overcome, e.g. TODO #2232
+
+
+def trans(psy):
+    ''' Apply all possible LFRic transformations. '''
+    rtrans = Dynamo0p3RedundantComputationTrans()
+    ctrans = Dynamo0p3ColourTrans()
+    otrans = Dynamo0p3OMPLoopTrans()
+    oregtrans = OMPParallelTrans()
+    inline_trans = KernelModuleInlineTrans()
+    matmul_trans = Matmul2CodeTrans()
+    const = LFRicConstants()
+    ahex_trans = Dynamo0p3AsyncHaloExchangeTrans()
+    mtrans = MoveTrans()
+
+    # Loop over all of the Invokes in the PSy object
+    for invoke in psy.invokes.invoke_list:
+        schedule = invoke.schedule
+
+        if ENABLE_REDUNDANT_COMPUTATION:
+            # Make setval_* compute redundantly to the level 1 halo if it
+            # is in its own loop
+            for loop in schedule.loops():
+                if loop.iteration_space == "dof":
+                    if len(loop.kernels()) == 1:
+                        if loop.kernels()[0].name in ["setval_c", "setval_x"]:
+                            rtrans.apply(loop, options={"depth": 1})
+
+        if ENABLE_ASYNC_HALOS:
+            # This transformation splits all synchronous halo exchanges
+            for h_ex in schedule.walk(DynHaloExchange):
+                ahex_trans.apply(h_ex)
+
+            # This transformation moves the start of the halo exchanges as
+            # far as possible offering the potential for overlap between
+            # communication and computation
+            location_cursor = 0
+            for ahex in schedule.walk(DynHaloExchangeStart):
+                if ahex.position <= location_cursor:
+                    continue
+                try:
+                    mtrans.apply(ahex, schedule.children[location_cursor])
+                    location_cursor += 1
+                except TransformationError:
+                    pass
+
+        if ENABLE_OMP_COLOURING:
+            # Colour loops over cells unless they are on discontinuous
+            # spaces or over dofs
+            for loop in schedule.loops():
+                if loop.iteration_space == "cell_column" \
+                    and loop.field_space.orig_name \
+                        not in const.VALID_DISCONTINUOUS_NAMES:
+                    ctrans.apply(loop)
+
+            # Add OpenMP to loops unless they are over colours or are null
+            for loop in schedule.loops():
+                if loop.loop_type not in ["colours", "null"]:
+                    oregtrans.apply(loop)
+                    otrans.apply(loop, options={"reprod": True})
+
+        # Transformations that modify kernel code will need to have the
+        # kernels inlined first
+        if ENABLE_INTRINSIC_INLINING:
+            for kernel in schedule.coded_kernels():
+                try:
+                    inline_trans.apply(kernel)
+                except TransformationError:
+                    pass
+
+    # Then transform all the kernels inlined into the module
+    if psy.invokes.invoke_list:
+        root = psy.invokes.invoke_list[0].schedule.ancestor(Container)
+        for kschedule in root.walk(KernelSchedule):
+            if ENABLE_INTRINSIC_INLINING:
+                # Expand MATMUL intrinsic
+                for bop in kschedule.walk(BinaryOperation):
+                    if bop.operator == BinaryOperation.Operator.MATMUL:
+                        try:
+                            matmul_trans.apply(bop)
+                        except TransformationError:
+                            pass
+
+    return psy