Add some utils to support m-to-n restart.

.github/workflows/ci.yaml

@@ -119,12 +119,16 @@ jobs:
 
         - name: Run examples
           run: |
+            set -x
             MINIFORGE_INSTALL_DIR=.miniforge3
             . "$MINIFORGE_INSTALL_DIR/bin/activate" testing
             export XDG_CACHE_HOME=/tmp
             mamba install vtk  # needed for the accuracy comparison
             [[ $(hostname) == "porter" ]] && export PYOPENCL_TEST="port:nv" && unset XDG_CACHE_HOME
-            # && export POCL_DEBUG=cuda
+
+             # This is only possible because actions run sequentially on porter
+            [[ $(hostname) == "porter" ]] && rm -rf /tmp/githubrunner/pocl-scratch && rm -rf /tmp/githubrunner/xdg-scratch
+
             scripts/run-integrated-tests.sh --examples
 
     doc:
@@ -195,8 +199,13 @@ jobs:
             fetch-depth: '0'
         - name: Prepare production environment
           run: |
+            set -x
             [[ $(uname) == Linux ]] && [[ $(hostname) != "porter" ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev
             [[ $(uname) == Darwin ]] && brew upgrade && brew install mpich
+
+            # This is only possible because actions run sequentially on porter
+            [[ $(hostname) == "porter" ]] && rm -rf /tmp/githubrunner/pocl-scratch && rm -rf /tmp/githubrunner/xdg-scratch
+
             MIRGEDIR=$(pwd)
             cat scripts/production-testing-env.sh
             . scripts/production-testing-env.sh

.readthedocs.yml

@@ -2,7 +2,12 @@ version: 2
 
 conda:
   environment: .rtd-env-py3.yml
-
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "mambaforge-22.9"
+
 sphinx:
   fail_on_warning: true
 

.rtd-env-py3.yml

@@ -4,13 +4,12 @@ channels:
 - conda-forge
 - nodefaults
 dependencies:
-# readthedocs does not yet support Python 3.11
-# See e.g. https://readthedocs.org/api/v2/build/18650881.txt
-- python=3.10
+- python=3.11
 - mpi4py
 - islpy
 - pip
 - pyopencl
+- graphviz
 - scipy
 - pip:
   - "git+https://github.com/inducer/pymbolic.git#egg=pymbolic"

bin/mapdecomp.py

@@ -0,0 +1,106 @@
+"""Read gmsh mesh, partition it, and create a pkl file per mesh partition."""
+
+__copyright__ = """
+Copyright (C) 2020 University of Illinois Board of Trustees
+"""
+
+__license__ = """
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+"""
+import argparse
+import pickle
+import os
+
+from meshmode.distributed import get_connected_parts
+from grudge.discretization import PartID
+
+
+def main(mesh_filename=None, output_path=None):
+    """Do it."""
+    if output_path is None:
+        output_path = "./"
+    output_path.strip("'")
+
+    if mesh_filename is None:
+        # Try to detect the mesh filename
+        raise AssertionError("No mesh filename.")
+
+    intradecomp_map = {}
+    nranks = 0
+    nvolumes = 0
+    volumes = set()
+    for r in range(10000):
+        mesh_pkl_filename = mesh_filename + f"_rank{r}.pkl"
+        if os.path.exists(mesh_pkl_filename):
+            nranks = nranks + 1
+            with open(mesh_pkl_filename, "rb") as pkl_file:
+                global_nelements, volume_to_local_mesh_data = \
+                    pickle.load(pkl_file)
+            for vol, meshdat in volume_to_local_mesh_data.items():
+                local_partid = PartID(volume_tag=vol, rank=r)
+                volumes.add(vol)
+                connected_parts = get_connected_parts(meshdat[0])
+                if connected_parts:
+                    intradecomp_map[local_partid] = connected_parts
+        else:
+            break
+    nvolumes = len(volumes)
+    rank_rank_nbrs = {r: set() for r in range(nranks)}
+    for part, nbrs in intradecomp_map.items():
+        local_rank = part.rank
+        for nbr in nbrs:
+            if nbr.rank != local_rank:
+                rank_rank_nbrs[local_rank].add(nbr.rank)
+    min_rank_nbrs = nranks
+    max_rank_nbrs = 0
+    num_nbr_dist = {}
+    total_nnbrs = 0
+    for _, rank_nbrs in rank_rank_nbrs.items():
+        nrank_nbrs = len(rank_nbrs)
+        total_nnbrs += nrank_nbrs
+        if nrank_nbrs not in num_nbr_dist:
+            num_nbr_dist[nrank_nbrs] = 0
+        num_nbr_dist[nrank_nbrs] += 1
+        min_rank_nbrs = min(min_rank_nbrs, nrank_nbrs)
+        max_rank_nbrs = max(max_rank_nbrs, nrank_nbrs)
+
+    mean_nnbrs = (1.0*total_nnbrs) / (1.0*nranks)
+
+    print(f"Number of ranks: {nranks}")
+    print(f"Number of volumes: {nvolumes}")
+    print(f"Volumes: {volumes}")
+    print("Number of rank neighbors (min, max, mean): "
+          f"({min_rank_nbrs}, {max_rank_nbrs}, {mean_nnbrs})")
+    print(f"Distribution of num nbrs: {num_nbr_dist=}")
+
+    # print(f"{intradecomp_map=}")
+    with open(f"intradecomp_map_np{nranks}.pkl", "wb") as pkl_file:
+        pickle.dump(intradecomp_map, pkl_file)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        description="MIRGE-Com Intradecomp mapper")
+    parser.add_argument("-m", "--mesh", type=str, dest="mesh_filename",
+                        nargs="?", action="store", help="root filename for mesh")
+
+    args = parser.parse_args()
+
+    main(mesh_filename=args.mesh_filename)

bin/meshdist.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-"""mirgecom mesh distribution utility"""
+"""Read gmsh mesh, partition it, and create a pkl file per mesh partition."""
 
 __copyright__ = """
 Copyright (C) 2020 University of Illinois Board of Trustees
@@ -69,15 +68,18 @@ class MyRuntimeError(RuntimeError):
 
 
 @mpi_entry_point
-def main(actx_class, mesh_source=None, ndist=None,
+def main(actx_class, mesh_source=None, ndist=None, dim=None,
          output_path=None, log_path=None,
          casename=None, use_1d_part=None, use_wall=False):
-
+    """The main function."""
     if mesh_source is None:
         raise ApplicationOptionsError("Missing mesh source file.")
 
     mesh_source.strip("'")
 
+    if dim is None:
+        dim = 3
+
     if log_path is None:
         log_path = "log_data"
 
@@ -177,7 +179,7 @@ def main(actx_class, mesh_source=None, ndist=None,
     def get_mesh_data():
         from meshmode.mesh.io import read_gmsh
         mesh, tag_to_elements = read_gmsh(
-            mesh_source,
+            mesh_source, force_ambient_dim=dim,
             return_tag_to_elements_map=True)
         volume_to_tags = {
             "fluid": ["fluid"]}
@@ -200,7 +202,7 @@ def my_partitioner(mesh, tag_to_elements, num_ranks):
     if os.path.exists(output_path):
         if not os.path.isdir(output_path):
             raise ApplicationOptionsError(
-                "Mesh dist mode requires \"output\""
+                "Mesh dist mode requires 'output'"
                 " parameter to be a directory for output.")
     if rank == 0:
         if not os.path.exists(output_path):
@@ -218,7 +220,7 @@ def my_partitioner(mesh, tag_to_elements, num_ranks):
         partition_generator_func=part_func, logmgr=logmgr)
 
     comm.Barrier()
-    
+
     logmgr_set_time(logmgr, 0, 0)
     logmgr
     logmgr.tick_before()
@@ -239,12 +241,17 @@ def my_partitioner(mesh, tag_to_elements, num_ranks):
                         action="store_true", help="Include wall domain in mesh.")
     parser.add_argument("-1", "--1dpart", dest="one_d_part",
                         action="store_true", help="Use 1D partitioner.")
+    parser.add_argument("-d", "--dimen", type=int, dest="dim",
+                        nargs="?", action="store",
+                        help="Number dimensions")
     parser.add_argument("-n", "--ndist", type=int, dest="ndist",
-                        nargs="?", action="store", help="Number of distributed parts")
+                        nargs="?", action="store",
+                        help="Number of distributed parts")
     parser.add_argument("-s", "--source", type=str, dest="source",
                         nargs="?", action="store", help="Gmsh mesh source file")
     parser.add_argument("-o", "--ouput-path", type=str, dest="output_path",
-                        nargs="?", action="store", help="Output path for distributed mesh pkl files")
+                        nargs="?", action="store",
+                        help="Output path for distributed mesh pkl files")
     parser.add_argument("-c", "--casename", type=str, dest="casename", nargs="?",
                         action="store", help="Root name of distributed mesh pkl files.")
     parser.add_argument("-g", "--logpath", type=str, dest="log_path", nargs="?",
@@ -256,7 +263,7 @@ def my_partitioner(mesh, tag_to_elements, num_ranks):
     actx_class = get_reasonable_array_context_class(
         lazy=False, distributed=True, profiling=False, numpy=False)
 
-    main(actx_class, mesh_source=args.source,
+    main(actx_class, mesh_source=args.source, dim=args.dim,
          output_path=args.output_path, ndist=args.ndist,
          log_path=args.log_path, casename=args.casename,
          use_1d_part=args.one_d_part, use_wall=args.use_wall)