From 3f228c86459d4e4cf3b5f62a56a1a11a33f281c4 Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Wed, 3 Jul 2024 12:20:04 +0200 Subject: [PATCH 01/11] feature partitioning: add skip_local flag to domain --- src/fclaw2d_convenience.c | 1 + src/fclaw_initialize.c | 2 +- src/forestclaw.c | 6 +++--- src/forestclaw.h | 6 +++++- src/forestclaw2d.c | 4 +++- src/forestclaw2d.h | 9 ++++++++- src/forestclaw3d.h | 9 ++++++++- 7 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/fclaw2d_convenience.c b/src/fclaw2d_convenience.c index 9c174e2bf..df0d9bb09 100644 --- a/src/fclaw2d_convenience.c +++ b/src/fclaw2d_convenience.c @@ -151,6 +151,7 @@ fclaw2d_domain_new (p4est_wrap_t * wrap, sc_keyvalue_t * attributes) local_num_patches = 0; local_minlevel = domain->possible_maxlevel; local_maxlevel = -1; + domain->p.skip_local = 1; /* prepare propagation of refinement/coarsening marks */ domain->p.smooth_refine = 0; diff --git a/src/fclaw_initialize.c b/src/fclaw_initialize.c index 4fca9c294..80bb550e6 100644 --- a/src/fclaw_initialize.c +++ b/src/fclaw_initialize.c @@ -81,7 +81,7 @@ void fclaw_initialize(fclaw_global_t *glob) const fclaw_options_t *fclaw_opt = fclaw_get_options(glob); /* set partitioning */ - fclaw_domain_set_partitioning(*domain, fclaw_opt->partition_for_coarsening); + fclaw_domain_set_partitioning(*domain, fclaw_opt->partition_for_coarsening, 1); /* This mapping context is needed by fortran mapping functions */ fclaw_map_context_t *cont = glob->cont; diff --git a/src/forestclaw.c b/src/forestclaw.c index 35031d05b..bed31a88f 100644 --- a/src/forestclaw.c +++ b/src/forestclaw.c @@ -698,15 +698,15 @@ fclaw_domain_set_refinement (fclaw_domain_t * domain, } void -fclaw_domain_set_partitioning(fclaw_domain_t *domain, int partition_for_coarsening) +fclaw_domain_set_partitioning(fclaw_domain_t *domain, int partition_for_coarsening, int skip_local) { if(domain->refine_dim == 2) { - fclaw2d_domain_set_partitioning(domain->d2,partition_for_coarsening); + fclaw2d_domain_set_partitioning(domain->d2,partition_for_coarsening, skip_local); } else if (domain->refine_dim == 3) { - fclaw3d_domain_set_partitioning(domain->d3,partition_for_coarsening); + fclaw3d_domain_set_partitioning(domain->d3,partition_for_coarsening, skip_local); } else { diff --git a/src/forestclaw.h b/src/forestclaw.h index 218ac3106..c5a0e4a68 100644 --- a/src/forestclaw.h +++ b/src/forestclaw.h @@ -863,9 +863,13 @@ void fclaw_domain_set_refinement (fclaw_domain_t * domain, * \param [in] partition_for_coarsening Boolean: If true, all future partitions * of the domain allow one level of coarsening. * Suggested default: 1. + * \param [in] skip_local Boolean: If true, the patch data of patches that + * stay local are not packed during partitioning. + * Suggested default: 1. */ void fclaw_domain_set_partitioning (fclaw_domain_t * domain, - int partition_for_coarsening); + int partition_for_coarsening, + int skip_local); /** Mark a patch for refinement. * This must ONLY be called for local patches. diff --git a/src/forestclaw2d.c b/src/forestclaw2d.c index 13d2c3259..cbee24478 100644 --- a/src/forestclaw2d.c +++ b/src/forestclaw2d.c @@ -1400,11 +1400,13 @@ fclaw2d_domain_set_refinement (fclaw2d_domain_t * domain, void fclaw2d_domain_set_partitioning (fclaw2d_domain_t * domain, - int partition_for_coarsening) + int partition_for_coarsening, + int skip_local) { p4est_wrap_t *wrap = (p4est_wrap_t *) domain->pp; p4est_wrap_set_partitioning (wrap, partition_for_coarsening); + domain->p.skip_local = skip_local; } void diff --git a/src/forestclaw2d.h b/src/forestclaw2d.h index b2faa05f7..1f2bd782b 100644 --- a/src/forestclaw2d.h +++ b/src/forestclaw2d.h @@ -156,6 +156,9 @@ typedef struct fclaw2d_domain_persist the desired refinement level to neighbors. */ int smooth_level; /**< The minimum level that refinement smoothing is enabled on. Use 0 for al levels. */ + int skip_local; /**< Boolean: If true the patch data of patches + that stay local is not packed during + partitioning. */ } fclaw2d_domain_persist_t; @@ -675,9 +678,13 @@ void fclaw2d_domain_set_refinement (fclaw2d_domain_t * domain, * \param [in] partition_for_coarsening Boolean: If true, all future partitions * of the domain allow one level of coarsening. * Suggested default: 1. + * \param [in] skip_local Boolean: If true, the patch data of patches that + * stay local are not packed during partitioning. + * Suggested default: 1. */ void fclaw2d_domain_set_partitioning (fclaw2d_domain_t * domain, - int partition_for_coarsening); + int partition_for_coarsening, + int skip_local); /** Mark a patch for refinement. * This must ONLY be called for local patches. diff --git a/src/forestclaw3d.h b/src/forestclaw3d.h index 41e4d77c8..e56e863d5 100644 --- a/src/forestclaw3d.h +++ b/src/forestclaw3d.h @@ -159,6 +159,9 @@ typedef struct fclaw3d_domain_persist the desired refinement level to neighbors. */ int smooth_level; /**< The minimum level that refinement smoothing is enabled on. Use 0 for al levels. */ + int skip_local; /**< Boolean: If true the patch data of patches + that stay local is not packed during + partitioning. */ } fclaw3d_domain_persist_t; @@ -798,9 +801,13 @@ void fclaw3d_domain_set_refinement (fclaw3d_domain_t * domain, * \param [in] partition_for_coarsening Boolean: If true, all future partitions * of the domain allow one level of coarsening. * Suggested default: 1. + * \param [in] skip_local Boolean: If true, the patch data of patches that + * stay local are not packed during partitioning. + * Suggested default: 1. */ void fclaw3d_domain_set_partitioning (fclaw3d_domain_t * domain, - int partition_for_coarsening); + int partition_for_coarsening, + int skip_local); /** Mark a patch for refinement. * This must ONLY be called for local patches. From 836d038001a9366d01a31c41da52e72a7bf7888e Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Wed, 3 Jul 2024 12:22:21 +0200 Subject: [PATCH 02/11] feature partitioning: add demo for partitioning strategies --- .../demo/2d/partitioning/CMakeLists.txt | 18 ++ applications/demo/2d/partitioning/Makefile.am | 23 +++ .../demo/2d/partitioning/partitioning.cpp | 172 ++++++++++++++++++ applications/demo/demo.apps | 5 +- applications/demo/demo.cmake | 5 +- 5 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 applications/demo/2d/partitioning/CMakeLists.txt create mode 100644 applications/demo/2d/partitioning/Makefile.am create mode 100644 applications/demo/2d/partitioning/partitioning.cpp diff --git a/applications/demo/2d/partitioning/CMakeLists.txt b/applications/demo/2d/partitioning/CMakeLists.txt new file mode 100644 index 000000000..75137e3b9 --- /dev/null +++ b/applications/demo/2d/partitioning/CMakeLists.txt @@ -0,0 +1,18 @@ +set(all ${PROJECT_SOURCE_DIR}/demo/2d/all) + + +add_executable(partitioning + partitioning.cpp +) + +set_target_properties(partitioning PROPERTIES OUTPUT_NAME partitioning) + +target_include_directories(partitioning PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${all} +) + +target_link_libraries(partitioning PRIVATE + FORESTCLAW::CLAWPACK4.6 + FORESTCLAW::CLAWPACK5 +) diff --git a/applications/demo/2d/partitioning/Makefile.am b/applications/demo/2d/partitioning/Makefile.am new file mode 100644 index 000000000..744c872c0 --- /dev/null +++ b/applications/demo/2d/partitioning/Makefile.am @@ -0,0 +1,23 @@ +## Makefile.am in applications/demo/2d/partitioning + +bin_PROGRAMS += applications/demo/2d/partitioning/partitioning + +applications_demo_2d_partitioning_partitioning_SOURCES = \ + applications/demo/2d/partitioning/partitioning.cpp + +## Include headers and libraries needed to build this application +## -- Only really need AM_CPPFLAGS and LDADD (since these include all the other +## variables). But others are included here for completeness +applications_demo_2d_partitioning_partitioning_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + $(FCLAW_CLAWPACK46_CPPFLAGS) \ + $(FCLAW_CLAWPACK5_CPPFLAGS) \ + $(FCLAW_CLAWPATCH_CPPFLAGS) + + +applications_demo_2d_partitioning_partitioning_LDADD = \ + $(LDADD) \ + $(FCLAW_CLAWPACK46_LDADD) \ + $(FCLAW_CLAWPACK5_LDADD) \ + $(FCLAW_CLAWPATCH_LDADD) \ + $(FCLAW_LDADD) diff --git a/applications/demo/2d/partitioning/partitioning.cpp b/applications/demo/2d/partitioning/partitioning.cpp new file mode 100644 index 000000000..d3bc784cd --- /dev/null +++ b/applications/demo/2d/partitioning/partitioning.cpp @@ -0,0 +1,172 @@ +/* + Copyright (c) 2012-2023 Carsten Burstedde, Donna Calhoun, Scott Aiton + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "../all/advection_user.h" +#include /* just for testing */ +#include + +static +void mark_refine (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *user) +{ + if (domain->mpirank == 1 || domain->mpirank == 3) { + fclaw2d_patch_mark_refine (domain, blockno, patchno); + } +} + +static +void alloc_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *user) +{ + patch->user = FCLAW_ALLOC (double, 1); + double *patch_data = (double *) patch->user; + *patch_data = (double) -1; +} + +static +void set_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *user) +{ + double *patch_data = (double *) patch->user; + *patch_data = (double) domain->mpirank * 1000 + blockno * 100 + patchno; +}; + +static +void pack_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *pack_data_here, + void *user) +{ + double *pack_double_here = (double *) pack_data_here; + double *patch_data = (double *) patch->user; + + *pack_double_here = *patch_data; +} + +static +void transfer_patch_data (fclaw2d_domain_t * old_domain, + fclaw2d_patch_t * old_patch, + fclaw2d_domain_t * new_domain, + fclaw2d_patch_t * new_patch, int blockno, + int old_patchno, int new_patchno,void *user) +{ + double *old_patch_data = (double *) old_patch->user; + double *new_patch_data = (double *) new_patch->user; + + *new_patch_data = *old_patch_data; +} + +static +void unpack_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *unpack_data_from_here, + void *user) +{ + double *unpack_double_from_here = (double *) unpack_data_from_here; + double *patch_data = (double *) patch->user; + + *patch_data = *unpack_double_from_here; +} + +static +void print_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *user) +{ + double *patch_data = (double *) patch->user; +// printf ("%d: entering patch with patchno %d.\n", domain->mpirank, patchno); + if (patch_data == NULL) { + return; + } +// printf ("%d: we have patch_data %f.\n", domain->mpirank, *patch_data); +}; + +static +void delete_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *user) +{ + FCLAW_FREE (patch->user); +}; + +int +main (int argc, char **argv) +{ + /* Initialize application */ + fclaw_app_t *app = fclaw_app_new (&argc, &argv, NULL); + + /* Run the program */ + /* Create global structure which stores the domain, timers, etc */ + int size, rank; + sc_MPI_Comm mpicomm = fclaw_app_get_mpi_size_rank (app, &size, &rank); + fclaw_global_t *glob = fclaw_global_new_comm (mpicomm, size, rank); + + /* Create and store domain */ + fclaw2d_domain_t *domain, *refined_domain, *partitioned_domain; + + domain = fclaw2d_domain_new_brick (mpicomm, 2, 2, 0, 0, 2); + + /* test domain_set_partitioning */ + FCLAW_ASSERT (domain->p.skip_local == 1); + fclaw2d_domain_set_partitioning (domain, 1, 0); + FCLAW_ASSERT (domain->p.skip_local == 0); + fclaw2d_domain_set_partitioning (domain, 1, 1); + FCLAW_ASSERT (domain->p.skip_local == 1); + + fclaw2d_domain_iterate_patches (domain, mark_refine, NULL); + + refined_domain = fclaw2d_domain_adapt (domain); + + fclaw2d_domain_iterate_patches (refined_domain, alloc_patch_data, NULL); + fclaw2d_domain_iterate_patches (refined_domain, set_patch_data, NULL); + sleep (domain->mpirank); + fclaw2d_domain_iterate_patches (refined_domain, print_patch_data, NULL); + + partitioned_domain = fclaw2d_domain_partition (refined_domain, 0); + + fclaw_global_productionf ("Starting pack data transfer.\n"); + fclaw2d_domain_iterate_patches (partitioned_domain, alloc_patch_data, NULL); + fclaw2d_domain_partition_t *p; + p = fclaw2d_domain_iterate_pack (refined_domain, sizeof(double), + pack_patch_data, NULL); + fclaw2d_domain_iterate_unpack (partitioned_domain, p, unpack_patch_data, + NULL); + fclaw2d_domain_iterate_transfer (refined_domain, partitioned_domain, + transfer_patch_data, NULL); + fclaw2d_domain_partition_free (p); + + sleep (domain->mpirank); + fclaw2d_domain_iterate_patches (partitioned_domain, print_patch_data, NULL); + + fclaw2d_domain_complete (partitioned_domain); + + fclaw2d_domain_iterate_patches (refined_domain, delete_patch_data, NULL); + fclaw2d_domain_iterate_patches (partitioned_domain, delete_patch_data, NULL); + fclaw2d_domain_destroy (domain); + fclaw2d_domain_destroy (refined_domain); + fclaw2d_domain_destroy (partitioned_domain); + + fclaw_global_destroy(glob); + + fclaw_app_destroy (app); + + return 0; +} diff --git a/applications/demo/demo.apps b/applications/demo/demo.apps index 51f69bbe9..9fd56d665 100644 --- a/applications/demo/demo.apps +++ b/applications/demo/demo.apps @@ -11,4 +11,7 @@ include applications/demo/2d/swirl_restart/Makefile.am include applications/demo/2d/swirl_rays/Makefile.am ## filament and swirl multisolver with exchange demonstration -include applications/demo/2d/filament_swirl/Makefile.am \ No newline at end of file +include applications/demo/2d/filament_swirl/Makefile.am + +## partitioning mode demonstration +include applications/demo/2d/partitioning/Makefile.am diff --git a/applications/demo/demo.cmake b/applications/demo/demo.cmake index b3eec3dae..a7fb3b34d 100644 --- a/applications/demo/demo.cmake +++ b/applications/demo/demo.cmake @@ -11,4 +11,7 @@ add_subdirectory(demo/2d/swirl_restart) add_subdirectory(demo/2d/swirl_rays) ## filament and swirl multisolver with exchange demonstration -add_subdirectory(demo/2d/filament_swirl) \ No newline at end of file +add_subdirectory(demo/2d/filament_swirl) + +## partitioning mode demonstration +add_subdirectory(demo/2d/partitioning) From adee649387a003fede641cec08c0104638b2ad26 Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Mon, 8 Jul 2024 15:58:07 +0200 Subject: [PATCH 03/11] feature partitioning: skip local patches during packing --- .../demo/2d/partitioning/partitioning.cpp | 11 +- src/forestclaw2d.c | 178 ++++++++++++++---- src/forestclaw2d.h | 2 + src/forestclaw3d.h | 2 + 4 files changed, 157 insertions(+), 36 deletions(-) diff --git a/applications/demo/2d/partitioning/partitioning.cpp b/applications/demo/2d/partitioning/partitioning.cpp index d3bc784cd..0f52af3ab 100644 --- a/applications/demo/2d/partitioning/partitioning.cpp +++ b/applications/demo/2d/partitioning/partitioning.cpp @@ -53,11 +53,15 @@ void set_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *patch_data = (double) domain->mpirank * 1000 + blockno * 100 + patchno; }; +static int num_patches_packed; + static void pack_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *pack_data_here, void *user) { + FCLAW_ASSERT (patch != NULL); + num_patches_packed++; double *pack_double_here = (double *) pack_data_here; double *patch_data = (double *) patch->user; @@ -144,16 +148,21 @@ main (int argc, char **argv) fclaw_global_productionf ("Starting pack data transfer.\n"); fclaw2d_domain_iterate_patches (partitioned_domain, alloc_patch_data, NULL); + + num_patches_packed = 0; fclaw2d_domain_partition_t *p; p = fclaw2d_domain_iterate_pack (refined_domain, sizeof(double), pack_patch_data, NULL); + fclaw_infof ("Packed %d of %d local patches.\n", num_patches_packed, + refined_domain->local_num_patches); + fclaw2d_domain_iterate_unpack (partitioned_domain, p, unpack_patch_data, NULL); fclaw2d_domain_iterate_transfer (refined_domain, partitioned_domain, transfer_patch_data, NULL); fclaw2d_domain_partition_free (p); - sleep (domain->mpirank); +// sleep (domain->mpirank); fclaw2d_domain_iterate_patches (partitioned_domain, print_patch_data, NULL); fclaw2d_domain_complete (partitioned_domain); diff --git a/src/forestclaw2d.c b/src/forestclaw2d.c index cbee24478..0d458db17 100644 --- a/src/forestclaw2d.c +++ b/src/forestclaw2d.c @@ -1718,47 +1718,142 @@ fclaw2d_domain_iterate_pack (fclaw2d_domain_t * domain, size_t data_size, { fclaw2d_domain_partition_t *p; p4est_wrap_t *wrap = (p4est_wrap_t *) domain->pp; - int blockno, patchno; - size_t zz; + p4est_t *p4est = wrap->p4est; + int mpirank = p4est->mpirank; + int blockno, patchno, *size, i; + int num_dest, num_src, pul, old_puf, new_puf, old_lnp, new_lnp, bnpb; + p4est_gloidx_t *old_gfq, *new_gfq; fclaw2d_block_t *block; fclaw2d_patch_t *patch; /* this routine should only be called for changed partitions */ - P4EST_ASSERT (!domain->pp_owned); - P4EST_ASSERT (wrap->old_global_first_quadrant != NULL); - P4EST_ASSERT (domain->local_num_patches == (int) - wrap->old_global_first_quadrant[wrap->p4est->mpirank + 1] - - wrap->old_global_first_quadrant[wrap->p4est->mpirank]); + FCLAW_ASSERT (!domain->pp_owned); + FCLAW_ASSERT (wrap->old_global_first_quadrant != NULL); + old_gfq = wrap->old_global_first_quadrant; + old_lnp = domain->local_num_patches; + FCLAW_ASSERT (old_lnp == old_gfq[mpirank + 1] - old_gfq[mpirank]); + new_gfq = p4est->global_first_quadrant; + new_lnp = p4est->local_num_quadrants; + FCLAW_ASSERT (new_lnp == new_gfq[mpirank + 1] - new_gfq[mpirank]); + + p = FCLAW_ALLOC_ZERO (fclaw2d_domain_partition_t, 1); + + /* compute partition_unchanged data from gfq arrays */ + pul = (int) SC_MAX (SC_MIN (old_gfq[mpirank + 1], new_gfq[mpirank + 1]) - + SC_MAX (old_gfq[mpirank], new_gfq[mpirank]), 0); + old_puf = (int) SC_MAX (0, new_gfq[mpirank] - old_gfq[mpirank]); + new_puf = (int) SC_MAX (0, old_gfq[mpirank] - new_gfq[mpirank]); + + /* allocate destination arrays */ + if (domain->p.skip_local) + { + /* we only receive data for patches that were not local before partition */ + num_dest = new_lnp - pul; + p->dest_sizes = sc_array_new_count (sizeof (int), (size_t) new_lnp); + sc_array_memset (p->dest_sizes, 0); + /* set data size for newly received patches */ + for (i = 0; i < new_puf; i++) + { + size = (int *) sc_array_index_int (p->dest_sizes, i); + *size = data_size; + } + for (i = new_puf + pul; i < new_lnp; i++) + { + size = (int *) sc_array_index_int (p->dest_sizes, i); + *size = data_size; + } + } + else + { + num_dest = new_lnp; + } + p->dest_data = sc_array_new_count (data_size, num_dest); - p = FCLAW_ALLOC (fclaw2d_domain_partition_t, 1); - p->dest_data = - sc_array_new_count (data_size, - (size_t) wrap->p4est->local_num_quadrants); + /* allocate source arrays */ + if (domain->p.skip_local) + { + num_src = old_lnp - pul; + p->src_sizes = sc_array_new_count (sizeof (int), old_lnp); + sc_array_memset (p->src_sizes, 0); + /* set data size for patches that have to be sent */ + for (i = 0; i < old_puf; i++) + { + size = (int *) sc_array_index (p->src_sizes, i); + *size = data_size; + } + for (i = old_puf + pul; i < old_lnp; i++) + { + size = (int *) sc_array_index (p->src_sizes, i); + *size = data_size; + } + } + else + { + num_src = old_lnp; + } + p->src_data = sc_array_new_count (data_size, num_src); /* pack patches into src_data array */ - p->src_data = - sc_array_new_count (data_size, (size_t) domain->local_num_patches); - for (zz = 0, blockno = 0; blockno < domain->num_blocks; ++blockno) + for (i = 0, blockno = 0; blockno < domain->num_blocks; ++blockno) { block = domain->blocks + blockno; - for (patchno = 0; patchno < block->num_patches; ++zz, ++patchno) + bnpb = block->num_patches_before; + + /* iterate over patches before partition-unchanged range */ + for (patchno = 0; + patchno < SC_MIN (old_puf - bnpb, block->num_patches); + ++i, ++patchno) { - FCLAW_ASSERT (zz == - (size_t) (block->num_patches_before + patchno)); patch = block->patches + patchno; patch_pack (domain, patch, blockno, patchno, - sc_array_index (p->src_data, zz), user); + sc_array_index_int (p->src_data, i), user); + } + + if (!domain->p.skip_local) + { + /* iterate over patches in partition-unchanged range */ + for (patchno = SC_MAX (old_puf - bnpb, 0); + patchno < SC_MIN (old_puf + pul - bnpb, block->num_patches); + ++i, ++patchno) + { + patch = block->patches + patchno; + patch_pack (domain, patch, blockno, patchno, + sc_array_index_int (p->src_data, i), user); + } + } + + /* iterate over patches after partition-unchanged range */ + for (patchno = SC_MAX (old_puf + pul - bnpb, 0); + patchno < block->num_patches; ++i, ++patchno) + { + patch = block->patches + patchno; + patch_pack (domain, patch, blockno, patchno, + sc_array_index_int (p->src_data, i), user); } } - FCLAW_ASSERT (zz == (size_t) domain->local_num_patches); + FCLAW_ASSERT (i == num_src); /* start transfering patch data according to the new partition */ - p->async_state = (void *) - p4est_transfer_fixed_begin (wrap->p4est->global_first_quadrant, - wrap->old_global_first_quadrant, - domain->mpicomm, COMM_TAG_FIXED, - p->dest_data->array, p->src_data->array, - data_size); + if (!domain->p.skip_local) + { + /* we packed all patches resulting in a fixed data size */ + p->async_state = (void *) + p4est_transfer_fixed_begin (new_gfq, old_gfq, + domain->mpicomm, COMM_TAG_FIXED, + p->dest_data->array, + p->src_data->array, data_size); + } + else + { + /* we packed only patches that are sent, resulting in custom data sizes */ + p->async_state = (void *) + p4est_transfer_custom_begin (new_gfq, old_gfq, + domain->mpicomm, COMM_TAG_CUSTOM, + p->dest_data->array, + (const int *) p->dest_sizes->array, + p->src_data->array, + (const int *) p->src_sizes->array); + } p->inside_async = 1; return p; @@ -1776,8 +1871,8 @@ fclaw2d_domain_iterate_transfer (fclaw2d_domain_t * old_domain, int dpuf, dpul, bnpb; /* this routine should only be called for a changed partition */ - P4EST_ASSERT (!old_domain->pp_owned); - P4EST_ASSERT (new_domain->pp_owned); + FCLAW_ASSERT (!old_domain->pp_owned); + FCLAW_ASSERT (new_domain->pp_owned); /* unpack patches from dest_data array */ dpuf = new_domain->partition_unchanged_first; @@ -1820,11 +1915,19 @@ fclaw2d_domain_iterate_unpack (fclaw2d_domain_t * domain, /* this routine should only be called for the new domain of a changed * partition */ - P4EST_ASSERT (domain->pp_owned); - P4EST_ASSERT (p->inside_async); + FCLAW_ASSERT (domain->pp_owned); + FCLAW_ASSERT (p->inside_async); /* wait for transfer of patch data to complete */ - p4est_transfer_fixed_end ((p4est_transfer_context_t *) p->async_state); + if (!domain->p.skip_local) + { + p4est_transfer_fixed_end ((p4est_transfer_context_t *) p->async_state); + } + else + { + p4est_transfer_custom_end ((p4est_transfer_context_t *) p->async_state); + } + p->async_state = NULL; p->inside_async = 0; @@ -1846,8 +1949,8 @@ fclaw2d_domain_iterate_unpack (fclaw2d_domain_t * domain, sc_array_index (p->dest_data, zz), user); } - /* we skip the local patches for unpacking */ - if (zz == (size_t) dpuf) + /* if skip_local is disabled, we have to skip the local patches in dest_data */ + if (zz == (size_t) dpuf && !domain->p.skip_local) { zz += dpul; } @@ -1861,7 +1964,7 @@ fclaw2d_domain_iterate_unpack (fclaw2d_domain_t * domain, sc_array_index (p->dest_data, zz), user); } } - FCLAW_ASSERT (zz == (size_t) domain->local_num_patches); + FCLAW_ASSERT (zz == p->dest_data->elem_count); } void @@ -1870,7 +1973,12 @@ fclaw2d_domain_partition_free (fclaw2d_domain_partition_t * p) P4EST_ASSERT (!p->inside_async); sc_array_destroy (p->src_data); sc_array_destroy (p->dest_data); - + if (p->src_sizes != NULL) + { + FCLAW_ASSERT (p->dest_sizes != NULL); + sc_array_destroy_null (&p->src_sizes); + sc_array_destroy_null (&p->dest_sizes); + } FCLAW_FREE (p); } @@ -2626,7 +2734,7 @@ fclaw2d_domain_is_meta (fclaw2d_domain_t * domain) void fclaw2d_domain_init_meta (fclaw2d_domain_t * domain, int mpirank) { - FCLAW_ASSERT(domain != NULL); + FCLAW_ASSERT (domain != NULL); /* initialize to -1 and set pointers to NULL */ memset (domain, -1, sizeof (fclaw2d_domain_t)); diff --git a/src/forestclaw2d.h b/src/forestclaw2d.h index 1f2bd782b..d8ec7ba96 100644 --- a/src/forestclaw2d.h +++ b/src/forestclaw2d.h @@ -808,7 +808,9 @@ void fclaw2d_domain_iterate_partitioned (fclaw2d_domain_t * old_domain, typedef struct fclaw2d_domain_partition { sc_array_t *src_data; /**< The patch data to send */ + sc_array_t *src_sizes; /**< The patch data sizes to send */ sc_array_t *dest_data; /**< The patch data to receive */ + sc_array_t *dest_sizes; /**< The patch data sizes to receive */ /** Temporary storage required for asynchronous patch data transfer. * It is allocated and freed by the begin/end calls below. */ diff --git a/src/forestclaw3d.h b/src/forestclaw3d.h index e56e863d5..4a81b28f3 100644 --- a/src/forestclaw3d.h +++ b/src/forestclaw3d.h @@ -931,7 +931,9 @@ void fclaw3d_domain_iterate_partitioned (fclaw3d_domain_t * old_domain, typedef struct fclaw3d_domain_partition { sc_array_t *src_data; /**< The patch data to send */ + sc_array_t *src_sizes; /**< The patch data sizes to send */ sc_array_t *dest_data; /**< The patch data to receive */ + sc_array_t *dest_sizes; /**< The patch data sizes to receive */ /** Temporary storage required for asynchronous patch data transfer. * It is allocated and freed by the begin/end calls below. */ From 8488952cfb6b59d2e06919fe062ab2cd2fb084a4 Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Mon, 8 Jul 2024 16:00:11 +0200 Subject: [PATCH 04/11] feature partitioning: indent --- .../demo/2d/partitioning/partitioning.cpp | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/applications/demo/2d/partitioning/partitioning.cpp b/applications/demo/2d/partitioning/partitioning.cpp index 0f52af3ab..4840d9cd6 100644 --- a/applications/demo/2d/partitioning/partitioning.cpp +++ b/applications/demo/2d/partitioning/partitioning.cpp @@ -27,27 +27,28 @@ #include /* just for testing */ #include -static -void mark_refine (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, - int blockno, int patchno, void *user) +static void +mark_refine (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *user) { - if (domain->mpirank == 1 || domain->mpirank == 3) { + if (domain->mpirank == 1 || domain->mpirank == 3) + { fclaw2d_patch_mark_refine (domain, blockno, patchno); } } -static -void alloc_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, - int blockno, int patchno, void *user) +static void +alloc_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *user) { patch->user = FCLAW_ALLOC (double, 1); double *patch_data = (double *) patch->user; *patch_data = (double) -1; } -static -void set_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, - int blockno, int patchno, void *user) +static void +set_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *user) { double *patch_data = (double *) patch->user; *patch_data = (double) domain->mpirank * 1000 + blockno * 100 + patchno; @@ -55,10 +56,9 @@ void set_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, static int num_patches_packed; -static -void pack_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, - int blockno, int patchno, void *pack_data_here, - void *user) +static void +pack_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *pack_data_here, void *user) { FCLAW_ASSERT (patch != NULL); num_patches_packed++; @@ -68,12 +68,12 @@ void pack_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *pack_double_here = *patch_data; } -static -void transfer_patch_data (fclaw2d_domain_t * old_domain, - fclaw2d_patch_t * old_patch, - fclaw2d_domain_t * new_domain, - fclaw2d_patch_t * new_patch, int blockno, - int old_patchno, int new_patchno,void *user) +static void +transfer_patch_data (fclaw2d_domain_t * old_domain, + fclaw2d_patch_t * old_patch, + fclaw2d_domain_t * new_domain, + fclaw2d_patch_t * new_patch, int blockno, + int old_patchno, int new_patchno, void *user) { double *old_patch_data = (double *) old_patch->user; double *new_patch_data = (double *) new_patch->user; @@ -81,10 +81,10 @@ void transfer_patch_data (fclaw2d_domain_t * old_domain, *new_patch_data = *old_patch_data; } -static -void unpack_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, - int blockno, int patchno, void *unpack_data_from_here, - void *user) +static void +unpack_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *unpack_data_from_here, + void *user) { double *unpack_double_from_here = (double *) unpack_data_from_here; double *patch_data = (double *) patch->user; @@ -92,21 +92,22 @@ void unpack_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *patch_data = *unpack_double_from_here; } -static -void print_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, - int blockno, int patchno, void *user) +static void +print_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *user) { double *patch_data = (double *) patch->user; // printf ("%d: entering patch with patchno %d.\n", domain->mpirank, patchno); - if (patch_data == NULL) { + if (patch_data == NULL) + { return; } // printf ("%d: we have patch_data %f.\n", domain->mpirank, *patch_data); }; -static -void delete_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, - int blockno, int patchno, void *user) +static void +delete_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, + int blockno, int patchno, void *user) { FCLAW_FREE (patch->user); }; @@ -147,11 +148,12 @@ main (int argc, char **argv) partitioned_domain = fclaw2d_domain_partition (refined_domain, 0); fclaw_global_productionf ("Starting pack data transfer.\n"); - fclaw2d_domain_iterate_patches (partitioned_domain, alloc_patch_data, NULL); + fclaw2d_domain_iterate_patches (partitioned_domain, alloc_patch_data, + NULL); num_patches_packed = 0; fclaw2d_domain_partition_t *p; - p = fclaw2d_domain_iterate_pack (refined_domain, sizeof(double), + p = fclaw2d_domain_iterate_pack (refined_domain, sizeof (double), pack_patch_data, NULL); fclaw_infof ("Packed %d of %d local patches.\n", num_patches_packed, refined_domain->local_num_patches); @@ -163,17 +165,19 @@ main (int argc, char **argv) fclaw2d_domain_partition_free (p); // sleep (domain->mpirank); - fclaw2d_domain_iterate_patches (partitioned_domain, print_patch_data, NULL); + fclaw2d_domain_iterate_patches (partitioned_domain, print_patch_data, + NULL); fclaw2d_domain_complete (partitioned_domain); fclaw2d_domain_iterate_patches (refined_domain, delete_patch_data, NULL); - fclaw2d_domain_iterate_patches (partitioned_domain, delete_patch_data, NULL); + fclaw2d_domain_iterate_patches (partitioned_domain, delete_patch_data, + NULL); fclaw2d_domain_destroy (domain); fclaw2d_domain_destroy (refined_domain); fclaw2d_domain_destroy (partitioned_domain); - fclaw_global_destroy(glob); + fclaw_global_destroy (glob); fclaw_app_destroy (app); From 524e8ee95cb485a0f31a10642a4303227b8fce2f Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Mon, 8 Jul 2024 16:05:49 +0200 Subject: [PATCH 05/11] feature partitioning: fix test for mpisize == 1 --- .../demo/2d/partitioning/partitioning.cpp | 68 +++++++++++-------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/applications/demo/2d/partitioning/partitioning.cpp b/applications/demo/2d/partitioning/partitioning.cpp index 4840d9cd6..f2da61aba 100644 --- a/applications/demo/2d/partitioning/partitioning.cpp +++ b/applications/demo/2d/partitioning/partitioning.cpp @@ -136,46 +136,54 @@ main (int argc, char **argv) fclaw2d_domain_set_partitioning (domain, 1, 1); FCLAW_ASSERT (domain->p.skip_local == 1); - fclaw2d_domain_iterate_patches (domain, mark_refine, NULL); + if (domain->mpisize != 1) + { + fclaw2d_domain_iterate_patches (domain, mark_refine, NULL); + + refined_domain = fclaw2d_domain_adapt (domain); - refined_domain = fclaw2d_domain_adapt (domain); + fclaw2d_domain_iterate_patches (refined_domain, alloc_patch_data, + NULL); + fclaw2d_domain_iterate_patches (refined_domain, set_patch_data, NULL); + sleep (domain->mpirank); + fclaw2d_domain_iterate_patches (refined_domain, print_patch_data, + NULL); - fclaw2d_domain_iterate_patches (refined_domain, alloc_patch_data, NULL); - fclaw2d_domain_iterate_patches (refined_domain, set_patch_data, NULL); - sleep (domain->mpirank); - fclaw2d_domain_iterate_patches (refined_domain, print_patch_data, NULL); + partitioned_domain = fclaw2d_domain_partition (refined_domain, 0); - partitioned_domain = fclaw2d_domain_partition (refined_domain, 0); + fclaw_global_productionf ("Starting pack data transfer.\n"); + fclaw2d_domain_iterate_patches (partitioned_domain, alloc_patch_data, + NULL); - fclaw_global_productionf ("Starting pack data transfer.\n"); - fclaw2d_domain_iterate_patches (partitioned_domain, alloc_patch_data, - NULL); + num_patches_packed = 0; + fclaw2d_domain_partition_t *p; + p = fclaw2d_domain_iterate_pack (refined_domain, sizeof (double), + pack_patch_data, NULL); + fclaw_infof ("Packed %d of %d local patches.\n", num_patches_packed, + refined_domain->local_num_patches); - num_patches_packed = 0; - fclaw2d_domain_partition_t *p; - p = fclaw2d_domain_iterate_pack (refined_domain, sizeof (double), - pack_patch_data, NULL); - fclaw_infof ("Packed %d of %d local patches.\n", num_patches_packed, - refined_domain->local_num_patches); + fclaw2d_domain_iterate_unpack (partitioned_domain, p, + unpack_patch_data, NULL); + fclaw2d_domain_iterate_transfer (refined_domain, partitioned_domain, + transfer_patch_data, NULL); + fclaw2d_domain_partition_free (p); - fclaw2d_domain_iterate_unpack (partitioned_domain, p, unpack_patch_data, - NULL); - fclaw2d_domain_iterate_transfer (refined_domain, partitioned_domain, - transfer_patch_data, NULL); - fclaw2d_domain_partition_free (p); +// sleep (domain->mpirank); + fclaw2d_domain_iterate_patches (partitioned_domain, print_patch_data, + NULL); -// sleep (domain->mpirank); - fclaw2d_domain_iterate_patches (partitioned_domain, print_patch_data, - NULL); + fclaw2d_domain_complete (partitioned_domain); - fclaw2d_domain_complete (partitioned_domain); + fclaw2d_domain_iterate_patches (refined_domain, delete_patch_data, + NULL); + fclaw2d_domain_iterate_patches (partitioned_domain, delete_patch_data, + NULL); + + fclaw2d_domain_destroy (partitioned_domain); + fclaw2d_domain_destroy (refined_domain); + } - fclaw2d_domain_iterate_patches (refined_domain, delete_patch_data, NULL); - fclaw2d_domain_iterate_patches (partitioned_domain, delete_patch_data, - NULL); fclaw2d_domain_destroy (domain); - fclaw2d_domain_destroy (refined_domain); - fclaw2d_domain_destroy (partitioned_domain); fclaw_global_destroy (glob); From 0b60cf4ef01ffd4cb797fa661ea36db84f3e701e Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Mon, 15 Jul 2024 16:09:52 +0200 Subject: [PATCH 06/11] feature partitioning: add comments to demo --- applications/demo/2d/partitioning/partitioning.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/applications/demo/2d/partitioning/partitioning.cpp b/applications/demo/2d/partitioning/partitioning.cpp index f2da61aba..5e6a21e11 100644 --- a/applications/demo/2d/partitioning/partitioning.cpp +++ b/applications/demo/2d/partitioning/partitioning.cpp @@ -31,6 +31,7 @@ static void mark_refine (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { + /* refine only some processes to ensure repartitioning */ if (domain->mpirank == 1 || domain->mpirank == 3) { fclaw2d_patch_mark_refine (domain, blockno, patchno); @@ -50,6 +51,7 @@ static void set_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { + /* set artificial patch data that allows tracking rank of origin and treeid */ double *patch_data = (double *) patch->user; *patch_data = (double) domain->mpirank * 1000 + blockno * 100 + patchno; }; @@ -78,6 +80,7 @@ transfer_patch_data (fclaw2d_domain_t * old_domain, double *old_patch_data = (double *) old_patch->user; double *new_patch_data = (double *) new_patch->user; + /* simply copy patch data from old to new location */ *new_patch_data = *old_patch_data; } From c7f6793d1de8b5dcfb69db930bc1276c85d5e840 Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Mon, 15 Jul 2024 16:11:04 +0200 Subject: [PATCH 07/11] feature partitioning: update documentation of iterate_pack --- src/forestclaw2d.h | 9 ++++++--- src/forestclaw3d.h | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/forestclaw2d.h b/src/forestclaw2d.h index d8ec7ba96..382a315ba 100644 --- a/src/forestclaw2d.h +++ b/src/forestclaw2d.h @@ -836,10 +836,13 @@ typedef void (*fclaw2d_pack_callback_t) (fclaw2d_domain_t * domain, void *user); /** Start asynchronous transfer of patch data after partition. - * The function iterates over all local patches of old partition and determines + * The function iterates over the local patches of old partition and determines * patches that have to be packed by use of \b patch_pack and send to the new - * partition. - * It must be followed by a call to \ref fclaw2d_domain_iterate_unpack. + * partition. If skip_local is set to false in the domain, \b patch_pack is + * called for every local patch. If skip_local is true (default), \b patch_pack + * is called only for local patches that are sent to a different process during + * partitioning. + * This function must be followed by a call to \ref fclaw2d_domain_iterate_unpack. * \param [in,out] domain The domain before partitioning. * \param [in] data_size The number of bytes of user data that has to be * packed and send per patch. diff --git a/src/forestclaw3d.h b/src/forestclaw3d.h index 4a81b28f3..eb2d0f077 100644 --- a/src/forestclaw3d.h +++ b/src/forestclaw3d.h @@ -959,10 +959,13 @@ typedef void (*fclaw3d_pack_callback_t) (fclaw3d_domain_t * domain, void *user); /** Start asynchronous transfer of patch data after partition. - * The function iterates over all local patches of old partition and determines + * The function iterates over the local patches of old partition and determines * patches that have to be packed by use of \b patch_pack and send to the new - * partition. - * It must be followed by a call to \ref fclaw3d_domain_iterate_unpack. + * partition. If skip_local is set to false in the domain, \b patch_pack is + * called for every local patch. If skip_local is true (default), \b patch_pack + * is called only for local patches that are sent to a different process during + * partitioning. + * This function must be followed by a call to \ref fclaw3d_domain_iterate_unpack. * \param [in,out] domain The domain before partitioning. * \param [in] data_size The number of bytes of user data that has to be * packed and send per patch. From d6be6bf6b7aba73c5d90f4181740884d9da5f139 Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Mon, 15 Jul 2024 16:16:16 +0200 Subject: [PATCH 08/11] feature partitioning: iterate through different test cases in demo --- .../demo/2d/partitioning/partitioning.cpp | 114 +++++++++--------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/applications/demo/2d/partitioning/partitioning.cpp b/applications/demo/2d/partitioning/partitioning.cpp index 5e6a21e11..5f968a322 100644 --- a/applications/demo/2d/partitioning/partitioning.cpp +++ b/applications/demo/2d/partitioning/partitioning.cpp @@ -127,67 +127,67 @@ main (int argc, char **argv) sc_MPI_Comm mpicomm = fclaw_app_get_mpi_size_rank (app, &size, &rank); fclaw_global_t *glob = fclaw_global_new_comm (mpicomm, size, rank); - /* Create and store domain */ fclaw2d_domain_t *domain, *refined_domain, *partitioned_domain; - domain = fclaw2d_domain_new_brick (mpicomm, 2, 2, 0, 0, 2); - - /* test domain_set_partitioning */ - FCLAW_ASSERT (domain->p.skip_local == 1); - fclaw2d_domain_set_partitioning (domain, 1, 0); - FCLAW_ASSERT (domain->p.skip_local == 0); - fclaw2d_domain_set_partitioning (domain, 1, 1); - FCLAW_ASSERT (domain->p.skip_local == 1); - - if (domain->mpisize != 1) + for (int test_case = 0; test_case < 2; test_case++) { - fclaw2d_domain_iterate_patches (domain, mark_refine, NULL); - - refined_domain = fclaw2d_domain_adapt (domain); - - fclaw2d_domain_iterate_patches (refined_domain, alloc_patch_data, - NULL); - fclaw2d_domain_iterate_patches (refined_domain, set_patch_data, NULL); - sleep (domain->mpirank); - fclaw2d_domain_iterate_patches (refined_domain, print_patch_data, - NULL); - - partitioned_domain = fclaw2d_domain_partition (refined_domain, 0); - - fclaw_global_productionf ("Starting pack data transfer.\n"); - fclaw2d_domain_iterate_patches (partitioned_domain, alloc_patch_data, - NULL); - - num_patches_packed = 0; - fclaw2d_domain_partition_t *p; - p = fclaw2d_domain_iterate_pack (refined_domain, sizeof (double), - pack_patch_data, NULL); - fclaw_infof ("Packed %d of %d local patches.\n", num_patches_packed, - refined_domain->local_num_patches); - - fclaw2d_domain_iterate_unpack (partitioned_domain, p, - unpack_patch_data, NULL); - fclaw2d_domain_iterate_transfer (refined_domain, partitioned_domain, - transfer_patch_data, NULL); - fclaw2d_domain_partition_free (p); - -// sleep (domain->mpirank); - fclaw2d_domain_iterate_patches (partitioned_domain, print_patch_data, - NULL); - - fclaw2d_domain_complete (partitioned_domain); - - fclaw2d_domain_iterate_patches (refined_domain, delete_patch_data, - NULL); - fclaw2d_domain_iterate_patches (partitioned_domain, delete_patch_data, - NULL); - - fclaw2d_domain_destroy (partitioned_domain); - fclaw2d_domain_destroy (refined_domain); + domain = fclaw2d_domain_new_brick (mpicomm, 2, 2, 0, 0, 2); + + /* set partitioning options */ + fclaw2d_domain_set_partitioning (domain, 1, (test_case % 2)); + + if (domain->mpisize != 1) + { + fclaw2d_domain_iterate_patches (domain, mark_refine, NULL); + + refined_domain = fclaw2d_domain_adapt (domain); + + fclaw2d_domain_iterate_patches (refined_domain, alloc_patch_data, + NULL); + fclaw2d_domain_iterate_patches (refined_domain, set_patch_data, + NULL); + fclaw2d_domain_iterate_patches (refined_domain, print_patch_data, + NULL); + + partitioned_domain = fclaw2d_domain_partition (refined_domain, 0); + + fclaw_global_productionf + ("Starting partitioning with skip_local = %d.\n", + domain->p.skip_local); + fclaw2d_domain_iterate_patches (partitioned_domain, + alloc_patch_data, NULL); + + num_patches_packed = 0; + fclaw2d_domain_partition_t *p; + p = fclaw2d_domain_iterate_pack (refined_domain, sizeof (double), + pack_patch_data, NULL); + fclaw_infof ("Packed %d of %d local patches.\n", + num_patches_packed, + refined_domain->local_num_patches); + + fclaw2d_domain_iterate_unpack (partitioned_domain, p, + unpack_patch_data, NULL); + fclaw2d_domain_iterate_transfer (refined_domain, + partitioned_domain, + transfer_patch_data, NULL); + fclaw2d_domain_partition_free (p); + + fclaw2d_domain_iterate_patches (partitioned_domain, + print_patch_data, NULL); + + fclaw2d_domain_complete (partitioned_domain); + + fclaw2d_domain_iterate_patches (refined_domain, delete_patch_data, + NULL); + fclaw2d_domain_iterate_patches (partitioned_domain, + delete_patch_data, NULL); + + fclaw2d_domain_destroy (partitioned_domain); + fclaw2d_domain_destroy (refined_domain); + } + + fclaw2d_domain_destroy (domain); } - - fclaw2d_domain_destroy (domain); - fclaw_global_destroy (glob); fclaw_app_destroy (app); From 087a1c50c0394f0361702b168fad22022af2235c Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Tue, 16 Jul 2024 15:48:23 +0200 Subject: [PATCH 09/11] feature partitioning: replace call to deprecated wrap_set_partitioning --- src/forestclaw2d.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/forestclaw2d.c b/src/forestclaw2d.c index 0d458db17..acda5a0f7 100644 --- a/src/forestclaw2d.c +++ b/src/forestclaw2d.c @@ -1405,7 +1405,7 @@ fclaw2d_domain_set_partitioning (fclaw2d_domain_t * domain, { p4est_wrap_t *wrap = (p4est_wrap_t *) domain->pp; - p4est_wrap_set_partitioning (wrap, partition_for_coarsening); + wrap->params.partition_for_coarsening = partition_for_coarsening; domain->p.skip_local = skip_local; } From fae1c915f15a803cb5cb4fe33e70f72d7a6b843e Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Wed, 17 Jul 2024 11:26:56 +0200 Subject: [PATCH 10/11] feature partitioning: add output_patch_data option to demo --- .../demo/2d/partitioning/partitioning.cpp | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/applications/demo/2d/partitioning/partitioning.cpp b/applications/demo/2d/partitioning/partitioning.cpp index 5f968a322..cd992174b 100644 --- a/applications/demo/2d/partitioning/partitioning.cpp +++ b/applications/demo/2d/partitioning/partitioning.cpp @@ -100,12 +100,11 @@ print_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { double *patch_data = (double *) patch->user; -// printf ("%d: entering patch with patchno %d.\n", domain->mpirank, patchno); if (patch_data == NULL) { return; } -// printf ("%d: we have patch_data %f.\n", domain->mpirank, *patch_data); + fclaw_infof ("Patch %d has patch data %f.\n", patchno, *patch_data); }; static void @@ -118,6 +117,10 @@ delete_patch_data (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int main (int argc, char **argv) { + /* This demo highlights the effects of the different partitioning options + * on the patch data partitioning. In particular, they differ in the amount + * of patch_pack callback calls that have to be performed. */ + /* Initialize application */ fclaw_app_t *app = fclaw_app_new (&argc, &argv, NULL); @@ -129,6 +132,8 @@ main (int argc, char **argv) fclaw2d_domain_t *domain, *refined_domain, *partitioned_domain; + /* iterate through the different partitioning strategies for patch data */ + int output_patch_data = 0; for (int test_case = 0; test_case < 2; test_case++) { domain = fclaw2d_domain_new_brick (mpicomm, 2, 2, 0, 0, 2); @@ -146,9 +151,11 @@ main (int argc, char **argv) NULL); fclaw2d_domain_iterate_patches (refined_domain, set_patch_data, NULL); - fclaw2d_domain_iterate_patches (refined_domain, print_patch_data, - NULL); - + if (output_patch_data) + { + fclaw2d_domain_iterate_patches (refined_domain, + print_patch_data, NULL); + } partitioned_domain = fclaw2d_domain_partition (refined_domain, 0); fclaw_global_productionf @@ -172,8 +179,11 @@ main (int argc, char **argv) transfer_patch_data, NULL); fclaw2d_domain_partition_free (p); - fclaw2d_domain_iterate_patches (partitioned_domain, - print_patch_data, NULL); + if (output_patch_data) + { + fclaw2d_domain_iterate_patches (partitioned_domain, + print_patch_data, NULL); + } fclaw2d_domain_complete (partitioned_domain); From 782311ee161ed4591f32935de93d24a3f3602a9a Mon Sep 17 00:00:00 2001 From: Hannes Brandt Date: Wed, 17 Jul 2024 11:27:16 +0200 Subject: [PATCH 11/11] feature partitioning: remove trailing whitespaces --- applications/demo/2d/partitioning/CMakeLists.txt | 8 ++++---- applications/demo/2d/partitioning/Makefile.am | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/applications/demo/2d/partitioning/CMakeLists.txt b/applications/demo/2d/partitioning/CMakeLists.txt index 75137e3b9..a655f1d98 100644 --- a/applications/demo/2d/partitioning/CMakeLists.txt +++ b/applications/demo/2d/partitioning/CMakeLists.txt @@ -1,14 +1,14 @@ set(all ${PROJECT_SOURCE_DIR}/demo/2d/all) -add_executable(partitioning - partitioning.cpp +add_executable(partitioning + partitioning.cpp ) set_target_properties(partitioning PROPERTIES OUTPUT_NAME partitioning) -target_include_directories(partitioning PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} +target_include_directories(partitioning PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} ${all} ) diff --git a/applications/demo/2d/partitioning/Makefile.am b/applications/demo/2d/partitioning/Makefile.am index 744c872c0..bb5cadd9c 100644 --- a/applications/demo/2d/partitioning/Makefile.am +++ b/applications/demo/2d/partitioning/Makefile.am @@ -6,7 +6,7 @@ applications_demo_2d_partitioning_partitioning_SOURCES = \ applications/demo/2d/partitioning/partitioning.cpp ## Include headers and libraries needed to build this application -## -- Only really need AM_CPPFLAGS and LDADD (since these include all the other +## -- Only really need AM_CPPFLAGS and LDADD (since these include all the other ## variables). But others are included here for completeness applications_demo_2d_partitioning_partitioning_CPPFLAGS = \ $(AM_CPPFLAGS) \