Matrix free (#59)

- Added matrix free capabilities
- Some Trilinos packages are now optional
- Refactored solver interface

.github/workflows/ci.yml

@@ -7,7 +7,6 @@ on:
   pull_request:
 
 env:
-  DOCKER_BASE: "${{ secrets.DOCKER_HUB_USERNAME }}/${{ secrets.DOCKER_HUB_IMAGE }}:latest"
   OMPI_ALLOW_RUN_AS_ROOT: 1
   OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
 
@@ -16,25 +15,25 @@ jobs:
     name: Build base container
     runs-on: self-hosted
     steps:
-      - uses: actions/checkout@v3.3.0
-      - uses: whoan/docker-build-with-cache-action@v6
+      - uses: actions/checkout@v4.2.0
+      - uses: whoan/docker-build-with-cache-action@v8.0.2
         id: docker-base-cache
         with:
-          username: kubagalecki
+          username: "${{ secrets.DOCKER_HUB_USERNAME }}"
           password: "${{ secrets.DOCKER_HUB_TOKEN }}"
-          image_name: l3ster-base
+          image_name: l3ster-ci
           dockerfile: scripts/ci/Dockerfile.base
 
   unit_tests_and_static_analysis:
     name: Static analysis and unit tests
     needs: build_base_container
     runs-on: self-hosted
-    container: kubagalecki/l3ster-base:latest
+    container: kubagalecki/l3ster-ci:latest
     env:
       BUILD_TYPE: Release
       TOOLCHAIN_PATH: ../cmake/toolchains/StaticAnalysis.cmake
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.2.0
       - name: Build and run tests
         run: scripts/ci/build-and-run-tests.sh
       - name: Cleanup
@@ -45,7 +44,7 @@ jobs:
     name: Sanitizers
     needs: unit_tests_and_static_analysis
     runs-on: self-hosted
-    container: kubagalecki/l3ster-base:latest
+    container: kubagalecki/l3ster-ci:latest
     env:
       BUILD_TYPE: ${{ matrix.build_type }}
       TOOLCHAIN_PATH: "../cmake/toolchains/${{ matrix.sanitizer }}Sanitizer.cmake"
@@ -54,9 +53,10 @@ jobs:
         build_type: [ Debug, Release ]
         sanitizer: [ Address, UndefinedBehavior ] # Thread disabled, too many false positives with TBB
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.2.0
       - name: Build and run tests
         run: scripts/ci/build-and-run-tests.sh
+        continue-on-error: true # Allow sanitizers to fail
       - name: Cleanup
         if: always()
         run: rm -rf build/
@@ -65,13 +65,13 @@ jobs:
     name: Deployment tests
     needs: unit_tests_and_static_analysis
     runs-on: self-hosted
-    container: kubagalecki/l3ster-base:latest
+    container: kubagalecki/l3ster-ci:latest
     env:
       DEPLOYMENT_TESTS: true
       BUILD_TYPE: Release
       TOOLCHAIN_PATH: ../cmake/toolchains/StaticAnalysis.cmake
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.2.0
       - name: Build and run tests
         run: scripts/ci/build-and-run-tests.sh
       - name: Cleanup
@@ -82,27 +82,26 @@ jobs:
     name: Code coverage
     needs: [ sanitizers, deployment ]
     runs-on: self-hosted
-    container: kubagalecki/l3ster-base:latest
+    container: kubagalecki/l3ster-ci:latest
     env:
       BUILD_TYPE: Debug
       TOOLCHAIN_PATH: ../cmake/toolchains/Coverage.cmake
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.2.0
       - name: Build and run tests
         run: scripts/ci/build-and-run-tests.sh
       - name: Gather coverage results via gcovr
         run: |
           gcovr --verbose --xml --print-summary --exclude-unreachable-branches --exclude-throw-branches \
-          --gcov-executable gcov-12 \
           --root "$GITHUB_WORKSPACE" \
           --exclude "$GITHUB_WORKSPACE/tests/" \
           --output "$GITHUB_WORKSPACE/coverage_report.xml" \
           "$GITHUB_WORKSPACE/build/tests/"
       - name: Cleanup
         if: always()
         run: rm -rf build/
-      - uses: codecov/codecov-action@v3.1.1
+      - uses: codecov/codecov-action@v4.5.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
-          verbose: true
+          verbose: true

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required( VERSION 3.19 )
+cmake_minimum_required( VERSION 3.24 )
 
 #######################################################################################################################
 
@@ -21,9 +21,6 @@ endif ()
 
 ### L3STER options ###
 
-# Verbosity (mainly relating to finding dependencies)
-option( L3STER_ENABLE_VERBOSITY "enable verbose config" ON )
-
 # Build tests
 option( L3STER_ENABLE_TESTS "enable tests" )
 
@@ -43,35 +40,52 @@ include( "cmake/ImportLibrary.cmake" )
 
 # Import libraries
 # Trilinos
-define_trilinos_target( ${L3STER_ENABLE_VERBOSITY} 13.4 Tpetra Belos MueLu )
-list( APPEND L3STER_INTERFACE_DEPENDENCIES Trilinos )
+set( L3STER_REQUIRED_Trilinos_PACKAGES "Kokkos;Tpetra" )
+set( L3STER_OPTIONAL_Trilinos_PACKAGES "Belos;Ifpack2;Amesos2;MueLu" )
+find_trilinos( 14.0 "${L3STER_REQUIRED_Trilinos_PACKAGES}" "${L3STER_OPTIONAL_Trilinos_PACKAGES}" )
+list( APPEND L3STER_DEPENDENCIES Trilinos::all_libs )
 
 # Eigen
 find_package( Eigen3 3.4 REQUIRED )
-list( APPEND L3STER_CMAKE_DEPENDENCIES Eigen3::Eigen )
+list( APPEND L3STER_DEPENDENCIES Eigen3::Eigen )
+
+# TBB
+find_package( TBB REQUIRED )
+list( APPEND L3STER_DEPENDENCIES TBB::tbb )
 
 # Hwloc
-importlibrary( hwloc.h "${L3STER_ENABLE_VERBOSITY}" )
+importlibrary( hwloc REQUIRED )
+list( APPEND L3STER_IMPORTED_DEPENDENCIES hwloc-import )
+list( APPEND L3STER_DEPENDENCIES hwloc-import )
 
 # Metis
-importlibrary( metis.h "${L3STER_ENABLE_VERBOSITY}" )
-
-# TBB
-find_package( TBB REQUIRED )
-list( APPEND L3STER_CMAKE_DEPENDENCIES TBB::tbb )
+importlibrary( metis REQUIRED )
+list( APPEND L3STER_IMPORTED_DEPENDENCIES metis-import )
+list( APPEND L3STER_DEPENDENCIES metis-import )
 
 #######################################################################################################################
 
 ### Define the L3STER target ###
 
+include( "cmake/HardwareInfo.cmake" )
+hardwareInfo()
+list( JOIN L3STER_CACHE_SIZES "uz,\n  " L3STER_CACHE_SIZES )
+string( APPEND L3STER_CACHE_SIZES "uz" )
+configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/include/l3ster/util/CacheSizesAtCompileTime.hpp.in"
+                "${CMAKE_CURRENT_SOURCE_DIR}/include/l3ster/util/CacheSizesAtCompileTime.hpp"
+                )
+
 add_library( L3STER INTERFACE )
 target_include_directories( L3STER INTERFACE
                             $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
                             $<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>
                             )
-set_target_properties( L3STER PROPERTIES INTERFACE_COMPILE_FEATURES cxx_std_20 )
-target_link_libraries( L3STER INTERFACE ${L3STER_CMAKE_DEPENDENCIES} )
-target_link_libraries( L3STER INTERFACE ${L3STER_INTERFACE_DEPENDENCIES} )
+set_target_properties( L3STER PROPERTIES INTERFACE_COMPILE_FEATURES cxx_std_23 )
+target_link_libraries( L3STER INTERFACE ${L3STER_DEPENDENCIES} )
+foreach ( pkg IN LISTS L3STER_OPTIONAL_Trilinos_PACKAGES )
+    string( TOUPPER "${pkg}" PKG )
+    target_compile_definitions( L3STER INTERFACE "L3STER_TRILINOS_HAS_${PKG}" )
+endforeach ()
 
 # Explicitly state L3STER sources - this is helpful to the IDE, but not required for a correct build
 file( GLOB_RECURSE L3STER_SOURCES
@@ -87,6 +101,7 @@ target_sources( L3STER INTERFACE $<BUILD_INTERFACE:${L3STER_SOURCES}> )
 ### Tests ###
 
 if ( L3STER_ENABLE_TESTS )
+    enable_testing()
     add_subdirectory( tests )
 endif ()
 
@@ -95,6 +110,7 @@ endif ()
 ### Benchmarks ###
 
 if ( L3STER_ENABLE_BENCHMARKS )
+    enable_testing()
     add_subdirectory( benchmarks )
 endif ()
 
@@ -105,9 +121,8 @@ endif ()
 include( GNUInstallDirs )
 include( CMakePackageConfigHelpers )
 
-# Mark L3STER and interface targets for export
-set( L3STER_EXPORT_TARGETS L3STER ${L3STER_INTERFACE_DEPENDENCIES} )
-install( TARGETS ${L3STER_EXPORT_TARGETS} EXPORT L3STERTargets )
+# Mark L3STER for export
+install( TARGETS L3STER ${L3STER_IMPORTED_DEPENDENCIES} EXPORT L3STERTargets )
 
 # Export L3STER
 install( EXPORT L3STERTargets

benchmarks/CMakeLists.txt

@@ -34,7 +34,7 @@ function( addMpiBenchmark source name nprocs )
         set( args ${ARGV4} )
     endif ()
     foreach ( np ${nprocs} )
-        set( bench_args "--nooversubscribe;--display-map;--bind-to;socket;--map-by;ppr:1:socket;-np;${np};$<TARGET_FILE:${bench_target}>;${args}" )
+        set( bench_args "--nooversubscribe;--display-map;--bind-to;socket;--map-by;ppr:1:package;-np;${np};$<TARGET_FILE:${bench_target}>;${args}" )
         set( bench_name ${name}_np_${np} )
         set( bench_dir "${CMAKE_CURRENT_BINARY_DIR}/${bench_name}" )
         file( MAKE_DIRECTORY ${bench_dir} )
@@ -48,4 +48,5 @@ function( addMpiBenchmark source name nprocs )
     set( L3STER_MPI_BENCHMARKS ${L3STER_MPI_BENCHMARKS} PARENT_SCOPE )
 endfunction()
 
-addmpibenchmark( Diffusion3DBenchmark.cpp Solve_diffusion_problem_3D "1" )
+addmpibenchmark( Diffusion3DBenchmark.cpp Solve_diffusion_problem_3D "1" )
+addmpibenchmark( Diffusion3DBenchmarkMatrixFree.cpp Solve_diffusion_problem_3D_Matrix_Free "1" )

benchmarks/Common.hpp

@@ -12,7 +12,11 @@ auto getExampleHexElement()
 {
     using el_t = mesh::Element< mesh::ElementType::Hex, O >;
     std::array< n_id_t, el_t::n_nodes > nodes;
-    std::iota(begin(nodes), end(nodes), 0);
+    n_id_t                              node = 0;
+    for (auto i : mesh::ElementTraits< mesh::Element< mesh::ElementType::Hex, O > >::boundary_node_inds)
+        nodes.at(i) = node++;
+    for (auto i : mesh::ElementTraits< mesh::Element< mesh::ElementType::Hex, O > >::internal_node_inds)
+        nodes.at(i) = node++;
     el_t element{nodes,
                  mesh::ElementData< mesh::ElementType::Hex, O >{{Point{0., 0., 0.},
                                                                  Point{1., 0., 0.},

benchmarks/Diffusion3DBenchmark.cpp

@@ -26,7 +26,7 @@ auto makeMesh(const lstr::MpiComm& comm, auto probdef_ctwrpr)
 int main(int argc, char* argv[])
 {
     L3sterScopeGuard scope_guard{argc, argv};
-    const MpiComm    comm{MPI_COMM_WORLD};
+    const auto       comm = std::make_shared< MpiComm >(MPI_COMM_WORLD);
 
     constexpr d_id_t      domain_id           = 0;
     static constexpr auto boundary_ids        = util::makeIotaArray< d_id_t, 6 >(1);
@@ -43,7 +43,7 @@ int main(int argc, char* argv[])
     constexpr auto        probdef_ctwrpr      = util::ConstexprValue< problem_def >{};
     constexpr auto        dirichletdef_ctwrpr = util::ConstexprValue< dirichlet_def >{};
 
-    const auto my_partition = makeMesh(comm, probdef_ctwrpr);
+    const auto my_partition = makeMesh(*comm, probdef_ctwrpr);
 
     constexpr auto field_inds  = util::makeIotaArray< size_t, problem_def.n_fields >();
     constexpr auto T_inds      = std::array< size_t, 1 >{0};
@@ -113,37 +113,35 @@ int main(int argc, char* argv[])
     alg_system.assembleProblem(diffusion_kernel3d, {domain_id});
     alg_system.setDirichletBCValues(dirichlet_bc_kernel, boundary_ids, T_inds);
     alg_system.endAssembly();
-    alg_system.describe(comm);
+    alg_system.describe();
 
     constexpr auto solver_opts  = IterSolverOpts{.verbosity = {.summary = true, .timing = true}};
-    constexpr auto precond_opts = ChebyshevOpts{.degree = 3};
+    constexpr auto precond_opts = Ifpack2JacobiOpts{};
     auto           solver       = CG{solver_opts, precond_opts};
-    auto           solution     = alg_system.initSolution();
-    alg_system.solve(solver, solution);
+    alg_system.solve(solver);
 
     L3STER_PROFILE_REGION_BEGIN("Solution management");
     auto solution_manager = SolutionManager{*my_partition, problem_def.n_fields};
-    alg_system.updateSolution(solution, dof_inds, solution_manager, field_inds);
+    alg_system.updateSolution(dof_inds, solution_manager, field_inds);
     L3STER_PROFILE_REGION_END("Solution management");
 
     L3STER_PROFILE_REGION_BEGIN("Compute solution error");
     const auto fval_getter = solution_manager.makeFieldValueGetter(field_inds);
-    const auto error       = computeNormL2(comm, error_kernel, *my_partition, {domain_id}, fval_getter);
+    const auto error       = computeNormL2(*comm, error_kernel, *my_partition, {domain_id}, fval_getter);
     L3STER_PROFILE_REGION_END("Compute solution error");
 
-    if (comm.getRank() == 0)
-    {
-        std::cout << "The L2 error components are:";
-        for (int i = 0; i < error.size(); ++i)
-            std::cout << "\n\t" << error[i];
-        std::cout << std::endl;
-    }
+    if (comm->getRank() == 0)
+        std::cout << std::format("\nThe L2 error components are:\n  {:.3e}\n  {:.3e}\n  {:.3e}\n  {:.3e}\n\n",
+                                 error[0],
+                                 error[1],
+                                 error[2],
+                                 error[3]);
 
     L3STER_PROFILE_REGION_BEGIN("Export results to VTK");
-    auto           exporter    = PvtuExporter{*my_partition};
+    auto           exporter    = PvtuExporter{comm, *my_partition};
     const auto     export_inds = util::gatherAsCommon(T_inds, T_grad_inds);
     constexpr auto field_names = std::array{"T"sv, "gradT"sv};
-    exporter.exportSolution("Cube_Diffusion.pvtu", comm, solution_manager, field_names, export_inds);
+    exporter.exportSolution("Cube_Diffusion.pvtu", solution_manager, field_names, export_inds);
     exporter.flushWriteQueue();
     L3STER_PROFILE_REGION_END("Export results to VTK");
 }