diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b81b769..6b8cf593 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,7 +75,7 @@ to turn off this check") endif() set(GINKGO_CHECKOUT_VERSION - "ogl_0600_gko190" + "ogl_0600_gko190" CACHE STRING "Use specific version of ginkgo") if(OGL_DEVEL_TOOLS) @@ -216,13 +216,14 @@ target_sources( src/MatrixWrapper/SparsityPattern.cpp src/MatrixWrapper/Distributed.cpp src/MatrixWrapper/Combination.cpp + src/Preconditioner/CoarseSolver.cpp src/Repartitioner.cpp src/Solver/CG.cpp - src/Solver/PipeCG.cpp - src/Solver/BiCGStab.cpp - src/Solver/GMRES.cpp - src/Solver/Multigrid.cpp - # src/Solver/IR.cpp + src/Solver/PipeCG.cpp + src/Solver/BiCGStab.cpp + src/Solver/GMRES.cpp + src/Solver/Multigrid.cpp + # src/Solver/IR.cpp ) enable_sanitizers( diff --git a/include/OGL/DevicePersistent/Vector.hpp b/include/OGL/DevicePersistent/Vector.hpp index c8610f59..7b0c6334 100644 --- a/include/OGL/DevicePersistent/Vector.hpp +++ b/include/OGL/DevicePersistent/Vector.hpp @@ -212,6 +212,7 @@ class PersistentVector tmp.set_executor(ref_exec); MPI_Request copy_back_req; + MPI_Iscatterv(tmp.get_data(), repartAllToAll.send_counts.data(), repartAllToAll.send_offsets.data(), MPI_DOUBLE, const_cast(memory_), repartAllToAll.recv_counts[0], diff --git a/include/OGL/Preconditioner.hpp b/include/OGL/Preconditioner.hpp index 2bd3d5f1..c4194f51 100644 --- a/include/OGL/Preconditioner.hpp +++ b/include/OGL/Preconditioner.hpp @@ -8,11 +8,19 @@ #include "OGL/DevicePersistent/Base.hpp" #include "OGL/MatrixWrapper/Distributed.hpp" +#include "OGL/Preconditioner/Cholesky.hpp" +#include "OGL/Preconditioner/ISAI.hpp" +#include "OGL/Preconditioner/Jacobi.hpp" +#include "OGL/Preconditioner/LU.hpp" +#include "OGL/Preconditioner/Multigrid.hpp" +#include "OGL/Preconditioner/Schwarz.hpp" #include "fvCFD.H" #include "regIOobject.H" namespace Foam { + + class Preconditioner { using mtx = gko::matrix::Csr; using bj = gko::preconditioner::Jacobi; @@ -52,110 +60,6 @@ class Preconditioner { verbose_(verbose) {} - template - std::shared_ptr wrap_multi_level_schwarz( - std::shared_ptr gkomatrix, - std::shared_ptr device_exec, - std::shared_ptr precond, const dictionary &d, - label local_rows) const - { - using pgm = gko::multigrid::Pgm; - using fc = gko::multigrid::FixedCoarsening; - using solver = gko::solver::Cg; - - auto selCoarseRows = d.lookupOrDefault("selCoarseRows", label(5)); - auto fixedCoarsening = - d.lookupOrDefault("fixedCoarsening", false); - auto coarseWeight = d.lookupOrDefault("coarseWeight", scalar(0.01)); - auto solveNormC = - d.lookupOrDefault("reductionCoarseSolver", label(1e-6)); - auto maxIterCoarse = d.lookupOrDefault("maxIterCoarse", label(50)); - - word msg = "Generate multi level schwarz:\n\tfixedCoarsening " + - std::to_string(fixedCoarsening) + "\n\tselCoarseRows " + - std::to_string(selCoarseRows) + "\n\trelTolCoarse " + - std::to_string(solveNormC) + "\n\tmaxIterCoarse " + - std::to_string(maxIterCoarse) + "\n\tcoarseWeigth" + - std::to_string(coarseWeight); - MLOG_0(verbose_, msg) - - auto pre_factory = ras::build().with_local_solver( - bj::build().with_skip_sorting(true).with_max_block_size(1u).on( - device_exec)); - - auto coarse_solver = gko::share( - solver::build() - .with_preconditioner(pre_factory) - .with_criteria( - gko::stop::Iteration::build().with_max_iters(maxIterCoarse), - gko::stop::ResidualNorm::build() - .with_reduction_factor(solveNormC)) - .on(device_exec)); - - if (fixedCoarsening) { - auto n_rows = local_rows / selCoarseRows; - auto sel_rows = - gko::array