1#ifndef AMREX_HYPRE_ML_ABECLAP_H_
2#define AMREX_HYPRE_ML_ABECLAP_H_
3#include <AMReX_Config.H>
14#include <HYPRE_sstruct_ls.h>
17#include <unordered_map>
91 std::string a_parmparse_prefix =
"hypre_mlabeclap");
102 std::string a_parmparse_prefix =
"hypre_mlabeclap");
150 m_alg_gather.clear();
151 m_alg_row_offset.clear();
152 m_alg_col_index.clear();
174 if (m_ij_solver) { m_ij_solver->setNeedSetup(); }
175 m_algmg_reset =
true;
180 [[nodiscard]]
int getNumIters () const noexcept {
return m_num_iters; }
207 std::pair<MultiFab const*, IntVect>
const& a_coarse_bc = {
nullptr,
IntVect(0)});
217 void solve (Vector<MultiFab*>
const& a_sol, Vector<MultiFab const*>
const& a_rhs,
237 Vector<FabArrayBase::CopyComTag>
const& tags);
243 void solveIJ (Vector<MultiFab*>
const& a_sol, Vector<MultiFab const*>
const& a_rhs,
260 void addNonStencilEntriesToGraph ();
272 [[nodiscard]]
Long cellId (
int lev,
IntVect const& iv)
const;
276 bool m_is_singular =
false;
281 std::string m_parmparse_prefix;
288 Real m_ascalar = std::numeric_limits<Real>::max();
289 Real m_bscalar = std::numeric_limits<Real>::max();
297 Vector<std::unique_ptr<MLMGBndry>> m_bndry;
298 Vector<std::unique_ptr<BndryRegister>> m_bndry_rhs;
299 Vector<iMultiFab> m_fine_masks;
300 Vector<iMultiFab> m_crse_masks;
304 Vector<iMultiFab> m_c2f_offset_from;
305 Vector<LayoutData<int>> m_c2f_total_from;
306 Vector<iMultiFab> m_c2f_nentries;
307 Vector<iMultiFab> m_c2f_offset_to;
308 Vector<LayoutData<int>> m_c2f_total_to;
311 Vector<Array<iMultiFab,AMREX_SPACEDIM>> m_offset_cf_bcoefs;
312 Vector<Array<LayoutData<std::unique_ptr<Gpu::DeviceVector<Real>>>,AMREX_SPACEDIM>> m_cf_bcoefs;
317 template <
class T>
using HostVector = Vector<T>;
322 Vector<HostVector<int>> m_f2c_bno;
323 Vector<HostVector<IntVect>> m_f2c_cell;
324 Vector<Vector<HYPRE_Int>> m_f2c_nentries;
325 Vector<HostVector<std::size_t>> m_f2c_offset;
326 Vector<HostVector<Real>> m_f2c_values;
328 HYPRE_SStructGrid m_ss_grid =
nullptr;
329 HYPRE_SStructStencil m_ss_stencil =
nullptr;
330 HYPRE_SStructGraph m_ss_graph =
nullptr;
331 HYPRE_SStructSolver m_ss_solver =
nullptr;
332 HYPRE_SStructSolver m_ss_precond =
nullptr;
333 HYPRE_SStructMatrix m_ss_A =
nullptr;
334 HYPRE_SStructVector m_ss_x =
nullptr;
335 HYPRE_SStructVector m_ss_b =
nullptr;
337 HYPRE_Solver m_solver =
nullptr;
341 std::unique_ptr<HypreIJIface> m_ij_solver;
345 bool m_use_ij =
false;
346 bool m_reuse_setup =
false;
347 bool m_throw_exception =
false;
349 bool m_ij_vectors_assembled =
false;
350 bool m_ij_gather_failed =
false;
352 bool m_algmg_reset =
false;
354 bool m_algmg_kept =
false;
355 int m_reuse_max_iter = 0;
358 composite::SetupReuse m_reuse;
362 Vector<Vector<Long>> m_box_offset;
364 Long m_ij_upper = -1;
366 std::unordered_map<Long,Vector<Long>> m_ns_targets;
370 Vector<std::unique_ptr<FabArray<BaseFab<Long>>>> m_cell_id;
371 Vector<LayoutData<Gpu::DeviceVector<HYPRE_BigInt>>> m_c2f_targets;
372 Vector<Gpu::DeviceVector<HYPRE_BigInt>> m_f2c_rows;
373 Vector<Gpu::DeviceVector<HYPRE_BigInt>> m_f2c_cols;
381 bool m_eb_scaling =
true;
382 bool m_beta_on_centroid =
false;
383 Vector<MultiFab> m_eb_scale;
384 Vector<std::unique_ptr<FabArray<BaseFab<Long>>>> m_crse_id;
385 Vector<MultiFab> m_crse_scale;
386 Vector<MultiFab> m_eb_bcrhs;
387 Vector<MultiFab> m_eb_beta;
388 Vector<MultiFab> m_eb_phi;
391 using EntryGather = composite::EntryGather;
394 EntryGather m_ij_gather;
399 enum class Backend { hypre, algmg };
400 Backend m_backend = Backend::hypre;
402 Real m_final_res = 0.0;
403 Vector<Long> m_rank_offset;
404 composite::OwnedCSR m_csr;
405 std::unique_ptr<SpMatrix<Real>> m_spA;
406 std::unique_ptr<AlgMG<Real>> m_algmg;
411 EntryGather m_alg_gather;
421 std::string m_dump_prefix;
422 Long m_dump_gather_max = 50000000;
425 void fail (std::string
const& msg)
const;
428 HYPRE_Int m_hypre_object_type = HYPRE_PARCSR;
Abstract factory interface for creating and destroying FAB objects.
Problem-domain geometry: maps between index space and physical space.
Constants describing linear-operator boundary condition types.
MultiFab-style container that stores Mask objects per grid.
#define AMREX_D_DECL(a, b, c)
Definition AMReX_SPACE.H:171
Abstract factory interface for creating, aliasing, and destroying FAB objects.
Definition AMReX_FabFactory.H:73
void setBCoeffsOnCentroids(bool v) noexcept
EB operator: the b coefficients of the following setups are at the face centroids,...
Definition AMReX_HypreMLABecLap.H:133
void solveIJ(Vector< MultiFab * > const &a_sol, Vector< MultiFab const * > const &a_rhs, Real a_reltol, Real a_abstol)
Solve with the IJ system (see m_use_ij); public for cuda.
Definition AMReX_HypreMLABecLap.cpp:2057
void solve(Vector< MultiFab * > const &a_sol, Vector< MultiFab const * > const &a_rhs, Real a_reltol, Real a_abstol)
Solve the assembled system with HYPRE (single-component).
Definition AMReX_HypreMLABecLap.cpp:1412
void setMaxIter(int v)
Override the maximum number of HYPRE iterations (v).
Definition AMReX_HypreMLABecLap.H:109
void setReuseSolverSetup(bool v)
IJ mode only: let setup() be called again with new coefficients (same grids), updating the matrix val...
Definition AMReX_HypreMLABecLap.H:142
HypreMLABecLap(HypreMLABecLap const &)=delete
void commBCoefs_local(int flev, Array< MultiFab const *, 3 > const &a_bcoefs, Vector< FabArrayBase::CopyComTag > const &tags)
Local portion of commBCoefs used to issue device-friendly copies.
Definition AMReX_HypreMLABecLap.cpp:2753
void buildEBTriplets(int ilev, Real a_ascalar, Real a_bscalar, MultiFab const *a_acoef, Array< MultiFab, 3 > const &a_bcoef, Vector< Triplets > &chunks)
Definition AMReX_HypreMLABecLap.cpp:1865
void resetSolverSetup()
IJ mode only: redo the solver/preconditioner setup at the next solve.
Definition AMReX_HypreMLABecLap.H:173
void setVerbose(int v)
Set HYPRE's logging verbosity (v).
Definition AMReX_HypreMLABecLap.H:107
~HypreMLABecLap()
Definition AMReX_HypreMLABecLap.cpp:315
void commBCoefs(int flev, Array< MultiFab const *, 3 > const &a_bcoefs)
Communicate b coefficients needed by level flev at coarse/fine interfaces.
Definition AMReX_HypreMLABecLap.cpp:2423
int getNumIters() const noexcept
Iterations and final relative residual of the last solve.
Definition AMReX_HypreMLABecLap.H:180
void setReuseSetupMaxIter(int n) noexcept
Definition AMReX_HypreMLABecLap.H:163
void setIsSingular(bool v)
Flag whether the assembled operator is singular (v).
Definition AMReX_HypreMLABecLap.H:111
void setEBHomogDirichlet(Vector< MultiFab const * > const &a_beta)
EB operator: homogeneous Dirichlet boundary condition on the EB.
Definition AMReX_HypreMLABecLap.cpp:1797
void setup(Real a_ascalar, Real a_bscalar, Vector< MultiFab const * > const &a_acoefs, Vector< Array< MultiFab const *, 3 > > const &a_bcoefs, Array< LinOpBCType, 3 > const &a_lobc, Array< LinOpBCType, 3 > const &a_hibc, Vector< MultiFab const * > const &a_levelbcdata, std::pair< MultiFab const *, IntVect > const &a_coarse_bc={nullptr, IntVect(0)})
Assemble the HYPRE operators/storage for the provided coefficients.
Definition AMReX_HypreMLABecLap.cpp:653
Real getMatrixScale() const noexcept
Definition AMReX_HypreMLABecLap.H:159
HypreMLABecLap(HypreMLABecLap &&)=delete
void setEBDirichlet(Vector< MultiFab const * > const &a_phi, Vector< MultiFab const * > const &a_beta)
EB operator: Dirichlet boundary condition on the EB for the following setups (the data are copied; th...
Definition AMReX_HypreMLABecLap.cpp:1790
void setReuseSetupMaxChange(Real f)
Definition AMReX_HypreMLABecLap.H:171
void buildIJMaps()
Build the device maps of the IJ assembly; public for cuda.
Definition AMReX_HypreMLABecLap.cpp:1648
Real getFinalResidualNorm() const noexcept
Definition AMReX_HypreMLABecLap.H:181
void setThrowException(bool t) noexcept
Throw std::runtime_error instead of aborting when the solve fails.
Definition AMReX_HypreMLABecLap.H:178
HypreMLABecLap & operator=(HypreMLABecLap const &)=delete
A collection (stored as an array) of FArrayBox objects.
Definition AMReX_MultiFab.H:40
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
amrex_real Real
Floating Point Type for Fields.
Definition AMReX_REAL.H:80
amrex_long Long
Definition AMReX_INT.H:30
PODVector< T, PinnedArenaAllocator< T > > PinnedVector
A PODVector that uses the pinned memory arena.
Definition AMReX_GpuContainers.H:48
std::array< T, N > Array
Definition AMReX_Array.H:31
PODVector< T, ArenaAllocator< T > > DeviceVector
A PODVector that uses the standard memory Arena. Note that the memory might or might not be managed d...
Definition AMReX_GpuContainers.H:24
int MPI_Comm
Definition AMReX_ccse-mpi.H:51
static constexpr int MPI_COMM_NULL
Definition AMReX_ccse-mpi.H:59
Definition AMReX_Amr.cpp:50
@ bogus
Definition AMReX_LO_BCTYPES.H:39
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
HypreSolverID
Definition AMReX_HypreMLABecLap.H:61
@ BoomerAMG
Definition AMReX_HypreMLABecLap.H:62
@ SSAMG
Definition AMReX_HypreMLABecLap.H:62
Per-box (row, column, value) triplets of the IJ assembly.
Definition AMReX_HypreMLABecLap.H:247
Gpu::DeviceVector< HYPRE_BigInt > rows
Definition AMReX_HypreMLABecLap.H:248
Gpu::DeviceVector< Real > vals
Definition AMReX_HypreMLABecLap.H:249
Gpu::DeviceVector< HYPRE_BigInt > cols
Definition AMReX_HypreMLABecLap.H:248