Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_HypreMLABecLap.H
Go to the documentation of this file.
1#ifndef AMREX_HYPRE_ML_ABECLAP_H_
2#define AMREX_HYPRE_ML_ABECLAP_H_
3#include <AMReX_Config.H>
4
6#include <AMReX_FabFactory.H>
7#include <AMReX_Geometry.H>
9#include <AMReX_LO_BCTYPES.H>
10#include <AMReX_MLMGBndry.H>
11#include <AMReX_MultiFab.H>
12#include <AMReX_MultiMask.H>
13
14#include <HYPRE_sstruct_ls.h>
15
16#include <limits>
17#include <unordered_map>
18#include <utility>
19
58
59namespace amrex {
60
61enum struct HypreSolverID {
63};
64
65// single component only, cell centered only
66
71{
72public:
73 HypreMLABecLap (HypreMLABecLap const&) = delete;
77
88 Vector<BoxArray> a_grids,
90 HypreSolverID a_hypre_solver_id,
91 std::string a_parmparse_prefix = "hypre_mlabeclap");
92
98 Vector<BoxArray> a_grids,
100 HypreSolverID a_hypre_solver_id,
101 Vector<FabFactory<FArrayBox> const*> a_factory,
102 std::string a_parmparse_prefix = "hypre_mlabeclap");
103
105
107 void setVerbose (int v) { m_verbose = v; }
109 void setMaxIter (int v) { m_maxiter = v; }
111 void setIsSingular (bool v) { m_is_singular = v; }
112
121 void setEBDirichlet (Vector<MultiFab const*> const& a_phi,
122 Vector<MultiFab const*> const& a_beta);
124 void setEBHomogDirichlet (Vector<MultiFab const*> const& a_beta);
133 void setBCoeffsOnCentroids (bool v) noexcept { m_beta_on_centroid = v; }
134
143 {
144 m_reuse_setup = v;
145 if (!v) {
146 // The slots of the in-place updates are of a setup with reuse on
147 // (the hypre solver is made anew anyway); the next setup with
148 // reuse on finds them again
149 m_ij_gather.clear();
150 m_alg_gather.clear();
151 m_alg_row_offset.clear();
152 m_alg_col_index.clear();
153 }
154 }
155
159 [[nodiscard]] Real getMatrixScale () const noexcept { return m_reuse.scale; }
163 void setReuseSetupMaxIter (int n) noexcept { m_reuse_max_iter = n; }
171 void setReuseSetupMaxChange (Real f) { m_reuse.setMaxChange(f); }
174 if (m_ij_solver) { m_ij_solver->setNeedSetup(); }
175 m_algmg_reset = true;
176 }
177
178 void setThrowException (bool t) noexcept { m_throw_exception = t; }
180 [[nodiscard]] int getNumIters () const noexcept { return m_num_iters; }
181 [[nodiscard]] Real getFinalResidualNorm () const noexcept { return m_final_res; }
182
201 void setup (Real a_ascalar, Real a_bscalar,
202 Vector<MultiFab const*> const& a_acoefs,
206 Vector<MultiFab const*> const& a_levelbcdata,
207 std::pair<MultiFab const*, IntVect> const& a_coarse_bc = {nullptr, IntVect(0)});
208
217 void solve (Vector<MultiFab*> const& a_sol, Vector<MultiFab const*> const& a_rhs,
218 Real a_reltol, Real a_abstol);
219
220 // public for cuda
221
228 void commBCoefs (int flev, Array<MultiFab const*,AMREX_SPACEDIM> const& a_bcoefs);
236 void commBCoefs_local (int flev, Array<MultiFab const*,AMREX_SPACEDIM> const& a_bcoefs,
237 Vector<FabArrayBase::CopyComTag> const& tags);
238
240 void buildIJMaps ();
241
243 void solveIJ (Vector<MultiFab*> const& a_sol, Vector<MultiFab const*> const& a_rhs,
244 Real a_reltol, Real a_abstol);
245
251
254 void buildEBTriplets (int ilev, Real a_ascalar, Real a_bscalar, MultiFab const* a_acoef,
255 Array<MultiFab,AMREX_SPACEDIM> const& a_bcoef,
256 Vector<Triplets>& chunks);
257
258private:
259
260 void addNonStencilEntriesToGraph ();
261
263 void copyEBData (Vector<MultiFab const*> const& src, Vector<MultiFab>& dst) const;
266 ebBCoefs (Vector<Array<MultiFab const*,AMREX_SPACEDIM>> const& a_bcoefs) const;
268 [[nodiscard]] Vector<Real> rowScaling () const;
269
272 [[nodiscard]] Long cellId (int lev, IntVect const& iv) const;
273
274 int m_verbose = 0;
275 int m_maxiter = 200;
276 bool m_is_singular = false;
277
278 Vector<Geometry> m_geom;
279 Vector<BoxArray> m_grids;
281 std::string m_parmparse_prefix;
282 Vector<FabFactory<FArrayBox> const*> m_factory;
283 int m_nlevels = 0;
284 MPI_Comm m_comm = MPI_COMM_NULL;
285
286 Vector<IntVect> m_ref_ratio;
287
288 Real m_ascalar = std::numeric_limits<Real>::max();
289 Real m_bscalar = std::numeric_limits<Real>::max();
296
297 Vector<std::unique_ptr<MLMGBndry>> m_bndry;
298 Vector<std::unique_ptr<BndryRegister>> m_bndry_rhs;
299 Vector<iMultiFab> m_fine_masks;
300 Vector<iMultiFab> m_crse_masks;
301
302 // For coarse cells at coarse/fine interface. The vector is for AMR
303 // levels.
304 Vector<iMultiFab> m_c2f_offset_from; // offset for sparse coarse from-cells
305 Vector<LayoutData<int>> m_c2f_total_from; // # of coarse from-cells w/ c2f entries
306 Vector<iMultiFab> m_c2f_nentries; // # of non-stencil entries
307 Vector<iMultiFab> m_c2f_offset_to; // offset for sparse to-cells, including fine (and coarse in 3d) cells
308 Vector<LayoutData<int>> m_c2f_total_to; // total sum of non-stencil entries in a Box
309
310 // B coefficients at coarse/fine interface
311 Vector<Array<iMultiFab,AMREX_SPACEDIM>> m_offset_cf_bcoefs;
312 Vector<Array<LayoutData<std::unique_ptr<Gpu::DeviceVector<Real>>>,AMREX_SPACEDIM>> m_cf_bcoefs;
313
314#ifdef AMREX_USE_GPU
315 template <class T> using HostVector = Gpu::PinnedVector<T>;
316#else
317 template <class T> using HostVector = Vector<T>;
318#endif
319
320 // For fine cells at coarse/fine interface. The non-stencil entries are
321 // from fine to coarse. The outer vector is for AMR levels.
322 Vector<HostVector<int>> m_f2c_bno; // local box number
323 Vector<HostVector<IntVect>> m_f2c_cell; // fine cell
324 Vector<Vector<HYPRE_Int>> m_f2c_nentries; // # of non-stencil entries
325 Vector<HostVector<std::size_t>> m_f2c_offset; // offset into m_f2c_values
326 Vector<HostVector<Real>> m_f2c_values; // values for non-stencil entries
327
328 HYPRE_SStructGrid m_ss_grid = nullptr;
329 HYPRE_SStructStencil m_ss_stencil = nullptr;
330 HYPRE_SStructGraph m_ss_graph = nullptr;
331 HYPRE_SStructSolver m_ss_solver = nullptr;
332 HYPRE_SStructSolver m_ss_precond = nullptr;
333 HYPRE_SStructMatrix m_ss_A = nullptr;
334 HYPRE_SStructVector m_ss_x = nullptr;
335 HYPRE_SStructVector m_ss_b = nullptr;
336
337 HYPRE_Solver m_solver = nullptr;
338
341 std::unique_ptr<HypreIJIface> m_ij_solver;
342
345 bool m_use_ij = false;
346 bool m_reuse_setup = false;
347 bool m_throw_exception = false;
349 bool m_ij_vectors_assembled = false;
350 bool m_ij_gather_failed = false; // hypre dropped an entry: no in-place updates
352 bool m_algmg_reset = false;
354 bool m_algmg_kept = false;
355 int m_reuse_max_iter = 0;
358 composite::SetupReuse m_reuse;
361 Gpu::DeviceVector<char> m_fixed_rows;
362 Vector<Vector<Long>> m_box_offset; // first row id of each box, per level
363 Long m_ij_lower = 0;
364 Long m_ij_upper = -1;
366 std::unordered_map<Long,Vector<Long>> m_ns_targets;
370 Vector<std::unique_ptr<FabArray<BaseFab<Long>>>> m_cell_id;
371 Vector<LayoutData<Gpu::DeviceVector<HYPRE_BigInt>>> m_c2f_targets;
372 Vector<Gpu::DeviceVector<HYPRE_BigInt>> m_f2c_rows;
373 Vector<Gpu::DeviceVector<HYPRE_BigInt>> m_f2c_cols;
374 Gpu::DeviceVector<HYPRE_BigInt> m_ij_rowids; // the local rows
375
380 bool m_eb = false;
381 bool m_eb_scaling = true;
382 bool m_beta_on_centroid = false;
383 Vector<MultiFab> m_eb_scale;
384 Vector<std::unique_ptr<FabArray<BaseFab<Long>>>> m_crse_id;
385 Vector<MultiFab> m_crse_scale;
386 Vector<MultiFab> m_eb_bcrhs;
387 Vector<MultiFab> m_eb_beta;
388 Vector<MultiFab> m_eb_phi;
391 using EntryGather = composite::EntryGather;
394 EntryGather m_ij_gather;
395
399 enum class Backend { hypre, algmg };
400 Backend m_backend = Backend::hypre;
401 int m_num_iters = 0; // last solve (IJ mode)
402 Real m_final_res = 0.0;
403 Vector<Long> m_rank_offset; // first row of each rank, plus the total
404 composite::OwnedCSR m_csr; // owned rows (AlgMG backend, dumps)
405 std::unique_ptr<SpMatrix<Real>> m_spA;
406 std::unique_ptr<AlgMG<Real>> m_algmg;
409 Gpu::DeviceVector<Long> m_alg_row_offset;
410 Gpu::DeviceVector<Long> m_alg_col_index;
411 EntryGather m_alg_gather;
412
420 int m_dump_max = 0;
421 std::string m_dump_prefix;
422 Long m_dump_gather_max = 50000000;
423 void dumpSystem (int k, Gpu::DeviceVector<Real> const& v, bool solution);
425 void fail (std::string const& msg) const;
426
427 HypreSolverID m_hypre_solver_id = HypreSolverID::BoomerAMG;
428 HYPRE_Int m_hypre_object_type = HYPRE_PARCSR;
429};
430
431}
432
433#endif
Abstract factory interface for creating and destroying FAB objects.
Problem-domain geometry: maps between index space and physical space.
Constants describing linear-operator boundary condition types.
MultiFab-style container that stores Mask objects per grid.
#define AMREX_D_DECL(a, b, c)
Definition AMReX_SPACE.H:171
Abstract factory interface for creating, aliasing, and destroying FAB objects.
Definition AMReX_FabFactory.H:73
void setBCoeffsOnCentroids(bool v) noexcept
EB operator: the b coefficients of the following setups are at the face centroids,...
Definition AMReX_HypreMLABecLap.H:133
void solveIJ(Vector< MultiFab * > const &a_sol, Vector< MultiFab const * > const &a_rhs, Real a_reltol, Real a_abstol)
Solve with the IJ system (see m_use_ij); public for cuda.
Definition AMReX_HypreMLABecLap.cpp:2057
void solve(Vector< MultiFab * > const &a_sol, Vector< MultiFab const * > const &a_rhs, Real a_reltol, Real a_abstol)
Solve the assembled system with HYPRE (single-component).
Definition AMReX_HypreMLABecLap.cpp:1412
void setMaxIter(int v)
Override the maximum number of HYPRE iterations (v).
Definition AMReX_HypreMLABecLap.H:109
void setReuseSolverSetup(bool v)
IJ mode only: let setup() be called again with new coefficients (same grids), updating the matrix val...
Definition AMReX_HypreMLABecLap.H:142
HypreMLABecLap(HypreMLABecLap const &)=delete
void commBCoefs_local(int flev, Array< MultiFab const *, 3 > const &a_bcoefs, Vector< FabArrayBase::CopyComTag > const &tags)
Local portion of commBCoefs used to issue device-friendly copies.
Definition AMReX_HypreMLABecLap.cpp:2753
void buildEBTriplets(int ilev, Real a_ascalar, Real a_bscalar, MultiFab const *a_acoef, Array< MultiFab, 3 > const &a_bcoef, Vector< Triplets > &chunks)
Definition AMReX_HypreMLABecLap.cpp:1865
void resetSolverSetup()
IJ mode only: redo the solver/preconditioner setup at the next solve.
Definition AMReX_HypreMLABecLap.H:173
void setVerbose(int v)
Set HYPRE's logging verbosity (v).
Definition AMReX_HypreMLABecLap.H:107
~HypreMLABecLap()
Definition AMReX_HypreMLABecLap.cpp:315
void commBCoefs(int flev, Array< MultiFab const *, 3 > const &a_bcoefs)
Communicate b coefficients needed by level flev at coarse/fine interfaces.
Definition AMReX_HypreMLABecLap.cpp:2423
int getNumIters() const noexcept
Iterations and final relative residual of the last solve.
Definition AMReX_HypreMLABecLap.H:180
void setReuseSetupMaxIter(int n) noexcept
Definition AMReX_HypreMLABecLap.H:163
void setIsSingular(bool v)
Flag whether the assembled operator is singular (v).
Definition AMReX_HypreMLABecLap.H:111
void setEBHomogDirichlet(Vector< MultiFab const * > const &a_beta)
EB operator: homogeneous Dirichlet boundary condition on the EB.
Definition AMReX_HypreMLABecLap.cpp:1797
void setup(Real a_ascalar, Real a_bscalar, Vector< MultiFab const * > const &a_acoefs, Vector< Array< MultiFab const *, 3 > > const &a_bcoefs, Array< LinOpBCType, 3 > const &a_lobc, Array< LinOpBCType, 3 > const &a_hibc, Vector< MultiFab const * > const &a_levelbcdata, std::pair< MultiFab const *, IntVect > const &a_coarse_bc={nullptr, IntVect(0)})
Assemble the HYPRE operators/storage for the provided coefficients.
Definition AMReX_HypreMLABecLap.cpp:653
Real getMatrixScale() const noexcept
Definition AMReX_HypreMLABecLap.H:159
HypreMLABecLap(HypreMLABecLap &&)=delete
void setEBDirichlet(Vector< MultiFab const * > const &a_phi, Vector< MultiFab const * > const &a_beta)
EB operator: Dirichlet boundary condition on the EB for the following setups (the data are copied; th...
Definition AMReX_HypreMLABecLap.cpp:1790
void setReuseSetupMaxChange(Real f)
Definition AMReX_HypreMLABecLap.H:171
void buildIJMaps()
Build the device maps of the IJ assembly; public for cuda.
Definition AMReX_HypreMLABecLap.cpp:1648
Real getFinalResidualNorm() const noexcept
Definition AMReX_HypreMLABecLap.H:181
void setThrowException(bool t) noexcept
Throw std::runtime_error instead of aborting when the solve fails.
Definition AMReX_HypreMLABecLap.H:178
HypreMLABecLap & operator=(HypreMLABecLap const &)=delete
A collection (stored as an array) of FArrayBox objects.
Definition AMReX_MultiFab.H:40
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
amrex_real Real
Floating Point Type for Fields.
Definition AMReX_REAL.H:80
amrex_long Long
Definition AMReX_INT.H:30
PODVector< T, PinnedArenaAllocator< T > > PinnedVector
A PODVector that uses the pinned memory arena.
Definition AMReX_GpuContainers.H:48
std::array< T, N > Array
Definition AMReX_Array.H:31
PODVector< T, ArenaAllocator< T > > DeviceVector
A PODVector that uses the standard memory Arena. Note that the memory might or might not be managed d...
Definition AMReX_GpuContainers.H:24
int MPI_Comm
Definition AMReX_ccse-mpi.H:51
static constexpr int MPI_COMM_NULL
Definition AMReX_ccse-mpi.H:59
Definition AMReX_Amr.cpp:50
@ bogus
Definition AMReX_LO_BCTYPES.H:39
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
HypreSolverID
Definition AMReX_HypreMLABecLap.H:61
@ BoomerAMG
Definition AMReX_HypreMLABecLap.H:62
@ SSAMG
Definition AMReX_HypreMLABecLap.H:62
Per-box (row, column, value) triplets of the IJ assembly.
Definition AMReX_HypreMLABecLap.H:247
Gpu::DeviceVector< HYPRE_BigInt > rows
Definition AMReX_HypreMLABecLap.H:248
Gpu::DeviceVector< Real > vals
Definition AMReX_HypreMLABecLap.H:249
Gpu::DeviceVector< HYPRE_BigInt > cols
Definition AMReX_HypreMLABecLap.H:248