Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_AlgMG.H
Go to the documentation of this file.
1#ifndef AMREX_ALGMG_H_
2#define AMREX_ALGMG_H_
3#include <AMReX_Config.H>
4
5#include <AMReX_Algebra.H>
6#include <AMReX_Enum.H>
7#include <AMReX_BiCGStab_MV.H>
8#include <AMReX_BLProfiler.H>
9#include <AMReX_GMRES_MV.H>
10#include <AMReX_PCG_MV.H>
12#include <AMReX_Scan.H>
13#include <AMReX_Smoother_MV.H>
14#include <AMReX_Vector.H>
15
16#include <cmath>
17#include <cstdint>
18#include <iomanip>
19#include <limits>
20#include <numeric>
21#include <memory>
22#include <stdexcept>
23#include <string>
24#include <type_traits>
25#include <utility>
26
27namespace amrex {
28
29namespace detail {
30
34template <typename TC, typename Index, typename T>
36bool pmis_blocked (CsrView<TC const,Index> const& csr, Long row, T wi, Long ig,
37 T const* AMREX_RESTRICT w, int const* AMREX_RESTRICT state,
38 Long const* AMREX_RESTRICT gmap, Long goff, int undecided)
39{
40 for (Long idx = csr.row_offset[row]; idx < csr.row_offset[row+1]; ++idx) {
41 Long j = csr.col_index[idx];
42 Long jg = gmap ? gmap[j] : j + goff;
43 if (jg != ig && state[j] == undecided &&
44 (w[j] > wi || (w[j] == wi && jg > ig))) {
45 return true;
46 }
47 }
48 return false;
49}
50
53template <typename T>
55T pmis_tiebreak (Long ig)
56{
57 auto z = static_cast<std::uint64_t>(ig) + 0x9E3779B97F4A7C15ULL;
58 z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
59 z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
60 z ^= (z >> 31);
61 constexpr int digits = std::numeric_limits<T>::digits;
62 static_assert(digits < 64, "pmis_tiebreak: T must be float or double");
63 return T(z >> (64 - digits)) / T(std::uint64_t(1) << digits);
64}
65
67template <typename T, typename Index>
69bool csr_row_has_state (CsrView<T const,Index> const& csr, Long row,
70 int const* AMREX_RESTRICT state, int c)
71{
72 for (Long idx = csr.row_offset[row]; idx < csr.row_offset[row+1]; ++idx) {
73 if (state[csr.col_index[idx]] == c) { return true; }
74 }
75 return false;
76}
77
80template <typename T, typename TS>
81struct DirectInterpRow
82{
83 ParCsr<T const> a;
84 ParCsr<TS const> s;
85 int const* AMREX_RESTRICT state; // local rows
86 Long const* AMREX_RESTRICT cidx; // local coarse index, local rows
87 int const* AMREX_RESTRICT state_r; // A's remote columns
88 int nlocal_c; // number of local coarse columns
89 int coarse;
90
94 template <typename F>
95 AMREX_GPU_HOST_DEVICE void walk (Long i, T& diag, F const& f) const
96 {
97 diag = T(0);
98 {
99 auto const& ac = a.csr0;
100 auto const& sc = s.csr0;
101 Long sp = sc.row_offset[i];
102 Long const se = sc.row_offset[i+1];
103 for (Long idx = ac.row_offset[i]; idx < ac.row_offset[i+1]; ++idx) {
104 Long const j = ac.col_index[idx];
105 T const v = ac.mat[idx];
106 if (j == i) { diag = v; continue; }
107 while (sp < se && sc.col_index[sp] < j) { ++sp; }
108 bool const strong_c = (sp < se && sc.col_index[sp] == j
109 && state[j] == coarse);
110 f(int(cidx[j]), v, strong_c);
111 }
112 }
113 if (has_remote_row(a, i)) {
114 auto const& ac = a.csr1;
115 auto const& sc = s.csr1;
116 Long const ii = a.row_map[i];
117 Long sp = 0, se = 0;
118 if (has_remote_row(s, i)) {
119 Long const jj = s.row_map[i];
120 sp = sc.row_offset[jj];
121 se = sc.row_offset[jj+1];
122 }
123 for (Long idx = ac.row_offset[ii]; idx < ac.row_offset[ii+1]; ++idx) {
124 Long const j = ac.col_index[idx];
125 Long const jg = a.col_map[j];
126 T const v = ac.mat[idx];
127 while (sp < se && s.col_map[sc.col_index[sp]] < jg) { ++sp; }
128 bool const strong_c = (sp < se && s.col_map[sc.col_index[sp]] == jg
129 && state_r[j] == coarse);
130 f(int(nlocal_c + j), v, strong_c);
131 }
132 }
133 }
134};
135
140template <typename T, typename TS>
141struct MMInterpRow
142{
143 ParCsr<TS const> s;
144 ParCsr<TS const> st;
145 int const* AMREX_RESTRICT state;
146 T const* AMREX_RESTRICT dbeta;
147 Long const* AMREX_RESTRICT cidx; // local coarse index, local rows
148 int const* AMREX_RESTRICT state_r; // S's remote columns
149 T const* AMREX_RESTRICT dbeta_r;
150 int nlocal_f; // local fine columns
151 int nlocal_c; // local coarse columns
152
156 template <typename F>
157 AMREX_GPU_HOST_DEVICE void walk (Long i, F const& f) const
158 {
159 {
160 auto const& sc = s.csr0;
161 auto const& tc = st.csr0;
162 Long tp = tc.row_offset[i];
163 Long const te = tc.row_offset[i+1];
164 for (Long idx = sc.row_offset[i]; idx < sc.row_offset[i+1]; ++idx) {
165 Long const j = sc.col_index[idx];
166 while (tp < te && tc.col_index[tp] < j) { ++tp; }
167 T const aji = (tp < te && tc.col_index[tp] == j) ? T(tc.mat[tp]) : T(0);
168 f(int(j), int(cidx[j]), T(sc.mat[idx]), aji, state[j], dbeta[j]);
169 }
170 }
171 if (has_remote_row(s, i)) {
172 auto const& sc = s.csr1;
173 auto const& tc = st.csr1;
174 Long const ii = s.row_map[i];
175 Long tp = 0, te = 0;
176 if (has_remote_row(st, i)) {
177 Long const jj = st.row_map[i];
178 tp = tc.row_offset[jj];
179 te = tc.row_offset[jj+1];
180 }
181 for (Long idx = sc.row_offset[ii]; idx < sc.row_offset[ii+1]; ++idx) {
182 Long const j = sc.col_index[idx];
183 Long const jg = s.col_map[j];
184 while (tp < te && st.col_map[tc.col_index[tp]] < jg) { ++tp; }
185 T const aji = (tp < te && st.col_map[tc.col_index[tp]] == jg)
186 ? T(tc.mat[tp]) : T(0);
187 f(int(nlocal_f + j), int(nlocal_c + j), T(sc.mat[idx]), aji,
188 state_r[j], dbeta_r[j]);
189 }
190 }
191 }
192};
193
194}
195
196
201
207
211
252template <typename T>
253class AlgMG
254{
255public:
258 using TS = std::conditional_t<std::is_same_v<T,double>, float, T>;
259
264
265 AlgMG () = default;
266 explicit AlgMG (SpMatrix<T>& a_mat);
267
269 void define (SpMatrix<T>& a_mat);
270
286
288 void solve (AlgVector<T>& a_sol, AlgVector<T> const& a_rhs);
289
290 void setVerbose (int v) { m_verbose = v; }
292 void setPrintIndentation (std::string s) { m_print_ident = std::move(s); }
293 void setMaxIter (int n) { m_maxiter = n; }
295 void setFixedIter (int n) { m_fixediter = n; }
296 void setRelTol (T t) { m_reltol = t; }
298 void setAbsTol (T t) { m_abstol = t; }
300 void setSmoother (Smoother sm) { if (m_smoother_type != sm) { m_smoother_type = sm; m_smoothers_stale = true; } }
302 void setRelaxWeight (T w) { if (m_relax_weight != w) { m_relax_weight = w; m_smoothers_stale = true; } }
304 void setChebyshevDegree (int d) { if (m_cheby_degree != d) { m_cheby_degree = d; m_smoothers_stale = true; } }
306 void setChebyshevRatio (T r) {
307 AMREX_ALWAYS_ASSERT_WITH_MESSAGE(r > T(1), "AlgMG::setChebyshevRatio: the ratio must exceed 1");
308 if (m_cheby_ratio != r) { m_cheby_ratio = r; m_smoothers_stale = true; }
309 }
310
311 void setPreSmooth (int nu) { m_nu1 = nu; }
313 void setPostSmooth (int nu) { m_nu2 = nu; }
316 void setBottomSmooth (int nu) { m_nu_bottom = nu; }
321 void setStrongThreshold (T theta) { if (m_strong_threshold != theta) { m_strong_threshold = theta; m_prepared = false; } }
324 void setMaxLevels (int n) { if (m_max_levels != n) { m_max_levels = n; m_prepared = false; } }
326 void setMaxCoarseSize (Long n) { if (m_max_coarse_size != n) { m_max_coarse_size = n; m_prepared = false; } }
328 void setAggressiveNumLevels (int n) { if (m_aggressive_num_levels != n) { m_aggressive_num_levels = n; m_prepared = false; } }
331 void setAggressiveDirectInterp (bool b) { if (m_aggressive_direct_interp != b) { m_aggressive_direct_interp = b; m_prepared = false; } }
336 void setSingular (bool b) {
337 if (m_singular != b) { m_singular = b; m_prepared = false; }
338 }
339
340 void setInterpType (InterpType it) { if (m_interp_type != it) { m_interp_type = it; m_prepared = false; } }
343 void setPMaxElmts (int n) {
344 AMREX_ALWAYS_ASSERT_WITH_MESSAGE(n <= max_p_elmts, "AlgMG::setPMaxElmts: at most 32");
345 int const p = (n < 0) ? default_p_max_elmts : n;
346 if (m_p_max_elmts != p) { m_p_max_elmts = p; m_prepared = false; }
347 }
348 static constexpr int max_p_elmts = 32;
349 static constexpr int default_p_max_elmts = 4;
351 void setTruncFactor (T f) { if (m_trunc_factor != f) { m_trunc_factor = f; m_prepared = false; } }
354 if (m_krylov == a) { return; }
355 m_krylov = a;
356 m_krylov_bicgstab.reset();
357 m_krylov_gmres.reset();
358 m_krylov_pcg.reset();
359 }
360
367 if (m_bottom_solver == bs) { return; }
368 m_bottom_solver = bs;
369 m_gmres.reset();
370 m_bicgstab.reset();
371 m_bottom_n = -1;
372 }
373 void setBottomTol (T t) { m_bottom_reltol = t; }
374 void setBottomMaxIter (int n) { m_bottom_maxiter = n; }
375 void setBottomVerbose (int v) { m_bottom_verbose = v; }
376 void setThrowException (bool b) { m_throw_exception = b; }
377
378 [[nodiscard]] int getNumIters () const { return m_niters; }
380 [[nodiscard]] int getNumSetups () const { return m_num_setups; }
381 [[nodiscard]] T getResidualNorm () const { return m_rnorm; }
383 [[nodiscard]] int numLevels () const { return int(m_a.size()); }
385 [[nodiscard]] std::size_t getSetupPeakBytes () const { return m_setup_peak_bytes; }
387 [[nodiscard]] Long numGlobalRows (int lev) const { return m_a[lev]->numGlobalRows(); }
388 [[nodiscard]] SpMatrix<T> const& getMatrix (int lev) const { return *m_a[lev]; }
389 [[nodiscard]] SpMatrix<T> const& getInterp (int lev) const { return m_P[lev]; }
390
391private:
392
393 enum PointType : int { undecided = 0, fine = 1, coarse = 2 };
394
395 bool m_prepared = false;
396 int m_num_setups = 0;
397 bool m_smoothers_stale = false;
398 bool m_throw_exception = false;
399 int m_verbose = 0;
400 int m_maxiter = 200;
401 int m_fixediter = 0;
402 int m_niters = 0;
403
404 Smoother m_smoother_type = Smoother::chebyshev;
405 T m_relax_weight = T(-1);
406 int m_cheby_degree = 2;
407 T m_cheby_ratio = T(6);
408 int m_nu1 = -1;
409 int m_nu2 = -1;
410 int m_nu_bottom = -1;
411
412 [[nodiscard]] int numPreSmooth () const {
413 return (m_nu1 >= 0) ? m_nu1 : ((m_smoother_type == Smoother::chebyshev) ? 1 : 2);
414 }
415 [[nodiscard]] int numPostSmooth () const {
416 return (m_nu2 >= 0) ? m_nu2 : ((m_smoother_type == Smoother::chebyshev) ? 1 : 2);
417 }
418 [[nodiscard]] int numBottomSmooth () const {
419 return (m_nu_bottom >= 0) ? m_nu_bottom : ((m_smoother_type == Smoother::chebyshev) ? 4 : 8);
420 }
421
422 int m_max_levels = 25;
423 Long m_max_coarse_size = 9;
424 int m_aggressive_num_levels = 0;
425 bool m_aggressive_direct_interp = false;
426 bool m_singular = false;
427 std::unique_ptr<AlgVector<T>> m_bproj; // rhs with the mean removed
428 T m_strong_threshold = T(0.25);
429
430 InterpType m_interp_type = InterpType::mm_ext_i;
431 int m_p_max_elmts = default_p_max_elmts;
432 T m_trunc_factor = T(0);
433 BottomSolver m_bottom_solver = BottomSolver::direct;
434 // Safety limit for the dense LU on every process; the coarsest level
435 // normally has a few rows (max_coarse_size). Beyond it smoother sweeps
436 // are used, which keeps the cycle linear for PCG and GMRES.
437 static constexpr Long max_direct_size = 1024;
438 Long m_bottom_n = -1; // rows of the factored coarsest matrix; -1: not built, 0: not applicable
439 Vector<double> m_bottom_lu; // dense LU, row-major, pivoted
440 Vector<int> m_bottom_piv;
441 T m_bottom_reltol = T(1.e-4);
442 int m_bottom_maxiter = 200;
443 int m_bottom_verbose = 0;
444
445 T m_reltol = std::is_same_v<T,float> ? T(1.e-4) : T(1.e-8);
446 T m_abstol = T(0);
447 T m_init_rnorm = std::numeric_limits<T>::max();
448 T m_rnorm = std::numeric_limits<T>::max();
449 T m_bnorm = std::numeric_limits<T>::max();
450
451 std::string m_print_ident;
452 std::size_t m_setup_peak_bytes = 0;
453
455 void note_memory (int lev, char const* stage)
456 {
457 auto const [bytes, reserved] = detail::arena_bytes();
458 m_setup_peak_bytes = std::max(m_setup_peak_bytes, bytes);
459 if (m_verbose >= 2 && bytes > 0) {
460 amrex::Print() << m_print_ident << "AlgMG: level " << lev << ": after " << stage
461 << ": arena in use " << double(bytes)/double(1<<20)
462 << " MB, reserved " << double(reserved)/double(1<<20) << " MB"
463#ifdef AMREX_USE_GPU
464 << ", device free " << double(Gpu::Device::freeMemAvailable())/double(1<<20) << " MB"
465#endif
466 << "\n";
467 }
468 }
469
470 SpMatrix<T>* m_mat = nullptr;
471 Vector<SpMatrix<T>*> m_a; // matrix on each level
472 Vector<std::unique_ptr<SpMatrix<T>>> m_crsemat; // owns levels 1..n-1
473 Vector<SpMatrix<T>> m_P; // m_P[lev] interpolates lev+1 -> lev
474 Vector<SpMatrix<T>> m_R; // m_R[lev] restricts lev -> lev+1
475
476 Vector<AlgVector<T>> m_res; // rhs - A(sol)
477 Vector<AlgVector<T>> m_cor; // cor = A^-1 (res)
478 Vector<AlgVector<T>> m_rescor; // res - A(cor)
479
480 Vector<std::unique_ptr<JacobiSmoother<T>>> m_smoother;
481 Vector<std::unique_ptr<ChebyshevSmoother<T>>> m_cheby;
482 Vector<std::unique_ptr<L1GaussSeidelSmoother<T>>> m_l1gs;
483 std::unique_ptr<GMRES_MV<T>> m_gmres;
484 std::unique_ptr<BiCGStab_MV<T>> m_bicgstab;
485
486 KrylovSolver m_krylov = KrylovSolver::none;
487 std::unique_ptr<BiCGStab_MV<T>> m_krylov_bicgstab;
488 std::unique_ptr<GMRES_MV<T>> m_krylov_gmres;
489 std::unique_ptr<PCG_MV<T>> m_krylov_pcg;
490
491 void clear_levels ();
492 void build_smoothers ();
494 void build_direct_bottom ();
495 void direct_bottom_solve ();
496 // In-place all-gather of a row-distributed dense array (stride doubles
497 // per row): each rank's rows already sit at their global offset in buf.
498 static void allgather_rows (double* buf, AlgPartition const& part, Long stride);
500 void prepare ()
501 {
502 if (!m_prepared) { setup(); }
503 else if (m_smoothers_stale) { build_smoothers(); }
504 }
506 void vcycle (AlgVector<T>& x, AlgVector<T> const& b);
507 void blew_up () const;
508 void solve_krylov (AlgVector<T>& a_sol, AlgVector<T> const& a_rhs, T res_target);
509 void smooth (int lev, AlgVector<T>& xvec, AlgVector<T> const& bvec, int nu,
510 bool with_initial_guess, bool backward = false);
511 void bottom_solve ();
512
513 static std::pair<T,T> compute_rbnorms (SpMatrix<T> const& mat,
514 AlgVector<T> const& xvec,
515 AlgVector<T> const& bvec,
516 AlgVector<T>& rvec);
517
518 static T compute_rnorm (SpMatrix<T> const& mat, AlgVector<T> const& xvec,
519 AlgVector<T> const& bvec, AlgVector<T>& rvec);
520
521//Public for CUDA
522public: // NOLINT
523
524 void setup ();
525
531 bool coarsen_level (int lev, SpMatrix<T>& A, SpMatrix<T>& P, AlgPartition& cpart,
532 bool& aggressive, Long& ncoarse, bool& truncated,
533 Long& nnz_untruncated);
534
537 [[nodiscard]] SpMatrix<TS> create_soc (SpMatrix<T> const& A, T scale) const;
538
540 static T strength_scale (SpMatrix<T> const& A);
541
545 bool isolated_as_coarse = false) const;
546
550 int which = PointType::coarse);
551
555 {
556 if (!m_singular) { return; }
557 T const mean = x.sum() / T(x.partition().numGlobalRows());
558 auto* AMREX_RESTRICT px = x.data();
559 ParallelForOMP(x.numLocalRows(), [=] AMREX_GPU_DEVICE (Long i) noexcept { px[i] -= mean; });
561 }
562
565 Gpu::DeviceVector<int> const& state,
566 Gpu::DeviceVector<Long> const& cidx,
567 AlgPartition const& cpart);
568
573 int max_elmts, T trunc_factor);
575 static void normalize_rows (SpMatrix<T>& P);
576
580 Gpu::DeviceVector<int> const& state,
581 Gpu::DeviceVector<Long> const& cidx,
582 AlgPartition const& cpart);
583
586 Gpu::DeviceVector<int> const& state,
587 Gpu::DeviceVector<Long> const& cidx,
588 AlgPartition const& cpart);
589
592 Gpu::DeviceVector<int> const& state,
593 Gpu::DeviceVector<Long> const& cidx,
594 AlgPartition const& cpart, T scale) const;
595
603 SpMatrix<TS> const& ST,
604 Gpu::DeviceVector<int> const& state,
605 Gpu::DeviceVector<Long> const& cidx,
606 AlgPartition const& cpart, T scale,
607 bool plus_i, int const* row_state = nullptr,
608 int max_elmts = 0, T trunc_factor = T(0),
609 Long* nnz_untruncated = nullptr);
610
611#ifndef AMREX_USE_GPU
615 static SpMatrix<T> build_interp_mm_host (SpMatrix<T>& A, SpMatrix<TS>& S,
616 SpMatrix<TS> const& ST,
617 Gpu::DeviceVector<int> const& state,
618 Gpu::DeviceVector<Long> const& cidx,
619 AlgPartition const& cpart, T scale, bool plus_i,
620 int max_elmts, T trunc_factor,
621 Long& nnz_untruncated);
622#endif
623};
624
625template <typename T>
627{
628 define(a_mat);
629}
630
631template <typename T>
633{
634 m_mat = &a_mat;
635 m_prepared = false;
636 clear_levels();
637 m_bproj.reset();
638 m_krylov_bicgstab.reset();
639 m_krylov_gmres.reset();
640 m_krylov_pcg.reset();
641}
642
643template <typename T>
645{
646 if (!m_prepared || m_a.empty()) {
647 define(a_mat);
648 return false;
649 }
650 AMREX_ALWAYS_ASSERT_WITH_MESSAGE(a_mat.partition() == m_mat->partition(),
651 "AlgMG::updateMatrix: the matrix must have the same row partition");
652 m_mat = &a_mat;
653 m_mat->setColumnPartition(m_mat->partition()); // as in setup
654 m_a[0] = m_mat;
655 // The Krylov solvers hold the matrix
656 m_krylov_bicgstab.reset();
657 m_krylov_gmres.reset();
658 m_krylov_pcg.reset();
659 if (numLevels() == 1) { // the finest level is the bottom
660 m_bottom_n = -1;
661 m_bottom_lu.clear();
662 m_bottom_piv.clear();
663 }
664 m_smoothers_stale = true;
665 return true;
666}
667
668template <typename T>
669void AlgMG<T>::clear_levels ()
670{
671 m_a.clear();
672 m_crsemat.clear();
673 m_P.clear();
674 m_R.clear();
675 m_res.clear();
676 m_cor.clear();
677 m_rescor.clear();
678 m_smoother.clear();
679 m_cheby.clear();
680 m_l1gs.clear();
681 m_gmres.reset();
682 m_bicgstab.reset();
683 m_bottom_n = -1;
684 m_bottom_lu.clear();
685 m_bottom_piv.clear();
686}
687
688template <typename T>
689void AlgMG<T>::solve (AlgVector<T>& a_sol, AlgVector<T> const& a_rhs)
690{
691 BL_PROFILE("AlgMG::solve");
692
693 prepare();
694
695 // Singular: iterate on the rhs with its mean removed.
696 AlgVector<T> const* prhs = &a_rhs;
697 if (m_singular) {
698 if (!m_bproj) { m_bproj = std::make_unique<AlgVector<T>>(m_mat->partition()); }
699 m_bproj->copy(a_rhs);
700 project_out(*m_bproj);
701 prhs = m_bproj.get();
702 }
703 AlgVector<T> const& rhs = *prhs;
704
705 auto norms = compute_rbnorms(*m_mat, a_sol, rhs, m_res[0]);
706 m_bnorm = norms.first;
707 m_init_rnorm = m_rnorm = norms.second;
708 m_niters = 0;
709
710 if (m_bnorm == 0) {
711 a_sol.setVal(0); // also the zero-mean solution of a singular system
712 m_rnorm = 0;
713 return;
714 }
715
716 if (m_verbose >= 1) {
717 amrex::Print() << m_print_ident << "AlgMG: Initial rhs 2-norm = " << m_bnorm << "\n"
718 << m_print_ident << "AlgMG: Initial residual 2-norm = " << m_rnorm << "\n";
719 }
720
721 T res_target = std::max(m_bnorm * m_reltol, m_abstol);
722 if (m_fixediter <= 0 && m_rnorm <= res_target) {
723 if (m_verbose >= 1) {
724 amrex::Print() << m_print_ident << "AlgMG: No iterations needed\n";
725 }
726 project_out(a_sol);
727 return;
728 }
729
730 if (m_krylov != KrylovSolver::none) {
731 solve_krylov(a_sol, rhs, res_target);
732 project_out(a_sol);
733 return;
734 }
735
736 int const niters = (m_fixediter > 0) ? m_fixediter : m_maxiter;
737 for (int iter = 0; iter < niters; ++iter)
738 {
739 vcycle(m_cor[0], m_res[0]);
740 a_sol.plusAsync(m_cor[0]);
741
742 m_rnorm = compute_rnorm(*m_mat, a_sol, rhs, m_res[0]);
743 m_niters = iter+1;
744
745 if (m_verbose >= 2) {
746 amrex::Print() << m_print_ident << "AlgMG: Iteration " << std::setw(3)
747 << iter+1 << ": rnorm, rnorm/bnorm = "
748 << m_rnorm << ", " << m_rnorm/m_bnorm << "\n";
749 }
750
751 bool converged = (m_fixediter <= 0) && (m_rnorm <= res_target);
752 if (converged) {
753 if (m_verbose >= 1) {
754 amrex::Print() << m_print_ident << "AlgMG: Final Iter. " << iter+1
755 << ": rnorm, rnorm/bnorm = "
756 << m_rnorm << ", " << m_rnorm/m_bnorm << "\n";
757 }
758 break;
759 } else if (m_rnorm > T(1.e20) * m_init_rnorm || amrex::isnan(m_rnorm)) {
760 if (m_verbose > 0) {
761 amrex::Print() << m_print_ident << "AlgMG: Failing to converge after " << iter+1 << " iterations."
762 << " rnorm, rnorm/bnorm = "
763 << m_rnorm << ", " << m_rnorm/m_bnorm << "\n";
764 }
765 blew_up();
766 }
767 }
768
769 if (m_verbose >= 1 && m_fixediter <= 0 && m_rnorm > res_target) {
770 amrex::Print() << m_print_ident << "AlgMG: Not converged after " << m_niters
771 << " iterations. rnorm/bnorm = " << m_rnorm/m_bnorm << "\n";
772 }
773 project_out(a_sol);
774}
775
776template <typename T>
778{
779 prepare();
780 vcycle(x, b);
781 project_out(x);
782}
783
784template <typename T>
785void AlgMG<T>::blew_up () const
786{
787 if (m_throw_exception) {
788 throw std::runtime_error("AlgMG blew up.");
789 } else {
790 amrex::Abort("AlgMG failing so lets stop here");
791 }
792}
793
794template <typename T>
795void AlgMG<T>::solve_krylov (AlgVector<T>& a_sol, AlgVector<T> const& a_rhs, T res_target)
796{
797 BL_PROFILE("AlgMG::solve_krylov");
798
799 // Bound on every solve so that a moved AlgMG object stays usable.
800 auto pc = [this] (AlgVector<T>& x, AlgVector<T> const& b) { this->precond(x, b); };
801 int const niters = (m_fixediter > 0) ? m_fixediter : m_maxiter;
802 T const atol = (m_fixediter > 0) ? T(0) : res_target;
803 int const verbose = std::max(0, m_verbose-1);
804 T krylov_rnorm = T(0);
805
806 // Converge on the absolute target reltol*|b|, as the V-cycle iteration.
807 // Restart if the recurrence residual converged before the true one.
808 auto run = [&] (auto& s) {
809 s.setVerbose(verbose);
810 s.setInitialGuessNonzero(true);
811 m_niters = 0;
812 while (true) {
813 s.setMaxIters(niters - m_niters);
814 s.solve(a_sol, a_rhs, T(0), atol);
815 m_niters += s.getNumIters();
816 krylov_rnorm = s.getResidualNorm();
817 m_rnorm = compute_rnorm(*m_mat, a_sol, a_rhs, m_res[0]);
818 if (m_fixediter > 0 || m_rnorm <= res_target || krylov_rnorm > res_target ||
819 m_niters >= niters || s.getNumIters() == 0 || amrex::isnan(m_rnorm)) {
820 break;
821 }
822 if (m_verbose >= 2) {
823 amrex::Print() << m_print_ident << "AlgMG: restarting Krylov solver at iteration "
824 << m_niters << ", rnorm/bnorm = " << m_rnorm/m_bnorm << "\n";
825 }
826 }
827 };
828
829 char const* name = "";
830 if (m_krylov == KrylovSolver::bicgstab) {
831 name = "BiCGStab";
832 if (!m_krylov_bicgstab) {
833 m_krylov_bicgstab = std::make_unique<BiCGStab_MV<T>>(m_mat);
834 }
835 m_krylov_bicgstab->setPrecond(pc);
836 run(m_krylov_bicgstab->getSolver());
837 } else if (m_krylov == KrylovSolver::gmres) {
838 name = "GMRES";
839 if (!m_krylov_gmres) {
840 m_krylov_gmres = std::make_unique<GMRES_MV<T>>(m_mat);
841 }
842 m_krylov_gmres->setPrecond(pc);
843 run(m_krylov_gmres->getSolver());
844 } else {
845 name = "PCG";
846 if (!m_krylov_pcg) {
847 m_krylov_pcg = std::make_unique<PCG_MV<T>>(m_mat);
848 }
849 m_krylov_pcg->setPrecond(pc);
850 run(m_krylov_pcg->getSolver());
851 }
852
853 if (amrex::isnan(m_rnorm) || m_rnorm > T(1.e20) * m_init_rnorm) { blew_up(); }
854
855 if (m_verbose >= 1) {
856 amrex::Print() << m_print_ident << "AlgMG(" << name << "): Final Iter. " << m_niters
857 << ": rnorm, rnorm/bnorm = " << m_rnorm << ", " << m_rnorm/m_bnorm;
858 if (m_verbose >= 2) {
859 amrex::Print() << " (Krylov recurrence " << krylov_rnorm << ")";
860 }
861 amrex::Print() << "\n";
862 if (m_fixediter <= 0 && m_rnorm > res_target) {
863 amrex::Print() << m_print_ident << "AlgMG(" << name << "): Not converged after "
864 << m_niters << " iterations.\n";
865 }
866 }
867}
868
869template <typename T>
870std::pair<T,T> AlgMG<T>::compute_rbnorms (SpMatrix<T> const& mat,
871 AlgVector<T> const& xvec,
872 AlgVector<T> const& bvec,
873 AlgVector<T>& rvec)
874{
875 auto bnorm = bvec.norm2(true);
876
877 amrex::computeResidual(rvec, mat, xvec, bvec);
878 auto rnorm = rvec.norm2(true);
879
880 if (ParallelContext::NProcsSub() > 1) {
882 rnorm = Math::powi<2>(rnorm);
884 bnorm = std::sqrt(bnorm);
885 rnorm = std::sqrt(rnorm);
886 }
887
888 return {bnorm, rnorm};
889}
890
891template <typename T>
892T AlgMG<T>::compute_rnorm (SpMatrix<T> const& mat, AlgVector<T> const& xvec,
893 AlgVector<T> const& bvec, AlgVector<T>& rvec)
894{
895 amrex::computeResidual(rvec, mat, xvec, bvec);
896 return rvec.norm2();
897}
898
899template <typename T>
900void AlgMG<T>::vcycle (AlgVector<T>& x, AlgVector<T> const& b)
901{
902 BL_PROFILE("AlgMG::vcycle");
903
904 int const nlev = numLevels();
905 int const bottom = nlev-1;
906
907 if (nlev == 1) { // bottom_solve works on the level vectors
908 if (&b != &m_res[0]) { m_res[0].copyAsync(b); }
909 bottom_solve();
910 if (&x != &m_cor[0]) { x.copyAsync(m_cor[0]); }
911 return;
912 }
913
914 // Level 0 uses the caller's vectors, the coarse levels their own.
915 auto cor = [&] (int lev) -> AlgVector<T>& { return (lev == 0) ? x : m_cor[lev]; };
916 auto res = [&] (int lev) -> AlgVector<T> const& { return (lev == 0) ? b : m_res[lev]; };
917
918 smooth(0, x, b, numPreSmooth(), false);
919
920 for (int lev = 0; lev < bottom; ++lev) {
921 computeResidual(m_rescor[lev], *m_a[lev], cor(lev), res(lev));
922 SpMV(m_res[lev+1], m_R[lev], m_rescor[lev]);
923 if (lev+1 == bottom) {
924 bottom_solve();
925 } else {
926 smooth(lev+1, m_cor[lev+1], m_res[lev+1], numPreSmooth(), false);
927 }
928 }
929
930 for (int lev = bottom-1; lev >= 0; --lev) {
931 SpMV(m_rescor[lev], m_P[lev], m_cor[lev+1]);
932 cor(lev).plusAsync(m_rescor[lev]);
933 smooth(lev, cor(lev), res(lev), numPostSmooth(), true, true);
934 }
935}
936
937template <typename T>
938void AlgMG<T>::smooth (int lev, AlgVector<T>& xvec, AlgVector<T> const& bvec, int nu,
939 bool with_initial_guess, bool backward)
940{
941 if (m_smoother_type == Smoother::l1_gauss_seidel) {
942 auto& smoother = *m_l1gs[lev];
943 smoother.setNumIters(nu);
944 smoother(xvec, bvec, with_initial_guess, backward);
945 } else if (m_smoother_type == Smoother::chebyshev) {
946 auto& smoother = *m_cheby[lev];
947 smoother.setNumIters(nu);
948 smoother(xvec, bvec, with_initial_guess);
949 } else {
950 auto& smoother = *m_smoother[lev];
951 smoother.setNumIters(nu);
952 smoother(xvec, bvec, with_initial_guess);
953 }
954}
955
956template <typename T>
957void AlgMG<T>::bottom_solve ()
958{
959 BL_PROFILE("AlgMG::bottom_solve");
960
961 int const lev = numLevels()-1;
962 project_out(m_res[lev]); // keep the coarsest solve consistent
963 auto smooth_bottom = [&] () {
964 if (m_smoother_type == Smoother::l1_gauss_seidel) {
965 // Forward then backward sweeps keep the cycle symmetric.
966 int const nu = (numBottomSmooth()+1)/2;
967 smooth(lev, m_cor[lev], m_res[lev], nu, false);
968 smooth(lev, m_cor[lev], m_res[lev], nu, true, true);
969 } else {
970 smooth(lev, m_cor[lev], m_res[lev], numBottomSmooth(), false);
971 }
972 };
973 // As in MLMG, a Krylov solve that does not reduce the residual is
974 // discarded; its iterate can be far worse than no correction. The
975 // solver starts from zero unless told otherwise.
976 auto krylov_bottom = [&] (auto& krylov) {
977 krylov.setVerbose(m_bottom_verbose);
978 krylov.getSolver().setMaxIters(m_bottom_maxiter);
979 krylov.solve(m_cor[lev], m_res[lev], m_bottom_reltol, T(0));
980 auto const& solver = krylov.getSolver();
981 T const rnorm0 = solver.getInitialResidualNorm();
982 if (rnorm0 > T(0) && !(solver.getResidualNorm() < rnorm0)) {
983 if (m_verbose >= 2) {
984 amrex::Print() << m_print_ident << "AlgMG: bottom solve failed, smoothing instead\n";
985 }
986 smooth_bottom();
987 }
988 };
989 auto bottom_solver = m_bottom_solver;
990 if (bottom_solver == BottomSolver::direct) {
991 if (m_bottom_n < 0) { build_direct_bottom(); }
992 if (m_bottom_n == 0) { bottom_solver = BottomSolver::jacobi; }
993 }
994 switch (bottom_solver) {
995 case BottomSolver::direct:
996 direct_bottom_solve();
997 break;
998 case BottomSolver::jacobi:
999 smooth_bottom();
1000 break;
1001 case BottomSolver::gmres:
1002 if (!m_gmres) {
1003 m_gmres = std::make_unique<GMRES_MV<T>>(m_a[lev]);
1004 m_gmres->setPrecond(JacobiSmoother<T>(m_a[lev], m_smoother_type != Smoother::jacobi));
1005 }
1006 krylov_bottom(*m_gmres);
1007 break;
1009 if (!m_bicgstab) {
1010 m_bicgstab = std::make_unique<BiCGStab_MV<T>>(m_a[lev]);
1011 m_bicgstab->setPrecond(JacobiSmoother<T>(m_a[lev], m_smoother_type != Smoother::jacobi));
1012 }
1013 krylov_bottom(*m_bicgstab);
1014 break;
1015 }
1016 project_out(m_cor[lev]);
1017}
1018
1019template <typename T>
1021{
1022 BL_PROFILE("AlgMG::setup");
1023
1024 ++m_num_setups;
1025 clear_levels();
1026 m_a.push_back(m_mat);
1027 m_mat->setColumnPartition(m_mat->partition()); // square: local column indexing
1028 m_setup_peak_bytes = 0;
1029 note_memory(0, "start");
1030
1031 // The coarsening follows De Sterck, Yang & Heys [2].
1032
1033 for (int lev = 0; lev+1 < m_max_levels; ++lev)
1034 {
1035 auto& A = *m_a[lev];
1036 if (A.numGlobalRows() <= m_max_coarse_size) { break; }
1037
1038 SpMatrix<T> P;
1039 AlgPartition cpart;
1040 bool aggressive = false;
1041 Long ncoarse = 0;
1042 bool truncated = false;
1043 Long nnz0 = 0;
1044 if (!coarsen_level(lev, A, P, cpart, aggressive, ncoarse, truncated, nnz0)) { break; }
1045 note_memory(lev, "strength release");
1046 if (m_p_max_elmts > 0 || m_trunc_factor > T(0)) {
1047 if (!truncated) {
1048 nnz0 = P.numLocalNonZeros();
1049 P = truncate_interp(P, cpart, m_p_max_elmts, m_trunc_factor);
1050 note_memory(lev, "truncation");
1051 }
1052 if (m_verbose >= 2) {
1053 Long nnz[2] = {nnz0, P.numLocalNonZeros()};
1055 amrex::Print() << m_print_ident << "AlgMG: level " << lev
1056 << ": P truncated from " << nnz[0] << " to "
1057 << nnz[1] << " nonzeros\n";
1058 }
1059 }
1060 // P 1 = 1 only to single precision, since the weights come from the
1061 // strength matrix; singular problems need it exactly.
1062 if (m_singular) { normalize_rows(P); }
1063 if (m_verbose >= 2 && aggressive) {
1064 amrex::Print() << m_print_ident << "AlgMG: level " << lev
1065 << ": aggressive coarsening, " << ncoarse << " -> "
1066 << cpart.numGlobalRows() << " C-points\n";
1067 }
1068 if (m_verbose >= 3) {
1069 // P should reproduce constants: row sums close to one.
1070 AlgVector<T> ones(cpart), rs(A.partition());
1071 ones.setVal(T(1));
1072 SpMV(rs, P, ones);
1073 Long nzero = Reduce::Sum<Long>(rs.numLocalRows(),
1074 [q = rs.data()] AMREX_GPU_DEVICE (Long i) -> Long { return q[i] == T(0); });
1075 T rmin = Reduce::Min<T>(rs.numLocalRows(),
1076 [q = rs.data()] AMREX_GPU_DEVICE (Long i) -> T { return q[i]; });
1077 T rmax = Reduce::Max<T>(rs.numLocalRows(),
1078 [q = rs.data()] AMREX_GPU_DEVICE (Long i) -> T { return q[i]; });
1082 amrex::Print() << m_print_ident << "AlgMG: level " << lev << ": P row sums in ["
1083 << rmin << ", " << rmax << "], " << nzero << " empty rows\n";
1084 }
1085 auto R = amrex::transpose(P, cpart);
1086 note_memory(lev, "R");
1087 SpMatrix<T> Ac;
1088#ifndef AMREX_USE_GPU
1089 Ac = amrex::RAP(R, A, P, cpart);
1090#else
1091 {
1092 // R (A P): measured smaller and faster than (R A) P, whose
1093 // intermediate gathers several fine rows per coarse row.
1094 auto AP = amrex::SpGEMM(A, P, cpart);
1095 note_memory(lev, "A*P");
1096 Ac = amrex::SpGEMM(R, AP, cpart);
1097 }
1098#endif
1099 note_memory(lev, "coarse operator");
1100
1101 m_P.push_back(std::move(P));
1102 m_R.push_back(std::move(R));
1103 m_crsemat.push_back(std::make_unique<SpMatrix<T>>(std::move(Ac)));
1104 m_a.push_back(m_crsemat.back().get());
1105 }
1106
1107 int const nlev = numLevels();
1108 for (int lev = 0; lev < nlev; ++lev) {
1109 m_res.emplace_back(m_a[lev]->partition());
1110 m_cor.emplace_back(m_a[lev]->partition());
1111 if (lev < nlev-1) {
1112 m_rescor.emplace_back(m_a[lev]->partition());
1113 }
1114 }
1115 build_smoothers();
1116
1117 if (m_verbose >= 1) {
1118 Long nnz0 = 0, nnz_total = 0;
1119 for (int lev = 0; lev < nlev; ++lev) {
1120 Long nnz = m_a[lev]->numLocalNonZeros();
1122 if (lev == 0) { nnz0 = nnz; }
1123 nnz_total += nnz;
1124 amrex::Print() << m_print_ident << "AlgMG: level " << lev << ": "
1125 << m_a[lev]->numGlobalRows() << " rows, "
1126 << nnz << " nonzeros\n";
1127 }
1128 amrex::Print() << m_print_ident << "AlgMG: operator complexity = "
1129 << double(nnz_total)/double(std::max(nnz0,Long(1))) << "\n";
1130 if (m_setup_peak_bytes > 0) {
1131 auto peak = m_setup_peak_bytes;
1133 amrex::Print() << m_print_ident << "AlgMG: setup peak arena use = "
1134 << double(peak)/double(1<<20) << " MB ("
1135 << double(peak)/double(m_mat->numGlobalRows())
1136 << " B per row)\n";
1137 }
1138 }
1139
1140 m_prepared = true;
1141}
1142
1143template <typename T>
1144void AlgMG<T>::build_direct_bottom ()
1145{
1146 BL_PROFILE("AlgMG::build_direct_bottom");
1147 int const lev = numLevels()-1;
1148 auto const& A = *m_a[lev];
1149 Long const n = A.numGlobalRows();
1150 m_bottom_n = 0;
1151 m_bottom_lu.clear();
1152 m_bottom_piv.clear();
1153 if (n < 1 || n > max_direct_size) {
1154 if (m_verbose >= 1 && n > max_direct_size) {
1155 amrex::Print() << m_print_ident << "AlgMG: coarsest level has " << n
1156 << " rows, using smoother sweeps instead of the direct bottom solver\n";
1157 }
1158 return;
1159 }
1160
1161 // Dense copy of the local rows at their global offsets; gathered below.
1162 Vector<double> dense(n*n, 0.0);
1163 auto const pc = A.const_parcsr();
1164 auto to_host = [] (auto const* dptr, Long cnt) {
1165 Vector<std::remove_const_t<std::remove_pointer_t<decltype(dptr)>>> h(cnt);
1166 if (cnt > 0) { Gpu::copy(Gpu::deviceToHost, dptr, dptr+cnt, h.begin()); }
1167 return h;
1168 };
1169 {
1170 auto const& c = pc.csr0;
1171 auto ro = to_host(c.row_offset, (c.nrows > 0) ? c.nrows+1 : 0);
1172 auto ci = to_host(c.col_index, c.nnz);
1173 auto va = to_host(c.mat, c.nnz);
1174 for (Long i = 0; i < c.nrows; ++i) {
1175 Long const ig = pc.row_begin + i;
1176 for (int idx = ro[i]; idx < ro[i+1]; ++idx) {
1177 dense[ig*n + pc.col_begin + ci[idx]] += double(va[idx]);
1178 }
1179 }
1180 }
1181 if (pc.csr1.nrows > 0 && pc.csr1.nnz > 0) {
1182 auto const& c = pc.csr1;
1183 auto ro = to_host(c.row_offset, c.nrows+1);
1184 auto ci = to_host(c.col_index, c.nnz);
1185 auto va = to_host(c.mat, c.nnz);
1186 auto rmap = to_host(pc.row_map, pc.csr0.nrows); // local row -> csr1 row or -1
1187 int ncols = 0;
1188 for (auto j : ci) { ncols = std::max(ncols, j+1); }
1189 auto cmap = to_host(pc.col_map, ncols);
1190 for (Long i = 0; i < pc.csr0.nrows; ++i) {
1191 Long const r = rmap[i];
1192 if (r < 0) { continue; }
1193 Long const ig = pc.row_begin + i;
1194 for (int idx = ro[r]; idx < ro[r+1]; ++idx) {
1195 dense[ig*n + cmap[ci[idx]]] += double(va[idx]);
1196 }
1197 }
1198 }
1199 allgather_rows(dense.data(), A.partition(), n);
1200
1201 // Singular: the constant vector is in the null space. Adding (1/n) 1 1^T
1202 // makes the matrix regular; the coarsest residual and correction have
1203 // their mean removed, so the added term never acts on them.
1204 if (m_singular) {
1205 double dsum = 0.0;
1206 for (Long i = 0; i < n; ++i) { dsum += std::abs(dense[i*n+i]); }
1207 double const w = (dsum/double(n))/double(n);
1208 for (auto& v : dense) { v += w; }
1209 }
1210
1211 // LU with partial pivoting, in place (Doolittle). A pivot at roundoff
1212 // level relative to the matrix means a singular matrix.
1213 double amat = 0.0;
1214 for (auto v : dense) { amat = std::max(amat, std::abs(v)); }
1215 double const pivot_tol = double(n) * std::numeric_limits<double>::epsilon() * amat;
1216 Vector<int> piv(n);
1217 std::iota(piv.begin(), piv.end(), 0);
1218 for (Long i = 0; i < n; ++i) {
1219 Long imax = i;
1220 double amax = std::abs(dense[i*n+i]);
1221 for (Long k = i+1; k < n; ++k) {
1222 double const a = std::abs(dense[k*n+i]);
1223 if (a > amax) { amax = a; imax = k; }
1224 }
1225 if (amax <= pivot_tol) {
1226 if (m_verbose >= 1) {
1227 amrex::Print() << m_print_ident << "AlgMG: coarsest matrix is singular, "
1228 << "using smoother sweeps instead of the direct bottom solver\n";
1229 }
1230 return;
1231 }
1232 if (imax != i) {
1233 std::swap(piv[i], piv[imax]);
1234 for (Long j = 0; j < n; ++j) { std::swap(dense[i*n+j], dense[imax*n+j]); }
1235 }
1236 double const dinv = 1.0/dense[i*n+i];
1237 for (Long j = i+1; j < n; ++j) {
1238 double const f = dense[j*n+i]*dinv;
1239 dense[j*n+i] = f;
1240 for (Long k = i+1; k < n; ++k) { dense[j*n+k] -= f*dense[i*n+k]; }
1241 }
1242 }
1243 m_bottom_lu = std::move(dense);
1244 m_bottom_piv = std::move(piv);
1245 m_bottom_n = n;
1246}
1247
1248template <typename T>
1249void AlgMG<T>::allgather_rows (double* buf, AlgPartition const& part, Long stride)
1250{
1251#ifdef AMREX_USE_MPI
1252 int const nprocs = ParallelContext::NProcsSub();
1253 if (nprocs == 1) { return; }
1254 Vector<int> counts(nprocs), displs(nprocs);
1255 for (int p = 0; p < nprocs; ++p) {
1256 displs[p] = int(part[p]*stride);
1257 counts[p] = int((part[p+1]-part[p])*stride);
1258 }
1259 BL_MPI_REQUIRE(MPI_Allgatherv(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
1260 buf, counts.data(), displs.data(), MPI_DOUBLE,
1262#else
1263 amrex::ignore_unused(buf, part, stride);
1264#endif
1265}
1266
1267template <typename T>
1268void AlgMG<T>::direct_bottom_solve ()
1269{
1270 int const lev = numLevels()-1;
1271 auto const& r = m_res[lev];
1272 auto& x = m_cor[lev];
1273 Long const n = m_bottom_n;
1274 Long const nl = r.numLocalRows();
1275 Long const b0 = r.globalBegin();
1276
1277 // Gather the residual.
1278 PODVector<double> rhs(n); // uninitialized; every entry is written below
1279 Gpu::PinnedVector<T> loc(nl);
1280 if (nl > 0) {
1281 Gpu::copy(Gpu::deviceToHost, r.data(), r.data()+nl, loc.begin());
1282 for (Long i = 0; i < nl; ++i) { rhs[b0+i] = double(loc[i]); }
1283 }
1284 allgather_rows(rhs.data(), r.partition(), 1);
1285
1286 // P A = L U: forward substitution with the unit lower part, then back.
1287 Vector<double> y(n);
1288 auto const& lu = m_bottom_lu;
1289 for (Long i = 0; i < n; ++i) {
1290 double v = rhs[m_bottom_piv[i]];
1291 for (Long k = 0; k < i; ++k) { v -= lu[i*n+k]*y[k]; }
1292 y[i] = v;
1293 }
1294 for (Long i = n-1; i >= 0; --i) {
1295 double v = y[i];
1296 for (Long k = i+1; k < n; ++k) { v -= lu[i*n+k]*y[k]; }
1297 y[i] = v/lu[i*n+i];
1298 }
1299
1300 if (nl > 0) {
1301 for (Long i = 0; i < nl; ++i) { loc[i] = T(y[b0+i]); }
1302 Gpu::copy(Gpu::hostToDevice, loc.begin(), loc.end(), x.data());
1303 }
1304}
1305
1306template <typename T>
1307void AlgMG<T>::build_smoothers ()
1308{
1309 m_smoother.clear();
1310 m_cheby.clear();
1311 m_l1gs.clear();
1312 m_gmres.reset(); // their preconditioner depends on the smoother type
1313 m_bicgstab.reset();
1314 int const nlev = numLevels();
1315 for (int lev = 0; lev < nlev; ++lev) {
1316 if (m_smoother_type == Smoother::l1_gauss_seidel) {
1317#ifdef AMREX_USE_GPU
1318 amrex::Abort("AlgMG: L1GaussSeidel smoother is only available in CPU builds");
1319#endif
1320 m_l1gs.push_back(std::make_unique<L1GaussSeidelSmoother<T>>(m_a[lev]));
1321 } else if (m_smoother_type == Smoother::chebyshev) {
1322 m_cheby.push_back(std::make_unique<ChebyshevSmoother<T>>(m_a[lev], true));
1323 m_cheby.back()->setDegree(m_cheby_degree);
1324 m_cheby.back()->setEigRatio(m_cheby_ratio);
1325 } else {
1326 bool const l1 = (m_smoother_type == Smoother::l1_jacobi);
1327 m_smoother.push_back(std::make_unique<JacobiSmoother<T>>(m_a[lev], l1));
1328 T const w = (m_relax_weight >= T(0)) ? m_relax_weight : (l1 ? T(4./3.) : T(2./3.));
1329 m_smoother.back()->setWeight(w);
1330 }
1331 }
1332 m_smoothers_stale = false;
1333}
1334
1335template <typename T>
1337 bool& aggressive, Long& ncoarse, bool& truncated,
1338 Long& nnz_untruncated)
1339{
1340#ifdef AMREX_USE_GPU
1341 amrex::ignore_unused(truncated, nnz_untruncated); // set by the host paths only
1342#endif
1343 T const sscale = strength_scale(A);
1344 auto S = create_soc(A, sscale);
1345 S.setColumnPartition(S.partition());
1346 note_memory(lev, "S");
1347 auto ST = amrex::transpose(S, S.partition());
1348 ST.setColumnPartition(ST.partition());
1349
1350 note_memory(lev, "strength");
1352 // Singular: an isolated point becomes C, so every row of P sums to one.
1353 pmis(S, ST, state, m_singular);
1354 note_memory(lev, "PMIS");
1355
1357 cpart = coarse_numbering(state, cidx);
1358 ncoarse = cpart.numGlobalRows();
1359 if (ncoarse == 0 || ncoarse == A.numGlobalRows()) { return false; }
1360#ifndef AMREX_USE_GPU
1361 bool const host_interp = m_interp_type != InterpType::direct
1362 && detail::spmat_comm_is_local(A.partition(), cpart);
1363#endif
1364
1365 if (lev < m_aggressive_num_levels) {
1366 // Second PMIS pass on the C-points connected at distance two.
1367 auto S2 = second_pass_strength(S, state, cidx, cpart);
1368 S2.setColumnPartition(cpart);
1369 auto S2T = amrex::transpose(S2, cpart);
1370 S2T.setColumnPartition(cpart);
1371 note_memory(lev, "second-pass strength");
1373 pmis(S2, S2T, state2, true); // isolated C-points stay coarse
1375 auto cpart2 = coarse_numbering(state2, cidx2);
1376 Long const nc2 = cpart2.numGlobalRows();
1377 if (nc2 > 0 && nc2 < ncoarse) {
1378 // C/F split of the fine points with respect to C2.
1379 Long const nrows = A.numLocalRows();
1380 Gpu::DeviceVector<int> statef(nrows);
1381 Gpu::DeviceVector<Long> cidxf(nrows);
1382 auto const* p1 = state.data();
1383 auto const* c1 = cidx.data();
1384 auto const* p2 = state2.data();
1385 auto const* c2 = cidx2.data();
1386 auto* pf = statef.data();
1387 auto* cf = cidxf.data();
1388 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
1389 {
1390 bool const c = (p1[i] == PointType::coarse) &&
1391 (p2[c1[i]] == PointType::coarse);
1392 pf[i] = c ? PointType::coarse : PointType::fine;
1393 cf[i] = (p1[i] == PointType::coarse) ? c2[c1[i]] : Long(0);
1394 });
1396
1397 bool const trunc = (m_p_max_elmts > 0 || m_trunc_factor > T(0));
1398 SpMatrix<T> P1;
1399#ifndef AMREX_USE_GPU
1400 if (!m_aggressive_direct_interp && host_interp) {
1401 Long unused;
1402 P1 = build_interp_mm_host(A, S, ST, state, cidx, cpart, sscale,
1403 m_interp_type == InterpType::mm_ext_i,
1404 m_p_max_elmts, m_trunc_factor, unused);
1405 } else if (!m_aggressive_direct_interp && m_interp_type != InterpType::direct) {
1406 P1 = build_interp_mm(A, S, ST, state, cidx, cpart, sscale,
1407 m_interp_type == InterpType::mm_ext_i, nullptr,
1408 m_p_max_elmts, m_trunc_factor);
1409 } else
1410#endif
1411 {
1412 P1 = m_aggressive_direct_interp ? build_interp(A, S, state, cidx, cpart)
1413 : interpolation(A, S, ST, state, cidx, cpart, sscale);
1414 if (trunc) { P1 = truncate_interp(P1, cpart, m_p_max_elmts, m_trunc_factor); }
1415 }
1416 note_memory(lev, "P1");
1417 // P2 needs only the rows of the first-pass C-points.
1418#ifndef AMREX_USE_GPU
1419 auto Pf = build_interp_mm(A, S, ST, statef, cidxf, cpart2, sscale,
1420 m_interp_type != InterpType::mm_ext,
1421 state.data(), m_p_max_elmts, m_trunc_factor);
1422#else
1423 auto Pf = build_interp_mm(A, S, ST, statef, cidxf, cpart2, sscale,
1424 m_interp_type != InterpType::mm_ext,
1425 state.data());
1426 if (trunc) { Pf = truncate_interp(Pf, cpart2, m_p_max_elmts, m_trunc_factor); }
1427#endif
1428 auto P2 = select_coarse_rows(Pf, state, cidx, cpart);
1429 note_memory(lev, "P2");
1430 P = amrex::SpGEMM(P1, P2, cpart2);
1431 cpart = cpart2;
1432 aggressive = true;
1433 }
1434 }
1435 if (!aggressive) {
1436#ifndef AMREX_USE_GPU
1437 if (host_interp) {
1438 P = build_interp_mm_host(A, S, ST, state, cidx, cpart, sscale,
1439 m_interp_type == InterpType::mm_ext_i,
1440 m_p_max_elmts, m_trunc_factor, nnz_untruncated);
1441 truncated = true;
1442 } else if (m_interp_type != InterpType::direct) {
1443 P = build_interp_mm(A, S, ST, state, cidx, cpart, sscale,
1444 m_interp_type == InterpType::mm_ext_i, nullptr,
1445 m_p_max_elmts, m_trunc_factor, &nnz_untruncated);
1446 truncated = true;
1447 } else
1448#endif
1449 {
1450 P = interpolation(A, S, ST, state, cidx, cpart, sscale);
1451 }
1452 }
1453 note_memory(lev, "interpolation");
1454 return true;
1455}
1456
1457template <typename T>
1459{
1460 auto const rs = P.rowSum();
1461 auto const* AMREX_RESTRICT prs = rs.data();
1462 Long nzero = Reduce::Sum<Long>(rs.numLocalRows(),
1463 [=] AMREX_GPU_DEVICE (Long i) -> Long { return prs[i] == T(0); });
1465 if (nzero > 0) {
1466 amrex::Abort("AlgMG: singular mode needs P 1 = 1, but " + std::to_string(nzero)
1467 + " rows of P are empty");
1468 }
1469 auto a = P.parcsr();
1470 ParallelForOMP(rs.numLocalRows(), [=] AMREX_GPU_DEVICE (Long i) noexcept
1471 {
1472 T const f = T(1) / prs[i];
1473 for (Long idx = a.csr0.row_offset[i]; idx < a.csr0.row_offset[i+1]; ++idx) {
1474 a.csr0.mat[idx] *= f;
1475 }
1476 if (detail::has_remote_row(a, i)) {
1477 Long const ii = a.row_map[i];
1478 for (Long idx = a.csr1.row_offset[ii]; idx < a.csr1.row_offset[ii+1]; ++idx) {
1479 a.csr1.mat[idx] *= f;
1480 }
1481 }
1482 });
1484}
1485
1486template <typename T>
1488{
1489 auto const& Ad = A.diagonalVector();
1490 auto const* AMREX_RESTRICT p = Ad.data();
1491 T dmax = Reduce::Max<T>(Ad.numLocalRows(),
1492 [=] AMREX_GPU_DEVICE (Long i) -> T { return std::abs(p[i]); });
1494 return (dmax > T(0)) ? dmax : T(1);
1495}
1496
1497template <typename T>
1499{
1500 BL_PROFILE("AlgMG::create_soc");
1501
1502 auto const& a = A.const_parcsr();
1503 Long const nrows = a.csr0.nrows;
1504 int const nlocal = int(A.numLocalRows());
1505
1506 auto const& Ad = A.diagonalVector();
1507 auto const* AMREX_RESTRICT A_ii = Ad.data();
1508 auto const threshold = m_strong_threshold;
1509
1510 // Threshold of row i: theta times the strongest connection, relaxed
1511 // by a few ulps so that entries nominally on the threshold are strong
1512 // in every row [1]. Rows dominated by their diagonal (row sum above
1513 // 0.9 |a_ii|) have no strong connection.
1514 T const tol = T(64) * std::numeric_limits<T>::epsilon();
1515
1516#ifndef AMREX_USE_GPU
1517 {
1518 // One pass per row on the host.
1519 auto const& c0 = a.csr0;
1520 auto const& c1 = a.csr1;
1521 auto rlen1 = [&] (Long i) -> Long {
1522 if (!detail::has_remote_row(a, i)) { return 0; }
1523 Long const ii = a.row_map[i];
1524 return c1.row_offset[ii+1] - c1.row_offset[ii];
1525 };
1526 auto csr = detail::csr_from_rows_cpu<TS,SpMatrix<TS>::template container_type,int>
1527 (nrows, [&] (Long i) { return c0.row_offset[i+1] - c0.row_offset[i] + rlen1(i); },
1528 [&] (int) {
1529 return [&, col = Vector<int>(), val = Vector<TS>()]
1530 (Long i, int const*& rc, TS const*& rv) mutable -> Long
1531 {
1532 Long const b = c0.row_offset[i];
1533 Long const e = c0.row_offset[i+1];
1534 Long const n1 = rlen1(i);
1535 Long const b1 = (n1 > 0) ? Long(c1.row_offset[a.row_map[i]]) : 0;
1536 T const sgn = std::copysign(T(1), A_ii[i]);
1537 T rs = T(0);
1538 T amin = std::numeric_limits<T>::max();
1539 for (Long idx = b; idx < e; ++idx) {
1540 rs += c0.mat[idx];
1541 if (c0.col_index[idx] != i) { amin = std::min(amin, sgn*c0.mat[idx]); }
1542 }
1543 for (Long idx = b1; idx < b1+n1; ++idx) {
1544 rs += c1.mat[idx];
1545 amin = std::min(amin, sgn*c1.mat[idx]);
1546 }
1547 Long n = 0;
1548 if (!(std::abs(rs) > T(0.9)*std::abs(A_ii[i])) && amin < T(0)) {
1549 T const thr = amin * threshold * (T(1) - tol);
1550 if (Long(col.size()) < e-b+n1) {
1551 col.resize(e-b+n1);
1552 val.resize(e-b+n1);
1553 }
1554 for (Long idx = b; idx < e; ++idx) {
1555 if (c0.col_index[idx] != i && sgn*c0.mat[idx] <= thr) {
1556 col[n] = c0.col_index[idx];
1557 val[n] = TS(c0.mat[idx]/scale);
1558 ++n;
1559 }
1560 }
1561 for (Long idx = b1; idx < b1+n1; ++idx) {
1562 if (sgn*c1.mat[idx] <= thr) {
1563 col[n] = nlocal + c1.col_index[idx];
1564 val[n] = TS(c1.mat[idx]/scale);
1565 ++n;
1566 }
1567 }
1568 }
1569 rc = col.data();
1570 rv = val.data();
1571 return n;
1572 };
1573 });
1574 SpMatrix<TS> S;
1575 S.define_split(A.partition(), A.partition(), std::move(csr), nlocal, a.col_map, 0);
1576 return S;
1577 }
1578#else
1579
1580 auto const rsA = A.rowSum();
1581 auto const* AMREX_RESTRICT prs = rsA.data();
1582 auto row_thresh = [=] AMREX_GPU_DEVICE (Long i) -> T {
1583 if (std::abs(prs[i]) > T(0.9)*std::abs(A_ii[i])) {
1584 return std::numeric_limits<T>::lowest();
1585 }
1586 auto const& c0 = a.csr0;
1587 T const sgn = std::copysign(T(1), A_ii[i]);
1588 T amin = std::numeric_limits<T>::max();
1589 for (Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
1590 if (c0.col_index[idx] != i) {
1591 amin = std::min(amin, sgn*c0.mat[idx]);
1592 }
1593 }
1594 if (detail::has_remote_row(a, i)) {
1595 auto const& c1 = a.csr1;
1596 Long const ii = a.row_map[i];
1597 for (Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
1598 amin = std::min(amin, sgn*c1.mat[idx]);
1599 }
1600 }
1601 if (amin >= T(0)) { return std::numeric_limits<T>::lowest(); }
1602 return amin * threshold * (T(1) - tol);
1603 };
1604 Gpu::DeviceVector<T> rthr(nrows);
1605 auto* AMREX_RESTRICT pthr = rthr.data();
1606 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept { pthr[i] = row_thresh(i); });
1607
1608 using local_csr_type = typename SpMatrix<TS>::local_csr_type;
1609 local_csr_type csr;
1610 csr.row_offset.resize(nrows+1);
1611 auto* AMREX_RESTRICT psrow = csr.row_offset.data();
1612 Long const nnz = Scan::PrefixSum<Long>
1613 (nrows+1,
1614 [=] AMREX_GPU_DEVICE (Long i) -> Long {
1615 if (i >= nrows) { return 0; }
1616 T const thr = pthr[i];
1617 T const sgn = std::copysign(T(1), A_ii[i]);
1618 Long n = 0;
1619 auto const& c0 = a.csr0;
1620 for (Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
1621 if (c0.col_index[idx] != i && sgn*c0.mat[idx] <= thr) { ++n; }
1622 }
1623 if (detail::has_remote_row(a, i)) {
1624 auto const& c1 = a.csr1;
1625 Long const ii = a.row_map[i];
1626 for (Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
1627 if (sgn*c1.mat[idx] <= thr) { ++n; }
1628 }
1629 }
1630 return n;
1631 },
1632 [=] AMREX_GPU_DEVICE (Long i, Long const& x) { psrow[i] = x; },
1634
1635 csr.resize(nrows, nnz);
1636 auto* AMREX_RESTRICT ps = csr.mat.data();
1637 auto* AMREX_RESTRICT pc = csr.col_index.data();
1638 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
1639 {
1640 T const thr = pthr[i];
1641 T const sgn = std::copysign(T(1), A_ii[i]);
1642 Long p = psrow[i];
1643 auto const& c0 = a.csr0;
1644 for (Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
1645 if (c0.col_index[idx] != i && sgn*c0.mat[idx] <= thr) {
1646 ps[p] = TS(c0.mat[idx]/scale);
1647 pc[p] = c0.col_index[idx];
1648 ++p;
1649 }
1650 }
1651 if (detail::has_remote_row(a, i)) {
1652 auto const& c1 = a.csr1;
1653 Long const ii = a.row_map[i];
1654 for (Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
1655 if (sgn*c1.mat[idx] <= thr) {
1656 ps[p] = TS(c1.mat[idx]/scale);
1657 pc[p] = nlocal + c1.col_index[idx];
1658 ++p;
1659 }
1660 }
1661 }
1662 });
1664
1665 SpMatrix<TS> S;
1666 S.define_split(A.partition(), A.partition(), std::move(csr), nlocal, a.col_map, 0);
1667 return S;
1668#endif
1669}
1670
1671template <typename T>
1673 bool isolated_as_coarse) const
1674{
1675 BL_PROFILE("AlgMG::pmis");
1676
1677 Long const nrows = S.numLocalRows();
1678 int const U = PointType::undecided;
1679 int const C = PointType::coarse;
1680
1681 // Measure: number of points that strongly depend on i. A negative
1682 // measure marks a point with no strong connection at all.
1683 auto const& s = S.const_parcsr();
1684 auto const& st = ST.const_parcsr();
1685 Gpu::DeviceVector<Long> measure(nrows);
1686 auto* AMREX_RESTRICT pm = measure.data();
1687 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
1688 {
1689 Long n = st.csr0.row_offset[i+1] - st.csr0.row_offset[i];
1690 if (detail::has_remote_row(st, i)) {
1691 Long const ii = st.row_map[i];
1692 n += st.csr1.row_offset[ii+1] - st.csr1.row_offset[ii];
1693 }
1694 Long nout = s.csr0.row_offset[i+1] - s.csr0.row_offset[i];
1695 if (detail::has_remote_row(s, i)) {
1696 Long const ii = s.row_map[i];
1697 nout += s.csr1.row_offset[ii+1] - s.csr1.row_offset[ii];
1698 }
1699 pm[i] = (n == 0 && nout == 0) ? Long(-1) : n;
1700 });
1701
1702 Gpu::DeviceVector<T> weight(nrows);
1703 state.resize(nrows);
1704 Gpu::DeviceVector<int> state2(nrows);
1705 auto* AMREX_RESTRICT pw = weight.data();
1706 auto* AMREX_RESTRICT ps = state.data();
1707 auto* AMREX_RESTRICT ps2 = state2.data();
1708
1709 Long const row_begin = s.row_begin;
1710
1711 // A point nobody depends on gets a weight below one, so it becomes C
1712 // only if all its dependencies are F.
1713 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
1714 {
1715 if (pm[i] < 0) {
1716 pw[i] = T(0);
1717 ps[i] = isolated_as_coarse ? PointType::coarse : PointType::fine;
1718 } else {
1719 pw[i] = T(pm[i]) + detail::pmis_tiebreak<T>(i + row_begin);
1720 ps[i] = PointType::undecided;
1721 }
1722 });
1723
1724 // Ghost weights are fixed; ghost states are gathered every sweep.
1725 auto const wS = S.gatherRemote(pw);
1726 auto const wST = ST.gatherRemote(pw);
1727 auto const* AMREX_RESTRICT pwS = wS.data();
1728 auto const* AMREX_RESTRICT pwST = wST.data();
1729
1730 Gpu::DeviceVector<int> sS, sST, s2S;
1731
1732 while (true) {
1733 S.gatherRemote(ps, sS);
1734 ST.gatherRemote(ps, sST);
1735 auto const* AMREX_RESTRICT psS = sS.data();
1736 auto const* AMREX_RESTRICT psST = sST.data();
1737
1738 // Independent set in S union S^T: keep only own index writes.
1739 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
1740 {
1741 int st_i = ps[i];
1742 if (st_i == U) {
1743 T const wi = pw[i];
1744 Long const ig = i + row_begin;
1745 bool blocked =
1746 detail::pmis_blocked(s.csr0, i, wi, ig, pw, ps, nullptr, s.col_begin, U) ||
1747 detail::pmis_blocked(st.csr0, i, wi, ig, pw, ps, nullptr, st.col_begin, U);
1748 if (!blocked && detail::has_remote_row(s, i)) {
1749 blocked = detail::pmis_blocked(s.csr1, s.row_map[i], wi, ig,
1750 pwS, psS, s.col_map, 0, U);
1751 }
1752 if (!blocked && detail::has_remote_row(st, i)) {
1753 blocked = detail::pmis_blocked(st.csr1, st.row_map[i], wi, ig,
1754 pwST, psST, st.col_map, 0, U);
1755 }
1756 if (!blocked) { st_i = C; }
1757 }
1758 ps2[i] = st_i;
1759 });
1760
1761 S.gatherRemote(ps2, s2S);
1762 auto const* AMREX_RESTRICT ps2S = s2S.data();
1763
1764 // Undecided points that strongly depend on a C-point become F.
1765 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
1766 {
1767 int st_i = ps2[i];
1768 if (st_i == U) {
1769 if (detail::csr_row_has_state(s.csr0, i, ps2, C) ||
1770 (detail::has_remote_row(s, i) &&
1771 detail::csr_row_has_state(s.csr1, s.row_map[i], ps2S, C))) {
1772 st_i = PointType::fine;
1773 }
1774 }
1775 ps[i] = st_i;
1776 });
1777
1778 Long nund = Reduce::Sum<Long>(nrows, [=] AMREX_GPU_DEVICE (Long i) -> Long {
1779 return (ps[i] == U) ? 1 : 0;
1780 });
1782 if (nund == 0) { break; }
1783 }
1784
1785 if (m_verbose >= 2) {
1786 S.gatherRemote(ps, sS);
1787 auto const* AMREX_RESTRICT psS = sS.data();
1788 Long nc = Reduce::Sum<Long>(nrows, [=] AMREX_GPU_DEVICE (Long i) -> Long {
1789 return (ps[i] == C) ? 1 : 0;
1790 });
1791 // C-points that strongly depend on another C-point. A point becomes
1792 // F only through its own dependencies, so a few one-directional
1793 // C-C connections remain where the strength is asymmetric.
1794 Long ncc = Reduce::Sum<Long>(nrows, [=] AMREX_GPU_DEVICE (Long i) -> Long {
1795 if (ps[i] != C) { return 0; }
1796 return (detail::csr_row_has_state(s.csr0, i, ps, C) ||
1797 (detail::has_remote_row(s, i) &&
1798 detail::csr_row_has_state(s.csr1, s.row_map[i], psS, C))) ? 1 : 0;
1799 });
1800 // F-points without a strong C neighbor get no interpolation.
1801 Long nf0 = Reduce::Sum<Long>(nrows, [=] AMREX_GPU_DEVICE (Long i) -> Long {
1802 if (ps[i] != PointType::fine) { return 0; }
1803 return (detail::csr_row_has_state(s.csr0, i, ps, C) ||
1804 (detail::has_remote_row(s, i) &&
1805 detail::csr_row_has_state(s.csr1, s.row_map[i], psS, C))) ? 0 : 1;
1806 });
1807 Long counts[3] = {nc, ncc, nf0};
1809 amrex::Print() << m_print_ident << "AlgMG: PMIS: " << counts[0] << " C-points of "
1810 << S.numGlobalRows() << ", C-C strong pairs " << counts[1]
1811 << ", F-points w/o C neighbor " << counts[2] << "\n";
1812 }
1813}
1814
1815template <typename T>
1817 Gpu::DeviceVector<Long>& cidx, int which)
1818{
1819 Long const nrows = Long(state.size());
1820 auto const* AMREX_RESTRICT ps = state.data();
1821 cidx.resize(nrows);
1822 auto* AMREX_RESTRICT pc = cidx.data();
1823 Long const nc = Scan::PrefixSum<Long>
1824 (nrows,
1825 [=] AMREX_GPU_DEVICE (Long i) -> Long { return (ps[i] == which) ? 1 : 0; },
1826 [=] AMREX_GPU_DEVICE (Long i, Long const& x) { pc[i] = x; },
1828
1829 int const nprocs = ParallelContext::NProcsSub();
1830 Vector<Long> counts(nprocs, nc);
1831#ifdef AMREX_USE_MPI
1832 // One value per rank: an all-gather, not a reduction of a P-vector.
1834#endif
1835 Vector<Long> rows(nprocs+1, 0); // exclusive prefix sum of counts
1836 std::partial_sum(counts.begin(), counts.end(), rows.begin()+1);
1837 return AlgPartition(std::move(rows));
1838}
1839
1840template <typename T>
1842 Gpu::DeviceVector<int> const& state,
1843 Gpu::DeviceVector<Long> const& cidx,
1844 AlgPartition const& cpart)
1845{
1846 BL_PROFILE("AlgMG::build_interp");
1847
1848 // Classical direct interpolation [1].
1849
1850 auto const& a = A.const_parcsr();
1851 Long const nrows = a.csr0.nrows;
1852 auto const* AMREX_RESTRICT ps = state.data();
1853 auto const* AMREX_RESTRICT pc = cidx.data();
1854 Long const cbegin = cpart.globalRowBegin();
1855 int const nlocal_c = int(cpart.numLocalRows());
1856
1857 // Global coarse index of local rows, and ghost copies for A's remote
1858 // columns (the remote column list of P).
1859 Gpu::DeviceVector<Long> gcidx(nrows);
1860 auto* AMREX_RESTRICT pgc = gcidx.data();
1861 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept { pgc[i] = pc[i] + cbegin; });
1862 auto const state_r = A.gatherRemote(ps);
1863 auto const gcidx_r = A.gatherRemote(pgc);
1864
1865 detail::DirectInterpRow<T,TS> const w{a, S.const_parcsr(), ps, pc,
1866 state_r.data(), nlocal_c,
1867 int(PointType::coarse)};
1868
1869 using local_csr_type = typename SpMatrix<T>::local_csr_type;
1870 local_csr_type csr;
1871 csr.row_offset.resize(nrows+1);
1872 auto* AMREX_RESTRICT prow = csr.row_offset.data();
1873
1874 Long const nnz = Scan::PrefixSum<Long>
1875 (nrows+1,
1876 [=] AMREX_GPU_DEVICE (Long i) -> Long {
1877 if (i >= nrows) { return 0; }
1878 if (ps[i] == PointType::coarse) { return 1; }
1879 Long n = 0;
1880 T diag;
1881 w.walk(i, diag, [&] (int, T, bool strong_c) { if (strong_c) { ++n; } });
1882 return n;
1883 },
1884 [=] AMREX_GPU_DEVICE (Long i, Long const& x) { prow[i] = x; },
1886
1887 csr.resize(nrows, nnz);
1888 auto* AMREX_RESTRICT pmat = csr.mat.data();
1889 auto* AMREX_RESTRICT pcol = csr.col_index.data();
1890
1891 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
1892 {
1893 Long p = prow[i];
1894 if (ps[i] == PointType::coarse) {
1895 pcol[p] = int(pc[i]);
1896 pmat[p] = T(1);
1897 return;
1898 }
1899
1900 T sum_n_neg = T(0), sum_n_pos = T(0), sum_p_neg = T(0), sum_p_pos = T(0);
1901 T diag;
1902 w.walk(i, diag, [&] (int, T v, bool strong_c) {
1903 if (v < T(0)) {
1904 sum_n_neg += v;
1905 if (strong_c) { sum_p_neg += v; }
1906 } else {
1907 sum_n_pos += v;
1908 if (strong_c) { sum_p_pos += v; }
1909 }
1910 });
1911
1912 T alfa = T(0), beta = T(0);
1913 if (sum_p_neg != T(0)) { alfa = sum_n_neg / sum_p_neg; } else { diag += sum_n_neg; }
1914 if (sum_p_pos != T(0)) { beta = sum_n_pos / sum_p_pos; } else { diag += sum_n_pos; }
1915 if (diag == T(0)) { diag = T(1); } // no interpolation weights anyway
1916
1917 T unused;
1918 w.walk(i, unused, [&] (int cj, T v, bool strong_c) {
1919 if (strong_c) {
1920 pcol[p] = cj;
1921 pmat[p] = ((v < T(0)) ? -alfa*v : -beta*v) / diag;
1922 ++p;
1923 }
1924 });
1925 });
1927
1928 SpMatrix<T> P;
1929 P.define_split(A.partition(), cpart, std::move(csr), nlocal_c, gcidx_r.data(), 0);
1930 return P;
1931}
1932
1933namespace detail {
1934
1938template <typename T, int KMAX>
1939struct TruncRow
1940{
1942 int max_elmts;
1943 T trunc_factor;
1944 int nlocal_cols;
1945
1947 template <typename F>
1948 AMREX_GPU_HOST_DEVICE void walk (Long i, F const& f) const
1949 {
1950 auto const& c0 = p.csr0;
1951 for (Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
1952 f(c0.col_index[idx], c0.mat[idx]);
1953 }
1954 if (has_remote_row(p, i)) {
1955 auto const& c1 = p.csr1;
1956 Long const ii = p.row_map[i];
1957 for (Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
1958 f(nlocal_cols + c1.col_index[idx], c1.mat[idx]);
1959 }
1960 }
1961 }
1962
1964 AMREX_GPU_HOST_DEVICE T threshold (Long i, T& sum_pos, T& sum_neg) const
1965 {
1966 T amax = T(0);
1967 sum_pos = T(0);
1968 sum_neg = T(0);
1969 walk(i, [&] (int, T v) {
1970 amax = std::max(amax, std::abs(v));
1971 if (v > T(0)) { sum_pos += v; } else { sum_neg += v; }
1972 });
1973 return trunc_factor * amax;
1974 }
1975
1977 [[nodiscard]] AMREX_GPU_HOST_DEVICE int count (Long i, T thresh) const
1978 {
1979 int n = 0;
1980 walk(i, [&] (int, T v) { if (std::abs(v) >= thresh) { ++n; } });
1981 return n;
1982 }
1983
1988 int select (Long i, T thresh, int* cols, T* vals) const
1989 {
1990 int const kmax = std::min(max_elmts, KMAX);
1991 int n = 0;
1992 walk(i, [&] (int col, T v) {
1993 T const av = std::abs(v);
1994 if (av < thresh) { return; }
1995 if (n == kmax && av <= std::abs(vals[n-1])) { return; }
1996 int pos = (n < kmax) ? n : n-1;
1997 while (pos > 0 && std::abs(vals[pos-1]) < av) {
1998 cols[pos] = cols[pos-1];
1999 vals[pos] = vals[pos-1];
2000 --pos;
2001 }
2002 cols[pos] = col;
2003 vals[pos] = v;
2004 if (n < kmax) { ++n; }
2005 });
2006 return n;
2007 }
2008};
2009
2010#ifndef AMREX_USE_GPU
2013template <typename T, int KMAX>
2014Long truncate_row_host (int* col, T* val, Long n, int max_elmts, T trunc_factor, int nc)
2015{
2016 // The row as a one-row CSR for TruncRow.
2017 int ro[2] = {0, int(n)};
2018 ParCsr<T const> prow;
2019 prow.csr0 = CsrView<T const,int>{val, col, ro, n, 1};
2020 TruncRow<T,KMAX> const tr{prow, max_elmts, trunc_factor, nc};
2021 T sum_pos, sum_neg;
2022 T const thresh = tr.threshold(0, sum_pos, sum_neg);
2023 int tcol[KMAX];
2024 T tval[KMAX];
2025 Long nk = 0;
2026 T kept_pos = T(0), kept_neg = T(0);
2027 if (max_elmts > 0) {
2028 nk = tr.select(0, thresh, tcol, tval);
2029 for (Long k = 0; k < nk; ++k) {
2030 if (tval[k] > T(0)) { kept_pos += tval[k]; } else { kept_neg += tval[k]; }
2031 }
2032 } else {
2033 for (Long k = 0; k < n; ++k) {
2034 if (std::abs(val[k]) >= thresh) {
2035 if (val[k] > T(0)) { kept_pos += val[k]; } else { kept_neg += val[k]; }
2036 }
2037 }
2038 }
2039 T const fpos = (kept_pos != T(0)) ? sum_pos/kept_pos : T(1);
2040 T const fneg = (kept_neg != T(0)) ? sum_neg/kept_neg : T(1);
2041 if (max_elmts > 0) {
2042 // Kept by decreasing magnitude; back to column order.
2043 for (Long k = 0; k < nk; ++k) {
2044 Long m = k;
2045 for (; m > 0 && col[m-1] > tcol[k]; --m) {
2046 col[m] = col[m-1];
2047 val[m] = val[m-1];
2048 }
2049 col[m] = tcol[k];
2050 val[m] = tval[k] * ((tval[k] > T(0)) ? fpos : fneg);
2051 }
2052 } else {
2053 for (Long k = 0; k < n; ++k) {
2054 if (std::abs(val[k]) >= thresh) {
2055 col[nk] = col[k];
2056 val[nk] = val[k] * ((val[k] > T(0)) ? fpos : fneg);
2057 ++nk;
2058 }
2059 }
2060 }
2061 return nk;
2062}
2063#endif
2064
2065}
2066
2067namespace detail {
2068
2073template <typename T, typename F>
2075compact_rows (ParCsr<T const> const& p, Long nc, int const* AMREX_RESTRICT state,
2076 Long const* AMREX_RESTRICT cidx, int c, F const& colmap)
2077{
2078 Long const nrows = p.csr0.nrows;
2079
2080 // Offsets by fine row, then gathered at the selected rows.
2081 Gpu::DeviceVector<Long> off(nrows+1);
2082 auto* AMREX_RESTRICT poff = off.data();
2083 Long const nnz = Scan::PrefixSum<Long>
2084 (nrows+1,
2085 [=] AMREX_GPU_DEVICE (Long i) -> Long {
2086 if (i >= nrows || state[i] != c) { return 0; }
2087 Long n = 0;
2088 auto const& c0 = p.csr0;
2089 for (Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
2090 if (colmap(c0.col_index[idx], false) >= 0) { ++n; }
2091 }
2092 if (has_remote_row(p, i)) {
2093 auto const& c1 = p.csr1;
2094 Long const ii = p.row_map[i];
2095 for (Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
2096 if (colmap(c1.col_index[idx], true) >= 0) { ++n; }
2097 }
2098 }
2099 return n;
2100 },
2101 [=] AMREX_GPU_DEVICE (Long i, Long const& x) { poff[i] = x; },
2103
2104 typename SpMatrix<T>::local_csr_type csr;
2105 csr.resize(nc, nnz);
2106 auto* AMREX_RESTRICT prow = csr.row_offset.data();
2107 auto* AMREX_RESTRICT pcol = csr.col_index.data();
2108 auto* AMREX_RESTRICT pmat = csr.mat.data();
2109 ParallelForOMP(nrows+1, [=] AMREX_GPU_DEVICE (Long i) noexcept
2110 {
2111 if (i == nrows) { prow[nc] = nnz; return; }
2112 if (state[i] != c) { return; }
2113 Long q = poff[i];
2114 prow[cidx[i]] = q;
2115 auto const& c0 = p.csr0;
2116 for (Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
2117 int const g = colmap(c0.col_index[idx], false);
2118 if (g >= 0) { pcol[q] = g; pmat[q] = c0.mat[idx]; ++q; }
2119 }
2120 if (has_remote_row(p, i)) {
2121 auto const& c1 = p.csr1;
2122 Long const ii = p.row_map[i];
2123 for (Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
2124 int const g = colmap(c1.col_index[idx], true);
2125 if (g >= 0) { pcol[q] = g; pmat[q] = c1.mat[idx]; ++q; }
2126 }
2127 }
2128 });
2130 return csr;
2131}
2132
2133}
2134
2135template <typename T>
2137 Gpu::DeviceVector<int> const& state,
2138 Gpu::DeviceVector<Long> const& cidx,
2139 AlgPartition const& cpart, T scale) const
2140{
2141 if (m_interp_type == InterpType::direct) {
2142 return build_interp(A, S, state, cidx, cpart);
2143 } else {
2144 return build_interp_mm(A, S, ST, state, cidx, cpart, scale,
2145 m_interp_type == InterpType::mm_ext_i);
2146 }
2147}
2148
2149template <typename T>
2151 Gpu::DeviceVector<int> const& state,
2152 Gpu::DeviceVector<Long> const& cidx,
2153 AlgPartition const& cpart)
2154{
2155 BL_PROFILE("AlgMG::second_pass_strength");
2156
2157 // Distance-two C-C connections: S_CF S_FC. The few direct C-C
2158 // connections PMIS leaves are ignored. Both factors are built in
2159 // compact C and F numberings, so the product never forms the full S^2.
2160 auto const& s = S.const_parcsr();
2161 Long const nrows = s.csr0.nrows;
2162 Long const nc = cpart.numLocalRows();
2163 int const C = PointType::coarse;
2164 int const F = PointType::fine;
2165
2167 auto const fpart = coarse_numbering(state, fidx, PointType::fine);
2168 Long const nf = fpart.numLocalRows();
2169 int const nlocal_c = int(nc);
2170 int const nlocal_f = int(nf);
2171
2172 auto const* AMREX_RESTRICT ps = state.data();
2173 auto const* AMREX_RESTRICT pc = cidx.data();
2174 auto const* AMREX_RESTRICT pf = fidx.data();
2175 Long const cbegin = cpart.globalRowBegin();
2176 Long const fbegin = fpart.globalRowBegin();
2177 Gpu::DeviceVector<Long> gcidx(nrows);
2178 Gpu::DeviceVector<Long> gfidx(nrows);
2179 auto* AMREX_RESTRICT pgc = gcidx.data();
2180 auto* AMREX_RESTRICT pgf = gfidx.data();
2181 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
2182 {
2183 pgc[i] = pc[i] + cbegin;
2184 pgf[i] = pf[i] + fbegin;
2185 });
2186 auto const state_r = S.gatherRemote(ps);
2187 auto const gcidx_r = S.gatherRemote(pgc);
2188 auto const gfidx_r = S.gatherRemote(pgf);
2189 auto const* AMREX_RESTRICT ps_r = state_r.data();
2190
2191 // S_CF: C rows, F columns in F numbering (remote: nlocal_f + remote index).
2192 auto csr_cf = detail::compact_rows<TS>
2193 (s, nc, ps, pc, C,
2194 [=] AMREX_GPU_DEVICE (int j, bool remote) -> int {
2195 if (remote) {
2196 return (ps_r[j] == F) ? nlocal_f + j : -1;
2197 } else {
2198 return (ps[j] == F) ? int(pf[j]) : -1;
2199 }
2200 });
2201 SpMatrix<TS> SCF;
2202 SCF.define_split(cpart, fpart, std::move(csr_cf), nlocal_f, gfidx_r.data(), 0);
2203
2204 // S_FC: F rows, C columns in C numbering.
2205 auto csr_fc = detail::compact_rows<TS>
2206 (s, nf, ps, pf, F,
2207 [=] AMREX_GPU_DEVICE (int j, bool remote) -> int {
2208 if (remote) {
2209 return (ps_r[j] == C) ? nlocal_c + j : -1;
2210 } else {
2211 return (ps[j] == C) ? int(pc[j]) : -1;
2212 }
2213 });
2214 SpMatrix<TS> SFC;
2215 SFC.define_split(fpart, cpart, std::move(csr_fc), nlocal_c, gcidx_r.data(), 0);
2216
2217 auto S2 = amrex::SpGEMM(SCF, SFC, cpart);
2218
2219 // Drop the diagonal: a C-point is not its own distance-two neighbor.
2220 auto const& s2 = S2.const_parcsr();
2221 typename SpMatrix<TS>::local_csr_type out;
2222 out.row_offset.resize(nc+1);
2223 auto* AMREX_RESTRICT orow = out.row_offset.data();
2224 Long const nnz = Scan::PrefixSum<Long>
2225 (nc+1,
2226 [=] AMREX_GPU_DEVICE (Long r) -> Long {
2227 if (r >= nc) { return 0; }
2228 Long n = 0;
2229 auto const& c0 = s2.csr0;
2230 for (Long idx = c0.row_offset[r]; idx < c0.row_offset[r+1]; ++idx) {
2231 n += (c0.col_index[idx] != int(r));
2232 }
2233 if (detail::has_remote_row(s2, r)) {
2234 Long const rr = s2.row_map[r];
2235 n += s2.csr1.row_offset[rr+1] - s2.csr1.row_offset[rr];
2236 }
2237 return n;
2238 },
2239 [=] AMREX_GPU_DEVICE (Long r, Long const& x) { orow[r] = x; },
2241 out.resize(nc, nnz);
2242 auto* AMREX_RESTRICT ocol = out.col_index.data();
2243 auto* AMREX_RESTRICT omat = out.mat.data();
2244 ParallelForOMP(nc, [=] AMREX_GPU_DEVICE (Long r) noexcept
2245 {
2246 Long q = orow[r];
2247 auto const& c0 = s2.csr0;
2248 for (Long idx = c0.row_offset[r]; idx < c0.row_offset[r+1]; ++idx) {
2249 if (c0.col_index[idx] != int(r)) {
2250 ocol[q] = c0.col_index[idx];
2251 omat[q] = c0.mat[idx];
2252 ++q;
2253 }
2254 }
2255 if (detail::has_remote_row(s2, r)) {
2256 auto const& c1 = s2.csr1;
2257 Long const rr = s2.row_map[r];
2258 for (Long idx = c1.row_offset[rr]; idx < c1.row_offset[rr+1]; ++idx) {
2259 ocol[q] = nlocal_c + c1.col_index[idx];
2260 omat[q] = c1.mat[idx];
2261 ++q;
2262 }
2263 }
2264 });
2266 SpMatrix<TS> S2d;
2267 S2d.define_split(cpart, cpart, std::move(out), nlocal_c, s2.col_map, 0);
2268 return S2d;
2269}
2270
2271template <typename T>
2273 Gpu::DeviceVector<int> const& state,
2274 Gpu::DeviceVector<Long> const& cidx,
2275 AlgPartition const& cpart)
2276{
2277 auto const& p = P.const_parcsr();
2278 int const nlocal = int(P.columnPartition().numLocalRows());
2279 auto csr = detail::compact_rows<T>
2280 (p, cpart.numLocalRows(), state.data(), cidx.data(), int(PointType::coarse),
2281 [=] AMREX_GPU_DEVICE (int j, bool remote) -> int {
2282 return remote ? nlocal + j : j;
2283 });
2284 SpMatrix<T> Q;
2285 Q.define_split(cpart, P.columnPartition(), std::move(csr), nlocal, p.col_map, 0);
2286 return Q;
2287}
2288
2289template <typename T>
2291 int max_elmts, T trunc_factor)
2292{
2293 BL_PROFILE("AlgMG::truncate_interp");
2294
2295 constexpr int KMAX = max_p_elmts; // rows of P are short
2296 AMREX_ALWAYS_ASSERT(max_elmts <= KMAX);
2297
2298 P.setColumnPartition(cpart);
2299 auto const& p = P.const_parcsr();
2300 Long const nrows = p.csr0.nrows;
2301 int const nlocal = int(cpart.numLocalRows());
2302 detail::TruncRow<T,KMAX> const tr{p, max_elmts, trunc_factor, nlocal};
2303
2304 // Drop threshold and signed sums of each row, computed once.
2305 Gpu::DeviceVector<T> rthr(nrows), rpos(nrows), rneg(nrows);
2306 auto* AMREX_RESTRICT pthr = rthr.data();
2307 auto* AMREX_RESTRICT ppos = rpos.data();
2308 auto* AMREX_RESTRICT pneg = rneg.data();
2309 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
2310 {
2311 pthr[i] = tr.threshold(i, ppos[i], pneg[i]);
2312 });
2313
2314 using local_csr_type = typename SpMatrix<T>::local_csr_type;
2315 local_csr_type csr;
2316 csr.row_offset.resize(nrows+1);
2317 auto* AMREX_RESTRICT prow = csr.row_offset.data();
2318 Long const nnz = Scan::PrefixSum<Long>
2319 (nrows+1,
2320 [=] AMREX_GPU_DEVICE (Long i) -> Long {
2321 if (i >= nrows) { return 0; }
2322 int const n = tr.count(i, pthr[i]);
2323 return (max_elmts > 0) ? std::min(n, max_elmts) : n;
2324 },
2325 [=] AMREX_GPU_DEVICE (Long i, Long const& x) { prow[i] = x; },
2327
2328 csr.resize(nrows, nnz);
2329 auto* AMREX_RESTRICT pmat = csr.mat.data();
2330 auto* AMREX_RESTRICT pcol = csr.col_index.data();
2331 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
2332 {
2333 T const thresh = pthr[i];
2334 T const sum_pos = ppos[i];
2335 T const sum_neg = pneg[i];
2336 Long q = prow[i];
2337 if (max_elmts > 0) {
2338 int cols[KMAX];
2339 T vals[KMAX];
2340 int const n = tr.select(i, thresh, cols, vals);
2341 T kept_pos = T(0), kept_neg = T(0);
2342 for (int k = 0; k < n; ++k) {
2343 if (vals[k] > T(0)) { kept_pos += vals[k]; } else { kept_neg += vals[k]; }
2344 }
2345 T const fpos = (kept_pos != T(0)) ? sum_pos/kept_pos : T(1);
2346 T const fneg = (kept_neg != T(0)) ? sum_neg/kept_neg : T(1);
2347 for (int k = 0; k < n; ++k) {
2348 pcol[q] = cols[k];
2349 pmat[q] = vals[k] * ((vals[k] > T(0)) ? fpos : fneg);
2350 ++q;
2351 }
2352 } else { // threshold only, no limit on the count
2353 T kept_pos = T(0), kept_neg = T(0);
2354 tr.walk(i, [&] (int, T v) {
2355 if (std::abs(v) >= thresh) {
2356 if (v > T(0)) { kept_pos += v; } else { kept_neg += v; }
2357 }
2358 });
2359 T const fpos = (kept_pos != T(0)) ? sum_pos/kept_pos : T(1);
2360 T const fneg = (kept_neg != T(0)) ? sum_neg/kept_neg : T(1);
2361 tr.walk(i, [&] (int col, T v) {
2362 if (std::abs(v) >= thresh) {
2363 pcol[q] = col;
2364 pmat[q] = v * ((v > T(0)) ? fpos : fneg);
2365 ++q;
2366 }
2367 });
2368 }
2369 });
2371
2372 // Compact columns sort like the global ones within each block.
2373 csr.sort();
2374 SpMatrix<T> Pt;
2375 Pt.define_split(P.partition(), cpart, std::move(csr), nlocal, p.col_map, 0);
2376 return Pt;
2377}
2378
2379template <typename T>
2381 SpMatrix<TS> const& ST,
2382 Gpu::DeviceVector<int> const& state,
2383 Gpu::DeviceVector<Long> const& cidx,
2384 AlgPartition const& cpart, T sscale,
2385 bool plus_i, int const* row_state,
2386 int max_elmts, T trunc_factor, Long* nnz_untruncated)
2387{
2388 BL_PROFILE("AlgMG::build_interp_mm");
2389
2390 // W = -M_F B_F of Li, Sjögreen & Yang (2021), Eqs. (3.4)-(3.12). Both
2391 // factors keep the natural row numbering with unit C rows, so P is one
2392 // product SpGEMM(M, B).
2393
2394 auto const& s = S.const_parcsr();
2395 Long const nrows = s.csr0.nrows;
2396 auto const* AMREX_RESTRICT ps = state.data();
2397 auto const* AMREX_RESTRICT pc = cidx.data();
2398 Long const cbegin = cpart.globalRowBegin();
2399 int const nlocal_f = int(A.numLocalRows());
2400 int const nlocal_c = int(cpart.numLocalRows());
2401 int const C = PointType::coarse;
2402 int const F = PointType::fine;
2403
2404 // alpha_i = a_ii + weak off-diagonal row sum = rowsum(A) - rowsum(S),
2405 // in the units of S. The weights do not depend on the scaling.
2406 auto const rsA = A.rowSum();
2407 auto const rsS = S.rowSum();
2408 auto const* AMREX_RESTRICT prsA = rsA.data();
2409 auto const* AMREX_RESTRICT prsS = rsS.data();
2410
2411 Gpu::DeviceVector<Long> gcidx(nrows);
2412 auto* AMREX_RESTRICT pgc = gcidx.data();
2413 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept { pgc[i] = pc[i] + cbegin; });
2414 auto const state_r = S.gatherRemote(ps);
2415 auto const gcidx_r = S.gatherRemote(pgc);
2416 auto const* AMREX_RESTRICT pstate_r = state_r.data();
2417
2418 // D_beta: row sum of strong connections to C-points.
2419 Gpu::DeviceVector<T> dbeta(nrows);
2420 auto* AMREX_RESTRICT pdb = dbeta.data();
2421 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
2422 {
2423 T d = T(0);
2424 auto const& sc = s.csr0;
2425 for (Long idx = sc.row_offset[i]; idx < sc.row_offset[i+1]; ++idx) {
2426 if (ps[sc.col_index[idx]] == C) { d += T(sc.mat[idx]); }
2427 }
2428 if (detail::has_remote_row(s, i)) {
2429 auto const& sc1 = s.csr1;
2430 Long const ii = s.row_map[i];
2431 for (Long idx = sc1.row_offset[ii]; idx < sc1.row_offset[ii+1]; ++idx) {
2432 if (pstate_r[sc1.col_index[idx]] == C) { d += T(sc1.mat[idx]); }
2433 }
2434 }
2435 pdb[i] = d;
2436 });
2438 auto const dbeta_r = S.gatherRemote(pdb);
2439
2440 detail::MMInterpRow<T,TS> const w{s, ST.const_parcsr(), ps, pdb, pc,
2441 pstate_r, dbeta_r.data(), nlocal_f, nlocal_c};
2442
2443 // Counts: M gets a diagonal plus the usable strong F neighbors; B gets
2444 // the strong C neighbors. C rows get one unit entry each.
2445 using local_csr_type = typename SpMatrix<T>::local_csr_type;
2446 local_csr_type mcsr, bcsr;
2447 mcsr.row_offset.resize(nrows+1);
2448 bcsr.row_offset.resize(nrows+1);
2449 auto* AMREX_RESTRICT pmrow = mcsr.row_offset.data();
2450 auto* AMREX_RESTRICT pbrow = bcsr.row_offset.data();
2451
2452 auto usable_f = [=] AMREX_GPU_DEVICE (T aji, int st_j, T db_j) -> bool {
2453 return st_j == F && (db_j + (plus_i ? aji : T(0))) != T(0);
2454 };
2455
2456 // Rows of P: all, or the C-points of `row_state` (the others stay
2457 // empty in M and so in P, but B needs every row).
2458 auto const* AMREX_RESTRICT prow_state = row_state;
2459 auto has_row = [=] AMREX_GPU_DEVICE (Long i) -> bool {
2460 return prow_state == nullptr || prow_state[i] == C;
2461 };
2462
2463 Long const mnnz = Scan::PrefixSum<Long>
2464 (nrows+1,
2465 [=] AMREX_GPU_DEVICE (Long i) -> Long {
2466 if (i >= nrows || !has_row(i)) { return 0; }
2467 Long n = 1;
2468 if (ps[i] == F) {
2469 w.walk(i, [&] (int, int, T, T aji, int st_j, T db_j) {
2470 if (usable_f(aji, st_j, db_j)) { ++n; }
2471 });
2472 }
2473 return n;
2474 },
2475 [=] AMREX_GPU_DEVICE (Long i, Long const& x) { pmrow[i] = x; },
2477
2478 Long const bnnz = Scan::PrefixSum<Long>
2479 (nrows+1,
2480 [=] AMREX_GPU_DEVICE (Long i) -> Long {
2481 if (i >= nrows) { return 0; }
2482 if (ps[i] == C) { return 1; }
2483 Long n = 0;
2484 w.walk(i, [&] (int, int, T, T, int st_j, T) {
2485 if (st_j == C) { ++n; }
2486 });
2487 return n;
2488 },
2489 [=] AMREX_GPU_DEVICE (Long i, Long const& x) { pbrow[i] = x; },
2491
2492 mcsr.resize(nrows, mnnz);
2493 bcsr.resize(nrows, bnnz);
2494 auto* AMREX_RESTRICT pmmat = mcsr.mat.data();
2495 auto* AMREX_RESTRICT pmcol = mcsr.col_index.data();
2496 auto* AMREX_RESTRICT pbmat = bcsr.mat.data();
2497 auto* AMREX_RESTRICT pbcol = bcsr.col_index.data();
2498
2499 ParallelForOMP(nrows, [=] AMREX_GPU_DEVICE (Long i) noexcept
2500 {
2501 Long mp = pmrow[i];
2502 Long bp = pbrow[i];
2503 bool const in_p = has_row(i);
2504 if (ps[i] != F) {
2505 if (in_p) {
2506 pmcol[mp] = int(i);
2507 pmmat[mp] = T(1);
2508 }
2509 pbcol[bp] = int(pc[i]);
2510 pbmat[bp] = T(1);
2511 return;
2512 }
2513 T const db_i = pdb[i];
2514 T const bscale = plus_i ? T(1) : ((db_i != T(0)) ? T(1)/db_i : T(0));
2515 if (!in_p) {
2516 w.walk(i, [&] (int, int jc, T aij, T, int st_j, T) {
2517 if (st_j == C) {
2518 pbcol[bp] = jc;
2519 pbmat[bp] = bscale*aij;
2520 ++bp;
2521 }
2522 });
2523 return;
2524 }
2525
2526 // Strong F neighbors without a usable denominator are lumped into
2527 // the diagonal as weak connections (remark after Eq. (3.8)).
2528 T alpha = prsA[i]/sscale - T(prsS[i]);
2529 T theta = T(0);
2530 w.walk(i, [&] (int, int, T aij, T aji, int st_j, T db_j) {
2531 if (st_j == F) {
2532 T const den = db_j + (plus_i ? aji : T(0));
2533 if (den != T(0)) {
2534 if (plus_i) { theta += aij * aji / den; }
2535 } else {
2536 alpha += aij;
2537 }
2538 }
2539 });
2540 T const denom = alpha + theta;
2541 T const scale = (denom != T(0)) ? T(-1)/denom : T(0);
2542
2543 pmcol[mp] = int(i);
2544 pmmat[mp] = plus_i ? scale : scale*db_i;
2545 ++mp;
2546 w.walk(i, [&] (int jf, int jc, T aij, T aji, int st_j, T db_j) {
2547 if (st_j == C) {
2548 pbcol[bp] = jc;
2549 pbmat[bp] = bscale*aij;
2550 ++bp;
2551 } else if (usable_f(aji, st_j, db_j)) {
2552 pmcol[mp] = jf;
2553 pmmat[mp] = plus_i ? scale*aij/(db_j + aji) : scale*aij;
2554 ++mp;
2555 }
2556 });
2557 });
2559
2560 // The diagonal of M is emitted first and must be sorted into place.
2561 mcsr.sort();
2562 SpMatrix<T> M, B;
2563 M.define_split(A.partition(), A.partition(), std::move(mcsr), nlocal_f, s.col_map, 0);
2564 B.define_split(A.partition(), cpart, std::move(bcsr), nlocal_c, gcidx_r.data(), 0);
2565#ifndef AMREX_USE_GPU
2566 if (max_elmts > 0 || trunc_factor > T(0)) {
2567 // Each row is truncated as the product makes it.
2568 constexpr int KMAX = max_p_elmts;
2569 AMREX_ALWAYS_ASSERT(max_elmts <= KMAX);
2570 Vector<Long> n0(nrows, 0);
2571 auto P = amrex::SpGEMM(M, B, cpart, [&] (Long i, int* col, T* val, Long n) {
2572 n0[i] = n;
2573 return detail::truncate_row_host<T,KMAX>(col, val, n, max_elmts, trunc_factor,
2574 nlocal_c);
2575 });
2576 if (nnz_untruncated) {
2577 *nnz_untruncated = std::accumulate(n0.begin(), n0.end(), Long(0));
2578 }
2579 return P;
2580 }
2581#else
2582 amrex::ignore_unused(max_elmts, trunc_factor, nnz_untruncated);
2583#endif
2584 return amrex::SpGEMM(M, B, cpart);
2585}
2586
2587#ifndef AMREX_USE_GPU
2588template <typename T>
2590 SpMatrix<TS> const& ST,
2591 Gpu::DeviceVector<int> const& state,
2592 Gpu::DeviceVector<Long> const& cidx,
2593 AlgPartition const& cpart, T sscale, bool plus_i,
2594 int max_elmts, T trunc_factor,
2595 Long& nnz_untruncated)
2596{
2597 BL_PROFILE("AlgMG::build_interp_mm_host");
2598
2599 // Row i of P = M B (see build_interp_mm) is summed from the rows of B
2600 // in the order of the SpGEMM, then truncated as in truncate_interp.
2601 constexpr int KMAX = max_p_elmts;
2602 AMREX_ALWAYS_ASSERT(max_elmts <= KMAX);
2603
2604 auto const& s = S.const_parcsr();
2605 Long const nrows = s.csr0.nrows;
2606 auto const* AMREX_RESTRICT ps = state.data();
2607 auto const* AMREX_RESTRICT pc = cidx.data();
2608 int const nc = int(cpart.numLocalRows());
2609 int const C = PointType::coarse;
2610 int const F = PointType::fine;
2611
2612 auto const rsA = A.rowSum();
2613 auto const rsS = S.rowSum();
2614 auto const* AMREX_RESTRICT prsA = rsA.data();
2615 auto const* AMREX_RESTRICT prsS = rsS.data();
2616
2617 // D_beta and the rows of B at the F-points: the strong C neighbors.
2618 Gpu::DeviceVector<T> dbeta(nrows);
2619 auto* AMREX_RESTRICT pdb = dbeta.data();
2620 Vector<Long> boff(nrows+1, 0);
2621 auto const& sc = s.csr0;
2622 ParallelForOMP(nrows, [&] (Long i) noexcept
2623 {
2624 T d = T(0);
2625 Long n = 0;
2626 for (Long idx = sc.row_offset[i]; idx < sc.row_offset[i+1]; ++idx) {
2627 if (ps[sc.col_index[idx]] == C) {
2628 d += T(sc.mat[idx]);
2629 ++n;
2630 }
2631 }
2632 pdb[i] = d;
2633 boff[i+1] = (ps[i] == F) ? n : 0;
2634 });
2635 std::partial_sum(boff.begin(), boff.end(), boff.begin());
2636 Vector<int> bcol(boff[nrows]);
2637 Vector<T> bval(boff[nrows]);
2638 ParallelForOMP(nrows, [&] (Long i) noexcept
2639 {
2640 if (ps[i] != F) { return; }
2641 T const bscale = plus_i ? T(1) : ((pdb[i] != T(0)) ? T(1)/pdb[i] : T(0));
2642 Long q = boff[i];
2643 for (Long idx = sc.row_offset[i]; idx < sc.row_offset[i+1]; ++idx) {
2644 int const j = sc.col_index[idx];
2645 if (ps[j] == C) {
2646 bcol[q] = int(pc[j]);
2647 bval[q] = bscale*T(sc.mat[idx]);
2648 ++q;
2649 }
2650 }
2651 });
2652
2653 detail::MMInterpRow<T,TS> const w{s, ST.const_parcsr(), ps, pdb, pc,
2654 nullptr, nullptr, int(nrows), nc};
2655 bool const trunc = (max_elmts > 0 || trunc_factor > T(0));
2656 struct Nbr { int j; T aij; T den; };
2657
2659 auto csr = detail::csr_from_rows_cpu<T,SpMatrix<T>::template container_type,int>
2660 (nrows, [&] (Long i) { return sc.row_offset[i+1] - sc.row_offset[i] + 1; },
2661 [&] (int t) {
2662 return [&, t, marker = Vector<int>(nc, -1), col = Vector<int>(1), val = Vector<T>(1),
2663 tmp = Vector<std::pair<int,T>>(), nbr = Vector<Nbr>()]
2664 (Long i, int const*& rc, T const*& rv) mutable -> Long
2665 {
2666 rc = col.data();
2667 rv = val.data();
2668 if (ps[i] != F) {
2669 col[0] = int(pc[i]);
2670 val[0] = T(1);
2671 ++nnz0[t];
2672 return 1;
2673 }
2674
2675 // Scale of row i, as in build_interp_mm, and the usable strong
2676 // F neighbors (column order) for M.
2677 T alpha = prsA[i]/sscale - T(prsS[i]);
2678 T theta = T(0);
2679 Long maxlen = boff[i+1] - boff[i];
2680 nbr.clear();
2681 w.walk(i, [&] (int jf, int, T aij, T aji, int st_j, T db_j) {
2682 if (st_j == F) {
2683 T const den = db_j + (plus_i ? aji : T(0));
2684 if (den != T(0)) {
2685 if (plus_i) { theta += aij * aji / den; }
2686 maxlen += boff[jf+1] - boff[jf];
2687 nbr.push_back({jf, aij, den});
2688 } else {
2689 alpha += aij;
2690 }
2691 }
2692 });
2693 T const denom = alpha + theta;
2694 T const scale = (denom != T(0)) ? T(-1)/denom : T(0);
2695 maxlen = std::min(maxlen, Long(nc));
2696 if (Long(col.size()) < maxlen) {
2697 col.resize(maxlen);
2698 val.resize(maxlen);
2699 rc = col.data();
2700 rv = val.data();
2701 }
2702
2703 // M_ik times row k of B, k in increasing order (M's row sorted).
2704 Long n = 0;
2705 auto add = [&] (Long k, T m) {
2706 for (Long bp = boff[k]; bp < boff[k+1]; ++bp) {
2707 int const j = bcol[bp];
2708 Long const pos = marker[j];
2709 if (pos >= 0 && pos < n && col[pos] == j) {
2710 val[pos] += m * bval[bp];
2711 } else {
2712 marker[j] = int(n);
2713 col[n] = j;
2714 val[n] = m * bval[bp];
2715 ++n;
2716 }
2717 }
2718 };
2719 T const mdiag = plus_i ? scale : scale*pdb[i];
2720 bool diag_done = false;
2721 for (auto const& nb : nbr) {
2722 if (!diag_done && nb.j > i) {
2723 add(i, mdiag);
2724 diag_done = true;
2725 }
2726 add(nb.j, plus_i ? scale*nb.aij/nb.den : scale*nb.aij);
2727 }
2728 if (!diag_done) { add(i, mdiag); }
2729 detail::sort_row_cpu(col.data(), val.data(), n, tmp);
2730 nnz0[t] += n;
2731 if (!trunc) { return n; }
2732
2733 return detail::truncate_row_host<T,KMAX>(col.data(), val.data(), n, max_elmts,
2734 trunc_factor, nc);
2735 };
2736 });
2737
2738 nnz_untruncated = std::accumulate(nnz0.begin(), nnz0.end(), Long(0));
2739 SpMatrix<T> P;
2740 P.define_split(A.partition(), cpart, std::move(csr), nc, nullptr, 0);
2741 return P;
2742}
2743#endif
2744
2745}
2746
2747#endif
#define BL_PROFILE(a)
Definition AMReX_BLProfiler.H:593
#define AMREX_ALWAYS_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:49
#define AMREX_ALWAYS_ASSERT(EX)
Definition AMReX_BLassert.H:50
Enum reflection utilities and the AMREX_ENUM macro.
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
#define AMREX_RESTRICT
Definition AMReX_Extension.H:37
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
while parser_exe_t p
Definition AMReX_Parser_Exe_Body.H:33
Algebraic multigrid solver for SpMatrix / AlgVector systems.
Definition AMReX_AlgMG.H:254
SpMatrix< TS > create_soc(SpMatrix< T > const &A, T scale) const
Definition AMReX_AlgMG.H:1498
void setAggressiveNumLevels(int n)
Number of levels with aggressive coarsening, starting at the finest. Next setup.
Definition AMReX_AlgMG.H:328
void setFixedIter(int n)
If positive, run exactly this many V-cycles regardless of tolerance.
Definition AMReX_AlgMG.H:295
void setMaxIter(int n)
Definition AMReX_AlgMG.H:293
void setChebyshevDegree(int d)
Chebyshev polynomial degree. Next solve.
Definition AMReX_AlgMG.H:304
SpMatrix< T > const & getMatrix(int lev) const
Definition AMReX_AlgMG.H:388
AlgMGInterpType InterpType
Definition AMReX_AlgMG.H:262
AlgMGKrylovSolver KrylovSolver
Definition AMReX_AlgMG.H:261
T getResidualNorm() const
Definition AMReX_AlgMG.H:381
void setMaxCoarseSize(Long n)
Stop coarsening when a level has at most this many rows. Next setup.
Definition AMReX_AlgMG.H:326
void solve(AlgVector< T > &a_sol, AlgVector< T > const &a_rhs)
Solve A x = b. a_sol holds the initial guess on entry.
Definition AMReX_AlgMG.H:689
static SpMatrix< TS > second_pass_strength(SpMatrix< TS > &S, Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > const &cidx, AlgPartition const &cpart)
Definition AMReX_AlgMG.H:2150
void setRelTol(T t)
Definition AMReX_AlgMG.H:296
std::conditional_t< std::is_same_v< T, double >, float, T > TS
Definition AMReX_AlgMG.H:258
void setBottomMaxIter(int n)
Definition AMReX_AlgMG.H:374
void setPreSmooth(int nu)
Number of pre-smoothing sweeps; negative restores the smoother's default.
Definition AMReX_AlgMG.H:311
static constexpr int default_p_max_elmts
Definition AMReX_AlgMG.H:349
void setBottomSolver(BottomSolver bs)
Definition AMReX_AlgMG.H:366
void define(SpMatrix< T > &a_mat)
Define the solver with a square matrix. Must be called before solve.
Definition AMReX_AlgMG.H:632
void setPMaxElmts(int n)
Definition AMReX_AlgMG.H:343
void setAbsTol(T t)
Converged when the residual 2-norm is below max(reltol*bnorm, abstol).
Definition AMReX_AlgMG.H:298
bool coarsen_level(int lev, SpMatrix< T > &A, SpMatrix< T > &P, AlgPartition &cpart, bool &aggressive, Long &ncoarse, bool &truncated, Long &nnz_untruncated)
Definition AMReX_AlgMG.H:1336
void setSingular(bool b)
Definition AMReX_AlgMG.H:336
void setPrintIndentation(std::string s)
Prefix of the lines the solver prints, e.g. to nest them in MLMG output.
Definition AMReX_AlgMG.H:292
AlgMG()=default
void setPostSmooth(int nu)
Number of post-smoothing sweeps; negative restores the smoother's default.
Definition AMReX_AlgMG.H:313
void setInterpType(InterpType it)
Takes effect at the next setup.
Definition AMReX_AlgMG.H:340
SpMatrix< T > const & getInterp(int lev) const
Definition AMReX_AlgMG.H:389
AlgMGBottomSolver BottomSolver
Definition AMReX_AlgMG.H:260
void setBottomSmooth(int nu)
Definition AMReX_AlgMG.H:316
bool updateMatrix(SpMatrix< T > &a_mat)
Replace the matrix, keeping the hierarchy.
Definition AMReX_AlgMG.H:644
void setTruncFactor(T f)
Drop interpolation weights below this fraction of the row maximum. Next setup.
Definition AMReX_AlgMG.H:351
static AlgPartition coarse_numbering(Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > &cidx, int which=PointType::coarse)
Local index among the rows of type which and their partition.
Definition AMReX_AlgMG.H:1816
void setBottomVerbose(int v)
Definition AMReX_AlgMG.H:375
static SpMatrix< T > select_coarse_rows(SpMatrix< T > const &P, Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > const &cidx, AlgPartition const &cpart)
Rows of P at the C-points, in C-point numbering.
Definition AMReX_AlgMG.H:2272
static T strength_scale(SpMatrix< T > const &A)
Largest |a_ii|, the unit of the strength matrix entries.
Definition AMReX_AlgMG.H:1487
void setMaxLevels(int n)
Definition AMReX_AlgMG.H:324
static SpMatrix< T > truncate_interp(SpMatrix< T > &P, AlgPartition const &cpart, int max_elmts, T trunc_factor)
Definition AMReX_AlgMG.H:2290
int numLevels() const
Number of levels; 0 before setup.
Definition AMReX_AlgMG.H:383
Long numGlobalRows(int lev) const
Global number of rows on a level. Valid after setup.
Definition AMReX_AlgMG.H:387
static constexpr int max_p_elmts
Definition AMReX_AlgMG.H:348
void precond(AlgVector< T > &x, AlgVector< T > const &b)
Definition AMReX_AlgMG.H:777
void setStrongThreshold(T theta)
Definition AMReX_AlgMG.H:321
static SpMatrix< T > build_interp_mm(SpMatrix< T > &A, SpMatrix< TS > &S, SpMatrix< TS > const &ST, Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > const &cidx, AlgPartition const &cpart, T scale, bool plus_i, int const *row_state=nullptr, int max_elmts=0, T trunc_factor=T(0), Long *nnz_untruncated=nullptr)
Definition AMReX_AlgMG.H:2380
void setRelaxWeight(T w)
Relaxation weight of the smoother; negative restores the default. Next solve.
Definition AMReX_AlgMG.H:302
void setup()
Definition AMReX_AlgMG.H:1020
void setAggressiveDirectInterp(bool b)
Definition AMReX_AlgMG.H:331
static SpMatrix< T > build_interp(SpMatrix< T > &A, SpMatrix< TS > const &S, Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > const &cidx, AlgPartition const &cpart)
Classical direct interpolation.
Definition AMReX_AlgMG.H:1841
SpMatrix< T > interpolation(SpMatrix< T > &A, SpMatrix< TS > &S, SpMatrix< TS > const &ST, Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > const &cidx, AlgPartition const &cpart, T scale) const
Interpolation of the selected type from the C-points in state.
Definition AMReX_AlgMG.H:2136
void pmis(SpMatrix< TS > &S, SpMatrix< TS > &ST, Gpu::DeviceVector< int > &state, bool isolated_as_coarse=false) const
Definition AMReX_AlgMG.H:1672
std::size_t getSetupPeakBytes() const
Peak bytes in use in The_Arena during setup (0 if not available).
Definition AMReX_AlgMG.H:385
void setSmoother(Smoother sm)
Takes effect at the next solve; the hierarchy is kept.
Definition AMReX_AlgMG.H:300
void setThrowException(bool b)
Definition AMReX_AlgMG.H:376
void setVerbose(int v)
Definition AMReX_AlgMG.H:290
static void normalize_rows(SpMatrix< T > &P)
Scales the rows of P to sum to one; aborts on an empty row.
Definition AMReX_AlgMG.H:1458
int getNumSetups() const
Number of setups run so far (for checks that the setup is reused).
Definition AMReX_AlgMG.H:380
void project_out(AlgVector< T > &x)
Definition AMReX_AlgMG.H:554
void setBottomTol(T t)
Definition AMReX_AlgMG.H:373
AlgMGSmoother Smoother
Definition AMReX_AlgMG.H:263
void setKrylovSolver(KrylovSolver a)
Krylov solver with the V-cycle as preconditioner (see KrylovSolver). Default None.
Definition AMReX_AlgMG.H:353
void setChebyshevRatio(T r)
Chebyshev eigenvalue ratio lambda_max / lambda_min. Next solve.
Definition AMReX_AlgMG.H:306
int getNumIters() const
Definition AMReX_AlgMG.H:378
Definition AMReX_AlgPartition.H:26
Long numGlobalRows() const
Total number of rows covered by the partition.
Definition AMReX_AlgPartition.H:55
Long globalRowBegin() const
Inclusive global index begin on this process.
Definition AMReX_AlgPartition.H:62
Long numLocalRows() const
Number of local rows.
Definition AMReX_AlgPartition.H:50
Distributed dense vector that mirrors the layout of an AlgPartition.
Definition AMReX_AlgVector.H:29
void setVal(T val)
Definition AMReX_AlgVector.H:273
Long numLocalRows() const
Number of entries stored on this rank.
Definition AMReX_AlgVector.H:74
T const * data() const
Definition AMReX_AlgVector.H:85
void plusAsync(AlgVector< T, Allocator > const &rhs)
Definition AMReX_AlgVector.H:312
Chebyshev polynomial smoother for AlgVector/SpMatrix systems.
Definition AMReX_Smoother_MV.H:163
static std::size_t freeMemAvailable()
Definition AMReX_GpuDevice.cpp:1481
Jacobi smoother for AlgVector/SpMatrix linear systems.
Definition AMReX_Smoother_MV.H:32
l1 hybrid Gauss-Seidel smoother for CPUs.
Definition AMReX_Smoother_MV.H:318
Dynamically allocated vector for trivially copyable data.
Definition AMReX_PODVector.H:308
size_type size() const noexcept
Definition AMReX_PODVector.H:654
void resize(size_type a_new_size, GrowthStrategy strategy=GrowthStrategy::Poisson)
Definition AMReX_PODVector.H:734
T * data() noexcept
Definition AMReX_PODVector.H:672
This class provides the user with a few print options.
Definition AMReX_Print.H:35
Distributed CSR matrix that manages storage and GPU-friendly partitions.
Definition AMReX_SpMatrix.H:65
void setColumnPartition(AlgPartition const &col_partition)
Set the column partition and split the matrix into local and remote blocks with 32-bit column indices...
Definition AMReX_SpMatrix.H:1019
Long numGlobalRows() const
Global row count.
Definition AMReX_SpMatrix.H:196
Gpu::DeviceVector< U > gatherRemote(U const *x)
Gather values at the columns of the off-diagonal block.
Definition AMReX_SpMatrix.H:1342
Long numLocalNonZeros() const
Number of nonzeros stored locally.
Definition AMReX_SpMatrix.H:198
AlgVector< T, AllocT > rowSum() const
Sum the values in each local row and return the result as an AlgVector.
Definition AMReX_SpMatrix.H:1164
void define_split(AlgPartition partition, AlgPartition const &col_partition, local_csr_type &&compact, Long nlocal, Long const *remote_cols, Long nremote)
Define directly in split form from a CSR with compact 32-bit columns: c < nlocal is the local column ...
Definition AMReX_SpMatrix.H:1695
AlgPartition const & columnPartition() const
Return the column partition used for matrix-vector and matrix-matrix multiplications.
Definition AMReX_SpMatrix.H:191
ParCsr< T const > const_parcsr() const
Const-qualified alias of parcsr() for convenience.
Definition AMReX_SpMatrix.H:1226
ParCsr< T > parcsr()
Build GPU-friendly CSR views split into diagonal/off-diagonal blocks.
Definition AMReX_SpMatrix.H:1201
Long numLocalRows() const
Number of rows owned by this rank.
Definition AMReX_SpMatrix.H:194
AlgPartition const & partition() const
Row partition describing how matrix rows are distributed across ranks.
Definition AMReX_SpMatrix.H:182
AlgVector< T, AllocT > const & diagonalVector() const
Return diagonal elements in a square matrix.
Definition AMReX_SpMatrix.H:1145
CSR< T, container_type, int > local_csr_type
Definition AMReX_SpMatrix.H:71
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
amrex_long Long
Definition AMReX_INT.H:30
T Min(N n, T const *v, T init_val=std::numeric_limits< T >::max())
Compute the minimum of an array of values.
Definition AMReX_Reduce.H:1105
T Max(N n, T const *v, T init_val=std::numeric_limits< T >::lowest())
Compute the maximum of an array of values.
Definition AMReX_Reduce.H:1128
T Sum(N n, T const *v, T init_val=0)
Compute the sum of an array of values.
Definition AMReX_Reduce.H:1082
void ParallelForOMP(T n, L const &f) noexcept
Performance-portable kernel launch function with optional OpenMP threading.
Definition AMReX_GpuLaunch.H:328
PODVector< T, PinnedArenaAllocator< T > > PinnedVector
A PODVector that uses the pinned memory arena.
Definition AMReX_GpuContainers.H:48
PODVector< T, ArenaAllocator< T > > DeviceVector
A PODVector that uses the standard memory Arena. Note that the memory might or might not be managed d...
Definition AMReX_GpuContainers.H:24
void Min(KeyValuePair< K, V > &vi, MPI_Comm comm)
Definition AMReX_ParallelReduce.H:161
void Sum(Gpu::DeviceVector< T > &v, MPI_Comm comm)
Definition AMReX_GpuParallelReduce.H:37
void AllGather(const T *v, int cnt, T *vs, MPI_Comm comm)
Definition AMReX_ParallelReduce.H:106
void Max(KeyValuePair< K, V > &vi, MPI_Comm comm)
Definition AMReX_ParallelReduce.H:133
#define AMREX_ENUM(CLASS,...)
Declare a scoped enum with reflection support.
Definition AMReX_Enum.H:270
void copy(HostToDevice, InIter begin, InIter end, OutIter result) noexcept
A host-to-device copy routine. Note this is just a wrapper around memcpy, so it assumes contiguous st...
Definition AMReX_GpuContainers.H:128
static constexpr DeviceToHost deviceToHost
Definition AMReX_GpuContainers.H:106
static constexpr HostToDevice hostToDevice
Definition AMReX_GpuContainers.H:105
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
std::string const & name()
Definition AMReX_Machine.cpp:46
constexpr T powi(T x) noexcept
Return pow(x, Power), where Power is an integer known at compile time.
Definition AMReX_Math.H:377
constexpr int get_max_threads()
Definition AMReX_OpenMP.H:36
MPI_Comm CommunicatorSub() noexcept
sub-communicator for current frame
Definition AMReX_ParallelContext.H:70
int NProcsSub() noexcept
number of ranks in current frame
Definition AMReX_ParallelContext.H:74
static constexpr struct amrex::Scan::Type::Exclusive exclusive
static constexpr RetSum retSum
Definition AMReX_Scan.H:34
T PrefixSum(N n, FIN const &fin, FOUT const &fout, TYPE, RetSum a_ret_sum=retSum)
Definition AMReX_Scan.H:815
static constexpr int MPI_DATATYPE_NULL
Definition AMReX_ccse-mpi.H:56
int verbose
Definition AMReX.cpp:113
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
AlgMGInterpType
Definition AMReX_AlgMG.H:210
@ mm_ext
Definition AMReX_AlgMG.H:210
@ mm_ext_i
Definition AMReX_AlgMG.H:210
@ C
Definition AMReX_Order.H:7
@ F
Definition AMReX_Order.H:7
@ none
Definition AMReX.H:558
void SpMV(Long nrows, Long ncols, T *__restrict__ py, CsrView< T const, Index > const &A, T const *__restrict__ px)
Perform y = A * x using CSR data (GPU/CPU aware).
Definition AMReX_SpMV.H:30
@ RAP
Definition AMReX_MLNodeLinOp.H:22
__host__ __device__ bool isnan(T x) noexcept
Return true if x is NaN. Works under fast math.
Definition AMReX_Math.H:117
AlgMGBottomSolver
Definition AMReX_AlgMG.H:200
@ jacobi
Definition AMReX_AlgMG.H:200
@ direct
Definition AMReX_AlgMG.H:200
@ gmres
Definition AMReX_AlgMG.H:200
@ bicgstab
Definition AMReX_AlgMG.H:200
AlgMGKrylovSolver
Definition AMReX_AlgMG.H:206
@ pcg
Definition AMReX_AlgMG.H:206
void computeResidual(AlgVector< T, AllocV > &res, SpMatrix< T, AllocM > const &A, AlgVector< T, AllocV > const &x, AlgVector< T, AllocV > const &b)
Compute the residual res = b - A * x.
Definition AMReX_SpMV.H:345
@ y
Definition AMReX_Orientation.H:14
@ x
Definition AMReX_Orientation.H:14
@ z
Definition AMReX_Orientation.H:14
@ smoother
Definition AMReX_MLLinOp.H:44
@ bicgstab
Definition AMReX_MLLinOp.H:44
__host__ __device__ constexpr IntVectND< dim > scale(const IntVectND< dim > &p, int s) noexcept
Returns a IntVectND obtained by multiplying each of the components of this IntVectND by s.
Definition AMReX_IntVect.H:1135
SpMatrix< T, Allocator > SpGEMM(SpMatrix< T, Allocator > const &A, SpMatrix< T, Allocator > const &B, AlgPartition const &col_partition, F const &row_post)
SpGEMM with a callback on each row of the product.
Definition AMReX_SpGEMM.H:1216
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
const int[]
Definition AMReX_BLProfiler.cpp:1665
@ bnorm
Definition AMReX_MLMG.H:23
CSR< T, V, IO > transpose(CSR< T, V, Index > const &csr, Long ncols)
Build the transpose CSR of csr.
Definition AMReX_SpMatUtil.H:25
AlgMGSmoother
Definition AMReX_AlgMG.H:219
@ chebyshev
Definition AMReX_AlgMG.H:219
@ l1_gauss_seidel
Definition AMReX_AlgMG.H:219
@ l1_jacobi
Definition AMReX_AlgMG.H:219
V< Index > col_index
Definition AMReX_CSR.H:62
void resize(Long num_rows, Long num_non_zeros)
Resize the storage to accommodate num_rows and num_non_zeros entries.
Definition AMReX_CSR.H:80
V< T > mat
Definition AMReX_CSR.H:61
V< Index > row_offset
Definition AMReX_CSR.H:63
Long nrows() const
Number of logical rows represented by the CSR offset array.
Definition AMReX_CSR.H:67
Definition AMReX_SpMatrix.H:39