3#include <AMReX_Config.H>
34template <
typename TC,
typename Index,
typename T>
36bool pmis_blocked (CsrView<TC const,Index>
const& csr,
Long row, T wi,
Long ig,
40 for (
Long idx = csr.row_offset[row]; idx < csr.row_offset[row+1]; ++idx) {
41 Long j = csr.col_index[idx];
42 Long jg = gmap ? gmap[j] : j + goff;
43 if (jg != ig && state[j] == undecided &&
44 (w[j] > wi || (w[j] == wi && jg > ig))) {
55T pmis_tiebreak (
Long ig)
57 auto z =
static_cast<std::uint64_t
>(ig) + 0x9E3779B97F4A7C15ULL;
58 z = (
z ^ (
z >> 30)) * 0xBF58476D1CE4E5B9ULL;
59 z = (
z ^ (
z >> 27)) * 0x94D049BB133111EBULL;
61 constexpr int digits = std::numeric_limits<T>::digits;
62 static_assert(digits < 64,
"pmis_tiebreak: T must be float or double");
63 return T(
z >> (64 - digits)) / T(std::uint64_t(1) << digits);
67template <
typename T,
typename Index>
69bool csr_row_has_state (CsrView<T const,Index>
const& csr,
Long row,
72 for (
Long idx = csr.row_offset[row]; idx < csr.row_offset[row+1]; ++idx) {
73 if (state[csr.col_index[idx]] == c) {
return true; }
80template <
typename T,
typename TS>
99 auto const& ac = a.csr0;
100 auto const& sc = s.csr0;
101 Long sp = sc.row_offset[i];
102 Long const se = sc.row_offset[i+1];
103 for (
Long idx = ac.row_offset[i]; idx < ac.row_offset[i+1]; ++idx) {
104 Long const j = ac.col_index[idx];
105 T
const v = ac.mat[idx];
106 if (j == i) { diag = v;
continue; }
107 while (sp < se && sc.col_index[sp] < j) { ++sp; }
108 bool const strong_c = (sp < se && sc.col_index[sp] == j
109 && state[j] == coarse);
110 f(
int(cidx[j]), v, strong_c);
113 if (has_remote_row(a, i)) {
114 auto const& ac = a.csr1;
115 auto const& sc = s.csr1;
116 Long const ii = a.row_map[i];
118 if (has_remote_row(s, i)) {
119 Long const jj = s.row_map[i];
120 sp = sc.row_offset[jj];
121 se = sc.row_offset[jj+1];
123 for (
Long idx = ac.row_offset[ii]; idx < ac.row_offset[ii+1]; ++idx) {
124 Long const j = ac.col_index[idx];
125 Long const jg = a.col_map[j];
126 T
const v = ac.mat[idx];
127 while (sp < se && s.col_map[sc.col_index[sp]] < jg) { ++sp; }
128 bool const strong_c = (sp < se && s.col_map[sc.col_index[sp]] == jg
129 && state_r[j] == coarse);
130 f(
int(nlocal_c + j), v, strong_c);
140template <
typename T,
typename TS>
156 template <
typename F>
160 auto const& sc = s.csr0;
161 auto const& tc = st.csr0;
162 Long tp = tc.row_offset[i];
163 Long const te = tc.row_offset[i+1];
164 for (
Long idx = sc.row_offset[i]; idx < sc.row_offset[i+1]; ++idx) {
165 Long const j = sc.col_index[idx];
166 while (tp < te && tc.col_index[tp] < j) { ++tp; }
167 T
const aji = (tp < te && tc.col_index[tp] == j) ? T(tc.mat[tp]) : T(0);
168 f(
int(j),
int(cidx[j]), T(sc.mat[idx]), aji, state[j], dbeta[j]);
171 if (has_remote_row(s, i)) {
172 auto const& sc = s.csr1;
173 auto const& tc = st.csr1;
174 Long const ii = s.row_map[i];
176 if (has_remote_row(st, i)) {
177 Long const jj = st.row_map[i];
178 tp = tc.row_offset[jj];
179 te = tc.row_offset[jj+1];
181 for (
Long idx = sc.row_offset[ii]; idx < sc.row_offset[ii+1]; ++idx) {
182 Long const j = sc.col_index[idx];
183 Long const jg = s.col_map[j];
184 while (tp < te && st.col_map[tc.col_index[tp]] < jg) { ++tp; }
185 T
const aji = (tp < te && st.col_map[tc.col_index[tp]] == jg)
186 ? T(tc.mat[tp]) : T(0);
187 f(
int(nlocal_f + j),
int(nlocal_c + j), T(sc.mat[idx]), aji,
188 state_r[j], dbeta_r[j]);
258 using TS = std::conditional_t<std::is_same_v<T,double>, float, T>;
300 void setSmoother (
Smoother sm) {
if (m_smoother_type != sm) { m_smoother_type = sm; m_smoothers_stale =
true; } }
302 void setRelaxWeight (T w) {
if (m_relax_weight != w) { m_relax_weight = w; m_smoothers_stale =
true; } }
304 void setChebyshevDegree (
int d) {
if (m_cheby_degree != d) { m_cheby_degree = d; m_smoothers_stale =
true; } }
308 if (m_cheby_ratio != r) { m_cheby_ratio = r; m_smoothers_stale =
true; }
321 void setStrongThreshold (T theta) {
if (m_strong_threshold != theta) { m_strong_threshold = theta; m_prepared =
false; } }
324 void setMaxLevels (
int n) {
if (m_max_levels != n) { m_max_levels = n; m_prepared =
false; } }
328 void setAggressiveNumLevels (
int n) {
if (m_aggressive_num_levels != n) { m_aggressive_num_levels = n; m_prepared =
false; } }
331 void setAggressiveDirectInterp (
bool b) {
if (m_aggressive_direct_interp != b) { m_aggressive_direct_interp = b; m_prepared =
false; } }
337 if (m_singular != b) { m_singular = b; m_prepared =
false; }
346 if (m_p_max_elmts !=
p) { m_p_max_elmts =
p; m_prepared =
false; }
351 void setTruncFactor (T f) {
if (m_trunc_factor != f) { m_trunc_factor = f; m_prepared =
false; } }
354 if (m_krylov == a) {
return; }
356 m_krylov_bicgstab.reset();
357 m_krylov_gmres.reset();
358 m_krylov_pcg.reset();
367 if (m_bottom_solver == bs) {
return; }
368 m_bottom_solver = bs;
393 enum PointType :
int { undecided = 0, fine = 1, coarse = 2 };
395 bool m_prepared =
false;
396 int m_num_setups = 0;
397 bool m_smoothers_stale =
false;
398 bool m_throw_exception =
false;
404 Smoother m_smoother_type = Smoother::chebyshev;
405 T m_relax_weight = T(-1);
406 int m_cheby_degree = 2;
407 T m_cheby_ratio = T(6);
410 int m_nu_bottom = -1;
412 [[nodiscard]]
int numPreSmooth ()
const {
413 return (m_nu1 >= 0) ? m_nu1 : ((m_smoother_type == Smoother::chebyshev) ? 1 : 2);
415 [[nodiscard]]
int numPostSmooth ()
const {
416 return (m_nu2 >= 0) ? m_nu2 : ((m_smoother_type == Smoother::chebyshev) ? 1 : 2);
418 [[nodiscard]]
int numBottomSmooth ()
const {
419 return (m_nu_bottom >= 0) ? m_nu_bottom : ((m_smoother_type == Smoother::chebyshev) ? 4 : 8);
422 int m_max_levels = 25;
423 Long m_max_coarse_size = 9;
424 int m_aggressive_num_levels = 0;
425 bool m_aggressive_direct_interp =
false;
426 bool m_singular =
false;
427 std::unique_ptr<AlgVector<T>> m_bproj;
428 T m_strong_threshold = T(0.25);
430 InterpType m_interp_type = InterpType::mm_ext_i;
432 T m_trunc_factor = T(0);
437 static constexpr Long max_direct_size = 1024;
438 Long m_bottom_n = -1;
439 Vector<double> m_bottom_lu;
440 Vector<int> m_bottom_piv;
441 T m_bottom_reltol = T(1.e-4);
442 int m_bottom_maxiter = 200;
443 int m_bottom_verbose = 0;
445 T m_reltol = std::is_same_v<T,float> ? T(1.e-4) : T(1.e-8);
447 T m_init_rnorm = std::numeric_limits<T>::max();
448 T m_rnorm = std::numeric_limits<T>::max();
449 T m_bnorm = std::numeric_limits<T>::max();
451 std::string m_print_ident;
452 std::size_t m_setup_peak_bytes = 0;
455 void note_memory (
int lev,
char const* stage)
457 auto const [bytes, reserved] = detail::arena_bytes();
458 m_setup_peak_bytes = std::max(m_setup_peak_bytes, bytes);
459 if (m_verbose >= 2 && bytes > 0) {
460 amrex::Print() << m_print_ident <<
"AlgMG: level " << lev <<
": after " << stage
461 <<
": arena in use " << double(bytes)/double(1<<20)
462 <<
" MB, reserved " << double(reserved)/double(1<<20) <<
" MB"
470 SpMatrix<T>* m_mat =
nullptr;
471 Vector<SpMatrix<T>*> m_a;
472 Vector<std::unique_ptr<SpMatrix<T>>> m_crsemat;
473 Vector<SpMatrix<T>> m_P;
474 Vector<SpMatrix<T>> m_R;
476 Vector<AlgVector<T>> m_res;
477 Vector<AlgVector<T>> m_cor;
478 Vector<AlgVector<T>> m_rescor;
480 Vector<std::unique_ptr<JacobiSmoother<T>>> m_smoother;
481 Vector<std::unique_ptr<ChebyshevSmoother<T>>> m_cheby;
482 Vector<std::unique_ptr<L1GaussSeidelSmoother<T>>> m_l1gs;
483 std::unique_ptr<GMRES_MV<T>> m_gmres;
484 std::unique_ptr<BiCGStab_MV<T>> m_bicgstab;
487 std::unique_ptr<BiCGStab_MV<T>> m_krylov_bicgstab;
488 std::unique_ptr<GMRES_MV<T>> m_krylov_gmres;
489 std::unique_ptr<PCG_MV<T>> m_krylov_pcg;
491 void clear_levels ();
492 void build_smoothers ();
494 void build_direct_bottom ();
495 void direct_bottom_solve ();
498 static void allgather_rows (
double* buf, AlgPartition
const& part,
Long stride);
502 if (!m_prepared) {
setup(); }
503 else if (m_smoothers_stale) { build_smoothers(); }
506 void vcycle (AlgVector<T>&
x, AlgVector<T>
const& b);
507 void blew_up ()
const;
508 void solve_krylov (AlgVector<T>& a_sol, AlgVector<T>
const& a_rhs, T res_target);
509 void smooth (
int lev, AlgVector<T>& xvec, AlgVector<T>
const& bvec,
int nu,
510 bool with_initial_guess,
bool backward =
false);
511 void bottom_solve ();
513 static std::pair<T,T> compute_rbnorms (SpMatrix<T>
const& mat,
514 AlgVector<T>
const& xvec,
515 AlgVector<T>
const& bvec,
518 static T compute_rnorm (SpMatrix<T>
const& mat, AlgVector<T>
const& xvec,
519 AlgVector<T>
const& bvec, AlgVector<T>& rvec);
532 bool& aggressive,
Long& ncoarse,
bool& truncated,
533 Long& nnz_untruncated);
545 bool isolated_as_coarse =
false)
const;
550 int which = PointType::coarse);
556 if (!m_singular) {
return; }
557 T
const mean =
x.sum() / T(
x.partition().numGlobalRows());
573 int max_elmts, T trunc_factor);
607 bool plus_i,
int const* row_state =
nullptr,
608 int max_elmts = 0, T trunc_factor = T(0),
609 Long* nnz_untruncated =
nullptr);
620 int max_elmts, T trunc_factor,
621 Long& nnz_untruncated);
638 m_krylov_bicgstab.reset();
639 m_krylov_gmres.reset();
640 m_krylov_pcg.reset();
646 if (!m_prepared || m_a.empty()) {
651 "AlgMG::updateMatrix: the matrix must have the same row partition");
656 m_krylov_bicgstab.reset();
657 m_krylov_gmres.reset();
658 m_krylov_pcg.reset();
662 m_bottom_piv.clear();
664 m_smoothers_stale =
true;
669void AlgMG<T>::clear_levels ()
685 m_bottom_piv.clear();
698 if (!m_bproj) { m_bproj = std::make_unique<AlgVector<T>>(m_mat->partition()); }
699 m_bproj->copy(a_rhs);
701 prhs = m_bproj.get();
705 auto norms = compute_rbnorms(*m_mat, a_sol, rhs, m_res[0]);
706 m_bnorm = norms.first;
707 m_init_rnorm = m_rnorm = norms.second;
716 if (m_verbose >= 1) {
717 amrex::Print() << m_print_ident <<
"AlgMG: Initial rhs 2-norm = " << m_bnorm <<
"\n"
718 << m_print_ident <<
"AlgMG: Initial residual 2-norm = " << m_rnorm <<
"\n";
721 T res_target = std::max(m_bnorm * m_reltol, m_abstol);
722 if (m_fixediter <= 0 && m_rnorm <= res_target) {
723 if (m_verbose >= 1) {
724 amrex::Print() << m_print_ident <<
"AlgMG: No iterations needed\n";
730 if (m_krylov != KrylovSolver::none) {
731 solve_krylov(a_sol, rhs, res_target);
736 int const niters = (m_fixediter > 0) ? m_fixediter : m_maxiter;
737 for (
int iter = 0; iter < niters; ++iter)
739 vcycle(m_cor[0], m_res[0]);
742 m_rnorm = compute_rnorm(*m_mat, a_sol, rhs, m_res[0]);
745 if (m_verbose >= 2) {
746 amrex::Print() << m_print_ident <<
"AlgMG: Iteration " << std::setw(3)
747 << iter+1 <<
": rnorm, rnorm/bnorm = "
748 << m_rnorm <<
", " << m_rnorm/m_bnorm <<
"\n";
751 bool converged = (m_fixediter <= 0) && (m_rnorm <= res_target);
753 if (m_verbose >= 1) {
754 amrex::Print() << m_print_ident <<
"AlgMG: Final Iter. " << iter+1
755 <<
": rnorm, rnorm/bnorm = "
756 << m_rnorm <<
", " << m_rnorm/m_bnorm <<
"\n";
759 }
else if (m_rnorm > T(1.e20) * m_init_rnorm ||
amrex::isnan(m_rnorm)) {
761 amrex::Print() << m_print_ident <<
"AlgMG: Failing to converge after " << iter+1 <<
" iterations."
762 <<
" rnorm, rnorm/bnorm = "
763 << m_rnorm <<
", " << m_rnorm/m_bnorm <<
"\n";
769 if (m_verbose >= 1 && m_fixediter <= 0 && m_rnorm > res_target) {
770 amrex::Print() << m_print_ident <<
"AlgMG: Not converged after " << m_niters
771 <<
" iterations. rnorm/bnorm = " << m_rnorm/m_bnorm <<
"\n";
785void AlgMG<T>::blew_up ()
const
787 if (m_throw_exception) {
788 throw std::runtime_error(
"AlgMG blew up.");
801 int const niters = (m_fixediter > 0) ? m_fixediter : m_maxiter;
802 T
const atol = (m_fixediter > 0) ? T(0) : res_target;
803 int const verbose = std::max(0, m_verbose-1);
804 T krylov_rnorm = T(0);
808 auto run = [&] (
auto& s) {
809 s.setVerbose(verbose);
810 s.setInitialGuessNonzero(
true);
813 s.setMaxIters(niters - m_niters);
814 s.solve(a_sol, a_rhs, T(0), atol);
815 m_niters += s.getNumIters();
816 krylov_rnorm = s.getResidualNorm();
817 m_rnorm = compute_rnorm(*m_mat, a_sol, a_rhs, m_res[0]);
818 if (m_fixediter > 0 || m_rnorm <= res_target || krylov_rnorm > res_target ||
819 m_niters >= niters || s.getNumIters() == 0 ||
amrex::isnan(m_rnorm)) {
822 if (m_verbose >= 2) {
823 amrex::Print() << m_print_ident <<
"AlgMG: restarting Krylov solver at iteration "
824 << m_niters <<
", rnorm/bnorm = " << m_rnorm/m_bnorm <<
"\n";
829 char const*
name =
"";
830 if (m_krylov == KrylovSolver::bicgstab) {
832 if (!m_krylov_bicgstab) {
833 m_krylov_bicgstab = std::make_unique<BiCGStab_MV<T>>(m_mat);
835 m_krylov_bicgstab->setPrecond(pc);
836 run(m_krylov_bicgstab->getSolver());
837 }
else if (m_krylov == KrylovSolver::gmres) {
839 if (!m_krylov_gmres) {
840 m_krylov_gmres = std::make_unique<GMRES_MV<T>>(m_mat);
842 m_krylov_gmres->setPrecond(pc);
843 run(m_krylov_gmres->getSolver());
847 m_krylov_pcg = std::make_unique<PCG_MV<T>>(m_mat);
849 m_krylov_pcg->setPrecond(pc);
850 run(m_krylov_pcg->getSolver());
853 if (
amrex::isnan(m_rnorm) || m_rnorm > T(1.e20) * m_init_rnorm) { blew_up(); }
855 if (m_verbose >= 1) {
856 amrex::Print() << m_print_ident <<
"AlgMG(" <<
name <<
"): Final Iter. " << m_niters
857 <<
": rnorm, rnorm/bnorm = " << m_rnorm <<
", " << m_rnorm/m_bnorm;
858 if (m_verbose >= 2) {
859 amrex::Print() <<
" (Krylov recurrence " << krylov_rnorm <<
")";
861 amrex::Print() <<
"\n";
862 if (m_fixediter <= 0 && m_rnorm > res_target) {
863 amrex::Print() << m_print_ident <<
"AlgMG(" <<
name <<
"): Not converged after "
864 << m_niters <<
" iterations.\n";
870std::pair<T,T> AlgMG<T>::compute_rbnorms (
SpMatrix<T> const& mat,
875 auto bnorm = bvec.norm2(
true);
878 auto rnorm = rvec.norm2(
true);
885 rnorm = std::sqrt(rnorm);
888 return {
bnorm, rnorm};
904 int const nlev = numLevels();
905 int const bottom = nlev-1;
908 if (&b != &m_res[0]) { m_res[0].copyAsync(b); }
910 if (&
x != &m_cor[0]) {
x.copyAsync(m_cor[0]); }
915 auto cor = [&] (
int lev) ->
AlgVector<T>& {
return (lev == 0) ?
x : m_cor[lev]; };
916 auto res = [&] (
int lev) ->
AlgVector<T> const& {
return (lev == 0) ? b : m_res[lev]; };
918 smooth(0,
x, b, numPreSmooth(),
false);
920 for (
int lev = 0; lev < bottom; ++lev) {
922 SpMV(m_res[lev+1], m_R[lev], m_rescor[lev]);
923 if (lev+1 == bottom) {
926 smooth(lev+1, m_cor[lev+1], m_res[lev+1], numPreSmooth(),
false);
930 for (
int lev = bottom-1; lev >= 0; --lev) {
931 SpMV(m_rescor[lev], m_P[lev], m_cor[lev+1]);
932 cor(lev).plusAsync(m_rescor[lev]);
933 smooth(lev, cor(lev), res(lev), numPostSmooth(),
true,
true);
939 bool with_initial_guess,
bool backward)
941 if (m_smoother_type == Smoother::l1_gauss_seidel) {
944 smoother(xvec, bvec, with_initial_guess, backward);
945 }
else if (m_smoother_type == Smoother::chebyshev) {
948 smoother(xvec, bvec, with_initial_guess);
952 smoother(xvec, bvec, with_initial_guess);
957void AlgMG<T>::bottom_solve ()
961 int const lev = numLevels()-1;
962 project_out(m_res[lev]);
963 auto smooth_bottom = [&] () {
964 if (m_smoother_type == Smoother::l1_gauss_seidel) {
966 int const nu = (numBottomSmooth()+1)/2;
967 smooth(lev, m_cor[lev], m_res[lev], nu,
false);
968 smooth(lev, m_cor[lev], m_res[lev], nu,
true,
true);
970 smooth(lev, m_cor[lev], m_res[lev], numBottomSmooth(),
false);
976 auto krylov_bottom = [&] (
auto& krylov) {
977 krylov.setVerbose(m_bottom_verbose);
978 krylov.getSolver().setMaxIters(m_bottom_maxiter);
979 krylov.solve(m_cor[lev], m_res[lev], m_bottom_reltol, T(0));
980 auto const& solver = krylov.getSolver();
981 T
const rnorm0 = solver.getInitialResidualNorm();
982 if (rnorm0 > T(0) && !(solver.getResidualNorm() < rnorm0)) {
983 if (m_verbose >= 2) {
984 amrex::Print() << m_print_ident <<
"AlgMG: bottom solve failed, smoothing instead\n";
989 auto bottom_solver = m_bottom_solver;
990 if (bottom_solver == BottomSolver::direct) {
991 if (m_bottom_n < 0) { build_direct_bottom(); }
992 if (m_bottom_n == 0) { bottom_solver = BottomSolver::jacobi; }
994 switch (bottom_solver) {
995 case BottomSolver::direct:
996 direct_bottom_solve();
998 case BottomSolver::jacobi:
1001 case BottomSolver::gmres:
1003 m_gmres = std::make_unique<GMRES_MV<T>>(m_a[lev]);
1004 m_gmres->setPrecond(
JacobiSmoother<T>(m_a[lev], m_smoother_type != Smoother::jacobi));
1006 krylov_bottom(*m_gmres);
1010 m_bicgstab = std::make_unique<BiCGStab_MV<T>>(m_a[lev]);
1011 m_bicgstab->setPrecond(
JacobiSmoother<T>(m_a[lev], m_smoother_type != Smoother::jacobi));
1013 krylov_bottom(*m_bicgstab);
1016 project_out(m_cor[lev]);
1019template <
typename T>
1026 m_a.push_back(m_mat);
1027 m_mat->setColumnPartition(m_mat->partition());
1028 m_setup_peak_bytes = 0;
1029 note_memory(0,
"start");
1033 for (
int lev = 0; lev+1 < m_max_levels; ++lev)
1035 auto& A = *m_a[lev];
1036 if (A.numGlobalRows() <= m_max_coarse_size) {
break; }
1040 bool aggressive =
false;
1042 bool truncated =
false;
1044 if (!
coarsen_level(lev, A, P, cpart, aggressive, ncoarse, truncated, nnz0)) {
break; }
1045 note_memory(lev,
"strength release");
1046 if (m_p_max_elmts > 0 || m_trunc_factor > T(0)) {
1050 note_memory(lev,
"truncation");
1052 if (m_verbose >= 2) {
1055 amrex::Print() << m_print_ident <<
"AlgMG: level " << lev
1056 <<
": P truncated from " << nnz[0] <<
" to "
1057 << nnz[1] <<
" nonzeros\n";
1063 if (m_verbose >= 2 && aggressive) {
1064 amrex::Print() << m_print_ident <<
"AlgMG: level " << lev
1065 <<
": aggressive coarsening, " << ncoarse <<
" -> "
1068 if (m_verbose >= 3) {
1082 amrex::Print() << m_print_ident <<
"AlgMG: level " << lev <<
": P row sums in ["
1083 << rmin <<
", " << rmax <<
"], " << nzero <<
" empty rows\n";
1086 note_memory(lev,
"R");
1088#ifndef AMREX_USE_GPU
1095 note_memory(lev,
"A*P");
1099 note_memory(lev,
"coarse operator");
1101 m_P.push_back(std::move(P));
1102 m_R.push_back(std::move(R));
1103 m_crsemat.push_back(std::make_unique<
SpMatrix<T>>(std::move(Ac)));
1104 m_a.push_back(m_crsemat.back().get());
1108 for (
int lev = 0; lev < nlev; ++lev) {
1109 m_res.emplace_back(m_a[lev]->partition());
1110 m_cor.emplace_back(m_a[lev]->partition());
1112 m_rescor.emplace_back(m_a[lev]->partition());
1117 if (m_verbose >= 1) {
1118 Long nnz0 = 0, nnz_total = 0;
1119 for (
int lev = 0; lev < nlev; ++lev) {
1120 Long nnz = m_a[lev]->numLocalNonZeros();
1122 if (lev == 0) { nnz0 = nnz; }
1124 amrex::Print() << m_print_ident <<
"AlgMG: level " << lev <<
": "
1125 << m_a[lev]->numGlobalRows() <<
" rows, "
1126 << nnz <<
" nonzeros\n";
1128 amrex::Print() << m_print_ident <<
"AlgMG: operator complexity = "
1129 << double(nnz_total)/double(std::max(nnz0,
Long(1))) <<
"\n";
1130 if (m_setup_peak_bytes > 0) {
1131 auto peak = m_setup_peak_bytes;
1133 amrex::Print() << m_print_ident <<
"AlgMG: setup peak arena use = "
1134 << double(peak)/double(1<<20) <<
" MB ("
1135 << double(peak)/double(m_mat->numGlobalRows())
1143template <
typename T>
1144void AlgMG<T>::build_direct_bottom ()
1147 int const lev = numLevels()-1;
1148 auto const& A = *m_a[lev];
1149 Long const n = A.numGlobalRows();
1151 m_bottom_lu.clear();
1152 m_bottom_piv.clear();
1153 if (n < 1 || n > max_direct_size) {
1154 if (m_verbose >= 1 && n > max_direct_size) {
1155 amrex::Print() << m_print_ident <<
"AlgMG: coarsest level has " << n
1156 <<
" rows, using smoother sweeps instead of the direct bottom solver\n";
1163 auto const pc = A.const_parcsr();
1164 auto to_host = [] (
auto const* dptr,
Long cnt) {
1165 Vector<std::remove_const_t<std::remove_pointer_t<
decltype(dptr)>>> h(cnt);
1170 auto const& c = pc.csr0;
1171 auto ro = to_host(c.row_offset, (c.nrows > 0) ? c.nrows+1 : 0);
1172 auto ci = to_host(c.col_index, c.nnz);
1173 auto va = to_host(c.mat, c.nnz);
1174 for (
Long i = 0; i < c.nrows; ++i) {
1175 Long const ig = pc.row_begin + i;
1176 for (
int idx = ro[i]; idx < ro[i+1]; ++idx) {
1177 dense[ig*n + pc.col_begin + ci[idx]] += double(va[idx]);
1181 if (pc.csr1.nrows > 0 && pc.csr1.nnz > 0) {
1182 auto const& c = pc.csr1;
1183 auto ro = to_host(c.row_offset, c.nrows+1);
1184 auto ci = to_host(c.col_index, c.nnz);
1185 auto va = to_host(c.mat, c.nnz);
1186 auto rmap = to_host(pc.row_map, pc.csr0.nrows);
1188 for (
auto j : ci) { ncols = std::max(ncols, j+1); }
1189 auto cmap = to_host(pc.col_map, ncols);
1190 for (
Long i = 0; i < pc.csr0.nrows; ++i) {
1191 Long const r = rmap[i];
1192 if (r < 0) {
continue; }
1193 Long const ig = pc.row_begin + i;
1194 for (
int idx = ro[r]; idx < ro[r+1]; ++idx) {
1195 dense[ig*n + cmap[ci[idx]]] += double(va[idx]);
1199 allgather_rows(dense.data(), A.partition(), n);
1206 for (
Long i = 0; i < n; ++i) { dsum += std::abs(dense[i*n+i]); }
1207 double const w = (dsum/double(n))/
double(n);
1208 for (
auto& v : dense) { v += w; }
1214 for (
auto v : dense) { amat = std::max(amat, std::abs(v)); }
1215 double const pivot_tol = double(n) * std::numeric_limits<double>::epsilon() * amat;
1217 std::iota(piv.begin(), piv.end(), 0);
1218 for (
Long i = 0; i < n; ++i) {
1220 double amax = std::abs(dense[i*n+i]);
1221 for (
Long k = i+1; k < n; ++k) {
1222 double const a = std::abs(dense[k*n+i]);
1223 if (a > amax) { amax = a; imax = k; }
1225 if (amax <= pivot_tol) {
1226 if (m_verbose >= 1) {
1227 amrex::Print() << m_print_ident <<
"AlgMG: coarsest matrix is singular, "
1228 <<
"using smoother sweeps instead of the direct bottom solver\n";
1233 std::swap(piv[i], piv[imax]);
1234 for (
Long j = 0; j < n; ++j) { std::swap(dense[i*n+j], dense[imax*n+j]); }
1236 double const dinv = 1.0/dense[i*n+i];
1237 for (
Long j = i+1; j < n; ++j) {
1238 double const f = dense[j*n+i]*dinv;
1240 for (
Long k = i+1; k < n; ++k) { dense[j*n+k] -= f*dense[i*n+k]; }
1243 m_bottom_lu = std::move(dense);
1244 m_bottom_piv = std::move(piv);
1248template <
typename T>
1249void AlgMG<T>::allgather_rows (
double* buf,
AlgPartition const& part,
Long stride)
1253 if (nprocs == 1) {
return; }
1255 for (
int p = 0;
p < nprocs; ++
p) {
1256 displs[
p] =
int(part[
p]*stride);
1257 counts[
p] =
int((part[
p+1]-part[
p])*stride);
1260 buf, counts.data(), displs.data(), MPI_DOUBLE,
1267template <
typename T>
1268void AlgMG<T>::direct_bottom_solve ()
1270 int const lev = numLevels()-1;
1271 auto const& r = m_res[lev];
1272 auto&
x = m_cor[lev];
1273 Long const n = m_bottom_n;
1274 Long const nl = r.numLocalRows();
1275 Long const b0 = r.globalBegin();
1282 for (
Long i = 0; i < nl; ++i) { rhs[b0+i] = double(loc[i]); }
1284 allgather_rows(rhs.data(), r.partition(), 1);
1288 auto const& lu = m_bottom_lu;
1289 for (
Long i = 0; i < n; ++i) {
1290 double v = rhs[m_bottom_piv[i]];
1291 for (
Long k = 0; k < i; ++k) { v -= lu[i*n+k]*
y[k]; }
1294 for (
Long i = n-1; i >= 0; --i) {
1296 for (
Long k = i+1; k < n; ++k) { v -= lu[i*n+k]*
y[k]; }
1301 for (
Long i = 0; i < nl; ++i) { loc[i] = T(
y[b0+i]); }
1306template <
typename T>
1307void AlgMG<T>::build_smoothers ()
1314 int const nlev = numLevels();
1315 for (
int lev = 0; lev < nlev; ++lev) {
1316 if (m_smoother_type == Smoother::l1_gauss_seidel) {
1318 amrex::Abort(
"AlgMG: L1GaussSeidel smoother is only available in CPU builds");
1321 }
else if (m_smoother_type == Smoother::chebyshev) {
1323 m_cheby.back()->setDegree(m_cheby_degree);
1324 m_cheby.back()->setEigRatio(m_cheby_ratio);
1326 bool const l1 = (m_smoother_type == Smoother::l1_jacobi);
1328 T
const w = (m_relax_weight >= T(0)) ? m_relax_weight : (l1 ? T(4./3.) : T(2./3.));
1329 m_smoother.back()->setWeight(w);
1332 m_smoothers_stale =
false;
1335template <
typename T>
1337 bool& aggressive,
Long& ncoarse,
bool& truncated,
1338 Long& nnz_untruncated)
1345 S.setColumnPartition(S.partition());
1346 note_memory(lev,
"S");
1348 ST.setColumnPartition(ST.partition());
1350 note_memory(lev,
"strength");
1353 pmis(S, ST, state, m_singular);
1354 note_memory(lev,
"PMIS");
1359 if (ncoarse == 0 || ncoarse == A.
numGlobalRows()) {
return false; }
1360#ifndef AMREX_USE_GPU
1361 bool const host_interp = m_interp_type != InterpType::direct
1362 && detail::spmat_comm_is_local(A.
partition(), cpart);
1365 if (lev < m_aggressive_num_levels) {
1368 S2.setColumnPartition(cpart);
1370 S2T.setColumnPartition(cpart);
1371 note_memory(lev,
"second-pass strength");
1373 pmis(S2, S2T, state2,
true);
1376 Long const nc2 = cpart2.numGlobalRows();
1377 if (nc2 > 0 && nc2 < ncoarse) {
1382 auto const* p1 = state.
data();
1383 auto const* c1 = cidx.
data();
1384 auto const* p2 = state2.
data();
1385 auto const* c2 = cidx2.
data();
1386 auto* pf = statef.
data();
1387 auto* cf = cidxf.
data();
1390 bool const c = (p1[i] == PointType::coarse) &&
1391 (p2[c1[i]] == PointType::coarse);
1392 pf[i] = c ? PointType::coarse : PointType::fine;
1393 cf[i] = (p1[i] == PointType::coarse) ? c2[c1[i]] :
Long(0);
1397 bool const trunc = (m_p_max_elmts > 0 || m_trunc_factor > T(0));
1399#ifndef AMREX_USE_GPU
1400 if (!m_aggressive_direct_interp && host_interp) {
1402 P1 = build_interp_mm_host(A, S, ST, state, cidx, cpart, sscale,
1403 m_interp_type == InterpType::mm_ext_i,
1404 m_p_max_elmts, m_trunc_factor, unused);
1405 }
else if (!m_aggressive_direct_interp && m_interp_type != InterpType::direct) {
1407 m_interp_type == InterpType::mm_ext_i,
nullptr,
1408 m_p_max_elmts, m_trunc_factor);
1412 P1 = m_aggressive_direct_interp ?
build_interp(A, S, state, cidx, cpart)
1414 if (trunc) { P1 =
truncate_interp(P1, cpart, m_p_max_elmts, m_trunc_factor); }
1416 note_memory(lev,
"P1");
1418#ifndef AMREX_USE_GPU
1420 m_interp_type != InterpType::mm_ext,
1421 state.
data(), m_p_max_elmts, m_trunc_factor);
1424 m_interp_type != InterpType::mm_ext,
1426 if (trunc) { Pf =
truncate_interp(Pf, cpart2, m_p_max_elmts, m_trunc_factor); }
1429 note_memory(lev,
"P2");
1436#ifndef AMREX_USE_GPU
1438 P = build_interp_mm_host(A, S, ST, state, cidx, cpart, sscale,
1439 m_interp_type == InterpType::mm_ext_i,
1440 m_p_max_elmts, m_trunc_factor, nnz_untruncated);
1442 }
else if (m_interp_type != InterpType::direct) {
1444 m_interp_type == InterpType::mm_ext_i,
nullptr,
1445 m_p_max_elmts, m_trunc_factor, &nnz_untruncated);
1453 note_memory(lev,
"interpolation");
1457template <
typename T>
1460 auto const rs = P.
rowSum();
1466 amrex::Abort(
"AlgMG: singular mode needs P 1 = 1, but " + std::to_string(nzero)
1467 +
" rows of P are empty");
1472 T
const f = T(1) / prs[i];
1473 for (
Long idx = a.csr0.row_offset[i]; idx < a.csr0.row_offset[i+1]; ++idx) {
1474 a.csr0.mat[idx] *= f;
1476 if (detail::has_remote_row(a, i)) {
1477 Long const ii = a.row_map[i];
1478 for (
Long idx = a.csr1.row_offset[ii]; idx < a.csr1.row_offset[ii+1]; ++idx) {
1479 a.csr1.mat[idx] *= f;
1486template <
typename T>
1494 return (dmax > T(0)) ? dmax : T(1);
1497template <
typename T>
1503 Long const nrows = a.csr0.nrows;
1508 auto const threshold = m_strong_threshold;
1514 T
const tol = T(64) * std::numeric_limits<T>::epsilon();
1516#ifndef AMREX_USE_GPU
1519 auto const& c0 = a.csr0;
1520 auto const& c1 = a.csr1;
1521 auto rlen1 = [&] (
Long i) ->
Long {
1522 if (!detail::has_remote_row(a, i)) {
return 0; }
1523 Long const ii = a.row_map[i];
1524 return c1.row_offset[ii+1] - c1.row_offset[ii];
1526 auto csr = detail::csr_from_rows_cpu<TS,SpMatrix<TS>::template container_type,
int>
1527 (nrows, [&] (
Long i) {
return c0.row_offset[i+1] - c0.row_offset[i] + rlen1(i); },
1530 (
Long i,
int const*& rc,
TS const*& rv)
mutable ->
Long
1532 Long const b = c0.row_offset[i];
1533 Long const e = c0.row_offset[i+1];
1534 Long const n1 = rlen1(i);
1535 Long const b1 = (n1 > 0) ?
Long(c1.row_offset[a.row_map[i]]) : 0;
1536 T
const sgn = std::copysign(T(1), A_ii[i]);
1538 T amin = std::numeric_limits<T>::max();
1539 for (
Long idx = b; idx < e; ++idx) {
1541 if (c0.col_index[idx] != i) { amin = std::min(amin, sgn*c0.mat[idx]); }
1543 for (
Long idx = b1; idx < b1+n1; ++idx) {
1545 amin = std::min(amin, sgn*c1.mat[idx]);
1548 if (!(std::abs(rs) > T(0.9)*std::abs(A_ii[i])) && amin < T(0)) {
1549 T
const thr = amin * threshold * (T(1) - tol);
1550 if (
Long(col.size()) < e-b+n1) {
1554 for (
Long idx = b; idx < e; ++idx) {
1555 if (c0.col_index[idx] != i && sgn*c0.mat[idx] <= thr) {
1556 col[n] = c0.col_index[idx];
1557 val[n] =
TS(c0.mat[idx]/
scale);
1561 for (
Long idx = b1; idx < b1+n1; ++idx) {
1562 if (sgn*c1.mat[idx] <= thr) {
1563 col[n] = nlocal + c1.col_index[idx];
1564 val[n] =
TS(c1.mat[idx]/
scale);
1580 auto const rsA = A.
rowSum();
1583 if (std::abs(prs[i]) > T(0.9)*std::abs(A_ii[i])) {
1584 return std::numeric_limits<T>::lowest();
1586 auto const& c0 = a.csr0;
1587 T
const sgn = std::copysign(T(1), A_ii[i]);
1588 T amin = std::numeric_limits<T>::max();
1589 for (
Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
1590 if (c0.col_index[idx] != i) {
1591 amin = std::min(amin, sgn*c0.mat[idx]);
1594 if (detail::has_remote_row(a, i)) {
1595 auto const& c1 = a.csr1;
1596 Long const ii = a.row_map[i];
1597 for (
Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
1598 amin = std::min(amin, sgn*c1.mat[idx]);
1601 if (amin >= T(0)) {
return std::numeric_limits<T>::lowest(); }
1602 return amin * threshold * (T(1) - tol);
1610 csr.row_offset.resize(nrows+1);
1615 if (i >= nrows) {
return 0; }
1616 T
const thr = pthr[i];
1617 T
const sgn = std::copysign(T(1), A_ii[i]);
1619 auto const& c0 = a.csr0;
1620 for (
Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
1621 if (c0.col_index[idx] != i && sgn*c0.mat[idx] <= thr) { ++n; }
1623 if (detail::has_remote_row(a, i)) {
1624 auto const& c1 = a.csr1;
1625 Long const ii = a.row_map[i];
1626 for (
Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
1627 if (sgn*c1.mat[idx] <= thr) { ++n; }
1635 csr.resize(nrows, nnz);
1640 T
const thr = pthr[i];
1641 T
const sgn = std::copysign(T(1), A_ii[i]);
1643 auto const& c0 = a.csr0;
1644 for (
Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
1645 if (c0.col_index[idx] != i && sgn*c0.mat[idx] <= thr) {
1647 pc[
p] = c0.col_index[idx];
1651 if (detail::has_remote_row(a, i)) {
1652 auto const& c1 = a.csr1;
1653 Long const ii = a.row_map[i];
1654 for (
Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
1655 if (sgn*c1.mat[idx] <= thr) {
1657 pc[
p] = nlocal + c1.col_index[idx];
1671template <
typename T>
1673 bool isolated_as_coarse)
const
1678 int const U = PointType::undecided;
1679 int const C = PointType::coarse;
1689 Long n = st.csr0.row_offset[i+1] - st.csr0.row_offset[i];
1690 if (detail::has_remote_row(st, i)) {
1691 Long const ii = st.row_map[i];
1692 n += st.csr1.row_offset[ii+1] - st.csr1.row_offset[ii];
1694 Long nout = s.csr0.row_offset[i+1] - s.csr0.row_offset[i];
1695 if (detail::has_remote_row(s, i)) {
1696 Long const ii = s.row_map[i];
1697 nout += s.csr1.row_offset[ii+1] - s.csr1.row_offset[ii];
1699 pm[i] = (n == 0 && nout == 0) ?
Long(-1) : n;
1709 Long const row_begin = s.row_begin;
1717 ps[i] = isolated_as_coarse ? PointType::coarse : PointType::fine;
1719 pw[i] = T(pm[i]) + detail::pmis_tiebreak<T>(i + row_begin);
1720 ps[i] = PointType::undecided;
1744 Long const ig = i + row_begin;
1746 detail::pmis_blocked(s.csr0, i, wi, ig, pw, ps,
nullptr, s.col_begin, U) ||
1747 detail::pmis_blocked(st.csr0, i, wi, ig, pw, ps,
nullptr, st.col_begin, U);
1748 if (!blocked && detail::has_remote_row(s, i)) {
1749 blocked = detail::pmis_blocked(s.csr1, s.row_map[i], wi, ig,
1750 pwS, psS, s.col_map, 0, U);
1752 if (!blocked && detail::has_remote_row(st, i)) {
1753 blocked = detail::pmis_blocked(st.csr1, st.row_map[i], wi, ig,
1754 pwST, psST, st.col_map, 0, U);
1756 if (!blocked) { st_i =
C; }
1769 if (detail::csr_row_has_state(s.csr0, i, ps2,
C) ||
1770 (detail::has_remote_row(s, i) &&
1771 detail::csr_row_has_state(s.csr1, s.row_map[i], ps2S,
C))) {
1772 st_i = PointType::fine;
1779 return (ps[i] == U) ? 1 : 0;
1782 if (nund == 0) {
break; }
1785 if (m_verbose >= 2) {
1789 return (ps[i] ==
C) ? 1 : 0;
1795 if (ps[i] !=
C) {
return 0; }
1796 return (detail::csr_row_has_state(s.csr0, i, ps,
C) ||
1797 (detail::has_remote_row(s, i) &&
1798 detail::csr_row_has_state(s.csr1, s.row_map[i], psS,
C))) ? 1 : 0;
1802 if (ps[i] != PointType::fine) {
return 0; }
1803 return (detail::csr_row_has_state(s.csr0, i, ps,
C) ||
1804 (detail::has_remote_row(s, i) &&
1805 detail::csr_row_has_state(s.csr1, s.row_map[i], psS,
C))) ? 0 : 1;
1807 Long counts[3] = {nc, ncc, nf0};
1809 amrex::Print() << m_print_ident <<
"AlgMG: PMIS: " << counts[0] <<
" C-points of "
1811 <<
", F-points w/o C neighbor " << counts[2] <<
"\n";
1815template <
typename T>
1836 std::partial_sum(counts.begin(), counts.end(), rows.begin()+1);
1840template <
typename T>
1851 Long const nrows = a.csr0.nrows;
1865 detail::DirectInterpRow<T,TS>
const w{a, S.
const_parcsr(), ps, pc,
1866 state_r.data(), nlocal_c,
1867 int(PointType::coarse)};
1871 csr.row_offset.resize(nrows+1);
1877 if (i >= nrows) {
return 0; }
1878 if (ps[i] == PointType::coarse) {
return 1; }
1881 w.walk(i, diag, [&] (
int, T,
bool strong_c) {
if (strong_c) { ++n; } });
1887 csr.resize(nrows, nnz);
1894 if (ps[i] == PointType::coarse) {
1895 pcol[
p] =
int(pc[i]);
1900 T sum_n_neg = T(0), sum_n_pos = T(0), sum_p_neg = T(0), sum_p_pos = T(0);
1902 w.walk(i, diag, [&] (
int, T v,
bool strong_c) {
1905 if (strong_c) { sum_p_neg += v; }
1908 if (strong_c) { sum_p_pos += v; }
1912 T alfa = T(0), beta = T(0);
1913 if (sum_p_neg != T(0)) { alfa = sum_n_neg / sum_p_neg; }
else { diag += sum_n_neg; }
1914 if (sum_p_pos != T(0)) { beta = sum_n_pos / sum_p_pos; }
else { diag += sum_n_pos; }
1915 if (diag == T(0)) { diag = T(1); }
1918 w.walk(i, unused, [&] (
int cj, T v,
bool strong_c) {
1921 pmat[
p] = ((v < T(0)) ? -alfa*v : -beta*v) / diag;
1938template <
typename T,
int KMAX>
1947 template <
typename F>
1950 auto const& c0 =
p.csr0;
1951 for (
Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
1952 f(c0.col_index[idx], c0.mat[idx]);
1954 if (has_remote_row(
p, i)) {
1955 auto const& c1 =
p.csr1;
1956 Long const ii =
p.row_map[i];
1957 for (
Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
1958 f(nlocal_cols + c1.col_index[idx], c1.mat[idx]);
1969 walk(i, [&] (
int, T v) {
1970 amax = std::max(amax, std::abs(v));
1971 if (v > T(0)) { sum_pos += v; }
else { sum_neg += v; }
1973 return trunc_factor * amax;
1980 walk(i, [&] (
int, T v) {
if (std::abs(v) >= thresh) { ++n; } });
1988 int select (
Long i, T thresh,
int* cols, T* vals)
const
1990 int const kmax = std::min(max_elmts, KMAX);
1992 walk(i, [&] (
int col, T v) {
1993 T
const av = std::abs(v);
1994 if (av < thresh) {
return; }
1995 if (n == kmax && av <= std::abs(vals[n-1])) {
return; }
1996 int pos = (n < kmax) ? n : n-1;
1997 while (pos > 0 && std::abs(vals[pos-1]) < av) {
1998 cols[pos] = cols[pos-1];
1999 vals[pos] = vals[pos-1];
2004 if (n < kmax) { ++n; }
2010#ifndef AMREX_USE_GPU
2013template <
typename T,
int KMAX>
2014Long truncate_row_host (
int* col, T* val,
Long n,
int max_elmts, T trunc_factor,
int nc)
2017 int ro[2] = {0,
int(n)};
2018 ParCsr<T const> prow;
2019 prow.csr0 = CsrView<T const,int>{val, col, ro, n, 1};
2020 TruncRow<T,KMAX>
const tr{prow, max_elmts, trunc_factor, nc};
2022 T
const thresh = tr.threshold(0, sum_pos, sum_neg);
2026 T kept_pos = T(0), kept_neg = T(0);
2027 if (max_elmts > 0) {
2028 nk = tr.select(0, thresh, tcol, tval);
2029 for (
Long k = 0; k < nk; ++k) {
2030 if (tval[k] > T(0)) { kept_pos += tval[k]; }
else { kept_neg += tval[k]; }
2033 for (
Long k = 0; k < n; ++k) {
2034 if (std::abs(val[k]) >= thresh) {
2035 if (val[k] > T(0)) { kept_pos += val[k]; }
else { kept_neg += val[k]; }
2039 T
const fpos = (kept_pos != T(0)) ? sum_pos/kept_pos : T(1);
2040 T
const fneg = (kept_neg != T(0)) ? sum_neg/kept_neg : T(1);
2041 if (max_elmts > 0) {
2043 for (
Long k = 0; k < nk; ++k) {
2045 for (; m > 0 && col[m-1] > tcol[k]; --m) {
2050 val[m] = tval[k] * ((tval[k] > T(0)) ? fpos : fneg);
2053 for (
Long k = 0; k < n; ++k) {
2054 if (std::abs(val[k]) >= thresh) {
2056 val[nk] = val[k] * ((val[k] > T(0)) ? fpos : fneg);
2073template <
typename T,
typename F>
2086 if (i >= nrows || state[i] != c) {
return 0; }
2088 auto const& c0 =
p.csr0;
2089 for (
Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
2090 if (colmap(c0.col_index[idx],
false) >= 0) { ++n; }
2092 if (has_remote_row(
p, i)) {
2093 auto const& c1 =
p.csr1;
2094 Long const ii =
p.row_map[i];
2095 for (
Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
2096 if (colmap(c1.col_index[idx],
true) >= 0) { ++n; }
2111 if (i == nrows) { prow[nc] = nnz;
return; }
2112 if (state[i] != c) {
return; }
2115 auto const& c0 =
p.csr0;
2116 for (
Long idx = c0.row_offset[i]; idx < c0.row_offset[i+1]; ++idx) {
2117 int const g = colmap(c0.col_index[idx],
false);
2118 if (g >= 0) { pcol[q] = g; pmat[q] = c0.mat[idx]; ++q; }
2120 if (has_remote_row(
p, i)) {
2121 auto const& c1 =
p.csr1;
2122 Long const ii =
p.row_map[i];
2123 for (
Long idx = c1.row_offset[ii]; idx < c1.row_offset[ii+1]; ++idx) {
2124 int const g = colmap(c1.col_index[idx],
true);
2125 if (g >= 0) { pcol[q] = g; pmat[q] = c1.mat[idx]; ++q; }
2135template <
typename T>
2141 if (m_interp_type == InterpType::direct) {
2145 m_interp_type == InterpType::mm_ext_i);
2149template <
typename T>
2161 Long const nrows = s.csr0.nrows;
2163 int const C = PointType::coarse;
2164 int const F = PointType::fine;
2168 Long const nf = fpart.numLocalRows();
2169 int const nlocal_c =
int(nc);
2170 int const nlocal_f =
int(nf);
2176 Long const fbegin = fpart.globalRowBegin();
2183 pgc[i] = pc[i] + cbegin;
2184 pgf[i] = pf[i] + fbegin;
2192 auto csr_cf = detail::compact_rows<TS>
2196 return (ps_r[j] ==
F) ? nlocal_f + j : -1;
2198 return (ps[j] ==
F) ?
int(pf[j]) : -1;
2202 SCF.
define_split(cpart, fpart, std::move(csr_cf), nlocal_f, gfidx_r.data(), 0);
2205 auto csr_fc = detail::compact_rows<TS>
2209 return (ps_r[j] ==
C) ? nlocal_c + j : -1;
2211 return (ps[j] ==
C) ?
int(pc[j]) : -1;
2215 SFC.
define_split(fpart, cpart, std::move(csr_fc), nlocal_c, gcidx_r.data(), 0);
2220 auto const& s2 = S2.const_parcsr();
2227 if (r >= nc) {
return 0; }
2229 auto const& c0 = s2.csr0;
2230 for (
Long idx = c0.row_offset[r]; idx < c0.row_offset[r+1]; ++idx) {
2231 n += (c0.col_index[idx] !=
int(r));
2233 if (detail::has_remote_row(s2, r)) {
2234 Long const rr = s2.row_map[r];
2235 n += s2.csr1.row_offset[rr+1] - s2.csr1.row_offset[rr];
2247 auto const& c0 = s2.csr0;
2248 for (
Long idx = c0.row_offset[r]; idx < c0.row_offset[r+1]; ++idx) {
2249 if (c0.col_index[idx] !=
int(r)) {
2250 ocol[q] = c0.col_index[idx];
2251 omat[q] = c0.mat[idx];
2255 if (detail::has_remote_row(s2, r)) {
2256 auto const& c1 = s2.csr1;
2257 Long const rr = s2.row_map[r];
2258 for (
Long idx = c1.row_offset[rr]; idx < c1.row_offset[rr+1]; ++idx) {
2259 ocol[q] = nlocal_c + c1.col_index[idx];
2260 omat[q] = c1.mat[idx];
2267 S2d.
define_split(cpart, cpart, std::move(out), nlocal_c, s2.col_map, 0);
2271template <
typename T>
2279 auto csr = detail::compact_rows<T>
2282 return remote ? nlocal + j : j;
2289template <
typename T>
2291 int max_elmts, T trunc_factor)
2300 Long const nrows =
p.csr0.nrows;
2302 detail::TruncRow<T,KMAX>
const tr{
p, max_elmts, trunc_factor, nlocal};
2311 pthr[i] = tr.threshold(i, ppos[i], pneg[i]);
2316 csr.row_offset.resize(nrows+1);
2321 if (i >= nrows) {
return 0; }
2322 int const n = tr.count(i, pthr[i]);
2323 return (max_elmts > 0) ? std::min(n, max_elmts) : n;
2328 csr.resize(nrows, nnz);
2333 T
const thresh = pthr[i];
2334 T
const sum_pos = ppos[i];
2335 T
const sum_neg = pneg[i];
2337 if (max_elmts > 0) {
2340 int const n = tr.select(i, thresh, cols, vals);
2341 T kept_pos = T(0), kept_neg = T(0);
2342 for (
int k = 0; k < n; ++k) {
2343 if (vals[k] > T(0)) { kept_pos += vals[k]; }
else { kept_neg += vals[k]; }
2345 T
const fpos = (kept_pos != T(0)) ? sum_pos/kept_pos : T(1);
2346 T
const fneg = (kept_neg != T(0)) ? sum_neg/kept_neg : T(1);
2347 for (
int k = 0; k < n; ++k) {
2349 pmat[q] = vals[k] * ((vals[k] > T(0)) ? fpos : fneg);
2353 T kept_pos = T(0), kept_neg = T(0);
2354 tr.walk(i, [&] (
int, T v) {
2355 if (std::abs(v) >= thresh) {
2356 if (v > T(0)) { kept_pos += v; }
else { kept_neg += v; }
2359 T
const fpos = (kept_pos != T(0)) ? sum_pos/kept_pos : T(1);
2360 T
const fneg = (kept_neg != T(0)) ? sum_neg/kept_neg : T(1);
2361 tr.walk(i, [&] (
int col, T v) {
2362 if (std::abs(v) >= thresh) {
2364 pmat[q] = v * ((v > T(0)) ? fpos : fneg);
2379template <
typename T>
2385 bool plus_i,
int const* row_state,
2386 int max_elmts, T trunc_factor,
Long* nnz_untruncated)
2395 Long const nrows = s.csr0.nrows;
2401 int const C = PointType::coarse;
2402 int const F = PointType::fine;
2406 auto const rsA = A.
rowSum();
2407 auto const rsS = S.
rowSum();
2424 auto const& sc = s.csr0;
2425 for (
Long idx = sc.row_offset[i]; idx < sc.row_offset[i+1]; ++idx) {
2426 if (ps[sc.col_index[idx]] ==
C) { d += T(sc.mat[idx]); }
2428 if (detail::has_remote_row(s, i)) {
2429 auto const& sc1 = s.csr1;
2430 Long const ii = s.row_map[i];
2431 for (
Long idx = sc1.row_offset[ii]; idx < sc1.row_offset[ii+1]; ++idx) {
2432 if (pstate_r[sc1.col_index[idx]] ==
C) { d += T(sc1.mat[idx]); }
2440 detail::MMInterpRow<T,TS>
const w{s, ST.
const_parcsr(), ps, pdb, pc,
2441 pstate_r, dbeta_r.data(), nlocal_f, nlocal_c};
2446 local_csr_type mcsr, bcsr;
2447 mcsr.row_offset.resize(nrows+1);
2448 bcsr.row_offset.resize(nrows+1);
2453 return st_j ==
F && (db_j + (plus_i ? aji : T(0))) != T(0);
2460 return prow_state ==
nullptr || prow_state[i] ==
C;
2466 if (i >= nrows || !has_row(i)) {
return 0; }
2469 w.walk(i, [&] (
int,
int, T, T aji,
int st_j, T db_j) {
2470 if (usable_f(aji, st_j, db_j)) { ++n; }
2481 if (i >= nrows) {
return 0; }
2482 if (ps[i] ==
C) {
return 1; }
2484 w.walk(i, [&] (
int,
int, T, T,
int st_j, T) {
2485 if (st_j ==
C) { ++n; }
2492 mcsr.resize(nrows, mnnz);
2493 bcsr.resize(nrows, bnnz);
2503 bool const in_p = has_row(i);
2509 pbcol[bp] =
int(pc[i]);
2513 T
const db_i = pdb[i];
2514 T
const bscale = plus_i ? T(1) : ((db_i != T(0)) ? T(1)/db_i : T(0));
2516 w.walk(i, [&] (
int,
int jc, T aij, T,
int st_j, T) {
2519 pbmat[bp] = bscale*aij;
2528 T alpha = prsA[i]/sscale - T(prsS[i]);
2530 w.walk(i, [&] (
int,
int, T aij, T aji,
int st_j, T db_j) {
2532 T
const den = db_j + (plus_i ? aji : T(0));
2534 if (plus_i) { theta += aij * aji / den; }
2540 T
const denom = alpha + theta;
2541 T
const scale = (denom != T(0)) ? T(-1)/denom : T(0);
2546 w.walk(i, [&] (
int jf,
int jc, T aij, T aji,
int st_j, T db_j) {
2549 pbmat[bp] = bscale*aij;
2551 }
else if (usable_f(aji, st_j, db_j)) {
2553 pmmat[mp] = plus_i ?
scale*aij/(db_j + aji) :
scale*aij;
2565#ifndef AMREX_USE_GPU
2566 if (max_elmts > 0 || trunc_factor > T(0)) {
2573 return detail::truncate_row_host<T,KMAX>(col, val, n, max_elmts, trunc_factor,
2576 if (nnz_untruncated) {
2577 *nnz_untruncated = std::accumulate(n0.begin(), n0.end(),
Long(0));
2587#ifndef AMREX_USE_GPU
2588template <
typename T>
2594 int max_elmts, T trunc_factor,
2595 Long& nnz_untruncated)
2601 constexpr int KMAX = max_p_elmts;
2605 Long const nrows = s.csr0.nrows;
2609 int const C = PointType::coarse;
2610 int const F = PointType::fine;
2612 auto const rsA = A.
rowSum();
2613 auto const rsS = S.
rowSum();
2621 auto const& sc = s.csr0;
2626 for (
Long idx = sc.row_offset[i]; idx < sc.row_offset[i+1]; ++idx) {
2627 if (ps[sc.col_index[idx]] ==
C) {
2628 d += T(sc.mat[idx]);
2633 boff[i+1] = (ps[i] ==
F) ? n : 0;
2635 std::partial_sum(boff.begin(), boff.end(), boff.begin());
2640 if (ps[i] !=
F) {
return; }
2641 T
const bscale = plus_i ? T(1) : ((pdb[i] != T(0)) ? T(1)/pdb[i] : T(0));
2643 for (
Long idx = sc.row_offset[i]; idx < sc.row_offset[i+1]; ++idx) {
2644 int const j = sc.col_index[idx];
2646 bcol[q] = int(pc[j]);
2647 bval[q] = bscale*T(sc.mat[idx]);
2653 detail::MMInterpRow<T,TS>
const w{s, ST.
const_parcsr(), ps, pdb, pc,
2654 nullptr,
nullptr,
int(nrows), nc};
2655 bool const trunc = (max_elmts > 0 || trunc_factor > T(0));
2656 struct Nbr {
int j; T aij; T den; };
2659 auto csr = detail::csr_from_rows_cpu<T,SpMatrix<T>::template container_type,
int>
2660 (nrows, [&] (
Long i) {
return sc.row_offset[i+1] - sc.row_offset[i] + 1; },
2664 (
Long i,
int const*& rc, T
const*& rv)
mutable ->
Long
2669 col[0] =
int(pc[i]);
2677 T alpha = prsA[i]/sscale - T(prsS[i]);
2679 Long maxlen = boff[i+1] - boff[i];
2681 w.walk(i, [&] (
int jf,
int, T aij, T aji,
int st_j, T db_j) {
2683 T
const den = db_j + (plus_i ? aji : T(0));
2685 if (plus_i) { theta += aij * aji / den; }
2686 maxlen += boff[jf+1] - boff[jf];
2687 nbr.push_back({jf, aij, den});
2693 T
const denom = alpha + theta;
2694 T
const scale = (denom != T(0)) ? T(-1)/denom : T(0);
2695 maxlen = std::min(maxlen,
Long(nc));
2696 if (
Long(col.size()) < maxlen) {
2705 auto add = [&] (
Long k, T m) {
2706 for (
Long bp = boff[k]; bp < boff[k+1]; ++bp) {
2707 int const j = bcol[bp];
2708 Long const pos = marker[j];
2709 if (pos >= 0 && pos < n && col[pos] == j) {
2710 val[pos] += m * bval[bp];
2714 val[n] = m * bval[bp];
2719 T
const mdiag = plus_i ?
scale :
scale*pdb[i];
2720 bool diag_done =
false;
2721 for (
auto const& nb : nbr) {
2722 if (!diag_done && nb.j > i) {
2726 add(nb.j, plus_i ?
scale*nb.aij/nb.den :
scale*nb.aij);
2728 if (!diag_done) { add(i, mdiag); }
2729 detail::sort_row_cpu(col.data(), val.data(), n, tmp);
2731 if (!trunc) {
return n; }
2733 return detail::truncate_row_host<T,KMAX>(col.data(), val.data(), n, max_elmts,
2738 nnz_untruncated = std::accumulate(nnz0.begin(), nnz0.end(),
Long(0));
#define BL_PROFILE(a)
Definition AMReX_BLProfiler.H:593
#define AMREX_ALWAYS_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:49
#define AMREX_ALWAYS_ASSERT(EX)
Definition AMReX_BLassert.H:50
Enum reflection utilities and the AMREX_ENUM macro.
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
#define AMREX_RESTRICT
Definition AMReX_Extension.H:37
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
while parser_exe_t p
Definition AMReX_Parser_Exe_Body.H:33
Algebraic multigrid solver for SpMatrix / AlgVector systems.
Definition AMReX_AlgMG.H:254
SpMatrix< TS > create_soc(SpMatrix< T > const &A, T scale) const
Definition AMReX_AlgMG.H:1498
void setAggressiveNumLevels(int n)
Number of levels with aggressive coarsening, starting at the finest. Next setup.
Definition AMReX_AlgMG.H:328
void setFixedIter(int n)
If positive, run exactly this many V-cycles regardless of tolerance.
Definition AMReX_AlgMG.H:295
void setMaxIter(int n)
Definition AMReX_AlgMG.H:293
void setChebyshevDegree(int d)
Chebyshev polynomial degree. Next solve.
Definition AMReX_AlgMG.H:304
SpMatrix< T > const & getMatrix(int lev) const
Definition AMReX_AlgMG.H:388
AlgMGInterpType InterpType
Definition AMReX_AlgMG.H:262
AlgMGKrylovSolver KrylovSolver
Definition AMReX_AlgMG.H:261
T getResidualNorm() const
Definition AMReX_AlgMG.H:381
void setMaxCoarseSize(Long n)
Stop coarsening when a level has at most this many rows. Next setup.
Definition AMReX_AlgMG.H:326
void solve(AlgVector< T > &a_sol, AlgVector< T > const &a_rhs)
Solve A x = b. a_sol holds the initial guess on entry.
Definition AMReX_AlgMG.H:689
static SpMatrix< TS > second_pass_strength(SpMatrix< TS > &S, Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > const &cidx, AlgPartition const &cpart)
Definition AMReX_AlgMG.H:2150
void setRelTol(T t)
Definition AMReX_AlgMG.H:296
std::conditional_t< std::is_same_v< T, double >, float, T > TS
Definition AMReX_AlgMG.H:258
void setBottomMaxIter(int n)
Definition AMReX_AlgMG.H:374
void setPreSmooth(int nu)
Number of pre-smoothing sweeps; negative restores the smoother's default.
Definition AMReX_AlgMG.H:311
static constexpr int default_p_max_elmts
Definition AMReX_AlgMG.H:349
void setBottomSolver(BottomSolver bs)
Definition AMReX_AlgMG.H:366
void define(SpMatrix< T > &a_mat)
Define the solver with a square matrix. Must be called before solve.
Definition AMReX_AlgMG.H:632
void setPMaxElmts(int n)
Definition AMReX_AlgMG.H:343
void setAbsTol(T t)
Converged when the residual 2-norm is below max(reltol*bnorm, abstol).
Definition AMReX_AlgMG.H:298
bool coarsen_level(int lev, SpMatrix< T > &A, SpMatrix< T > &P, AlgPartition &cpart, bool &aggressive, Long &ncoarse, bool &truncated, Long &nnz_untruncated)
Definition AMReX_AlgMG.H:1336
void setSingular(bool b)
Definition AMReX_AlgMG.H:336
void setPrintIndentation(std::string s)
Prefix of the lines the solver prints, e.g. to nest them in MLMG output.
Definition AMReX_AlgMG.H:292
void setPostSmooth(int nu)
Number of post-smoothing sweeps; negative restores the smoother's default.
Definition AMReX_AlgMG.H:313
void setInterpType(InterpType it)
Takes effect at the next setup.
Definition AMReX_AlgMG.H:340
SpMatrix< T > const & getInterp(int lev) const
Definition AMReX_AlgMG.H:389
AlgMGBottomSolver BottomSolver
Definition AMReX_AlgMG.H:260
void setBottomSmooth(int nu)
Definition AMReX_AlgMG.H:316
bool updateMatrix(SpMatrix< T > &a_mat)
Replace the matrix, keeping the hierarchy.
Definition AMReX_AlgMG.H:644
void setTruncFactor(T f)
Drop interpolation weights below this fraction of the row maximum. Next setup.
Definition AMReX_AlgMG.H:351
static AlgPartition coarse_numbering(Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > &cidx, int which=PointType::coarse)
Local index among the rows of type which and their partition.
Definition AMReX_AlgMG.H:1816
void setBottomVerbose(int v)
Definition AMReX_AlgMG.H:375
static SpMatrix< T > select_coarse_rows(SpMatrix< T > const &P, Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > const &cidx, AlgPartition const &cpart)
Rows of P at the C-points, in C-point numbering.
Definition AMReX_AlgMG.H:2272
static T strength_scale(SpMatrix< T > const &A)
Largest |a_ii|, the unit of the strength matrix entries.
Definition AMReX_AlgMG.H:1487
void setMaxLevels(int n)
Definition AMReX_AlgMG.H:324
static SpMatrix< T > truncate_interp(SpMatrix< T > &P, AlgPartition const &cpart, int max_elmts, T trunc_factor)
Definition AMReX_AlgMG.H:2290
int numLevels() const
Number of levels; 0 before setup.
Definition AMReX_AlgMG.H:383
Long numGlobalRows(int lev) const
Global number of rows on a level. Valid after setup.
Definition AMReX_AlgMG.H:387
static constexpr int max_p_elmts
Definition AMReX_AlgMG.H:348
void precond(AlgVector< T > &x, AlgVector< T > const &b)
Definition AMReX_AlgMG.H:777
void setStrongThreshold(T theta)
Definition AMReX_AlgMG.H:321
static SpMatrix< T > build_interp_mm(SpMatrix< T > &A, SpMatrix< TS > &S, SpMatrix< TS > const &ST, Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > const &cidx, AlgPartition const &cpart, T scale, bool plus_i, int const *row_state=nullptr, int max_elmts=0, T trunc_factor=T(0), Long *nnz_untruncated=nullptr)
Definition AMReX_AlgMG.H:2380
void setRelaxWeight(T w)
Relaxation weight of the smoother; negative restores the default. Next solve.
Definition AMReX_AlgMG.H:302
void setup()
Definition AMReX_AlgMG.H:1020
void setAggressiveDirectInterp(bool b)
Definition AMReX_AlgMG.H:331
static SpMatrix< T > build_interp(SpMatrix< T > &A, SpMatrix< TS > const &S, Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > const &cidx, AlgPartition const &cpart)
Classical direct interpolation.
Definition AMReX_AlgMG.H:1841
SpMatrix< T > interpolation(SpMatrix< T > &A, SpMatrix< TS > &S, SpMatrix< TS > const &ST, Gpu::DeviceVector< int > const &state, Gpu::DeviceVector< Long > const &cidx, AlgPartition const &cpart, T scale) const
Interpolation of the selected type from the C-points in state.
Definition AMReX_AlgMG.H:2136
void pmis(SpMatrix< TS > &S, SpMatrix< TS > &ST, Gpu::DeviceVector< int > &state, bool isolated_as_coarse=false) const
Definition AMReX_AlgMG.H:1672
std::size_t getSetupPeakBytes() const
Peak bytes in use in The_Arena during setup (0 if not available).
Definition AMReX_AlgMG.H:385
void setSmoother(Smoother sm)
Takes effect at the next solve; the hierarchy is kept.
Definition AMReX_AlgMG.H:300
void setThrowException(bool b)
Definition AMReX_AlgMG.H:376
void setVerbose(int v)
Definition AMReX_AlgMG.H:290
static void normalize_rows(SpMatrix< T > &P)
Scales the rows of P to sum to one; aborts on an empty row.
Definition AMReX_AlgMG.H:1458
int getNumSetups() const
Number of setups run so far (for checks that the setup is reused).
Definition AMReX_AlgMG.H:380
void project_out(AlgVector< T > &x)
Definition AMReX_AlgMG.H:554
void setBottomTol(T t)
Definition AMReX_AlgMG.H:373
AlgMGSmoother Smoother
Definition AMReX_AlgMG.H:263
void setKrylovSolver(KrylovSolver a)
Krylov solver with the V-cycle as preconditioner (see KrylovSolver). Default None.
Definition AMReX_AlgMG.H:353
void setChebyshevRatio(T r)
Chebyshev eigenvalue ratio lambda_max / lambda_min. Next solve.
Definition AMReX_AlgMG.H:306
int getNumIters() const
Definition AMReX_AlgMG.H:378
Definition AMReX_AlgPartition.H:26
Long numGlobalRows() const
Total number of rows covered by the partition.
Definition AMReX_AlgPartition.H:55
Long globalRowBegin() const
Inclusive global index begin on this process.
Definition AMReX_AlgPartition.H:62
Long numLocalRows() const
Number of local rows.
Definition AMReX_AlgPartition.H:50
Distributed dense vector that mirrors the layout of an AlgPartition.
Definition AMReX_AlgVector.H:29
void setVal(T val)
Definition AMReX_AlgVector.H:273
Long numLocalRows() const
Number of entries stored on this rank.
Definition AMReX_AlgVector.H:74
T const * data() const
Definition AMReX_AlgVector.H:85
void plusAsync(AlgVector< T, Allocator > const &rhs)
Definition AMReX_AlgVector.H:312
Chebyshev polynomial smoother for AlgVector/SpMatrix systems.
Definition AMReX_Smoother_MV.H:163
static std::size_t freeMemAvailable()
Definition AMReX_GpuDevice.cpp:1481
Jacobi smoother for AlgVector/SpMatrix linear systems.
Definition AMReX_Smoother_MV.H:32
l1 hybrid Gauss-Seidel smoother for CPUs.
Definition AMReX_Smoother_MV.H:318
Dynamically allocated vector for trivially copyable data.
Definition AMReX_PODVector.H:308
size_type size() const noexcept
Definition AMReX_PODVector.H:654
void resize(size_type a_new_size, GrowthStrategy strategy=GrowthStrategy::Poisson)
Definition AMReX_PODVector.H:734
T * data() noexcept
Definition AMReX_PODVector.H:672
This class provides the user with a few print options.
Definition AMReX_Print.H:35
Distributed CSR matrix that manages storage and GPU-friendly partitions.
Definition AMReX_SpMatrix.H:65
void setColumnPartition(AlgPartition const &col_partition)
Set the column partition and split the matrix into local and remote blocks with 32-bit column indices...
Definition AMReX_SpMatrix.H:1019
Long numGlobalRows() const
Global row count.
Definition AMReX_SpMatrix.H:196
Gpu::DeviceVector< U > gatherRemote(U const *x)
Gather values at the columns of the off-diagonal block.
Definition AMReX_SpMatrix.H:1342
Long numLocalNonZeros() const
Number of nonzeros stored locally.
Definition AMReX_SpMatrix.H:198
AlgVector< T, AllocT > rowSum() const
Sum the values in each local row and return the result as an AlgVector.
Definition AMReX_SpMatrix.H:1164
void define_split(AlgPartition partition, AlgPartition const &col_partition, local_csr_type &&compact, Long nlocal, Long const *remote_cols, Long nremote)
Define directly in split form from a CSR with compact 32-bit columns: c < nlocal is the local column ...
Definition AMReX_SpMatrix.H:1695
AlgPartition const & columnPartition() const
Return the column partition used for matrix-vector and matrix-matrix multiplications.
Definition AMReX_SpMatrix.H:191
ParCsr< T const > const_parcsr() const
Const-qualified alias of parcsr() for convenience.
Definition AMReX_SpMatrix.H:1226
ParCsr< T > parcsr()
Build GPU-friendly CSR views split into diagonal/off-diagonal blocks.
Definition AMReX_SpMatrix.H:1201
Long numLocalRows() const
Number of rows owned by this rank.
Definition AMReX_SpMatrix.H:194
AlgPartition const & partition() const
Row partition describing how matrix rows are distributed across ranks.
Definition AMReX_SpMatrix.H:182
AlgVector< T, AllocT > const & diagonalVector() const
Return diagonal elements in a square matrix.
Definition AMReX_SpMatrix.H:1145
CSR< T, container_type, int > local_csr_type
Definition AMReX_SpMatrix.H:71
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
amrex_long Long
Definition AMReX_INT.H:30
T Min(N n, T const *v, T init_val=std::numeric_limits< T >::max())
Compute the minimum of an array of values.
Definition AMReX_Reduce.H:1105
T Max(N n, T const *v, T init_val=std::numeric_limits< T >::lowest())
Compute the maximum of an array of values.
Definition AMReX_Reduce.H:1128
T Sum(N n, T const *v, T init_val=0)
Compute the sum of an array of values.
Definition AMReX_Reduce.H:1082
void ParallelForOMP(T n, L const &f) noexcept
Performance-portable kernel launch function with optional OpenMP threading.
Definition AMReX_GpuLaunch.H:328
PODVector< T, PinnedArenaAllocator< T > > PinnedVector
A PODVector that uses the pinned memory arena.
Definition AMReX_GpuContainers.H:48
PODVector< T, ArenaAllocator< T > > DeviceVector
A PODVector that uses the standard memory Arena. Note that the memory might or might not be managed d...
Definition AMReX_GpuContainers.H:24
void Min(KeyValuePair< K, V > &vi, MPI_Comm comm)
Definition AMReX_ParallelReduce.H:161
void Sum(Gpu::DeviceVector< T > &v, MPI_Comm comm)
Definition AMReX_GpuParallelReduce.H:37
void AllGather(const T *v, int cnt, T *vs, MPI_Comm comm)
Definition AMReX_ParallelReduce.H:106
void Max(KeyValuePair< K, V > &vi, MPI_Comm comm)
Definition AMReX_ParallelReduce.H:133
#define AMREX_ENUM(CLASS,...)
Declare a scoped enum with reflection support.
Definition AMReX_Enum.H:270
void copy(HostToDevice, InIter begin, InIter end, OutIter result) noexcept
A host-to-device copy routine. Note this is just a wrapper around memcpy, so it assumes contiguous st...
Definition AMReX_GpuContainers.H:128
static constexpr DeviceToHost deviceToHost
Definition AMReX_GpuContainers.H:106
static constexpr HostToDevice hostToDevice
Definition AMReX_GpuContainers.H:105
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
std::string const & name()
Definition AMReX_Machine.cpp:46
constexpr T powi(T x) noexcept
Return pow(x, Power), where Power is an integer known at compile time.
Definition AMReX_Math.H:377
constexpr int get_max_threads()
Definition AMReX_OpenMP.H:36
MPI_Comm CommunicatorSub() noexcept
sub-communicator for current frame
Definition AMReX_ParallelContext.H:70
int NProcsSub() noexcept
number of ranks in current frame
Definition AMReX_ParallelContext.H:74
static constexpr struct amrex::Scan::Type::Exclusive exclusive
static constexpr RetSum retSum
Definition AMReX_Scan.H:34
T PrefixSum(N n, FIN const &fin, FOUT const &fout, TYPE, RetSum a_ret_sum=retSum)
Definition AMReX_Scan.H:815
static constexpr int MPI_DATATYPE_NULL
Definition AMReX_ccse-mpi.H:56
int verbose
Definition AMReX.cpp:113
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
AlgMGInterpType
Definition AMReX_AlgMG.H:210
@ mm_ext
Definition AMReX_AlgMG.H:210
@ mm_ext_i
Definition AMReX_AlgMG.H:210
@ C
Definition AMReX_Order.H:7
@ F
Definition AMReX_Order.H:7
@ none
Definition AMReX.H:558
void SpMV(Long nrows, Long ncols, T *__restrict__ py, CsrView< T const, Index > const &A, T const *__restrict__ px)
Perform y = A * x using CSR data (GPU/CPU aware).
Definition AMReX_SpMV.H:30
@ RAP
Definition AMReX_MLNodeLinOp.H:22
__host__ __device__ bool isnan(T x) noexcept
Return true if x is NaN. Works under fast math.
Definition AMReX_Math.H:117
AlgMGBottomSolver
Definition AMReX_AlgMG.H:200
@ jacobi
Definition AMReX_AlgMG.H:200
@ direct
Definition AMReX_AlgMG.H:200
@ gmres
Definition AMReX_AlgMG.H:200
@ bicgstab
Definition AMReX_AlgMG.H:200
AlgMGKrylovSolver
Definition AMReX_AlgMG.H:206
@ pcg
Definition AMReX_AlgMG.H:206
void computeResidual(AlgVector< T, AllocV > &res, SpMatrix< T, AllocM > const &A, AlgVector< T, AllocV > const &x, AlgVector< T, AllocV > const &b)
Compute the residual res = b - A * x.
Definition AMReX_SpMV.H:345
@ y
Definition AMReX_Orientation.H:14
@ x
Definition AMReX_Orientation.H:14
@ z
Definition AMReX_Orientation.H:14
@ smoother
Definition AMReX_MLLinOp.H:44
@ bicgstab
Definition AMReX_MLLinOp.H:44
__host__ __device__ constexpr IntVectND< dim > scale(const IntVectND< dim > &p, int s) noexcept
Returns a IntVectND obtained by multiplying each of the components of this IntVectND by s.
Definition AMReX_IntVect.H:1135
SpMatrix< T, Allocator > SpGEMM(SpMatrix< T, Allocator > const &A, SpMatrix< T, Allocator > const &B, AlgPartition const &col_partition, F const &row_post)
SpGEMM with a callback on each row of the product.
Definition AMReX_SpGEMM.H:1216
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
const int[]
Definition AMReX_BLProfiler.cpp:1665
@ bnorm
Definition AMReX_MLMG.H:23
CSR< T, V, IO > transpose(CSR< T, V, Index > const &csr, Long ncols)
Build the transpose CSR of csr.
Definition AMReX_SpMatUtil.H:25
AlgMGSmoother
Definition AMReX_AlgMG.H:219
@ chebyshev
Definition AMReX_AlgMG.H:219
@ l1_gauss_seidel
Definition AMReX_AlgMG.H:219
@ l1_jacobi
Definition AMReX_AlgMG.H:219
V< Index > col_index
Definition AMReX_CSR.H:62
void resize(Long num_rows, Long num_non_zeros)
Resize the storage to accommodate num_rows and num_non_zeros entries.
Definition AMReX_CSR.H:80
V< T > mat
Definition AMReX_CSR.H:61
V< Index > row_offset
Definition AMReX_CSR.H:63
Long nrows() const
Number of logical rows represented by the CSR offset array.
Definition AMReX_CSR.H:67
Definition AMReX_SpMatrix.H:39