Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_FillPatchUtil_I.H
Go to the documentation of this file.
1#ifndef AMREX_FillPatchUtil_I_H_
2#define AMREX_FillPatchUtil_I_H_
3#include <AMReX_Config.H>
4
5namespace amrex {
6
8namespace detail {
9
10template <typename F, typename MF>
11auto call_interp_hook (F const& f, MF& mf, int icomp, int ncomp)
12 -> decltype(f(mf[0],Box(),icomp,ncomp))
13{
14#ifdef AMREX_USE_OMP
15#pragma omp parallel if (Gpu::notInLaunchRegion())
16#endif
17 for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
18 auto& dfab = mf[mfi];
19 const Box& dbx = dfab.box();
20 f(dfab, dbx, icomp, ncomp);
21 }
22}
23
24template <typename F, typename MF>
25auto call_interp_hook (F const& f, MF& mf, int icomp, int ncomp)
26 -> decltype(f(mf,icomp,ncomp))
27{
28 f(mf, icomp, ncomp);
29}
30
31}
33
34template <typename Interp>
35bool ProperlyNested (const IntVect& ratio, const IntVect& blocking_factor, int ngrow,
36 const IndexType& boxType, Interp* mapper)
37{
38 int ratio_max = ratio[0];
39#if (AMREX_SPACEDIM > 1)
40 ratio_max = std::max(ratio_max, ratio[1]);
41#endif
42#if (AMREX_SPACEDIM == 3)
43 ratio_max = std::max(ratio_max, ratio[2]);
44#endif
45 // There are at least this many coarse cells outside fine grids
46 // (except at physical boundaries).
47 const IntVect& nbuf = blocking_factor / ratio_max;
48
49 Box crse_box(IntVect(AMREX_D_DECL(0 ,0 ,0 )), IntVect(AMREX_D_DECL(4*nbuf[0]-1,
50 4*nbuf[1]-1,
51 4*nbuf[2]-1)));
52 crse_box.convert(boxType);
53 Box fine_box(nbuf, IntVect(AMREX_D_DECL(3*nbuf[0]-1,3*nbuf[1]-1,3*nbuf[2]-1)));
54 fine_box.convert(boxType);
55 fine_box.refine(ratio_max);
56 fine_box.grow(ngrow);
57 const Box& fine_box_coarsened = mapper->CoarseBox(fine_box, ratio_max);
58 return crse_box.contains(fine_box_coarsened);
59}
60
61template <FabArrayType MF, typename BC>
62void
64 const Vector<MF*>& smf, const Vector<Real>& stime,
65 int scomp, int dcomp, int ncomp,
66 const Geometry& geom,
67 BC& physbcf, int bcfcomp)
68{
69 FillPatchSingleLevel(mf, mf.nGrowVect(), time, smf, stime, scomp, dcomp, ncomp,
70 geom, physbcf, bcfcomp);
71}
72
73template <FabArrayType MF, typename BC>
74void
75FillPatchSingleLevel (MF& mf, IntVect const& nghost, Real time,
76 const Vector<MF*>& smf, const Vector<Real>& stime,
77 int scomp, int dcomp, int ncomp,
78 const Geometry& geom,
79 BC& physbcf, int bcfcomp)
80{
81 BL_PROFILE("FillPatchSingleLevel");
82
83 AMREX_ASSERT((!smf.empty()) &&
84 (scomp+ncomp <= smf[0]->nComp()) &&
85 (dcomp+ncomp <= mf.nComp()) &&
86 (smf.size() == stime.size()) &&
87 nghost.allLE(mf.nGrowVect()));
88
89 IntVect src_ghost(0);
90 if constexpr (std::is_same_v<BC,PhysBCFunctUseCoarseGhost>) {
91 src_ghost = physbcf.fp1_src_ghost;
92 }
93
94 if (smf.size() == 1)
95 {
96 if (&mf == smf[0] && scomp == dcomp) {
97 mf.FillBoundary(dcomp, ncomp, nghost, geom.periodicity());
98 } else {
99 mf.ParallelCopy(*smf[0], scomp, dcomp, ncomp, src_ghost, nghost, geom.periodicity());
100 }
101 }
102 else if (smf.size() == 2)
103 {
104 BL_ASSERT(smf[0]->boxArray() == smf[1]->boxArray());
105 const Real t0 = stime[0];
106 const Real t1 = stime[1];
107 const bool interp_in_time = (time != t0 && time != t1 &&
108 !amrex::almostEqual(t0,t1));
109 const bool use_src1 = (time != t0 && time == t1);
110 MF raii;
111 MF * dmf;
112 int destcomp;
113 bool sameba;
114 bool fill_dmf;
115 if (mf.boxArray() == smf[0]->boxArray() &&
116 mf.DistributionMap() == smf[0]->DistributionMap())
117 {
118 dmf = &mf;
119 destcomp = dcomp;
120 sameba = true;
121 fill_dmf = (&mf != smf[0] && &mf != smf[1]) || scomp != dcomp;
122 }
123 else if (!interp_in_time)
124 {
125 dmf = smf[use_src1 ? 1 : 0];
126 destcomp = scomp;
127 sameba = false;
128 fill_dmf = false;
129 }
130 else
131 {
132 raii.define(smf[0]->boxArray(), smf[0]->DistributionMap(), ncomp, src_ghost,
133 MFInfo().SetArena(The_Async_Arena()), smf[0]->Factory());
134
135 dmf = &raii;
136 destcomp = 0;
137 sameba = false;
138 fill_dmf = true;
139 }
140
141 if (fill_dmf)
142 {
143 IntVect interp_ghost(0);
144 if constexpr (std::is_same_v<BC,PhysBCFunctUseCoarseGhost>) {
145 interp_ghost = physbcf.fp1_src_ghost;
146 if (sameba) {
147 interp_ghost.min(nghost);
148 }
149 }
150
151 // Patch case: interpolate only the cells the ParallelCopy below reads,
152 // i.e. the hull of the local and send tags of each source FAB. Not with
153 // MPI teams, whose members work on each other's FABs but only have the
154 // send tags of their own.
155 const bool use_hull = !sameba && ParallelDescriptor::TeamSize() == 1;
156 Vector<Box> hull;
157 if (use_hull) {
158 hull.resize(raii.local_size());
159 if (!mf.empty() && !raii.empty()) {
160 auto const& cpc = mf.getCPC(nghost, raii, src_ghost, geom.periodicity());
161 for (auto const& tag : *cpc.m_LocTags) {
162 Box& h = hull[raii.localindex(tag.srcIndex)];
163 h = h.ok() ? amrex::minBox(h, tag.sbox) : tag.sbox;
164 }
165 for (auto const& kv : *cpc.m_SndTags) {
166 for (auto const& tag : kv.second) {
167 Box& h = hull[raii.localindex(tag.srcIndex)];
168 h = h.ok() ? amrex::minBox(h, tag.sbox) : tag.sbox;
169 }
170 }
171 }
172 }
173
174 // Keep the denominator nonzero; 0/0 would trap under fpe_trap_invalid.
175 const Real dt = interp_in_time ? (t1-t0) : Real(1.0);
176 const Real alpha = (t1-time)/dt;
177 const Real beta = (time-t0)/dt;
178
179#ifdef AMREX_USE_OMP
180#pragma omp parallel if (Gpu::notInLaunchRegion())
181#endif
182 for (MFIter mfi(*dmf,TilingIfNotGPU()); mfi.isValid(); ++mfi)
183 {
184 Box bx = mfi.growntilebox(interp_ghost);
185 if (use_hull) {
186 const Box& hbx = hull[mfi.LocalIndex()];
187 if (!hbx.ok()) { continue; }
188 bx &= hbx;
189 }
190 auto const sfab0 = smf[0]->array(mfi);
191 auto const sfab1 = smf[1]->array(mfi);
192 auto dfab = dmf->array(mfi);
193
194 if (interp_in_time)
195 {
196 AMREX_HOST_DEVICE_PARALLEL_FOR_4D ( bx, ncomp, i, j, k, n,
197 {
198 dfab(i,j,k,n+destcomp) = alpha*sfab0(i,j,k,n+scomp)
199 + beta*sfab1(i,j,k,n+scomp);
200 });
201 }
202 else if (use_src1)
203 {
204 AMREX_HOST_DEVICE_PARALLEL_FOR_4D ( bx, ncomp, i, j, k, n,
205 {
206 dfab(i,j,k,n+destcomp) = sfab1(i,j,k,n+scomp);
207 });
208 }
209 else
210 {
211 AMREX_HOST_DEVICE_PARALLEL_FOR_4D ( bx, ncomp, i, j, k, n,
212 {
213 dfab(i,j,k,n+destcomp) = sfab0(i,j,k,n+scomp);
214 });
215 }
216 }
217 }
218
219 if (sameba)
220 {
221 // Note that when sameba is true mf's BoxArray is nonoverlapping.
222 // So FillBoundary is safe.
223 mf.FillBoundary(dcomp, ncomp, nghost, geom.periodicity());
224 }
225 else
226 {
227 mf.ParallelCopy(*dmf, destcomp, dcomp, ncomp, src_ghost, nghost, geom.periodicity());
228 }
229 }
230 else {
231 amrex::Abort("FillPatchSingleLevel: high-order interpolation in time not implemented yet");
232 }
233
234 physbcf(mf, dcomp, ncomp, nghost, time, bcfcomp);
235}
236
254void FillPatchInterp (MultiFab& mf_fine_patch, int fcomp, MultiFab const& mf_crse_patch, int ccomp,
255 int ncomp, IntVect const& ng, const Geometry& cgeom, const Geometry& fgeom,
256 Box const& dest_domain, const IntVect& ratio,
257 MFInterpolater* mapper, const Vector<BCRec>& bcs, int bcscomp);
258
259template <FabArrayType MF, typename Interp>
260requires (!std::is_same_v<Interp, MFInterpolater>)
261void
262FillPatchInterp (MF& mf_fine_patch, int fcomp, MF const& mf_crse_patch, int ccomp,
263 int ncomp, IntVect const& ng, const Geometry& cgeom, const Geometry& fgeom,
264 Box const& dest_domain, const IntVect& ratio,
265 Interp* mapper, const Vector<BCRec>& bcs, int bcscomp)
266{
267 BL_PROFILE("FillPatchInterp(Fab)");
268
269 Box const& cdomain = amrex::convert(cgeom.Domain(), mf_fine_patch.ixType());
270 int idummy=0;
271#ifdef AMREX_USE_OMP
272#pragma omp parallel if (Gpu::notInLaunchRegion())
273#endif
274 {
275 Vector<BCRec> bcr(ncomp);
276 for (MFIter mfi(mf_fine_patch); mfi.isValid(); ++mfi)
277 {
278 auto& sfab = mf_crse_patch[mfi];
279 const Box& sbx = sfab.box();
280
281 auto& dfab = mf_fine_patch[mfi];
282 Box const& dbx = amrex::grow(mfi.validbox(),ng) & dest_domain;
283
284 amrex::setBC(sbx,cdomain,bcscomp,0,ncomp,bcs,bcr);
285 mapper->interp(sfab, ccomp, dfab, fcomp, ncomp, dbx, ratio,
286 cgeom, fgeom, bcr, idummy, idummy, RunOn::Gpu);
287 }
288 }
289}
290
291template <FabArrayType MF>
292void
293FillPatchInterp (MF& mf_fine_patch, int fcomp, MF const& mf_crse_patch, int ccomp,
294 int ncomp, IntVect const& ng, const Geometry& cgeom, const Geometry& fgeom,
295 Box const& dest_domain, const IntVect& ratio,
296 InterpBase* mapper, const Vector<BCRec>& bcs, int bcscomp)
297{
298 if (dynamic_cast<MFInterpolater*>(mapper)) {
299 FillPatchInterp(mf_fine_patch, fcomp, mf_crse_patch, ccomp,
300 ncomp, ng, cgeom, fgeom, dest_domain, ratio,
301 static_cast<MFInterpolater*>(mapper), bcs, bcscomp);
302 } else if (dynamic_cast<Interpolater*>(mapper)) {
303 FillPatchInterp(mf_fine_patch, fcomp, mf_crse_patch, ccomp,
304 ncomp, ng, cgeom, fgeom, dest_domain, ratio,
305 static_cast<Interpolater*>(mapper), bcs, bcscomp);
306 } else {
307 amrex::Abort("FillPatchInterp: unknown InterpBase");
308 }
309}
310
311template <FabArrayType MF, typename iMF, typename Interp>
312requires (!std::is_same_v<Interp, MFInterpolater>)
313void
314InterpFace (Interp *interp,
315 MF const& mf_crse_patch, int crse_comp,
316 MF& mf_refined_patch, int fine_comp,
317 int ncomp, const IntVect& ratio,
318 const iMF& solve_mask, const Geometry& crse_geom, const Geometry& fine_geom,
319 int bcscomp, RunOn gpu_or_cpu,
320 const Vector<BCRec>& bcs)
321{
322 Vector<BCRec> bcr(ncomp);
323 Box const& cdomain = amrex::convert(crse_geom.Domain(), mf_crse_patch.ixType());
324 for (MFIter mfi(mf_refined_patch);mfi.isValid(); ++mfi)
325 {
326 auto& sfab = mf_crse_patch[mfi];
327 const Box& sbx = sfab.box();
328 auto& dfab = mf_refined_patch[mfi];
329 Box const& dbx = dfab.box();
330 auto& ifab = solve_mask[mfi];
331 amrex::setBC(sbx,cdomain,bcscomp,0,ncomp,bcs,bcr);
332 interp->interp_face(sfab,crse_comp,dfab,fine_comp,ncomp,
333 dbx, ratio, ifab, crse_geom, fine_geom,
334 bcr, bcscomp, gpu_or_cpu);
335 }
336}
337
338template <FabArrayType MF, typename iMF>
339void
341 MF const& mf_crse_patch, int crse_comp,
342 MF& mf_refined_patch, int fine_comp,
343 int ncomp, const IntVect& ratio,
344 const iMF& solve_mask, const Geometry& crse_geom, const Geometry& fine_geom,
345 int bccomp, RunOn gpu_or_cpu,
346 const Vector<BCRec>& bcs)
347{
348 if (dynamic_cast<MFInterpolater*>(interp)){
349 InterpFace(static_cast<MFInterpolater*>(interp),
350 mf_crse_patch, crse_comp,mf_refined_patch, fine_comp,
351 ncomp, ratio, solve_mask, crse_geom, fine_geom, bccomp,
352 gpu_or_cpu, bcs);
353 }
354 else if (dynamic_cast<Interpolater*>(interp)){
355 InterpFace(static_cast<Interpolater*>(interp),
356 mf_crse_patch, crse_comp,mf_refined_patch, fine_comp,
357 ncomp, ratio, solve_mask, crse_geom, fine_geom, bccomp,
358 gpu_or_cpu, bcs);
359 }
360 else {
361 amrex::Abort("InterpFace: unknown InterpBase");
362 }
363}
364
365
367namespace detail {
368
369// ======== FArrayBox
370
371 template <MultiFabLike MF>
372 requires (std::same_as<typename MF::FABType::value_type, FArrayBox>)
373 MF make_mf_crse_patch (FabArrayBase::FPinfo const& fpc, int ncomp, Arena* arena = nullptr)
374 {
375 MF mf_crse_patch(fpc.ba_crse_patch, fpc.dm_patch, ncomp, 0,
376 MFInfo().SetArena(arena), *fpc.fact_crse_patch);
377 return mf_crse_patch;
378 }
379
380 template <MultiFabLike MF>
381 requires (std::same_as<typename MF::FABType::value_type, FArrayBox>)
382 MF make_mf_crse_patch (FabArrayBase::FPinfo const& fpc, int ncomp, IndexType idx_type, Arena* arena = nullptr)
383 {
384 MF mf_crse_patch(amrex::convert(fpc.ba_crse_patch, idx_type), fpc.dm_patch,
385 ncomp, 0, MFInfo().SetArena(arena), *fpc.fact_crse_patch);
386 return mf_crse_patch;
387 }
388
389 template <MultiFabLike MF>
390 requires (std::same_as<typename MF::FABType::value_type, FArrayBox>)
391 MF make_mf_fine_patch (FabArrayBase::FPinfo const& fpc, int ncomp, Arena* arena = nullptr)
392 {
393 MF mf_fine_patch(fpc.ba_fine_patch, fpc.dm_patch, ncomp, 0,
394 MFInfo().SetArena(arena), *fpc.fact_fine_patch);
395 return mf_fine_patch;
396 }
397
398 template <MultiFabLike MF>
399 requires (std::same_as<typename MF::FABType::value_type, FArrayBox>)
400 MF make_mf_fine_patch (FabArrayBase::FPinfo const& fpc, int ncomp, IndexType idx_type, Arena* arena = nullptr)
401 {
402 MF mf_fine_patch(amrex::convert(fpc.ba_fine_patch, idx_type), fpc.dm_patch,
403 ncomp, 0, MFInfo().SetArena(arena), *fpc.fact_fine_patch);
404 return mf_fine_patch;
405 }
406
407 template <MultiFabLike MF>
408 requires (std::same_as<typename MF::FABType::value_type, FArrayBox>)
409 MF make_mf_refined_patch (FabArrayBase::FPinfo const& fpc, int ncomp, IndexType idx_type, IntVect ratio, Arena* arena = nullptr)
410 {
411 MF mf_refined_patch(amrex::convert(fpc.refinedPatch(ratio), idx_type),
412 fpc.dm_patch, ncomp, 0, MFInfo().SetArena(arena),
413 *fpc.fact_fine_patch);
414 return mf_refined_patch;
415 }
416
417 template <MultiFabLike MF>
418 requires (std::same_as<typename MF::FABType::value_type, FArrayBox>)
419 MF make_mf_crse_mask (FabArrayBase::FPinfo const& fpc, int ncomp, IndexType idx_type, IntVect ratio, Arena* arena = nullptr)
420 {
421 MF mf_crse_mask(amrex::convert(amrex::coarsen(fpc.ba_fine_patch, ratio), idx_type),
422 fpc.dm_patch, ncomp, 0, MFInfo().SetArena(arena),
423 *fpc.fact_fine_patch);
424 return mf_crse_mask;
425 }
426
427 template <MultiFabLike MF>
428 requires (std::same_as<typename MF::FABType::value_type, FArrayBox>)
429 void mf_set_domain_bndry (MF &mf, Geometry const & geom)
430 {
431 mf.setDomainBndry(std::numeric_limits<Real>::quiet_NaN(), geom);
432 }
433
434
435// ======== Not FArrayBox
436
437 template <MultiFabLike MF>
438 requires (!std::same_as<typename MF::FABType::value_type, FArrayBox>)
439 MF make_mf_crse_patch (FabArrayBase::FPinfo const& fpc, int ncomp, Arena* arena = nullptr)
440 {
441 return MF(fpc.ba_crse_patch, fpc.dm_patch, ncomp, 0,
442 MFInfo().SetArena(arena));
443 }
444
445 template <MultiFabLike MF>
446 requires (!std::same_as<typename MF::FABType::value_type, FArrayBox>)
447 MF make_mf_crse_patch (FabArrayBase::FPinfo const& fpc, int ncomp, IndexType idx_type, Arena* arena = nullptr)
448 {
449 return MF(amrex::convert(fpc.ba_crse_patch, idx_type), fpc.dm_patch, ncomp, 0,
450 MFInfo().SetArena(arena));
451 }
452
453 template <MultiFabLike MF>
454 requires (!std::same_as<typename MF::FABType::value_type, FArrayBox>)
455 MF make_mf_fine_patch (FabArrayBase::FPinfo const& fpc, int ncomp, Arena* arena = nullptr)
456 {
457 return MF(fpc.ba_fine_patch, fpc.dm_patch, ncomp, 0,
458 MFInfo().SetArena(arena));
459 }
460
461 template <MultiFabLike MF>
462 requires (!std::same_as<typename MF::FABType::value_type, FArrayBox>)
463 MF make_mf_fine_patch (FabArrayBase::FPinfo const& fpc, int ncomp, IndexType idx_type, Arena* arena = nullptr)
464 {
465 return MF(amrex::convert(fpc.ba_fine_patch, idx_type), fpc.dm_patch, ncomp, 0,
466 MFInfo().SetArena(arena));
467 }
468
469 template <MultiFabLike MF>
470 requires (!std::same_as<typename MF::FABType::value_type, FArrayBox>)
471 MF make_mf_refined_patch (FabArrayBase::FPinfo const& fpc, int ncomp, IndexType idx_type, IntVect ratio, Arena* arena = nullptr)
472 {
473 return MF(amrex::convert(fpc.refinedPatch(ratio), idx_type), fpc.dm_patch, ncomp, 0,
474 MFInfo().SetArena(arena));
475 }
476
477 template <MultiFabLike MF>
478 requires (!std::same_as<typename MF::FABType::value_type, FArrayBox>)
479 MF make_mf_crse_mask (FabArrayBase::FPinfo const& fpc, int ncomp, IndexType idx_type, IntVect ratio, Arena* arena = nullptr)
480 {
481 return MF(amrex::convert(amrex::coarsen(fpc.ba_fine_patch, ratio), idx_type), fpc.dm_patch, ncomp, 0,
482 MFInfo().SetArena(arena));
483 }
484
485 template <MultiFabLike MF>
486 requires (!std::same_as<typename MF::FABType::value_type, FArrayBox>)
487 void mf_set_domain_bndry (MF &/*mf*/, Geometry const & /*geom*/)
488 {
489 // nothing
490 }
491
492 template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
493 int
494 FillPatchTwoLevels_doit (MF& mf, IntVect const& nghost, Real time,
495 const Vector<MF*>& cmf, const Vector<Real>& ct,
496 const Vector<MF*>& fmf, const Vector<Real>& ft,
497 int scomp, int dcomp, int ncomp,
498 const Geometry& cgeom, const Geometry& fgeom,
499 BC& cbc, int cbccomp,
500 BC& fbc, int fbccomp,
501 const IntVect& ratio,
502 Interp* mapper,
503 const Vector<BCRec>& bcs, int bcscomp,
504 const PreInterpHook& pre_interp,
505 const PostInterpHook& post_interp,
506 EB2::IndexSpace const* index_space,
507 bool return_error_code = false)
508 {
509 BL_PROFILE("FillPatchTwoLevels");
510
511 int success_code = return_error_code ? 0 : -1;
512 int failure_code = 1;
513
514 if (nghost.max() > 0 || mf.getBDKey() != fmf[0]->getBDKey())
515 {
516 const InterpolaterBoxCoarsener& coarsener = mapper->BoxCoarsener(ratio);
517
518 // Test for Face-centered data. In 1D, face and nodal index types
519 // are the same, and we treat the data as nodal.
520 if ( mf.ixType() != IndexType::TheNodeType() &&
521 AMREX_D_TERM( mf.ixType().nodeCentered(0),
522 + mf.ixType().nodeCentered(1),
523 + mf.ixType().nodeCentered(2) ) == 1 )
524 {
525 if ( !dynamic_cast<Interpolater*>(mapper) ){
526 amrex::Abort("This interpolater has not yet implemented a version for face-based data");
527 }
528
529 // Convert to cell-centered MF meta-data for FPInfo.
530 MF mf_cc_dummy( amrex::convert(mf.boxArray(), IntVect::TheZeroVector()),
531 mf.DistributionMap(), ncomp, nghost, MFInfo().SetAlloc(false) );
532 MF fmf_cc_dummy( amrex::convert(fmf[0]->boxArray(), IntVect::TheZeroVector()),
533 fmf[0]->DistributionMap(), ncomp, nghost, MFInfo().SetAlloc(false) );
534
535 const FabArrayBase::FPinfo& fpc = FabArrayBase::TheFPinfo(fmf_cc_dummy, mf_cc_dummy,
536 nghost,
537 coarsener,
538 fgeom,
539 cgeom,
540 index_space);
541
542 if ( ! fpc.ba_crse_patch.empty())
543 {
544 if (return_error_code) {
545 BoxArray const& cba = amrex::convert(cmf[0]->boxArray(), IntVect(0));
546 if (!cba.contains(fpc.ba_crse_patch,cgeom.periodicity())) {
547 return failure_code;
548 }
549 }
550
551 MF mf_crse_patch = make_mf_crse_patch<MF> (fpc, ncomp, mf.boxArray().ixType(), The_Async_Arena());
552 // Must make sure fine exists under needed coarse faces.
553 // It stores values for the final (interior) interpolation,
554 // which is done from this fine MF that's been partially filled
555 // (with only faces overlying coarse having valid data).
556 MF mf_refined_patch = make_mf_refined_patch<MF> (fpc, ncomp, mf.boxArray().ixType(), ratio, The_Async_Arena());
557 auto solve_mask = make_mf_crse_mask<iMultiFab>(fpc, ncomp, mf.boxArray().ixType(), ratio, The_Async_Arena());
558
559 mf_set_domain_bndry(mf_crse_patch, cgeom);
560 if constexpr (std::is_same_v<BC,PhysBCFunctUseCoarseGhost>) {
561 cbc.fp1_src_ghost = cbc.cghost;
562 }
563 FillPatchSingleLevel(mf_crse_patch, time, cmf, ct, scomp, 0, ncomp,
564 cgeom, cbc, cbccomp);
565
566 mf_set_domain_bndry(mf_refined_patch, fgeom);
567 if constexpr (std::is_same_v<BC,PhysBCFunctUseCoarseGhost>) {
568 fbc.fp1_src_ghost = IntVect(0);
569 }
570 FillPatchSingleLevel(mf_refined_patch, time, fmf, ft, scomp, 0, ncomp,
571 fgeom, fbc, fbccomp);
572
573 // Aliased MFs, used to allow CPC caching.
574 MF mf_known( amrex::coarsen(fmf[0]->boxArray(), ratio), fmf[0]->DistributionMap(),
575 ncomp, nghost, MFInfo().SetAlloc(false) );
576 MF mf_solution( amrex::coarsen(mf_refined_patch.boxArray(), ratio), mf_refined_patch.DistributionMap(),
577 ncomp, 0, MFInfo().SetAlloc(false) );
578
579 const FabArrayBase::CPC& mask_cpc = mf_solution.getCPC(IntVect::TheZeroVector(),
580 mf_known, IntVect::TheZeroVector(),
581 cgeom.periodicity());
582
583 solve_mask.setVal(1); // Values to solve.
584 solve_mask.setVal(0, mask_cpc, 0, ncomp); // Known values.
585
586 detail::call_interp_hook(pre_interp, mf_crse_patch, 0, ncomp);
587
588 InterpFace(mapper, mf_crse_patch, 0, mf_refined_patch, 0, ncomp,
589 ratio, solve_mask, cgeom, fgeom, bcscomp, RunOn::Gpu, bcs);
590
591 detail::call_interp_hook(post_interp, mf_refined_patch, 0, ncomp);
592
593 bool aliasing = false;
594 for (auto const& fmf_a : fmf) {
595 aliasing = aliasing || (&mf == fmf_a);
596 }
597 if (aliasing) {
598 mf.ParallelCopyToGhost(mf_refined_patch, 0, dcomp, ncomp,
599 IntVect{0}, nghost);
600 } else {
601 mf.ParallelCopy(mf_refined_patch, 0, dcomp, ncomp,
602 IntVect{0}, nghost);
603 }
604 }
605 }
606 else
607 {
608 const FabArrayBase::FPinfo& fpc = FabArrayBase::TheFPinfo(*fmf[0], mf,
609 nghost,
610 coarsener,
611 fgeom,
612 cgeom,
613 index_space);
614
615 if ( ! fpc.ba_crse_patch.empty())
616 {
617 if (return_error_code) {
618 BoxArray const& cba = cmf[0]->boxArray();
619 if (!cba.contains(fpc.ba_crse_patch,cgeom.periodicity())) {
620 return failure_code;
621 }
622 }
623
624 MF mf_crse_patch = make_mf_crse_patch<MF>(fpc, ncomp, The_Async_Arena());
625 mf_set_domain_bndry (mf_crse_patch, cgeom);
626
627 if constexpr (std::is_same_v<BC,PhysBCFunctUseCoarseGhost>) {
628 cbc.fp1_src_ghost = cbc.cghost;
629 }
630 FillPatchSingleLevel(mf_crse_patch, time, cmf, ct, scomp, 0, ncomp, cgeom, cbc, cbccomp);
631
632 MF mf_fine_patch = make_mf_fine_patch<MF>(fpc, ncomp, The_Async_Arena());
633
634 detail::call_interp_hook(pre_interp, mf_crse_patch, 0, ncomp);
635
636 Box fdomain_g( amrex::convert(fgeom.Domain(),mf.ixType()) );
637 for (int i = 0; i < AMREX_SPACEDIM; ++i) {
638 if (fgeom.isPeriodic(i)) {
639 fdomain_g.grow(i, nghost[i]);
640 } else {
641 if constexpr (std::is_same_v
642 <BC, PhysBCFunctUseCoarseGhost>) {
643 fdomain_g.grow(i, fbc.nghost_outside_domain[i]);
644 }
645 }
646 }
647 FillPatchInterp(mf_fine_patch, 0, mf_crse_patch, 0,
648 ncomp, IntVect(0), cgeom, fgeom,
649 fdomain_g, ratio, mapper, bcs, bcscomp);
650
651 detail::call_interp_hook(post_interp, mf_fine_patch, 0, ncomp);
652
653 mf.ParallelCopy(mf_fine_patch, 0, dcomp, ncomp, IntVect{0}, nghost);
654 }
655 }
656 }
657
658 if constexpr(std::is_same_v<BC, PhysBCFunctUseCoarseGhost>) {
659 fbc.fp1_src_ghost = IntVect(0);
660 }
661 FillPatchSingleLevel(mf, nghost, time, fmf, ft, scomp, dcomp, ncomp,
662 fgeom, fbc, fbccomp);
663
664 return success_code;
665 }
666
667 template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
668 void
669 FillPatchTwoLevels_doit (Array<MF*, AMREX_SPACEDIM> const& mf, IntVect const& nghost, Real time,
670 const Vector<Array<MF*, AMREX_SPACEDIM> >& cmf, const Vector<Real>& ct,
671 const Vector<Array<MF*, AMREX_SPACEDIM> >& fmf, const Vector<Real>& ft,
672 int scomp, int dcomp, int ncomp,
673 const Geometry& cgeom, const Geometry& fgeom,
676 const IntVect& ratio,
677 Interp* mapper,
678 const Array<Vector<BCRec>, AMREX_SPACEDIM>& bcs, const Array<int, AMREX_SPACEDIM>& bcscomp,
679 const PreInterpHook& pre_interp,
680 const PostInterpHook& post_interp,
681 EB2::IndexSpace const* index_space)
682 {
683 BL_PROFILE("FillPatchTwoLevels (Array<MF*>)");
684
685 using FAB = typename MF::FABType::value_type;
686 using iFAB = typename iMultiFab::FABType::value_type;
687
688 AMREX_ASSERT(AMREX_D_TERM(mf[0]->ixType().nodeCentered(0),
689 && mf[1]->ixType().nodeCentered(1),
690 && mf[2]->ixType().nodeCentered(2)));
691
692 // These need to be true: (ba[0] == ba[1] == ba[2]) & (dm[0] == dm[1] == dm[2]).
693 // Debatable whether these are required, or will be enforced elsewhere prior to this func.
695 && BoxArray::SameRefs(mf[0]->boxArray(), mf[1]->boxArray()),
696 && BoxArray::SameRefs(mf[0]->boxArray(), mf[2]->boxArray())));
697/*
698 AMREX_ASSERT(AMREX_D_TERM(true,
699 && DistributionMapping::SameRefs(mf[0]->DistributionMap(), mf[1]->DistributionMap()),
700 && DistributionMapping::SameRefs(mf[0]->DistributionMap(), mf[2]->DistributionMap())));
701*/
702
703
704 // Test all of them?
705 if (nghost.max() > 0 || mf[0]->getBDKey() != fmf[0][0]->getBDKey())
706 {
707 const InterpolaterBoxCoarsener& coarsener = mapper->BoxCoarsener(ratio);
708
709 // Convert to cell-centered MF meta-data for FPInfo.
710 MF mf_cc_dummy( amrex::convert(mf[0]->boxArray(), IntVect::TheZeroVector()),
711 mf[0]->DistributionMap(), ncomp, nghost, MFInfo().SetAlloc(false) );
712 MF fmf_cc_dummy( amrex::convert(fmf[0][0]->boxArray(), IntVect::TheZeroVector()),
713 fmf[0][0]->DistributionMap(), ncomp, nghost, MFInfo().SetAlloc(false) );
714
715 const FabArrayBase::FPinfo& fpc = FabArrayBase::TheFPinfo(fmf_cc_dummy, mf_cc_dummy,
716 nghost,
717 coarsener,
718 fgeom,
719 cgeom,
720 index_space);
721
722 if ( !fpc.ba_crse_patch.empty() )
723 {
724 Array<MF, AMREX_SPACEDIM> mf_crse_patch;
725 Array<MF, AMREX_SPACEDIM> mf_refined_patch;
727
728 for (int d=0; d<AMREX_SPACEDIM; ++d)
729 {
730 mf_crse_patch[d] = make_mf_crse_patch<MF> (fpc, ncomp, mf[d]->boxArray().ixType(), The_Async_Arena());
731 mf_refined_patch[d] = make_mf_refined_patch<MF> (fpc, ncomp, mf[d]->boxArray().ixType(), ratio, The_Async_Arena());
732 solve_mask[d] = make_mf_crse_mask<iMultiFab>(fpc, ncomp, mf[d]->boxArray().ixType(), ratio, The_Async_Arena());
733
734 mf_set_domain_bndry(mf_crse_patch[d], cgeom);
735 Vector<MF*> cmf_time;
736 for (const auto & mfab : cmf)
737 { cmf_time.push_back(mfab[d]); }
738
739 FillPatchSingleLevel(mf_crse_patch[d], time, cmf_time, ct, scomp, 0, ncomp,
740 cgeom, cbc[d], cbccomp[d]);
741
742 mf_set_domain_bndry(mf_refined_patch[d], fgeom);
743 Vector<MF*> fmf_time;
744 for (const auto & mfab : fmf)
745 { fmf_time.push_back(mfab[d]); }
746
747 FillPatchSingleLevel(mf_refined_patch[d], time, fmf_time, ft, scomp, 0, ncomp,
748 fgeom, fbc[d], fbccomp[d]);
749
750
751 // Aliased MFs, used to allow CPC caching.
752 MF mf_known( amrex::coarsen(fmf[0][d]->boxArray(), ratio), fmf[0][d]->DistributionMap(),
753 ncomp, nghost, MFInfo().SetAlloc(false) );
754 MF mf_solution( amrex::coarsen(mf_refined_patch[d].boxArray(), ratio), mf_refined_patch[d].DistributionMap(),
755 ncomp, 0, MFInfo().SetAlloc(false) );
756
757 const FabArrayBase::CPC& mask_cpc = mf_solution.getCPC(IntVect::TheZeroVector(),
758 mf_known, IntVect::TheZeroVector(),
759 cgeom.periodicity());
760
761 solve_mask[d].setVal(1); // Values to solve.
762 solve_mask[d].setVal(0, mask_cpc, 0, ncomp); // Known values.
763 }
764
765 int idummy=0;
766#ifdef AMREX_USE_OMP
767// bool cc = fpc.ba_crse_patch.ixType().cellCentered();
768 bool cc = false; // can anything be done to allow threading, or can the OpenMP just be removed?
769#pragma omp parallel if (cc && Gpu::notInLaunchRegion() )
770#endif
771 {
772 Vector<Array<BCRec, AMREX_SPACEDIM> > bcr(ncomp);
773 for (MFIter mfi(mf_refined_patch[0]); mfi.isValid(); ++mfi)
774 {
775 Array<FAB*, AMREX_SPACEDIM> sfab{ AMREX_D_DECL( &(mf_crse_patch[0][mfi]),
776 &(mf_crse_patch[1][mfi]),
777 &(mf_crse_patch[2][mfi]) )};
778 Array<FAB*, AMREX_SPACEDIM> dfab{ AMREX_D_DECL( &(mf_refined_patch[0][mfi]),
779 &(mf_refined_patch[1][mfi]),
780 &(mf_refined_patch[2][mfi]) )};
781 Array<iFAB*, AMREX_SPACEDIM> mfab{ AMREX_D_DECL( &(solve_mask[0][mfi]),
782 &(solve_mask[1][mfi]),
783 &(solve_mask[2][mfi]) )};
784
785 const Box& sbx_cc = amrex::convert(sfab[0]->box(), IntVect::TheZeroVector());
786 const Box& dbx_cc = amrex::convert(dfab[0]->box(), IntVect::TheZeroVector());
787
788 for (int d=0; d<AMREX_SPACEDIM; ++d)
789 {
790 Vector<BCRec> bcr_d(ncomp);
791
792 amrex::setBC(sfab[d]->box(), amrex::convert(cgeom.Domain(), mf[d]->ixType()),
793 bcscomp[d],0,ncomp,bcs[d],bcr_d);
794
795 for (int n=0; n<ncomp; ++n)
796 { bcr[n][d] = bcr_d[n]; }
797 }
798
799 pre_interp(sfab, sbx_cc, 0, ncomp);
800
801 mapper->interp_arr(sfab, 0, dfab, 0, ncomp, dbx_cc, ratio, mfab,
802 cgeom, fgeom, bcr, idummy, idummy, RunOn::Gpu);
803
804 post_interp(dfab, dbx_cc, 0, ncomp);
805 }
806 }
807
808 for (int d=0; d<AMREX_SPACEDIM; ++d)
809 {
810 bool aliasing = false;
811 for (auto const& fmf_a : fmf) {
812 aliasing = aliasing || (mf[d] == fmf_a[d]);
813 }
814 if (aliasing) {
815 mf[d]->ParallelCopyToGhost(mf_refined_patch[d], 0, dcomp, ncomp,
816 IntVect{0}, nghost);
817 } else {
818 mf[d]->ParallelCopy(mf_refined_patch[d], 0, dcomp, ncomp,
819 IntVect{0}, nghost);
820 }
821 }
822 }
823 }
824
825 for (int d=0; d<AMREX_SPACEDIM; ++d)
826 {
827 Vector<MF*> fmf_time;
828 for (auto const& ffab : fmf)
829 { fmf_time.push_back(ffab[d]); }
830
831 FillPatchSingleLevel(*mf[d], nghost, time, fmf_time, ft, scomp, dcomp, ncomp,
832 fgeom, fbc[d], fbccomp[d]);
833 }
834 }
835
836} // namespace detail
838
839template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
840void
841FillPatchTwoLevels (MF& mf, IntVect const& nghost, Real time,
842 const Vector<MF*>& cmf, const Vector<Real>& ct,
843 const Vector<MF*>& fmf, const Vector<Real>& ft,
844 int scomp, int dcomp, int ncomp,
845 const Geometry& cgeom, const Geometry& fgeom,
846 BC& cbc, int cbccomp,
847 BC& fbc, int fbccomp,
848 const IntVect& ratio,
849 Interp* mapper,
850 const Vector<BCRec>& bcs, int bcscomp,
851 const PreInterpHook& pre_interp,
852 const PostInterpHook& post_interp)
853{
854#ifdef AMREX_USE_EB
855 EB2::IndexSpace const* index_space = EB2::TopIndexSpaceIfPresent();
856#else
857 EB2::IndexSpace const* index_space = nullptr;
858#endif
859 detail::FillPatchTwoLevels_doit(mf,nghost,time,cmf,ct,fmf,ft,
860 scomp,dcomp,ncomp,cgeom,fgeom,
861 cbc,cbccomp,fbc,fbccomp,ratio,mapper,bcs,bcscomp,
862 pre_interp,post_interp,index_space);
863}
864
865template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
866void
868 const Vector<MF*>& cmf, const Vector<Real>& ct,
869 const Vector<MF*>& fmf, const Vector<Real>& ft,
870 int scomp, int dcomp, int ncomp,
871 const Geometry& cgeom, const Geometry& fgeom,
872 BC& cbc, int cbccomp,
873 BC& fbc, int fbccomp,
874 const IntVect& ratio,
875 Interp* mapper,
876 const Vector<BCRec>& bcs, int bcscomp,
877 const PreInterpHook& pre_interp,
878 const PostInterpHook& post_interp)
879{
880#ifdef AMREX_USE_EB
881 EB2::IndexSpace const* index_space = EB2::TopIndexSpaceIfPresent();
882#else
883 EB2::IndexSpace const* index_space = nullptr;
884#endif
885
886 detail::FillPatchTwoLevels_doit(mf,mf.nGrowVect(),time,cmf,ct,fmf,ft,
887 scomp,dcomp,ncomp,cgeom,fgeom,
888 cbc,cbccomp,fbc,fbccomp,ratio,mapper,bcs,bcscomp,
889 pre_interp,post_interp,index_space);
890}
891
892template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
893void
895 const Vector<Array<MF*, AMREX_SPACEDIM> >& cmf, const Vector<Real>& ct,
896 const Vector<Array<MF*, AMREX_SPACEDIM> >& fmf, const Vector<Real>& ft,
897 int scomp, int dcomp, int ncomp,
898 const Geometry& cgeom, const Geometry& fgeom,
901 const IntVect& ratio,
902 Interp* mapper,
903 const Array<Vector<BCRec>, AMREX_SPACEDIM>& bcs, const Array<int, AMREX_SPACEDIM>& bcscomp,
904 const PreInterpHook& pre_interp,
905 const PostInterpHook& post_interp)
906{
907#ifdef AMREX_USE_EB
908 EB2::IndexSpace const* index_space = EB2::TopIndexSpaceIfPresent();
909#else
910 EB2::IndexSpace const* index_space = nullptr;
911#endif
912
913 detail::FillPatchTwoLevels_doit(mf,nghost,time,cmf,ct,fmf,ft,
914 scomp,dcomp,ncomp,cgeom,fgeom,
915 cbc,cbccomp,fbc,fbccomp,ratio,mapper,bcs,bcscomp,
916 pre_interp,post_interp,index_space);
917}
918
919template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
920void
922 const Vector<Array<MF*, AMREX_SPACEDIM> >& cmf, const Vector<Real>& ct,
923 const Vector<Array<MF*, AMREX_SPACEDIM> >& fmf, const Vector<Real>& ft,
924 int scomp, int dcomp, int ncomp,
925 const Geometry& cgeom, const Geometry& fgeom,
926 Array<BC, AMREX_SPACEDIM>& cbc, int cbccomp,
927 Array<BC, AMREX_SPACEDIM>& fbc, int fbccomp,
928 const IntVect& ratio,
929 Interp* mapper,
930 const Array<Vector<BCRec>, AMREX_SPACEDIM>& bcs, int bcscomp,
931 const PreInterpHook& pre_interp,
932 const PostInterpHook& post_interp)
933{
934#ifdef AMREX_USE_EB
935 EB2::IndexSpace const* index_space = EB2::TopIndexSpaceIfPresent();
936#else
937 EB2::IndexSpace const* index_space = nullptr;
938#endif
939
940 Array<int, AMREX_SPACEDIM> cbccomp_arr = {AMREX_D_DECL(cbccomp,cbccomp,cbccomp)};
941 Array<int, AMREX_SPACEDIM> fbccomp_arr = {AMREX_D_DECL(fbccomp,fbccomp,fbccomp)};
942 Array<int, AMREX_SPACEDIM> bcscomp_arr = {AMREX_D_DECL(bcscomp,bcscomp,bcscomp)};
943
944 detail::FillPatchTwoLevels_doit(mf,nghost,time,cmf,ct,fmf,ft,
945 scomp,dcomp,ncomp,cgeom,fgeom,
946 cbc,cbccomp_arr,fbc,fbccomp_arr,ratio,mapper,bcs,bcscomp_arr,
947 pre_interp,post_interp,index_space);
948}
949
950template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
951void
953 const Vector<Array<MF*, AMREX_SPACEDIM> >& cmf, const Vector<Real>& ct,
954 const Vector<Array<MF*, AMREX_SPACEDIM> >& fmf, const Vector<Real>& ft,
955 int scomp, int dcomp, int ncomp,
956 const Geometry& cgeom, const Geometry& fgeom,
957 Array<BC, AMREX_SPACEDIM>& cbc, int cbccomp,
958 Array<BC, AMREX_SPACEDIM>& fbc, int fbccomp,
959 const IntVect& ratio,
960 Interp* mapper,
961 const Array<Vector<BCRec>, AMREX_SPACEDIM>& bcs, int bcscomp,
962 const PreInterpHook& pre_interp,
963 const PostInterpHook& post_interp)
964{
965#ifdef AMREX_USE_EB
966 EB2::IndexSpace const* index_space = EB2::TopIndexSpaceIfPresent();
967#else
968 EB2::IndexSpace const* index_space = nullptr;
969#endif
970
971 Array<int, AMREX_SPACEDIM> cbccomp_arr = {AMREX_D_DECL(cbccomp,cbccomp,cbccomp)};
972 Array<int, AMREX_SPACEDIM> fbccomp_arr = {AMREX_D_DECL(fbccomp,fbccomp,fbccomp)};
973 Array<int, AMREX_SPACEDIM> bcscomp_arr = {AMREX_D_DECL(bcscomp,bcscomp,bcscomp)};
974
975 detail::FillPatchTwoLevels_doit(mf,mf[0]->nGrowVect(),time,cmf,ct,fmf,ft,
976 scomp,dcomp,ncomp,cgeom,fgeom,
977 cbc,cbccomp_arr,fbc,fbccomp_arr,ratio,mapper,bcs,bcscomp_arr,
978 pre_interp,post_interp,index_space);
979}
980
981#ifdef AMREX_USE_EB
982template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
983void
984FillPatchTwoLevels (MF& mf, IntVect const& nghost, Real time,
985 const EB2::IndexSpace& index_space,
986 const Vector<MF*>& cmf, const Vector<Real>& ct,
987 const Vector<MF*>& fmf, const Vector<Real>& ft,
988 int scomp, int dcomp, int ncomp,
989 const Geometry& cgeom, const Geometry& fgeom,
990 BC& cbc, int cbccomp,
991 BC& fbc, int fbccomp,
992 const IntVect& ratio,
993 Interp* mapper,
994 const Vector<BCRec>& bcs, int bcscomp,
995 const PreInterpHook& pre_interp,
996 const PostInterpHook& post_interp)
997{
998 detail::FillPatchTwoLevels_doit(mf,nghost,time,cmf,ct,fmf,ft,
999 scomp,dcomp,ncomp,cgeom,fgeom,
1000 cbc,cbccomp,fbc,fbccomp,ratio,mapper,bcs,bcscomp,
1001 pre_interp,post_interp,&index_space);
1002}
1003
1004template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
1005void
1007 const EB2::IndexSpace& index_space,
1008 const Vector<MF*>& cmf, const Vector<Real>& ct,
1009 const Vector<MF*>& fmf, const Vector<Real>& ft,
1010 int scomp, int dcomp, int ncomp,
1011 const Geometry& cgeom, const Geometry& fgeom,
1012 BC& cbc, int cbccomp,
1013 BC& fbc, int fbccomp,
1014 const IntVect& ratio,
1015 Interp* mapper,
1016 const Vector<BCRec>& bcs, int bcscomp,
1017 const PreInterpHook& pre_interp,
1018 const PostInterpHook& post_interp)
1019{
1020 detail::FillPatchTwoLevels_doit(mf,mf.nGrowVect(),time,cmf,ct,fmf,ft,
1021 scomp,dcomp,ncomp,cgeom,fgeom,
1022 cbc,cbccomp,fbc,fbccomp,ratio,mapper,bcs,bcscomp,
1023 pre_interp,post_interp,&index_space);
1024}
1025#endif
1026
1027template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
1028void
1030 const MF& cmf, int scomp, int dcomp, int ncomp,
1031 const Geometry& cgeom, const Geometry& fgeom,
1032 BC& cbc, int cbccomp,
1033 BC& fbc, int fbccomp,
1034 const IntVect& ratio,
1035 Interp* mapper,
1036 const Vector<BCRec>& bcs, int bcscomp,
1037 const PreInterpHook& pre_interp,
1038 const PostInterpHook& post_interp)
1039{
1040#ifdef AMREX_USE_EB
1041 EB2::IndexSpace const* index_space = EB2::TopIndexSpaceIfPresent();
1042#else
1043 EB2::IndexSpace const* index_space = nullptr;
1044#endif
1045
1046 InterpFromCoarseLevel(mf,mf.nGrowVect(),time,index_space,cmf,scomp,dcomp,ncomp,cgeom,fgeom,
1047 cbc,cbccomp,fbc,fbccomp,ratio,mapper,bcs,bcscomp,
1048 pre_interp,post_interp);
1049}
1050
1051template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
1052void
1054 const Array<MF*, AMREX_SPACEDIM>& cmf, int scomp, int dcomp, int ncomp,
1055 const Geometry& cgeom, const Geometry& fgeom,
1056 Array<BC, AMREX_SPACEDIM>& cbc, int cbccomp,
1057 Array<BC, AMREX_SPACEDIM>& fbc, int fbccomp,
1058 const IntVect& ratio,
1059 Interp* mapper,
1060 const Array<Vector<BCRec>, AMREX_SPACEDIM>& bcs, int bcscomp,
1061 const PreInterpHook& pre_interp,
1062 const PostInterpHook& post_interp)
1063{
1064 InterpFromCoarseLevel(mf,mf[0]->nGrowVect(),time,cmf,scomp,dcomp,ncomp,cgeom,fgeom,
1065 cbc,cbccomp,fbc,fbccomp,ratio,mapper,bcs,bcscomp,
1066 pre_interp,post_interp);
1067}
1068
1069template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
1070void
1071InterpFromCoarseLevel (MF& mf, IntVect const& nghost, Real time,
1072 const MF& cmf, int scomp, int dcomp, int ncomp,
1073 const Geometry& cgeom, const Geometry& fgeom,
1074 BC& cbc, int cbccomp,
1075 BC& fbc, int fbccomp,
1076 const IntVect& ratio,
1077 Interp* mapper,
1078 const Vector<BCRec>& bcs, int bcscomp,
1079 const PreInterpHook& pre_interp,
1080 const PostInterpHook& post_interp)
1081{
1082#ifdef AMREX_USE_EB
1083 EB2::IndexSpace const* index_space = EB2::TopIndexSpaceIfPresent();
1084#else
1085 EB2::IndexSpace const* index_space = nullptr;
1086#endif
1087
1088 InterpFromCoarseLevel(mf,nghost,time,index_space,cmf,scomp,dcomp,ncomp,cgeom,fgeom,
1089 cbc,cbccomp,fbc,fbccomp,ratio,mapper,bcs,bcscomp,
1090 pre_interp,post_interp);
1091}
1092
1093template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
1094void
1095InterpFromCoarseLevel (MF& mf, IntVect const& nghost, Real time,
1096 const EB2::IndexSpace* index_space,
1097 const MF& cmf, int scomp, int dcomp, int ncomp,
1098 const Geometry& cgeom, const Geometry& fgeom,
1099 BC& cbc, int cbccomp,
1100 BC& fbc, int fbccomp,
1101 const IntVect& ratio,
1102 Interp* mapper,
1103 const Vector<BCRec>& bcs, int bcscomp,
1104 const PreInterpHook& pre_interp,
1105 const PostInterpHook& post_interp)
1106{
1107 BL_PROFILE("InterpFromCoarseLevel");
1108
1109 using FAB = typename MF::FABType::value_type;
1110
1111 const InterpolaterBoxCoarsener& coarsener = mapper->BoxCoarsener(ratio);
1112
1113 const BoxArray& ba = mf.boxArray();
1114 const DistributionMapping& dm = mf.DistributionMap();
1115
1116 const IndexType& typ = ba.ixType();
1117
1118 BL_ASSERT(typ == cmf.boxArray().ixType());
1119
1120 Box fdomain_g( amrex::convert(fgeom.Domain(),mf.ixType()) );
1121 for (int i = 0; i < AMREX_SPACEDIM; ++i) {
1122 if (fgeom.isPeriodic(i)) {
1123 fdomain_g.grow(i, nghost[i]);
1124 } else {
1125 if constexpr (std::is_same_v<BC, PhysBCFunctUseCoarseGhost>) {
1126 fdomain_g.grow(i, fbc.nghost_outside_domain[i]);
1127 }
1128 }
1129 }
1130
1131 MF mf_crse_patch;
1132 IntVect send_ghost(0), recv_ghost(0);
1133 if constexpr (std::is_same_v<BC, PhysBCFunctUseCoarseGhost>) {
1134 mf_crse_patch.define(amrex::coarsen(ba,ratio), dm, ncomp, fbc.src_ghost, MFInfo().SetArena(The_Async_Arena()));
1135 send_ghost = fbc.cghost;
1136 recv_ghost = fbc.src_ghost;
1137 } else {
1138 BoxArray ba_crse_patch(ba.size());
1139 { // TODO: later we might want to cache this
1140 for (int i = 0, N = ba.size(); i < N; ++i)
1141 {
1142 Box bx = amrex::convert(amrex::grow(ba[i],nghost), typ);
1143 bx &= fdomain_g;
1144 ba_crse_patch.set(i, coarsener.doit(bx));
1145 }
1146 }
1147
1148#ifndef AMREX_USE_EB
1149 amrex::ignore_unused(index_space);
1150#else
1151 if (index_space) {
1152 auto factory = makeEBFabFactory(index_space, cgeom, ba_crse_patch, dm,
1153 {0,0,0}, EBSupport::basic);
1154 mf_crse_patch.define(ba_crse_patch, dm, ncomp, 0, MFInfo().SetArena(The_Async_Arena()), *factory);
1155 } else
1156#endif
1157 {
1158 mf_crse_patch.define(ba_crse_patch, dm, ncomp, 0, MFInfo().SetArena(The_Async_Arena()));
1159 }
1160 detail::mf_set_domain_bndry (mf_crse_patch, cgeom);
1161 }
1162
1163 mf_crse_patch.ParallelCopy(cmf, scomp, 0, ncomp, send_ghost, recv_ghost,
1164 cgeom.periodicity());
1165
1166 cbc(mf_crse_patch, 0, ncomp, mf_crse_patch.nGrowVect(), time, cbccomp);
1167
1168 detail::call_interp_hook(pre_interp, mf_crse_patch, 0, ncomp);
1169
1170 FillPatchInterp(mf, dcomp, mf_crse_patch, 0, ncomp, nghost, cgeom, fgeom, fdomain_g,
1171 ratio, mapper, bcs, bcscomp);
1172
1173#ifdef AMREX_USE_OMP
1174#pragma omp parallel if (Gpu::notInLaunchRegion())
1175#endif
1176 for (MFIter mfi(mf); mfi.isValid(); ++mfi)
1177 {
1178 FAB& dfab = mf[mfi];
1179 Box dfab_bx = dfab.box();
1180 dfab_bx.grow(nghost-mf.nGrowVect());
1181 const Box& dbx = dfab_bx & fdomain_g;
1182
1183 post_interp(dfab, dbx, dcomp, ncomp);
1184 }
1185
1186 fbc(mf, dcomp, ncomp, nghost, time, fbccomp);
1187}
1188
1189template <FabArrayType MF, typename BC, typename Interp, typename PreInterpHook, typename PostInterpHook>
1190void
1192 const Array<MF*, AMREX_SPACEDIM>& cmf, int scomp, int dcomp, int ncomp,
1193 const Geometry& cgeom, const Geometry& fgeom,
1194 Array<BC, AMREX_SPACEDIM>& cbc, int cbccomp,
1195 Array<BC, AMREX_SPACEDIM>& fbc, int fbccomp,
1196 const IntVect& ratio,
1197 Interp* mapper,
1198 const Array<Vector<BCRec>, AMREX_SPACEDIM>& bcs, int bcscomp,
1199 const PreInterpHook& pre_interp,
1200 const PostInterpHook& post_interp)
1201{
1202 BL_PROFILE("InterpFromCoarseLevel(array)");
1203
1204 using FAB = typename MF::FABType::value_type;
1205 using iFAB = typename iMultiFab::FABType::value_type;
1206
1207 const InterpolaterBoxCoarsener& coarsener = mapper->BoxCoarsener(ratio);
1208 const BoxArray& ba = mf[0]->boxArray();
1209 const DistributionMapping& dm = mf[0]->DistributionMap();
1210
1211 AMREX_ASSERT(AMREX_D_TERM(mf[0]->ixType().nodeCentered(0),
1212 && mf[1]->ixType().nodeCentered(1),
1213 && mf[2]->ixType().nodeCentered(2)));
1214
1215 // These need to be true: (ba[0] == ba[1] == ba[2]) & (dm[0] == dm[1] == dm[2]).
1216 // Debatable whether these are required, or will be enforced elsewhere prior to this func.
1218 && BoxArray::SameRefs(mf[0]->boxArray(), mf[1]->boxArray()),
1219 && BoxArray::SameRefs(mf[0]->boxArray(), mf[2]->boxArray())));
1220/*
1221 AMREX_ASSERT(AMREX_D_TERM(true,
1222 && DistributionMapping::SameRefs(mf[0]->DistributionMap(), mf[1]->DistributionMap()),
1223 && DistributionMapping::SameRefs(mf[0]->DistributionMap(), mf[2]->DistributionMap())));
1224*/
1225
1226 // If needed, adjust to fully overlap the coarse cells.
1227 IntVect nghost_adj = nghost;
1228 for (int d=0; d<AMREX_SPACEDIM; ++d) {
1229 if (nghost[d] % ratio[d] != 0) {
1230 nghost_adj[d] += ratio[d] - (nghost[d] % ratio[d]);
1231 }
1232 }
1233
1234 Array<MF*, AMREX_SPACEDIM> mf_local = mf;
1235 int dcomp_adj = dcomp;
1236 Array<std::unique_ptr<MF>, AMREX_SPACEDIM> mf_temp;
1237 if (! nghost.allGE(nghost_adj)) {
1238 for (int d=0; d<AMREX_SPACEDIM; ++d) {
1239 mf_temp[d] = std::make_unique<MF>(mf[d]->boxArray(),
1240 mf[d]->DistributionMap(), ncomp, nghost_adj);
1241 mf_local[d] = mf_temp[d].get();
1242 }
1243 dcomp_adj = 0;
1244 }
1245
1246 // Create a cell-centered boxArray of the region to interp.
1247 // Convert this boxArray and domain as needed.
1248 Box fdomain = amrex::convert(fgeom.Domain(), IntVect::TheZeroVector());
1249 Box fdomain_g(fdomain);
1250 for (int d = 0; d < AMREX_SPACEDIM; ++d) {
1251 if (fgeom.isPeriodic(d)) {
1252 fdomain_g.grow(d,nghost_adj[d]);
1253 }
1254 }
1255
1256 // Build patches, using domain to account for periodic bcs.
1257 BoxArray ba_crse_patch(ba.size());
1258 { // TODO: later we might want to cache this
1259 for (int i = 0, N = ba.size(); i < N; ++i)
1260 {
1261 Box bx = amrex::convert(amrex::grow(ba[i], nghost_adj), IntVect::TheZeroVector());
1262 bx &= fdomain_g;
1263 ba_crse_patch.set(i, coarsener.doit(bx));
1264 }
1265 }
1266
1267 Array<MF, AMREX_SPACEDIM> mf_crse_patch;
1268 for (int d = 0; d<AMREX_SPACEDIM; ++d)
1269 {
1270 IndexType typ = mf[d]->boxArray().ixType();
1271 BoxArray ba_crse_idxed = amrex::convert(ba_crse_patch, typ);
1272
1273#ifdef AMREX_USE_EB
1274 auto crse_factory = makeEBFabFactory(cgeom, ba_crse_idxed, dm, {0,0,0}, EBSupport::basic);
1275 mf_crse_patch[d].define(ba_crse_idxed, dm, ncomp, 0, MFInfo().SetArena(The_Async_Arena()), *crse_factory);
1276#else
1277 mf_crse_patch[d].define(ba_crse_idxed, dm, ncomp, 0, MFInfo().SetArena(The_Async_Arena()));
1278#endif
1279 detail::mf_set_domain_bndry(mf_crse_patch[d], cgeom);
1280
1281 mf_crse_patch[d].ParallelCopy(*(cmf[d]), scomp, 0, ncomp, cgeom.periodicity());
1282 cbc[d](mf_crse_patch[d], 0, ncomp, mf_crse_patch[d].nGrowVect(), time, cbccomp);
1283 }
1284
1285 int idummy1=0, idummy2=0;
1286#ifdef AMREX_USE_OMP
1287#pragma omp parallel if (Gpu::notInLaunchRegion())
1288#endif
1289 {
1291
1292 // Empty containers describing that all points must be solved (no mask).
1293 Array<iFAB*, AMREX_SPACEDIM> mfab{ AMREX_D_DECL( nullptr, nullptr, nullptr ) };
1294
1295 for (MFIter mfi(mf_crse_patch[0]); mfi.isValid(); ++mfi)
1296 {
1297 Array<FAB*, AMREX_SPACEDIM> sfab{ AMREX_D_DECL( &(mf_crse_patch[0][mfi]),
1298 &(mf_crse_patch[1][mfi]),
1299 &(mf_crse_patch[2][mfi]) )};
1300 Array<FAB*, AMREX_SPACEDIM> dfab{ AMREX_D_DECL( &(*mf_local[0])[mfi],
1301 &(*mf_local[1])[mfi],
1302 &(*mf_local[2])[mfi] )};
1303
1304 const Box& sbx_cc = amrex::convert(sfab[0]->box(), IntVect::TheZeroVector());
1305 // Interpolate only over the region the coarse patch was built from.
1306 Box dfab_cc = amrex::convert(amrex::grow(ba[mfi.index()], nghost_adj),
1308 const Box& dbx_cc = dfab_cc & fdomain_g;
1309
1310 for (int d=0; d<AMREX_SPACEDIM; ++d)
1311 {
1312 Vector<BCRec> bcr_d(ncomp);
1313
1314 amrex::setBC(sfab[d]->box(),
1315 amrex::convert(cgeom.Domain(), sfab[d]->box().ixType()),
1316 bcscomp,0,ncomp,bcs[d],bcr_d);
1317
1318 for (int n=0; n<ncomp; ++n)
1319 { bcr[n][d] = bcr_d[n]; }
1320 }
1321
1322 pre_interp(sfab, sbx_cc, 0, ncomp);
1323
1324 mapper->interp_arr(sfab, 0, dfab, dcomp_adj, ncomp, dbx_cc, ratio, mfab,
1325 cgeom, fgeom, bcr, idummy1, idummy2, RunOn::Gpu);
1326
1327 post_interp(dfab, dbx_cc, dcomp_adj, ncomp);
1328 }
1329 }
1330
1331 for (int d=0; d<AMREX_SPACEDIM; ++d)
1332 {
1333 if (mf[d] != mf_local[d]) {
1334 amrex::Copy(*mf[d], *mf_local[d], 0, dcomp, ncomp, nghost);
1335 }
1336
1337 fbc[d](*mf[d], dcomp, ncomp, nghost, time, fbccomp);
1338 }
1339}
1340
1341template <FabArrayType MF, typename Interp>
1342void
1343InterpFromCoarseLevel (MF& mf, IntVect const& nghost,
1344 IntVect const& nghost_outside_domain,
1345 const MF& cmf, int scomp, int dcomp, int ncomp,
1346 const Geometry& cgeom, const Geometry& fgeom,
1347 const IntVect& ratio, Interp* mapper,
1348 const Vector<BCRec>& bcs, int bcscomp)
1349{
1350 PhysBCFunctUseCoarseGhost erfbc(cmf,nghost,nghost_outside_domain,ratio,mapper);
1351 InterpFromCoarseLevel(mf, nghost, Real(0.0), cmf, scomp, dcomp, ncomp,
1352 cgeom, fgeom, erfbc, 0, erfbc, 0, ratio, mapper,
1353 bcs, bcscomp);
1354}
1355
1356template <FabArrayType MF>
1357void
1358FillPatchSingleLevel (MF& mf, IntVect const& nghost, Real time,
1359 const Vector<MF*>& smf, IntVect const& snghost,
1360 const Vector<Real>& stime, int scomp, int dcomp, int ncomp,
1361 const Geometry& geom)
1362{
1363 PhysBCFunctUseCoarseGhost erfbc(snghost);
1364 FillPatchSingleLevel(mf, nghost, time, smf, stime, scomp, dcomp, ncomp, geom,
1365 erfbc, 0);
1366}
1367
1368template <FabArrayType MF, typename Interp>
1369void
1370FillPatchTwoLevels (MF& mf, IntVect const& nghost,
1371 IntVect const& nghost_outside_domain, Real time,
1372 const Vector<MF*>& cmf, const Vector<Real>& ct,
1373 const Vector<MF*>& fmf, const Vector<Real>& ft,
1374 int scomp, int dcomp, int ncomp,
1375 const Geometry& cgeom, const Geometry& fgeom,
1376 const IntVect& ratio, Interp* mapper,
1377 const Vector<BCRec>& bcs, int bcscomp)
1378{
1379 AMREX_ALWAYS_ASSERT_WITH_MESSAGE(nghost_outside_domain == 0, "TODO");
1380 PhysBCFunctUseCoarseGhost erfbc(*cmf[0], nghost, nghost_outside_domain, ratio,
1381 mapper);
1382 FillPatchTwoLevels(mf, nghost, time, cmf, ct, fmf, ft, scomp, dcomp, ncomp,
1383 cgeom, fgeom, erfbc, 0, erfbc, 0, ratio, mapper,
1384 bcs, bcscomp);
1385}
1386
1387template <FabArrayType MF, typename BC, typename Interp>
1388void
1389FillPatchNLevels (MF& mf, int level, const IntVect& nghost, Real time,
1390 const Vector<Vector<MF*>>& smf, const Vector<Vector<Real>>& st,
1391 int scomp, int dcomp, int ncomp,
1392 const Vector<Geometry>& geom,
1393 Vector<BC>& bc, int bccomp,
1394 const Vector<IntVect>& ratio,
1395 Interp* mapper,
1396 const Vector<BCRec>& bcr, int bcrcomp)
1397{
1398 BL_PROFILE("FillPatchNLevels");
1399
1400 // FillPatchTwolevels relies on that mf's valid region is inside the
1401 // domain at periodic boundaries. But when we create coarsen boxarray
1402 // using mapper->CoarseBox, the resulting boxarray might violate the
1403 // requirement. If that happens, we need to create a second version of
1404 // the boxarray that is safe for FillPatchTwolevels.
1405
1406 auto get_clayout = [&] () -> std::tuple<BoxArray,BoxArray,DistributionMapping>
1407 {
1408 if (level == 0) {
1409 return std::make_tuple(BoxArray(),BoxArray(),DistributionMapping());
1410 } else {
1411 BoxArray const& ba = mf.boxArray();
1412 auto const& typ = ba.ixType();
1413 std::map<int,Vector<Box>> extra_boxes_map;
1414 BoxList cbl(typ);
1415 cbl.reserve(ba.size());
1416 for (int i = 0, N = int(ba.size()); i < N; ++i) {
1417 Box const& cbox = mapper->CoarseBox(amrex::grow(ba[i],nghost),ratio[level-1]);
1418 cbl.push_back(cbox);
1419 Box gdomain = geom[level-1].growNonPeriodicDomain(cbox.length());
1420 gdomain.convert(typ);
1421 if (!gdomain.contains(cbox)) {
1422 auto& extra_boxes = extra_boxes_map[i];
1423 auto const& pshift = geom[level-1].periodicity().shiftIntVect();
1424 for (auto const& piv : pshift) {
1425 auto const& ibox = amrex::shift(cbox,piv) & gdomain;
1426 if (ibox.ok()) {
1427 extra_boxes.push_back(ibox);
1428 }
1429 }
1430 }
1431 }
1432
1433 BoxArray cba2;
1435 if (!extra_boxes_map.empty()) {
1436 BoxList cbl2 = cbl;
1437 auto& lbox = cbl2.data();
1438 DistributionMapping const& dm = mf.DistributionMap();
1439 Vector<int> procmap2 = dm.ProcessorMap();
1440 for (auto const& [i, vb] : extra_boxes_map) {
1441 lbox[i] = vb[0];
1442 for (int j = 1, nj = int(vb.size()); j < nj; ++j) {
1443 lbox.push_back(vb[j]);
1444 procmap2.push_back(dm[i]);
1445 }
1446 }
1447 cba2 = BoxArray(std::move(cbl2));
1448 dm2 = DistributionMapping(std::move(procmap2));
1449 }
1450
1451 return std::make_tuple(BoxArray(std::move(cbl)), cba2, dm2);
1452 }
1453 };
1454
1455#ifdef AMREX_USE_EB
1456 EB2::IndexSpace const* index_space = EB2::TopIndexSpaceIfPresent();
1457#else
1458 EB2::IndexSpace const* index_space = nullptr;
1459#endif
1460
1461 AMREX_ALWAYS_ASSERT(level < int(geom.size()) &&
1462 level < int(bc.size()) &&
1463 level < int(ratio.size()+1));
1464 if (level == 0) {
1465 FillPatchSingleLevel(mf, nghost, time, smf[0], st[0], scomp, dcomp, ncomp, geom[0],
1466 bc[0], bccomp);
1467 } else if (level >= int(smf.size()))
1468 {
1469 auto const& [ba1, ba2, dm2] = get_clayout();
1470 MF cmf1, cmf2;
1471#ifdef AMREX_USE_EB
1472 if (index_space) {
1473 auto factory = makeEBFabFactory(index_space, geom[level-1], ba1,
1474 mf.DistributionMap(), {0,0,0},
1476 cmf1.define(ba1, mf.DistributionMap(), ncomp, 0, MFInfo().SetArena(The_Async_Arena()), *factory);
1477 if (!ba2.empty()) {
1478 auto factory2 = makeEBFabFactory(index_space, geom[level-1], ba2,
1479 dm2, {0,0,0},
1481 cmf2.define(ba2, dm2, ncomp, 0, MFInfo().SetArena(The_Async_Arena()), *factory2);
1482 }
1483 } else
1484#endif
1485 {
1486 cmf1.define(ba1, mf.DistributionMap(), ncomp, 0, MFInfo().SetArena(The_Async_Arena()));
1487 if (!ba2.empty()) {
1488 cmf2.define(ba2, dm2, ncomp, 0, MFInfo().SetArena(The_Async_Arena()));
1489 }
1490 }
1491
1492 MF* p_mf_inside = (ba2.empty()) ? &cmf1 : &cmf2;
1493 FillPatchNLevels(*p_mf_inside, level-1, IntVect(0), time, smf, st, scomp, 0, ncomp,
1494 geom, bc, bccomp, ratio, mapper, bcr, bcrcomp);
1495 if (&cmf1 != p_mf_inside) {
1496 cmf1.ParallelCopy(*p_mf_inside, geom[level-1].periodicity());
1497 }
1498 Box domain_g = geom[level].growPeriodicDomain(nghost);
1499 domain_g.convert(mf.ixType());
1500 FillPatchInterp(mf, dcomp, cmf1, 0, ncomp, nghost, geom[level-1], geom[level],
1501 domain_g, ratio[level-1], mapper, bcr, bcrcomp);
1502 } else {
1504 int error_code = detail::FillPatchTwoLevels_doit(mf, nghost, time,
1505 smf[level-1], st[level-1],
1506 smf[level ], st[level ],
1507 scomp, dcomp, ncomp,
1508 geom[level-1], geom[level],
1509 bc[level-1], bccomp,
1510 bc[level ], bccomp,
1511 ratio[level-1], mapper, bcr, bcrcomp,
1512 hook, hook, index_space, true);
1513 if (error_code == 0) { return; }
1514
1515 auto const& [ba1, ba2, dm2] = get_clayout();
1516 MF cmf_tmp;
1517#ifdef AMREX_USE_EB
1518 if (index_space) {
1519 if (ba2.empty()) {
1520 auto factory = makeEBFabFactory(index_space, geom[level-1], ba1,
1521 mf.DistributionMap(), {0,0,0},
1523 cmf_tmp.define(ba1, mf.DistributionMap(), ncomp, 0, MFInfo().SetArena(The_Async_Arena()), *factory);
1524 } else {
1525 auto factory = makeEBFabFactory(index_space, geom[level-1], ba2,
1526 dm2, {0,0,0},
1528 cmf_tmp.define(ba2, dm2, ncomp, 0, MFInfo().SetArena(The_Async_Arena()), *factory);
1529 }
1530 } else
1531#endif
1532 {
1533 if (ba2.empty()) {
1534 cmf_tmp.define(ba1, mf.DistributionMap(), ncomp, 0, MFInfo().SetArena(The_Async_Arena()));
1535 } else {
1536 cmf_tmp.define(ba2, dm2, ncomp, 0, MFInfo().SetArena(The_Async_Arena()));
1537 }
1538 }
1539
1540 FillPatchNLevels(cmf_tmp, level-1, IntVect(0), time, smf, st, scomp, 0, ncomp,
1541 geom, bc, bccomp, ratio, mapper, bcr, bcrcomp);
1542
1543 Vector<MF*> cmf{&cmf_tmp};
1544 Vector<MF*> fmf = smf[level];
1545 Vector<MF> fmf_raii;
1546 if (scomp != 0) {
1547 for (auto const* p : fmf) {
1548 fmf_raii.emplace_back(*p, amrex::make_alias, scomp, ncomp);
1549 }
1550 fmf.clear();
1551 for (auto& a : fmf_raii) {
1552 fmf.push_back(&a);
1553 }
1554 }
1555
1556 detail::FillPatchTwoLevels_doit(mf, nghost, time,
1557 cmf, {time},
1558 fmf, st[level],
1559 0, dcomp, ncomp,
1560 geom[level-1], geom[level],
1561 bc[level-1], bccomp,
1562 bc[level ], bccomp,
1563 ratio[level-1], mapper, bcr, bcrcomp,
1564 hook, hook, index_space);
1565 }
1566}
1567
1568}
1569
1570#endif
#define BL_PROFILE(a)
Definition AMReX_BLProfiler.H:593
#define AMREX_ALWAYS_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:49
#define BL_ASSERT(EX)
Definition AMReX_BLassert.H:39
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_ALWAYS_ASSERT(EX)
Definition AMReX_BLassert.H:50
#define AMREX_HOST_DEVICE_PARALLEL_FOR_4D(...)
Definition AMReX_GpuLaunchMacrosC.nolint.H:111
while parser_exe_t p
Definition AMReX_Parser_Exe_Body.H:33
#define AMREX_D_TERM(a, b, c)
Definition AMReX_SPACE.H:172
#define AMREX_D_DECL(a, b, c)
Definition AMReX_SPACE.H:171
Reference-counted collection of Boxes.
Definition AMReX_BoxArray.H:681
IndexType ixType() const noexcept
Return index type of this BoxArray.
Definition AMReX_BoxArray.H:1269
static bool SameRefs(const BoxArray &lhs, const BoxArray &rhs)
whether two BoxArrays share the same data
Definition AMReX_BoxArray.H:1252
Long size() const noexcept
Return the number of boxes in the BoxArray.
Definition AMReX_BoxArray.H:758
void set(int i, const Box &ibox)
Set element i to ibox if no coarsening transformer is active.
Definition AMReX_BoxArray.cpp:897
A list of Boxes sharing a common IndexType.
Definition AMReX_BoxList.H:109
Vector< Box > & data() noexcept
Returns a reference to the Vector<Box>.
Definition AMReX_BoxList.H:373
void reserve(std::size_t n)
Reserve storage for at least n Boxes.
Definition AMReX_BoxList.H:148
void push_back(const Box &bn)
Append a Box to this BoxList.
Definition AMReX_BoxList.H:151
__host__ __device__ BoxND & grow(int i) noexcept
Grow in all directions by i cells (negative shrinks).
Definition AMReX_Box.H:668
__host__ __device__ IntVectND< dim > length() const noexcept
Return the length of the BoxND.
Definition AMReX_Box.H:167
__host__ __device__ BoxND & convert(IndexTypeND< dim > typ) noexcept
Convert the BoxND from the current type into the argument type. This may change the BoxND coordinates...
Definition AMReX_Box.H:1008
__host__ __device__ bool contains(const IntVectND< dim > &p) const noexcept
Return true if argument is contained within BoxND.
Definition AMReX_Box.H:233
__host__ __device__ IndexTypeND< dim > ixType() const noexcept
Return the indexing type.
Definition AMReX_Box.H:148
__host__ __device__ BoxND & refine(int ref_ratio) noexcept
Refine BoxND by given (positive) refinement ratio. NOTE: if type(dir) = CELL centered: lo <- lo*ratio...
Definition AMReX_Box.H:730
__host__ __device__ bool ok() const noexcept
Return true if high bounds are >= low bounds and the index type is valid.
Definition AMReX_Box.H:229
Calculates the distribution of FABs to MPI processes.
Definition AMReX_DistributionMapping.H:55
const Vector< int > & ProcessorMap() const noexcept
Returns a constant reference to the mapping of boxes in the underlying BoxArray to the CPU that holds...
Definition AMReX_DistributionMapping.cpp:49
Definition AMReX_EB2.H:28
static const FPinfo & TheFPinfo(const FabArrayBase &srcfa, const FabArrayBase &dstfa, const IntVect &dstng, const BoxConverter &coarsener, const Geometry &fgeom, const Geometry &cgeom, const EB2::IndexSpace *)
Definition AMReX_FabArrayBase.cpp:2116
Rectangular problem domain geometry.
Definition AMReX_Geometry.H:87
const Box & Domain() const noexcept
Returns our rectangular domain.
Definition AMReX_Geometry.H:246
Periodicity periodicity() const noexcept
Return the Periodicity based on the length of the domain.
Definition AMReX_Geometry.H:426
bool isPeriodic(int dir) const noexcept
Is the domain periodic in the specified direction?
Definition AMReX_Geometry.H:399
__host__ __device__ constexpr CellIndex ixType(int dir) const noexcept
Returns the CellIndex in direction dir.
Definition AMReX_IndexType.H:117
__host__ static __device__ constexpr IndexTypeND< dim > TheNodeType() noexcept
Definition AMReX_IndexType.H:160
__host__ __device__ constexpr int min() const noexcept
minimum (no absolute values) value
Definition AMReX_IntVect.H:331
__host__ __device__ constexpr bool allGE(const IntVectND< dim > &rhs) const noexcept
Returns true if this is greater than or equal to argument for all components. NOTE: This is NOT a str...
Definition AMReX_IntVect.H:549
__host__ static __device__ constexpr IntVectND< dim > TheZeroVector() noexcept
Definition AMReX_IntVect.H:778
__host__ __device__ constexpr bool allLE(const IntVectND< dim > &rhs) const noexcept
Returns true if this is less than or equal to argument for all components. NOTE: This is NOT a strict...
Definition AMReX_IntVect.H:499
Definition AMReX_InterpBase.H:39
Definition AMReX_InterpBase.H:22
Box doit(const Box &fine) const override
Apply the coarse-box logic to the supplied fine box.
Definition AMReX_InterpBase.cpp:12
Virtual base class for interpolaters.
Definition AMReX_Interpolater.H:32
Definition AMReX_MFInterpolater.H:20
Iterator for looping ever tiles and boxes of amrex::FabArray based containers.
Definition AMReX_MFIter.H:88
bool isValid() const noexcept
Is the iterator valid i.e. is it associated with a FAB?
Definition AMReX_MFIter.H:176
Definition AMReX_PhysBCFunct.H:135
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
Long size() const noexcept
Definition AMReX_Vector.H:54
amrex_real Real
Floating Point Type for Fields.
Definition AMReX_REAL.H:80
std::unique_ptr< EBFArrayBoxFactory > makeEBFabFactory(const Geometry &a_geom, const BoxArray &a_ba, const DistributionMapping &a_dm, const Vector< int > &a_ngrow, EBSupport a_support)
Definition AMReX_EBFabFactory.cpp:240
__host__ __device__ BoxND< dim > convert(const BoxND< dim > &b, const IntVectND< dim > &typ) noexcept
Return a copy of b converted to the nodal flags typ.
Definition AMReX_Box.H:1632
__host__ __device__ BoxND< dim > minBox(const BoxND< dim > &b1, const BoxND< dim > &b2) noexcept
Return the minimal BoxND containing both b1 and b2.
Definition AMReX_Box.H:1923
__host__ __device__ BoxND< dim > coarsen(const BoxND< dim > &b, int ref_ratio) noexcept
Return a copy of b coarsened by the isotropic ratio ref_ratio.
Definition AMReX_Box.H:1471
__host__ __device__ BoxND< dim > shift(const BoxND< dim > &b, int dir, int nzones) noexcept
Return a copy of b shifted by nzones cells in direction dir.
Definition AMReX_Box.H:1550
__host__ __device__ BoxND< dim > grow(const BoxND< dim > &b, int i) noexcept
Return a copy of b grown uniformly by i cells in every direction.
Definition AMReX_Box.H:1328
std::array< T, N > Array
Definition AMReX_Array.H:31
Arena * The_Async_Arena()
Definition AMReX_Arena.cpp:849
__host__ __device__ bool almostEqual(T x, T y, int ulp=2)
Definition AMReX_Algorithm.H:140
const IndexSpace * TopIndexSpaceIfPresent() noexcept
Return the top IndexSpace if one has been built (nullptr otherwise).
Definition AMReX_EB2.cpp:99
int TeamSize() noexcept
Definition AMReX_ParallelDescriptor.H:310
Definition AMReX_Amr.cpp:50
@ make_alias
Definition AMReX_MakeType.H:7
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
void Copy(FabArray< DFAB > &dst, FabArray< SFAB > const &src, int srccomp, int dstcomp, int numcomp, int nghost)
Definition AMReX_FabArray.H:219
@ F
Definition AMReX_Order.H:7
BoxArray const & boxArray(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.boxArray().
Definition AMReX_FabArrayBase.cpp:3038
DistributionMapping const & DistributionMap(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.DistributionMap().
Definition AMReX_FabArrayBase.cpp:3043
RunOn
Definition AMReX_GpuControl.H:65
@ Gpu
Definition AMReX_GpuControl.H:65
bool ProperlyNested(const IntVect &ratio, const IntVect &blocking_factor, int ngrow, const IndexType &boxType, Interp *mapper)
Test if AMR grids are properly nested.
Definition AMReX_FillPatchUtil_I.H:35
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
@ basic
EBCellFlag.
Definition AMReX_EBSupport.H:16
void InterpFace(Interp *interp, MF const &mf_crse_patch, int crse_comp, MF &mf_refined_patch, int fine_comp, int ncomp, const IntVect &ratio, const iMF &solve_mask, const Geometry &crse_geom, const Geometry &fine_geom, int bcscomp, RunOn gpu_or_cpu, const Vector< BCRec > &bcs)
Definition AMReX_FillPatchUtil_I.H:314
void FillPatchSingleLevel(MF &mf, IntVect const &nghost, Real time, const Vector< MF * > &smf, const Vector< Real > &stime, int scomp, int dcomp, int ncomp, const Geometry &geom, BC &physbcf, int bcfcomp)
FillPatch with data from the current level.
Definition AMReX_FillPatchUtil_I.H:75
IndexTypeND< 3 > IndexType
IndexType is an alias for amrex::IndexTypeND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:41
IntVect nGrowVect(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.nGrowVect().
Definition AMReX_FabArrayBase.cpp:3033
bool TilingIfNotGPU() noexcept
Definition AMReX_MFIter.H:12
void InterpFromCoarseLevel(MF &mf, Real time, const MF &cmf, int scomp, int dcomp, int ncomp, const Geometry &cgeom, const Geometry &fgeom, BC &cbc, int cbccomp, BC &fbc, int fbccomp, const IntVect &ratio, Interp *mapper, const Vector< BCRec > &bcs, int bcscomp, const PreInterpHook &pre_interp={}, const PostInterpHook &post_interp={})
Fill with interpolation of coarse level data.
Definition AMReX_FillPatchUtil_I.H:1029
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
void setBC(const Box &bx, const Box &domain, int src_comp, int dest_comp, int ncomp, const Vector< BCRec > &bc_dom, Vector< BCRec > &bcr) noexcept
Populate a span of BCRec objects based on a domain record.
Definition AMReX_BCRec.cpp:8
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
void FillPatchTwoLevels(MF &mf, IntVect const &nghost, Real time, const Vector< MF * > &cmf, const Vector< Real > &ct, const Vector< MF * > &fmf, const Vector< Real > &ft, int scomp, int dcomp, int ncomp, const Geometry &cgeom, const Geometry &fgeom, BC &cbc, int cbccomp, BC &fbc, int fbccomp, const IntVect &ratio, Interp *mapper, const Vector< BCRec > &bcs, int bcscomp, const PreInterpHook &pre_interp={}, const PostInterpHook &post_interp={})
FillPatch with data from the current level and the level below.
Definition AMReX_FillPatchUtil_I.H:841
void FillPatchInterp(MultiFab &mf_fine_patch, int fcomp, MultiFab const &mf_crse_patch, int ccomp, int ncomp, IntVect const &ng, const Geometry &cgeom, const Geometry &fgeom, Box const &dest_domain, const IntVect &ratio, MFInterpolater *mapper, const Vector< BCRec > &bcs, int bcscomp)
Helper that applies a MFInterpolater to fill a fine patch from a coarse patch.
Definition AMReX_FillPatchUtil.cpp:138
void FillPatchNLevels(MF &mf, int level, const IntVect &nghost, Real time, const Vector< Vector< MF * > > &smf, const Vector< Vector< Real > > &st, int scomp, int dcomp, int ncomp, const Vector< Geometry > &geom, Vector< BC > &bc, int bccomp, const Vector< IntVect > &ratio, Interp *mapper, const Vector< BCRec > &bcr, int bcrcomp)
FillPatch with data from AMR levels.
Definition AMReX_FillPatchUtil_I.H:1389
FabArray memory allocation information.
Definition AMReX_FabArray.H:73
Definition AMReX_FillPatchUtil.H:39