Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_FabArrayCommI.H
Go to the documentation of this file.
1
2#include <AMReX_FBI.H>
3#include <AMReX_PCI.H>
5
11
12namespace amrex {
13
14template <class FAB>
15template <typename BUF, class F>
16requires (BaseFabType<F>)
17void
18FabArray<FAB>::FBEP_nowait (int scomp, int ncomp, const IntVect& nghost,
19 const Periodicity& period, bool cross,
20 bool enforce_periodicity_only,
21 bool override_sync,
22 IntVect const& sumboundary_src_nghost,
23 bool deterministic) // so far the deterministic is for sumboundary only
24{
25 BL_PROFILE_SYNC_START_TIMED("SyncBeforeComms: FB");
26 BL_PROFILE("FillBoundary_nowait()");
27
28 AMREX_ASSERT_WITH_MESSAGE(!fbd, "FillBoundary_nowait() called when comm operation already in progress.");
29 AMREX_ASSERT(!enforce_periodicity_only || !override_sync);
30
31 bool sumboundary = sumboundary_src_nghost.allGE(0);
32
33 bool work_to_do;
34 if (enforce_periodicity_only) {
35 work_to_do = period.isAnyPeriodic();
36 } else if (override_sync) {
37 work_to_do = (nghost.max() > 0) || !is_cell_centered();
38 } else if (sumboundary) {
39 work_to_do = true;
40 } else {
41 work_to_do = nghost.max() > 0;
42 }
43
44 // Only a plain FillBoundary brings the halo up to date. The other
45 // operations here either touch valid data or only part of the halo, so
46 // they make no claim about it.
47 if (enforce_periodicity_only || override_sync || sumboundary) {
49 } else {
50 n_filled = nghost;
51 }
52
53 if (!work_to_do) { return; }
54
55 const FB& TheFB = getFB(nghost, period, cross, enforce_periodicity_only, override_sync, sumboundary_src_nghost);
56
58 {
59 //
60 // There can only be local work to do.
61 //
62 int N_locs = (*TheFB.m_LocTags).size();
63 if (N_locs == 0) { return; }
64#ifdef AMREX_USE_GPU
66 {
67#if defined(__CUDACC__) && defined(AMREX_USE_CUDA)
68 if (Gpu::inGraphRegion() && !sumboundary)
69 {
70 FB_local_copy_cuda_graph_1(TheFB, scomp, ncomp);
71 }
72 else
73#endif
74 {
75 if (sumboundary) {
77 FB_local_add_gpu(TheFB, scomp, ncomp, deterministic);
78 } else {
79 amrex::Abort("SumBoundary requires operator+=");
80 }
81 } else {
82 FB_local_copy_gpu(TheFB, scomp, ncomp);
83 if (!Gpu::inNoSyncRegion()) {
85 }
86 }
87 }
88 }
89 else
90#endif
91 {
92 if (sumboundary) {
94 FB_local_add_cpu(TheFB, scomp, ncomp);
95 } else {
96 amrex::Abort("SumBoundary requires operator+=");
97 }
98 } else {
99 FB_local_copy_cpu(TheFB, scomp, ncomp);
100 }
101 }
102
103 return;
104 }
105
106#ifdef BL_USE_MPI
107
108 //
109 // Do this before prematurely exiting if running in parallel.
110 // Otherwise sequence numbers will not match across MPI processes.
111 //
112 int SeqNum = ParallelDescriptor::SeqNum();
113
114 const int N_locs = TheFB.m_LocTags->size();
115 const int N_rcvs = TheFB.m_RcvTags->size();
116 const int N_snds = TheFB.m_SndTags->size();
117
118 if (N_locs == 0 && N_rcvs == 0 && N_snds == 0) {
119 // No work to do.
120 return;
121 }
122
123#ifdef AMREX_FB_UNPACK_ON_ARRIVAL
124 // Unpack on arrival: GPU-aware MPI, no CUDA graph, not deterministic.
125 // COPY needs thread-safe tags or atomic stores; ADD needs thread-safe
126 // tags or atomic add. Other cases take the path below.
127 if (!deterministic && ParallelDescriptor::UseGpuAwareMpi() &&
132 {
133 FB_on_arrival_nowait<BUF>(TheFB, scomp, ncomp, SeqNum, sumboundary);
134 return;
135 }
136#endif
137
138 fbd = std::make_unique<FBData<FAB>>();
139 fbd->fb = &TheFB;
140 fbd->scomp = scomp;
141 fbd->ncomp = ncomp;
142 fbd->tag = SeqNum;
143 fbd->deterministic = deterministic;
144
145 //
146 // Post rcvs. Allocate one chunk of space to hold'm all.
147 //
148
149 if (N_rcvs > 0) {
150 PostRcvs<BUF>(*TheFB.m_RcvTags, fbd->the_recv_data,
151 fbd->recv_data, fbd->recv_size, fbd->recv_from, fbd->recv_reqs,
152 ncomp, SeqNum);
153 fbd->recv_stat.resize(N_rcvs);
154 }
155
156 //
157 // Post send's
158 //
159 char*& the_send_data = fbd->the_send_data;
160 Vector<char*> & send_data = fbd->send_data;
161 Vector<std::size_t> send_size;
162 Vector<int> send_rank;
163 Vector<MPI_Request>& send_reqs = fbd->send_reqs;
165
166 if (N_snds > 0)
167 {
168 PrepareSendBuffers<BUF>(*TheFB.m_SndTags, the_send_data, send_data, send_size, send_rank,
169 send_reqs, send_cctc, ncomp);
170
171#ifdef AMREX_USE_GPU
173 {
174#if defined(__CUDACC__) && defined(AMREX_USE_CUDA)
175 if (std::is_same_v<BUF,value_type> && Gpu::inGraphRegion()) {
176 FB_pack_send_buffer_cuda_graph(TheFB, scomp, ncomp, send_data, send_size, send_cctc);
177 }
178 else
179#endif
180 {
181 pack_send_buffer_gpu<BUF>(*this, scomp, ncomp, send_data, send_size, send_cctc, TheFB.m_id);
182 }
183 }
184 else
185#endif
186 {
187 pack_send_buffer_cpu<BUF>(*this, scomp, ncomp, send_data, send_size, send_cctc);
188 }
189
190 AMREX_ASSERT(send_reqs.size() == N_snds);
191 PostSnds(send_data, send_size, send_rank, send_reqs, SeqNum);
192 }
193
195
196 //
197 // Do the local work. Hope for a bit of communication/computation overlap.
198 //
199 if (N_locs > 0)
200 {
201#ifdef AMREX_USE_GPU
203 {
204#if defined(__CUDACC__) && defined(AMREX_USE_CUDA)
205 if (Gpu::inGraphRegion() && !sumboundary) {
206 FB_local_copy_cuda_graph_n(TheFB, scomp, ncomp);
207 }
208 else
209#endif
210 {
211 if (sumboundary) {
213 FB_local_add_gpu(TheFB, scomp, ncomp, deterministic);
214 } else {
215 amrex::Abort("SumBoundary requires operator+=");
216 }
217 } else {
218 FB_local_copy_gpu(TheFB, scomp, ncomp);
219 if (!Gpu::inNoSyncRegion() && N_rcvs == 0) {
221 }
222 }
223 }
224 }
225 else
226#endif
227 {
228 if (sumboundary) {
230 FB_local_add_cpu(TheFB, scomp, ncomp);
231 } else {
232 amrex::Abort("SumBoundary requires operator+=");
233 }
234 } else {
235 FB_local_copy_cpu(TheFB, scomp, ncomp);
236 }
237 }
238
240 }
241
242#endif /*BL_USE_MPI*/
243
244#ifndef AMREX_USE_GPU
245 amrex::ignore_unused(deterministic);
246#endif
247}
248
249template <class FAB>
250template <typename BUF, class F>
251requires (BaseFabType<F>)
252void
254{
255#ifdef AMREX_USE_MPI
256
257 BL_PROFILE("FillBoundary_finish()");
259
260 if (!fbd) { return; }
261
262#ifdef AMREX_FB_UNPACK_ON_ARRIVAL
263 if (fbd->oa_meta) {
265 return;
266 }
267#endif
268
269 const FB* TheFB = fbd->fb;
270 bool sumboundary = TheFB->m_sb_snghost.allGE(0);
271 const auto N_rcvs = static_cast<int>(TheFB->m_RcvTags->size());
272 if (N_rcvs > 0)
273 {
274 Vector<const CopyComTagsContainer*> recv_cctc(N_rcvs,nullptr);
275 for (int k = 0; k < N_rcvs; k++)
276 {
277 if (fbd->recv_size[k] > 0)
278 {
279 auto const& cctc = TheFB->m_RcvTags->at(fbd->recv_from[k]);
280 recv_cctc[k] = &cctc;
281 }
282 }
283
284 int actual_n_rcvs = N_rcvs - std::ranges::count(fbd->recv_data, nullptr);
285
286 if (actual_n_rcvs > 0) {
287 ParallelDescriptor::Waitall(fbd->recv_reqs, fbd->recv_stat);
288#ifdef AMREX_DEBUG
289 if (!CheckRcvStats(fbd->recv_stat, fbd->recv_size, fbd->tag))
290 {
291 amrex::Abort("FillBoundary_finish failed with wrong message size");
292 }
293#endif
294 }
295
296 bool is_thread_safe = TheFB->m_threadsafe_rcv;
297 auto op = sumboundary ? FabArrayBase::ADD : FabArrayBase::COPY;
298
299#ifdef AMREX_USE_GPU
301 {
302#if defined(__CUDACC__) && defined(AMREX_USE_CUDA)
303 if (std::is_same_v<BUF,value_type> && Gpu::inGraphRegion() && !sumboundary)
304 {
305 FB_unpack_recv_buffer_cuda_graph(*TheFB, fbd->scomp, fbd->ncomp,
306 fbd->recv_data, fbd->recv_size,
307 recv_cctc, is_thread_safe);
308 }
309 else
310#endif
311 {
312 bool deterministic = fbd->deterministic;
313 unpack_recv_buffer_gpu<BUF>(*this, fbd->scomp, fbd->ncomp, fbd->recv_data, fbd->recv_size,
314 recv_cctc, op, is_thread_safe, TheFB->m_id, deterministic);
315 }
316 }
317 else
318#endif
319 {
320 unpack_recv_buffer_cpu<BUF>(*this, fbd->scomp, fbd->ncomp, fbd->recv_data, fbd->recv_size,
321 recv_cctc, op, is_thread_safe);
322 }
323
324 if (fbd->the_recv_data)
325 {
326 if (Gpu::inNoSyncRegion()) {
327 Gpu::freeAsync(amrex::The_Comms_Arena(), fbd->the_recv_data);
328 } else {
329 amrex::The_Comms_Arena()->free(fbd->the_recv_data);
330 }
331 fbd->the_recv_data = nullptr;
332 }
333 }
334
335 const auto N_snds = static_cast<int>(TheFB->m_SndTags->size());
336 if (N_snds > 0) {
337 Vector<MPI_Status> stats(fbd->send_reqs.size());
338 ParallelDescriptor::Waitall(fbd->send_reqs, stats);
339 amrex::The_Comms_Arena()->free(fbd->the_send_data);
340 fbd->the_send_data = nullptr;
341 }
342
343 fbd.reset();
344
345#endif
346}
347
348template <class FAB>
349void
351 int scomp,
352 int dcomp,
353 int ncomp,
354 const IntVect& snghost,
355 const IntVect& dnghost,
356 const Periodicity& period,
357 CpOp op,
358 const FabArrayBase::CPC * a_cpc,
359 bool deterministic)
360{
361 BL_PROFILE("FabArray::ParallelCopy()");
362
363 ParallelCopy_nowait(src, scomp, dcomp, ncomp, snghost, dnghost, period, op, a_cpc,
364 false, deterministic);
366}
367
368template <class FAB>
369void
370FabArray<FAB>::ParallelCopy (const FabArray<FAB>& src, int src_comp, int dest_comp,
371 int num_comp, const IntVect& snghost, const IntVect& dnghost,
372 const IntVect& offset, const Periodicity& period)
373{
374 BL_PROFILE("FabArray::ParallelCopy()");
375
376 ParallelCopy_nowait(src,src_comp,dest_comp,num_comp,snghost,dnghost,offset,period);
378}
379
380template <class FAB>
381void
382FabArray<FAB>::ParallelAdd (const FabArray<FAB>& src, int src_comp, int dest_comp,
383 int num_comp, const IntVect& snghost, const IntVect& dnghost,
384 const IntVect& offset, const Periodicity& period)
385{
386 BL_PROFILE("FabArray::ParallelAdd()");
387
388 ParallelCopy_nowait(src,src_comp,dest_comp,num_comp,snghost,dnghost,offset,period,
391}
392
393template <class FAB>
394void
396 int scomp,
397 int dcomp,
398 int ncomp,
399 const IntVect& snghost,
400 const IntVect& dnghost,
401 const Periodicity& period)
402{
403 BL_PROFILE("FabArray::ParallelCopyToGhost()");
404
405 ParallelCopy_nowait(src, scomp, dcomp, ncomp, snghost, dnghost, period,
406 FabArrayBase::COPY, nullptr, true);
408}
409
410template <class FAB>
411void
413 int scomp,
414 int dcomp,
415 int ncomp,
416 const IntVect& snghost,
417 const IntVect& dnghost,
418 const Periodicity& period)
419{
420 ParallelCopy_nowait(src, scomp, dcomp, ncomp, snghost, dnghost, period,
421 FabArrayBase::COPY, nullptr, true);
422}
423
424template <class FAB>
425void
430
431
432template <class FAB>
433void
435 int scomp,
436 int dcomp,
437 int ncomp,
438 const IntVect& snghost,
439 const IntVect& dnghost,
440 const Periodicity& period,
441 CpOp op,
442 const FabArrayBase::CPC * a_cpc,
443 bool to_ghost_cells_only,
444 bool deterministic)
445{
446 ParallelCopy_nowait(src,scomp,dcomp,ncomp,snghost,dnghost,IntVect(0),period,op,a_cpc,
447 to_ghost_cells_only, deterministic);
448}
449
450template <class FAB>
451void
453 int scomp,
454 int dcomp,
455 int ncomp,
456 const IntVect& snghost,
457 const IntVect& dnghost,
458 const IntVect& offset,
459 const Periodicity& period,
460 CpOp op,
461 const FabArrayBase::CPC * a_cpc,
462 bool to_ghost_cells_only,
463 bool deterministic)
464{
465 BL_PROFILE_SYNC_START_TIMED("SyncBeforeComms: PC");
466 BL_PROFILE("FabArray::ParallelCopy_nowait()");
467
468 AMREX_ASSERT_WITH_MESSAGE(!pcd, "ParallelCopy_nowait() called when comm operation already in progress.");
469
470 if (empty() || src.empty()) {
471 return;
472 }
473
475 BL_ASSERT(boxArray().ixType() == src.boxArray().ixType());
476 BL_ASSERT(src.nGrowVect().allGE(snghost));
477 BL_ASSERT( nGrowVect().allGE(dnghost));
478
479 n_filled = dnghost;
480
481 if ((ParallelDescriptor::NProcs() == 1) &&
482 (this->size() == 1) && (src.size() == 1) &&
483 !period.isAnyPeriodic() && !to_ghost_cells_only && (offset == 0))
484 {
485 if (this != &src) { // avoid self copy or plus
486 auto const& da = this->array(0, dcomp);
487 auto const& sa = src.const_array(0, scomp);
488 Box box = amrex::grow(src.box(0),snghost)
489 & amrex::grow(this->box(0),dnghost);
490 if (op == FabArrayBase::COPY) {
491#ifdef AMREX_USE_GPU
492 if (Gpu::inLaunchRegion()) {
493 ParallelFor(box, ncomp,
494 [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
495 da(i,j,k,n) = sa(i,j,k,n);
496 });
497 if (!Gpu::inNoSyncRegion()) {
499 }
500 } else
501#endif
502 {
503 auto const& lo = amrex::lbound(box);
504 auto const& hi = amrex::ubound(box);
505#ifdef AMREX_USE_OMP
506#pragma omp parallel for collapse(3)
507#endif
508 for (int n = 0; n < ncomp; ++n) {
509 for (int k = lo.z; k <= hi.z; ++k) {
510 for (int j = lo.y; j <= hi.y; ++j) {
512 for (int i = lo.x; i <= hi.x; ++i) {
513 da(i,j,k,n) = sa(i,j,k,n);
514 }}}}
515 }
516 } else {
517#ifdef AMREX_USE_GPU
518 if (Gpu::inLaunchRegion()) {
519 ParallelFor(box, ncomp,
520 [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
521 da(i,j,k,n) += sa(i,j,k,n);
522 });
523 if (!Gpu::inNoSyncRegion()) {
525 }
526 } else
527#endif
528 {
529 auto const& lo = amrex::lbound(box);
530 auto const& hi = amrex::ubound(box);
531#ifdef AMREX_USE_OMP
532#pragma omp parallel for collapse(3)
533#endif
534 for (int n = 0; n < ncomp; ++n) {
535 for (int k = lo.z; k <= hi.z; ++k) {
536 for (int j = lo.y; j <= hi.y; ++j) {
538 for (int i = lo.x; i <= hi.x; ++i) {
539 da(i,j,k,n) += sa(i,j,k,n);
540 }}}}
541 }
542 }
543 }
544 return;
545 }
546
547 if ((src.boxArray().ixType().cellCentered() || op == FabArrayBase::COPY) &&
549 snghost == IntVect::TheZeroVector() &&
550 dnghost == IntVect::TheZeroVector() &&
551 !period.isAnyPeriodic() && !to_ghost_cells_only && (offset == 0))
552 {
553 //
554 // Short-circuit full intersection code if we're doing copy()s or if
555 // we're doing plus()s on cell-centered data. Don't do plus()s on
556 // non-cell-centered data this simplistic way.
557 //
558 if (this != &src) { // avoid self copy or plus
559 if (op == FabArrayBase::COPY) {
560 Copy(*this, src, scomp, dcomp, ncomp, IntVect(0));
561 } else {
562 Add(*this, src, scomp, dcomp, ncomp, IntVect(0));
563 }
564 }
565 return;
566 }
567
568 const CPC& thecpc = (a_cpc) ? *a_cpc : getCPC(dnghost, src, snghost, period,
569 to_ghost_cells_only,offset);
570
572 {
573 //
574 // There can only be local work to do.
575 //
576
577 int N_locs = (*thecpc.m_LocTags).size();
578 if (N_locs == 0) { return; }
579#ifdef AMREX_USE_GPU
581 {
582 PC_local_gpu(thecpc, src, scomp, dcomp, ncomp, op,
583 deterministic && (op == FabArrayBase::ADD));
584 if (!Gpu::inNoSyncRegion()) {
586 }
587 }
588 else
589#endif
590 {
591 PC_local_cpu(thecpc, src, scomp, dcomp, ncomp, op);
592 }
593
594 return;
595 }
596
597#ifdef BL_USE_MPI
598
599 //
600 // Do this before prematurely exiting if running in parallel.
601 // Otherwise sequence numbers will not match across MPI processes.
602 //
603 int tag = ParallelDescriptor::SeqNum();
604
605 const int N_snds = thecpc.m_SndTags->size();
606 const int N_rcvs = thecpc.m_RcvTags->size();
607 const int N_locs = thecpc.m_LocTags->size();
608
609 if (N_locs == 0 && N_rcvs == 0 && N_snds == 0) {
610 //
611 // No work to do.
612 //
613
614 return;
615 }
616
617 //
618 // Send/Recv at most MaxComp components at a time to cut down memory usage.
619 //
620 int NCompLeft = ncomp;
621 int SC = scomp, DC = dcomp, NC;
622
623 for (int ipass = 0; ipass < ncomp; )
624 {
625 pcd = std::make_unique<PCData<FAB>>();
626 pcd->cpc = &thecpc;
627 pcd->src = &src;
628 pcd->op = op;
629 pcd->tag = tag;
630 // Deterministic GPU unpacking is currently implemented only for ADD.
631 pcd->deterministic = deterministic && (op == FabArrayBase::ADD);
632
633 NC = std::min(NCompLeft,FabArrayBase::MaxComp);
634 const bool last_iter = (NCompLeft == NC);
635
636 pcd->SC = SC;
637 pcd->DC = DC;
638 pcd->NC = NC;
639
640 //
641 // Post rcvs. Allocate one chunk of space to hold'm all.
642 //
643 pcd->the_recv_data = nullptr;
644
645 pcd->actual_n_rcvs = 0;
646 if (N_rcvs > 0) {
647 PostRcvs(*thecpc.m_RcvTags, pcd->the_recv_data,
648 pcd->recv_data, pcd->recv_size, pcd->recv_from, pcd->recv_reqs, NC, pcd->tag);
649 pcd->actual_n_rcvs = N_rcvs - std::ranges::count(pcd->recv_size, 0);
650 }
651
652 //
653 // Post send's
654 //
655 Vector<char*> send_data;
656 Vector<std::size_t> send_size;
657 Vector<int> send_rank;
659
660 if (N_snds > 0)
661 {
662 src.PrepareSendBuffers(*thecpc.m_SndTags, pcd->the_send_data, send_data, send_size,
663 send_rank, pcd->send_reqs, send_cctc, NC);
664
665#ifdef AMREX_USE_GPU
667 {
668 pack_send_buffer_gpu(src, SC, NC, send_data, send_size, send_cctc, thecpc.m_id);
669 }
670 else
671#endif
672 {
673 pack_send_buffer_cpu(src, SC, NC, send_data, send_size, send_cctc);
674 }
675
676 AMREX_ASSERT(pcd->send_reqs.size() == N_snds);
677 FabArray<FAB>::PostSnds(send_data, send_size, send_rank, pcd->send_reqs, pcd->tag);
678 }
679
680 //
681 // Do the local work. Hope for a bit of communication/computation overlap.
682 //
683 if (N_locs > 0)
684 {
685#ifdef AMREX_USE_GPU
687 {
688 PC_local_gpu(thecpc, src, SC, DC, NC, op,
689 deterministic && (op == FabArrayBase::ADD));
690 if (!Gpu::inNoSyncRegion() && N_rcvs == 0) {
692 }
693 }
694 else
695#endif
696 {
697 PC_local_cpu(thecpc, src, SC, DC, NC, op);
698 }
699 }
700
701 if (!last_iter)
702 {
704 BL_PROFILE_SYNC_START_TIMED("SyncBeforeComms: PC");
705
706 SC += NC;
707 DC += NC;
708 }
709
710 ipass += NC;
711 NCompLeft -= NC;
712 }
713
714#endif /*BL_USE_MPI*/
715
716#ifndef AMREX_USE_GPU
717 amrex::ignore_unused(deterministic);
718#endif
719}
720
721template <class FAB>
722void
724{
725
726#ifdef BL_USE_MPI
727
728 BL_PROFILE("FabArray::ParallelCopy_finish()");
730
731 if (!pcd) { return; }
732
733 const CPC* thecpc = pcd->cpc;
734
735 const auto N_snds = static_cast<int>(thecpc->m_SndTags->size());
736 const auto N_rcvs = static_cast<int>(thecpc->m_RcvTags->size());
737
738 if (N_rcvs > 0)
739 {
740 Vector<const CopyComTagsContainer*> recv_cctc(N_rcvs,nullptr);
741 for (int k = 0; k < N_rcvs; ++k)
742 {
743 if (pcd->recv_size[k] > 0)
744 {
745 auto const& cctc = thecpc->m_RcvTags->at(pcd->recv_from[k]);
746 recv_cctc[k] = &cctc;
747 }
748 }
749
750 if (pcd->actual_n_rcvs > 0) {
751 Vector<MPI_Status> stats(N_rcvs);
752 ParallelDescriptor::Waitall(pcd->recv_reqs, stats);
753#ifdef AMREX_DEBUG
754 if (!CheckRcvStats(stats, pcd->recv_size, pcd->tag))
755 {
756 amrex::Abort("ParallelCopy failed with wrong message size");
757 }
758#endif
759 }
760
761 bool is_thread_safe = thecpc->m_threadsafe_rcv;
762
763#ifdef AMREX_USE_GPU
765 {
766 unpack_recv_buffer_gpu(*this, pcd->DC, pcd->NC, pcd->recv_data, pcd->recv_size,
767 recv_cctc, pcd->op, is_thread_safe, thecpc->m_id,
768 pcd->deterministic);
769 }
770 else
771#endif
772 {
773 unpack_recv_buffer_cpu(*this, pcd->DC, pcd->NC, pcd->recv_data, pcd->recv_size,
774 recv_cctc, pcd->op, is_thread_safe);
775 }
776
777 if (pcd->the_recv_data)
778 {
779 if (Gpu::inNoSyncRegion()) {
780 Gpu::freeAsync(amrex::The_Comms_Arena(), pcd->the_recv_data);
781 } else {
782 amrex::The_Comms_Arena()->free(pcd->the_recv_data);
783 }
784 pcd->the_recv_data = nullptr;
785 }
786 }
787
788 if (N_snds > 0) {
789 if (! thecpc->m_SndTags->empty()) {
790 Vector<MPI_Status> stats(pcd->send_reqs.size());
791 ParallelDescriptor::Waitall(pcd->send_reqs, stats);
792 }
793 amrex::The_Comms_Arena()->free(pcd->the_send_data);
794 pcd->the_send_data = nullptr;
795 }
796
797 pcd.reset();
798
799#endif /*BL_USE_MPI*/
800}
801
802template <class FAB>
803void
804FabArray<FAB>::copyTo (FAB& dest, int scomp, int dcomp, int ncomp, int nghost) const
805{
806 BL_PROFILE("FabArray::copy(fab)");
807
808 BL_ASSERT(dcomp + ncomp <= dest.nComp());
809 BL_ASSERT(IntVect(nghost).allLE(nGrowVect()));
810
811 int root_proc = this->DistributionMap()[0];
812
813 BoxArray ba(dest.box());
814 DistributionMapping dm(Vector<int>{root_proc});
815 FabArray<FAB> destmf(ba, dm, ncomp, 0, MFInfo().SetAlloc(false));
816 if (ParallelDescriptor::MyProc() == root_proc) {
817 destmf.setFab(0, FAB(dest, amrex::make_alias, dcomp, ncomp));
818 }
819
820 destmf.ParallelCopy(*this, scomp, 0, ncomp, nghost, 0);
821
822#ifdef BL_USE_MPI
823 using T = typename FAB::value_type;
824 if (ParallelContext::NProcsSub() > 1) {
825 Long count = dest.numPts()*ncomp;
826 T* const p0 = dest.dataPtr(dcomp);
827 T* pb = p0;
828#ifdef AMREX_USE_GPU
829 if (dest.arena()->isDevice() && !ParallelDescriptor::UseGpuAwareMpi()) {
830 pb = (T*)The_Pinned_Arena()->alloc(sizeof(T)*count);
831 Gpu::dtoh_memcpy_async(pb, p0, sizeof(T)*count);
832 }
833 Gpu::streamSynchronize(); // MPI reads the data next
834#endif
837#ifdef AMREX_USE_GPU
838 if (pb != p0) {
839 Gpu::htod_memcpy_async(p0, pb, sizeof(T)*count);
841 The_Pinned_Arena()->free(pb);
842 }
843#endif
844 }
845#endif
846}
847
848#ifdef BL_USE_MPI
849template <class FAB>
850template <typename BUF>
851[[nodiscard]] TheFaArenaPointer
852FabArray<FAB>::PrepareSendBuffers (const MapOfCopyComTagContainers& SndTags,
853 Vector<char*>& send_data,
854 Vector<std::size_t>& send_size,
855 Vector<int>& send_rank,
856 Vector<MPI_Request>& send_reqs,
858 int ncomp)
859{
860 char* pointer = nullptr;
861 PrepareSendBuffers<BUF>(SndTags, pointer, send_data, send_size, send_rank, send_reqs, send_cctc, ncomp);
862 return TheFaArenaPointer(pointer);
863}
864
865template <class FAB>
866template <typename BUF>
867void
868FabArray<FAB>::PrepareSendBuffers (const MapOfCopyComTagContainers& SndTags,
869 char*& the_send_data,
870 Vector<char*>& send_data,
871 Vector<std::size_t>& send_size,
872 Vector<int>& send_rank,
873 Vector<MPI_Request>& send_reqs,
874 Vector<const CopyComTagsContainer*>& send_cctc,
875 int ncomp)
876{
877 send_data.clear();
878 send_size.clear();
879 send_rank.clear();
880 send_reqs.clear();
881 send_cctc.clear();
882 const auto N_snds = SndTags.size();
883 if (N_snds == 0) { return; }
884 send_data.reserve(N_snds);
885 send_size.reserve(N_snds);
886 send_rank.reserve(N_snds);
887 send_reqs.reserve(N_snds);
888 send_cctc.reserve(N_snds);
889
890 Vector<std::size_t> offset; offset.reserve(N_snds);
891 std::size_t total_volume = 0;
892 for (auto const& kv : SndTags)
893 {
894 auto const& cctc = kv.second;
895
896 std::size_t nbytes = 0;
897 for (auto const& cct : kv.second)
898 {
899 nbytes += cct.sbox.numPts() * ncomp * sizeof(BUF);
900 }
901
902 std::size_t acd = ParallelDescriptor::sizeof_selected_comm_data_type(nbytes);
903 nbytes = amrex::aligned_size(acd, nbytes); // so that bytes are aligned
904
905 // Also need to align the offset properly
906 total_volume = amrex::aligned_size(std::max(alignof(BUF), acd),
907 total_volume);
908
909 offset.push_back(total_volume);
910 total_volume += nbytes;
911
912 send_data.push_back(nullptr);
913 send_size.push_back(nbytes);
914 send_rank.push_back(kv.first);
915 send_reqs.push_back(MPI_REQUEST_NULL);
916 send_cctc.push_back(&cctc);
917 }
918
919 if (total_volume > 0)
920 {
921 the_send_data = static_cast<char*>(amrex::The_Comms_Arena()->alloc(total_volume));
922 for (int i = 0, N = static_cast<int>(send_size.size()); i < N; ++i) {
923 send_data[i] = the_send_data + offset[i];
924 }
925 } else {
926 the_send_data = nullptr;
927 }
928}
929
930template <class FAB>
931void
932FabArray<FAB>::PostSnds (Vector<char*> const& send_data,
933 Vector<std::size_t> const& send_size,
934 Vector<int> const& send_rank,
935 Vector<MPI_Request>& send_reqs,
936 int SeqNum)
937{
939
940 const auto N_snds = static_cast<int>(send_reqs.size());
941 for (int j = 0; j < N_snds; ++j)
942 {
943 if (send_size[j] > 0) {
944 const int rank = ParallelContext::global_to_local_rank(send_rank[j]);
945 send_reqs[j] = ParallelDescriptor::Asend
946 (send_data[j], send_size[j], rank, SeqNum, comm).req();
947 }
948 }
949}
950
951template <class FAB>
952template <typename BUF>
953TheFaArenaPointer FabArray<FAB>::PostRcvs (const MapOfCopyComTagContainers& RcvTags,
954 Vector<char*>& recv_data,
955 Vector<std::size_t>& recv_size,
956 Vector<int>& recv_from,
957 Vector<MPI_Request>& recv_reqs,
958 int ncomp,
959 int SeqNum)
960{
961 char* pointer = nullptr;
962 PostRcvs(RcvTags, pointer, recv_data, recv_size, recv_from, recv_reqs, ncomp, SeqNum);
963 return TheFaArenaPointer(pointer);
964}
965
966template <class FAB>
967template <typename BUF>
968void
969FabArray<FAB>::PostRcvs (const MapOfCopyComTagContainers& RcvTags,
970 char*& the_recv_data,
971 Vector<char*>& recv_data,
972 Vector<std::size_t>& recv_size,
973 Vector<int>& recv_from,
974 Vector<MPI_Request>& recv_reqs,
975 int ncomp,
976 int SeqNum)
977{
978 recv_data.clear();
979 recv_size.clear();
980 recv_from.clear();
981 recv_reqs.clear();
982
983 Vector<std::size_t> offset;
984 std::size_t TotalRcvsVolume = 0;
985 for (const auto& kv : RcvTags) // loop over senders
986 {
987 std::size_t nbytes = 0;
988 for (auto const& cct : kv.second)
989 {
990 nbytes += cct.dbox.numPts() * ncomp * sizeof(BUF);
991 }
992
993 std::size_t acd = ParallelDescriptor::sizeof_selected_comm_data_type(nbytes);
994 nbytes = amrex::aligned_size(acd, nbytes); // so that nbytes are aligned
995
996 // Also need to align the offset properly
997 TotalRcvsVolume = amrex::aligned_size(std::max(alignof(BUF),acd),
998 TotalRcvsVolume);
999
1000 offset.push_back(TotalRcvsVolume);
1001 TotalRcvsVolume += nbytes;
1002
1003 recv_data.push_back(nullptr);
1004 recv_size.push_back(nbytes);
1005 recv_from.push_back(kv.first);
1006 recv_reqs.push_back(MPI_REQUEST_NULL);
1007 }
1008
1009 const auto nrecv = static_cast<int>(recv_from.size());
1010
1012
1013 if (TotalRcvsVolume == 0)
1014 {
1015 the_recv_data = nullptr;
1016 }
1017 else
1018 {
1019 the_recv_data = static_cast<char*>(amrex::The_Comms_Arena()->alloc(TotalRcvsVolume));
1020
1021 for (int i = 0; i < nrecv; ++i)
1022 {
1023 recv_data[i] = the_recv_data + offset[i];
1024 if (recv_size[i] > 0)
1025 {
1026 const int rank = ParallelContext::global_to_local_rank(recv_from[i]);
1027 recv_reqs[i] = ParallelDescriptor::Arecv
1028 (recv_data[i], recv_size[i], rank, SeqNum, comm).req();
1029 }
1030 }
1031 }
1032}
1033#endif
1034
1035template <class FAB>
1036void
1038 int scomp,
1039 int dcomp,
1040 int ncomp,
1041 const IntVect& nghost)
1042{
1044 "FabArray::Redistribute: must have the same BoxArray");
1045
1046 if (ParallelContext::NProcsSub() == 1)
1047 {
1048 Copy(*this, src, scomp, dcomp, ncomp, nghost);
1049 return;
1050 }
1051
1052#ifdef BL_USE_MPI
1053
1055
1056 ParallelCopy(src, scomp, dcomp, ncomp, nghost, nghost, Periodicity::NonPeriodic(),
1057 FabArrayBase::COPY, &cpc);
1058
1059#endif
1060}
1061
1062template <class FAB>
1063void
1065{
1066#if defined(AMREX_USE_MPI) && !defined(AMREX_DEBUG)
1067 // We only test if no DEBUG because in DEBUG we check the status later.
1068 // If Test is done here, the status check will fail.
1069 int flag;
1070 ParallelDescriptor::Test(fbd->recv_reqs, flag, fbd->recv_stat);
1071#endif
1072}
1073
1075namespace detail {
1076template <class TagT>
1077void fbv_copy (Vector<TagT> const& tags)
1078{
1079 const int N = tags.size();
1080 if (N == 0) { return; }
1081#ifdef AMREX_USE_GPU
1082 if (Gpu::inLaunchRegion()) {
1083 ParallelFor(tags, 1,
1084 [=] AMREX_GPU_DEVICE (int i, int j, int k, int, TagT const& tag) noexcept
1085 {
1086 const int ncomp = tag.dfab.nComp();
1087 for (int n = 0; n < ncomp; ++n) {
1088 tag.dfab(i,j,k,n) = tag.sfab(i+tag.offset.x,j+tag.offset.y,k+tag.offset.z,n);
1089 }
1090 });
1091 } else
1092#endif
1093 {
1094#ifdef AMREX_USE_OMP
1095#pragma omp parallel for
1096#endif
1097 for (int itag = 0; itag < N; ++itag) {
1098 auto const& tag = tags[itag];
1099 const int ncomp = tag.dfab.nComp();
1100 AMREX_LOOP_4D(tag.dbox, ncomp, i, j, k, n,
1101 {
1102 tag.dfab(i,j,k,n) = tag.sfab(i+tag.offset.x,j+tag.offset.y,k+tag.offset.z,n);
1103 });
1104 }
1105 }
1106}
1107}
1109
1123template <FabArrayType MF>
1124void
1126 Vector<int> const& ncomp, Vector<IntVect> const& nghost,
1127 Vector<Periodicity> const& period,
1128 Vector<int> const& cross = {})
1129{
1130 BL_PROFILE("FillBoundary_nowait(Vector)");
1131 const int N = mf.size();
1132 for (int i = 0; i < N; ++i) {
1133 mf[i]->FillBoundary_nowait(scomp[i], ncomp[i], nghost[i], period[i],
1134 cross.empty() ? 0 : cross[i]);
1135 }
1136}
1137
1146template <FabArrayType MF>
1147void
1149 const Periodicity& a_period = Periodicity::NonPeriodic())
1150{
1151 Vector<int> scomp(mf.size(), 0);
1152 Vector<int> ncomp;
1153 Vector<IntVect> nghost;
1154 Vector<Periodicity> period(mf.size(), a_period);
1155 ncomp.reserve(mf.size());
1156 nghost.reserve(mf.size());
1157 for (auto const& x : mf) {
1158 ncomp.push_back(x->nComp());
1159 nghost.push_back(x->nGrowVect());
1160 }
1161 FillBoundary_nowait(mf, scomp, ncomp, nghost, period);
1162}
1163
1169template <FabArrayType MF>
1170void
1172{
1173 BL_PROFILE("FillBoundary_finish(Vector)");
1174 const int N = mf.size();
1175 for (int i = 0; i < N; ++i) {
1176 mf[i]->FillBoundary_finish();
1177 }
1178}
1179
1192template <FabArrayType MF>
1193void
1195 Vector<int> const& ncomp, Vector<IntVect> const& nghost,
1196 Vector<Periodicity> const& period)
1197{
1198 BL_PROFILE("FillBoundaryAndSync_nowait(Vector)");
1199 const int N = mf.size();
1200 for (int i = 0; i < N; ++i) {
1201 mf[i]->FillBoundaryAndSync_nowait(scomp[i], ncomp[i], nghost[i], period[i]);
1202 }
1203}
1204
1211template <FabArrayType MF>
1212void
1214 const Periodicity& a_period = Periodicity::NonPeriodic())
1215{
1216 Vector<int> scomp(mf.size(), 0);
1217 Vector<int> ncomp;
1218 Vector<IntVect> nghost;
1219 Vector<Periodicity> period(mf.size(), a_period);
1220 ncomp.reserve(mf.size());
1221 nghost.reserve(mf.size());
1222 for (auto const& x : mf) {
1223 ncomp.push_back(x->nComp());
1224 nghost.push_back(x->nGrowVect());
1225 }
1226 FillBoundaryAndSync_nowait(mf, scomp, ncomp, nghost, period);
1227}
1228
1234template <FabArrayType MF>
1235void
1237{
1238 BL_PROFILE("FillBoundaryAndSync_finish(Vector)");
1239 const int N = mf.size();
1240 for (int i = 0; i < N; ++i) {
1241 mf[i]->FillBoundaryAndSync_finish();
1242 }
1243}
1244
1260template <FabArrayType MF>
1261void
1262FillBoundary (Vector<MF*> const& mf, Vector<int> const& scomp,
1263 Vector<int> const& ncomp, Vector<IntVect> const& nghost,
1264 Vector<Periodicity> const& period, Vector<int> const& cross = {})
1265{
1266 BL_PROFILE("FillBoundary(Vector)");
1267#if 1
1268 const int N = mf.size();
1269 for (int i = 0; i < N; ++i) {
1270 mf[i]->FillBoundary_nowait(scomp[i], ncomp[i], nghost[i], period[i],
1271 cross.empty() ? 0 : cross[i]);
1272 }
1273 for (int i = 0; i < N; ++i) {
1274 mf[i]->FillBoundary_finish();
1275 }
1276
1277#else
1278 using FAB = typename MF::FABType::value_type;
1279 using T = typename FAB::value_type;
1280
1281 const int nmfs = mf.size();
1283 int N_locs = 0;
1284 int N_rcvs = 0;
1285 int N_snds = 0;
1286 for (int imf = 0; imf < nmfs; ++imf) {
1287 if (nghost[imf].max() > 0) {
1288 auto const& TheFB = mf[imf]->getFB(nghost[imf], period[imf],
1289 cross.empty() ? 0 : cross[imf]);
1290 // The FB is cached. Therefore it's safe take its address for later use.
1291 cmds.push_back(static_cast<FabArrayBase::CommMetaData const*>(&TheFB));
1292 N_locs += TheFB.m_LocTags->size();
1293 N_rcvs += TheFB.m_RcvTags->size();
1294 N_snds += TheFB.m_SndTags->size();
1295 } else {
1296 cmds.push_back(nullptr);
1297 }
1298 }
1299
1300 using TagT = Array4CopyTag<T>;
1301 Vector<TagT> local_tags;
1302 local_tags.reserve(N_locs);
1303 static_assert(amrex::IsStoreAtomic<T>::value, "FillBoundary(Vector): storing T is not atomic");
1304 for (int imf = 0; imf < nmfs; ++imf) {
1305 if (cmds[imf]) {
1306 auto const& tags = *(cmds[imf]->m_LocTags);
1307 for (auto const& tag : tags) {
1308 local_tags.push_back(TagT{.dfab = (*mf[imf])[tag.dstIndex].array(scomp[imf],ncomp[imf]),
1309 .dindex = tag.dstIndex,
1310 .sfab = (*mf[imf])[tag.srcIndex].const_array(scomp[imf],ncomp[imf]),
1311 .dbox = tag.dbox,
1312 .offset = (tag.sbox.smallEnd()-tag.dbox.smallEnd()).dim3()});
1314 }
1315 }
1316
1317 if (ParallelContext::NProcsSub() == 1) {
1318 detail::fbv_copy(local_tags);
1319 if (!Gpu::inNoSyncRegion()) {
1321 }
1322 return;
1323 }
1324
1325#ifdef AMREX_USE_MPI
1326 //
1327 // Do this before prematurely exiting if running in parallel.
1328 // Otherwise sequence numbers will not match across MPI processes.
1329 //
1330 int SeqNum = ParallelDescriptor::SeqNum();
1332
1333 if (N_locs == 0 && N_rcvs == 0 && N_snds == 0) { return; } // No work to do
1334
1335 char* the_recv_data = nullptr;
1336 Vector<int> recv_from;
1337 Vector<std::size_t> recv_size;
1338 Vector<MPI_Request> recv_reqs;
1339 Vector<MPI_Status> recv_stat;
1340 Vector<TagT> recv_tags;
1341
1342 if (N_rcvs > 0) {
1343
1344 for (int imf = 0; imf < nmfs; ++imf) {
1345 if (cmds[imf]) {
1346 auto const& tags = *(cmds[imf]->m_RcvTags);
1347 for (const auto& kv : tags) {
1348 recv_from.push_back(kv.first);
1349 }
1350 }
1351 }
1352 amrex::RemoveDuplicates(recv_from);
1353 const int nrecv = recv_from.size();
1354
1355 recv_reqs.resize(nrecv, MPI_REQUEST_NULL);
1356 recv_stat.resize(nrecv);
1357
1358 recv_tags.reserve(N_rcvs);
1359
1360 Vector<Vector<std::size_t> > recv_offset(nrecv);
1361 Vector<std::size_t> offset;
1362 recv_size.reserve(nrecv);
1363 offset.reserve(nrecv);
1364 std::size_t TotalRcvsVolume = 0;
1365 for (int i = 0; i < nrecv; ++i) {
1366 std::size_t nbytes = 0;
1367 for (int imf = 0; imf < nmfs; ++imf) {
1368 if (cmds[imf]) {
1369 auto const& tags = *(cmds[imf]->m_RcvTags);
1370 auto it = tags.find(recv_from[i]);
1371 if (it != tags.end()) {
1372 for (auto const& cct : it->second) {
1373 auto& dfab = (*mf[imf])[cct.dstIndex];
1374 recv_offset[i].push_back(nbytes);
1375 recv_tags.push_back(TagT{
1376 .dfab = dfab.array(scomp[imf],ncomp[imf]),
1377 .dindex = cct.dstIndex,
1378 .sfab = makeArray4<T const>(nullptr,cct.dbox,ncomp[imf]),
1379 .dbox = cct.dbox,
1380 .offset = Dim3{.x = 0, .y = 0, .z = 0}
1381 });
1382 nbytes += dfab.nBytes(cct.dbox,ncomp[imf]);
1383 }
1384 }
1385 }
1386 }
1387
1388 std::size_t acd = ParallelDescriptor::sizeof_selected_comm_data_type(nbytes);
1389 nbytes = amrex::aligned_size(acd, nbytes); // so that nbytes are aligned
1390
1391 // Also need to align the offset properly
1392 TotalRcvsVolume = amrex::aligned_size(std::max(alignof(T),acd), TotalRcvsVolume);
1393
1394 offset.push_back(TotalRcvsVolume);
1395 TotalRcvsVolume += nbytes;
1396
1397 recv_size.push_back(nbytes);
1398 }
1399
1400 the_recv_data = static_cast<char*>(amrex::The_Comms_Arena()->alloc(TotalRcvsVolume));
1401
1402 int k = 0;
1403 for (int i = 0; i < nrecv; ++i) {
1404 char* p = the_recv_data + offset[i];
1405 const int rank = ParallelContext::global_to_local_rank(recv_from[i]);
1406 recv_reqs[i] = ParallelDescriptor::Arecv
1407 (p, recv_size[i], rank, SeqNum, comm).req();
1408 for (int j = 0, nj = recv_offset[i].size(); j < nj; ++j) {
1409 recv_tags[k++].sfab.p = (T const*)(p + recv_offset[i][j]);
1410 }
1411 }
1412 }
1413
1414 char* the_send_data = nullptr;
1415 Vector<int> send_rank;
1416 Vector<char*> send_data;
1417 Vector<std::size_t> send_size;
1418 Vector<MPI_Request> send_reqs;
1419 if (N_snds > 0) {
1420 for (int imf = 0; imf < nmfs; ++imf) {
1421 if (cmds[imf]) {
1422 auto const& tags = *(cmds[imf]->m_SndTags);
1423 for (auto const& kv : tags) {
1424 send_rank.push_back(kv.first);
1425 }
1426 }
1427 }
1428 amrex::RemoveDuplicates(send_rank);
1429 const int nsend = send_rank.size();
1430
1431 send_data.resize(nsend, nullptr);
1432 send_reqs.resize(nsend, MPI_REQUEST_NULL);
1433
1434 Vector<TagT> send_tags;
1435 send_tags.reserve(N_snds);
1436
1437 Vector<Vector<std::size_t> > send_offset(nsend);
1438 Vector<std::size_t> offset;
1439 send_size.reserve(nsend);
1440 offset.reserve(nsend);
1441 std::size_t TotalSndsVolume = 0;
1442 for (int i = 0; i < nsend; ++i) {
1443 std::size_t nbytes = 0;
1444 for (int imf = 0; imf < nmfs; ++imf) {
1445 if (cmds[imf]) {
1446 auto const& tags = *(cmds[imf]->m_SndTags);
1447 auto it = tags.find(send_rank[i]);
1448 if (it != tags.end()) {
1449 for (auto const& cct : it->second) {
1450 auto const& sfab = (*mf[imf])[cct.srcIndex];
1451 send_offset[i].push_back(nbytes);
1452 send_tags.push_back(TagT{
1453 .dfab = amrex::makeArray4<T>(nullptr,cct.sbox,ncomp[imf]),
1454 .dindex = cct.dstIndex,
1455 .sfab = sfab.const_array(scomp[imf],ncomp[imf]),
1456 .dbox = cct.sbox,
1457 .offset = Dim3{.x = 0, .y = 0, .z = 0}
1458 });
1459 nbytes += sfab.nBytes(cct.sbox,ncomp[imf]);
1460 }
1461 }
1463 }
1465 std::size_t acd = ParallelDescriptor::sizeof_selected_comm_data_type(nbytes);
1466 nbytes = amrex::aligned_size(acd, nbytes); // so that bytes are aligned
1467
1468 // Also need to align the offset properly
1469 TotalSndsVolume = amrex::aligned_size(std::max(alignof(T),acd), TotalSndsVolume);
1470
1471 offset.push_back(TotalSndsVolume);
1472 TotalSndsVolume += nbytes;
1473
1474 send_size.push_back(nbytes);
1475 }
1476
1477 the_send_data = static_cast<char*>(amrex::The_Comms_Arena()->alloc(TotalSndsVolume));
1478 int k = 0;
1479 for (int i = 0; i < nsend; ++i) {
1480 send_data[i] = the_send_data + offset[i];
1481 for (int j = 0, nj = send_offset[i].size(); j < nj; ++j) {
1482 send_tags[k++].dfab.p = (T*)(send_data[i] + send_offset[i][j]);
1483 }
1484 }
1485
1486 detail::fbv_copy(send_tags);
1488
1489 FabArray<FAB>::PostSnds(send_data, send_size, send_rank, send_reqs, SeqNum);
1490 }
1491
1492#if !defined(AMREX_DEBUG)
1493 int recv_flag;
1494 ParallelDescriptor::Test(recv_reqs, recv_flag, recv_stat);
1495#endif
1496
1497 if (N_locs > 0) {
1498 detail::fbv_copy(local_tags);
1499 if (!Gpu::inNoSyncRegion() && N_rcvs == 0) {
1501 }
1502#if !defined(AMREX_DEBUG)
1503 ParallelDescriptor::Test(recv_reqs, recv_flag, recv_stat);
1504#endif
1505 }
1506
1507 if (N_rcvs > 0) {
1508 ParallelDescriptor::Waitall(recv_reqs, recv_stat);
1509#ifdef AMREX_DEBUG
1510 if (!FabArrayBase::CheckRcvStats(recv_stat, recv_size, SeqNum)) {
1511 amrex::Abort("FillBoundary(vector) failed with wrong message size");
1512 }
1513#endif
1514
1515 detail::fbv_copy(recv_tags);
1517
1518 amrex::The_Comms_Arena()->free(the_recv_data);
1519 }
1520
1521 if (N_snds > 0) {
1522 Vector<MPI_Status> stats(send_reqs.size());
1523 ParallelDescriptor::Waitall(send_reqs, stats);
1524 amrex::The_Comms_Arena()->free(the_send_data);
1525 }
1526
1527#endif // #ifdef AMREX_USE_MPI
1528#endif // #if 1 #else
1529}
1530
1543template <FabArrayType MF>
1544void
1546 Vector<int> const& ncomp, Vector<IntVect> const& nghost,
1547 Vector<Periodicity> const& period)
1548{
1549 BL_PROFILE("FillBoundaryAndSync(Vector)");
1550 const int N = mf.size();
1551 for (int i = 0; i < N; ++i) {
1552 mf[i]->FillBoundaryAndSync_nowait(scomp[i], ncomp[i], nghost[i], period[i]);
1553 }
1554 for (int i = 0; i < N; ++i) {
1555 mf[i]->FillBoundaryAndSync_finish();
1556 }
1557}
1558
1560template <FabArrayType MF>
1561void
1563{
1564 FillBoundary_nowait(mf, a_period);
1566}
1567
1569template <FabArrayType MF>
1570void
1576
1577}
#define BL_PROFILE(a)
Definition AMReX_BLProfiler.H:593
#define BL_PROFILE_SYNC_STOP()
Definition AMReX_BLProfiler.H:709
#define BL_PROFILE_SYNC_START_TIMED(fname)
Definition AMReX_BLProfiler.H:708
#define AMREX_ALWAYS_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:49
#define BL_ASSERT(EX)
Definition AMReX_BLassert.H:39
#define AMREX_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:37
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_PRAGMA_SIMD
Definition AMReX_Extension.H:84
Internal Fab copy tags and GPU helpers used by FabArray transfers.
GPU FillBoundary and SumBoundary with GPU-aware MPI. Messages are unpacked as they arrive when at lea...
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_LOOP_4D(bx, ncomp, i, j, k, n, block)
while parser_exe_t p
Definition AMReX_Parser_Exe_Body.H:33
virtual void free(void *pt)=0
Free a previously allocated block pointed to by pt.
virtual void * alloc(std::size_t sz)=0
Allocate sz bytes from this arena.
Reference-counted collection of Boxes.
Definition AMReX_BoxArray.H:681
IndexType ixType() const noexcept
Return index type of this BoxArray.
Definition AMReX_BoxArray.H:1269
Calculates the distribution of FABs to MPI processes.
Definition AMReX_DistributionMapping.H:55
IntVect nGrowVect() const noexcept
Definition AMReX_FabArrayBase.H:85
const FB & getFB(const IntVect &nghost, const Periodicity &period, bool cross=false, bool enforce_periodicity_only=false, bool override_sync=false, IntVect const &sumboundary_src_nghost=IntVect(-1)) const
Retrieve (and cache) the FillBoundary metadata for the requested halo.
Definition AMReX_FabArrayBase.cpp:1308
bool is_cell_centered() const noexcept
This tests on whether the FabArray is cell-centered.
Definition AMReX_FabArrayBase.cpp:2789
IndexType ixType() const noexcept
Return index type.
Definition AMReX_FabArrayBase.H:91
int size() const noexcept
Return the number of FABs in the FabArray.
Definition AMReX_FabArrayBase.H:115
const CPC & getCPC(const IntVect &dstng, const FabArrayBase &src, const IntVect &srcng, const Periodicity &period, bool to_ghost_cells_only=false, const IntVect &offset=IntVect(0)) const
Retrieve (and cache) the ParallelCopy metadata between two FabArrays.
Definition AMReX_FabArrayBase.cpp:583
const DistributionMapping & DistributionMap() const noexcept
Return constant reference to associated DistributionMapping.
Definition AMReX_FabArrayBase.H:135
bool empty() const noexcept
Definition AMReX_FabArrayBase.H:94
CpOp
parallel copy or add
Definition AMReX_FabArrayBase.H:432
@ ADD
Definition AMReX_FabArrayBase.H:432
@ COPY
Definition AMReX_FabArrayBase.H:432
IntVect n_filled
Definition AMReX_FabArrayBase.H:542
Box box(int K) const noexcept
Return the Kth Box in the BoxArray. That is, the valid region of the Kth grid.
Definition AMReX_FabArrayBase.H:106
DistributionMapping distributionMap
Definition AMReX_FabArrayBase.H:536
static int MaxComp
The maximum number of components to copy() at a time.
Definition AMReX_FabArrayBase.H:309
BoxArray boxarray
Definition AMReX_FabArrayBase.H:535
const BoxArray & boxArray() const noexcept
Return a constant reference to the BoxArray that defines the valid region associated with this FabArr...
Definition AMReX_FabArrayBase.H:100
An Array of FortranArrayBox(FAB)-like Objects.
Definition AMReX_FabArray.H:383
void ParallelCopyToGhost_finish()
Finish the current ParallelCopyToGhost_nowait() operation.
Definition AMReX_FabArrayCommI.H:426
void FillBoundary_finish()
Definition AMReX_FabArrayCommI.H:253
void ParallelCopy(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition AMReX_FabArray.H:999
void FBEP_nowait(int scomp, int ncomp, const IntVect &nghost, const Periodicity &period, bool cross, bool enforce_periodicity_only=false, bool override_sync=false, IntVect const &sumboundary_src_nghost=IntVect(-1), bool deterministic=false)
Definition AMReX_FabArrayCommI.H:18
void Redistribute(const FabArray< FAB > &src, int scomp, int dcomp, int ncomp, const IntVect &nghost)
Copy from src to this. this and src have the same BoxArray, but different DistributionMapping.
Definition AMReX_FabArrayCommI.H:1037
void ParallelCopyToGhost(const FabArray< FAB > &src, int scomp, int dcomp, int ncomp, const IntVect &snghost, const IntVect &dnghost, const Periodicity &period=Periodicity::NonPeriodic())
Copy source data to destination ghost cells only.
Definition AMReX_FabArrayCommI.H:395
static void unpack_recv_buffer_gpu(FabArray< FAB > &dst, int dcomp, int ncomp, Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< const CopyComTagsContainer * > const &recv_cctc, CpOp op, bool is_thread_safe, std::uint64_t id, bool deterministic)
Unpack received messages into dst on the GPU.
void ParallelCopy_finish()
Finish the current nowait ParallelCopy or ParallelAdd operation.
Definition AMReX_FabArrayCommI.H:723
std::unique_ptr< PCData< FAB > > pcd
Definition AMReX_FabArray.H:1967
void PC_local_cpu(const CPC &thecpc, FabArray< FAB > const &src, int scomp, int dcomp, int ncomp, CpOp op)
Definition AMReX_PCI.H:8
std::unique_ptr< FBData< FAB > > fbd
Definition AMReX_FabArray.H:1966
void ParallelAdd(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic())
This function copies data from src to this FabArray. Each FAB in fa is intersected with all FABs in t...
Definition AMReX_FabArray.H:996
void FB_local_add_cpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:467
void FB_on_arrival_nowait(const FB &TheFB, int scomp, int ncomp, int SeqNum, bool sumboundary)
void ParallelCopyToGhost_nowait(const FabArray< FAB > &src, int scomp, int dcomp, int ncomp, const IntVect &snghost, const IntVect &dnghost, const Periodicity &period=Periodicity::NonPeriodic())
Start an asynchronous version of ParallelCopyToGhost().
Definition AMReX_FabArrayCommI.H:412
static void unpack_recv_buffer_cpu(FabArray< FAB > &dst, int dcomp, int ncomp, Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< const CopyComTagsContainer * > const &recv_cctc, CpOp op, bool is_thread_safe)
void FB_local_add_gpu(const FB &TheFB, int scomp, int ncomp, bool deterministic)
Definition AMReX_FBI.H:630
void FB_local_copy_gpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:558
void FB_on_arrival_finish()
void clear()
Releases FAB memory in the FabArray.
Definition AMReX_FabArray.H:2283
static void pack_send_buffer_gpu(FabArray< FAB > const &src, int scomp, int ncomp, Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< const CopyComTagsContainer * > const &send_cctc, std::uint64_t id)
FabArray() noexcept
Constructs an empty FabArray<FAB>.
Definition AMReX_FabArray.H:2386
void FillBoundary_test()
Definition AMReX_FabArrayCommI.H:1064
void copyTo(FAB &dest, int nghost=0) const
Copy the values contained in the intersection of the valid + nghost region of this FabArray with the ...
Definition AMReX_FabArray.H:3059
Array4< typename FabArray< FAB >::value_type const > array(const MFIter &mfi) const noexcept
Read-only Array4 view for the FAB referenced by iterator mfi.
Definition AMReX_FabArray.H:649
void FB_local_copy_cpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:396
void PC_local_gpu(const CPC &thecpc, FabArray< FAB > const &src, int scomp, int dcomp, int ncomp, CpOp op, bool deterministic)
Definition AMReX_PCI.H:98
static void pack_send_buffer_cpu(FabArray< FAB > const &src, int scomp, int ncomp, Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< const CopyComTagsContainer * > const &send_cctc)
void ParallelCopy_nowait(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Start an asynchronous ParallelCopy that copies every component.
Definition AMReX_FabArray.H:1031
Array4< typename FabArray< FAB >::value_type const > const_array(const MFIter &mfi) const noexcept
Synonym for array(const MFIter&) that highlights read-only semantics.
Definition AMReX_FabArray.H:677
__host__ __device__ bool cellCentered() const noexcept
True if the IndexTypeND is CELL based in all directions.
Definition AMReX_IndexType.H:102
__host__ __device__ constexpr bool allGE(const IntVectND< dim > &rhs) const noexcept
Returns true if this is greater than or equal to argument for all components. NOTE: This is NOT a str...
Definition AMReX_IntVect.H:549
__host__ static __device__ constexpr IntVectND< dim > TheZeroVector() noexcept
Definition AMReX_IntVect.H:778
__host__ __device__ constexpr int max() const noexcept
maximum (no absolute values) value
Definition AMReX_IntVect.H:320
MPI_Request req() const
Definition AMReX_ParallelDescriptor.H:74
This provides length of period for periodic domains. 0 means it is not periodic in that direction....
Definition AMReX_Periodicity.H:17
static const Periodicity & NonPeriodic() noexcept
Definition AMReX_Periodicity.cpp:52
bool isAnyPeriodic() const noexcept
Definition AMReX_Periodicity.H:22
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
Long size() const noexcept
Definition AMReX_Vector.H:54
Checks if a type is derived from amrex::BaseFab.
Definition AMReX_Concepts.H:13
amrex_long Long
Definition AMReX_INT.H:30
__host__ __device__ Dim3 ubound(Array4< T > const &a) noexcept
Return the inclusive upper bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1377
__host__ __device__ Dim3 lbound(Array4< T > const &a) noexcept
Return the inclusive lower bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1363
__host__ __device__ BoxND< dim > grow(const BoxND< dim > &b, int i) noexcept
Return a copy of b grown uniformly by i cells in every direction.
Definition AMReX_Box.H:1328
Arena * The_Comms_Arena()
Definition AMReX_Arena.cpp:899
Arena * The_Pinned_Arena()
Definition AMReX_Arena.cpp:879
std::size_t aligned_size(std::size_t align_requirement, std::size_t size) noexcept
Return the smallest multiple of align_requirement that is >= size.
Definition AMReX_Arena.H:39
int MyProc() noexcept
Definition AMReX_ParallelDescriptor.H:128
void Bcast(Gpu::DeviceVector< T > &v, int root, MPI_Comm comm)
Definition AMReX_GpuParallelReduce.H:105
int NProcs() noexcept
Definition AMReX_ParallelDescriptor.H:255
__host__ __device__ constexpr const T & max(const T &a, const T &b) noexcept
Definition AMReX_Algorithm.H:54
void freeAsync(Arena *arena, void *mem) noexcept
Definition AMReX_GpuDevice.H:364
bool inGraphRegion()
Definition AMReX_GpuControl.H:117
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
void dtoh_memcpy_async(void *p_h, const void *p_d, const std::size_t sz) noexcept
Definition AMReX_GpuDevice.H:454
bool inLaunchRegion() noexcept
Definition AMReX_GpuControl.H:88
bool inNoSyncRegion() noexcept
Definition AMReX_GpuControl.H:148
void htod_memcpy_async(void *p_d, const void *p_h, const std::size_t sz) noexcept
Definition AMReX_GpuDevice.H:440
MPI_Comm CommunicatorSub() noexcept
sub-communicator for current frame
Definition AMReX_ParallelContext.H:70
int global_to_local_rank(int rank) noexcept
Definition AMReX_ParallelContext.H:101
int NProcsSub() noexcept
number of ranks in current frame
Definition AMReX_ParallelContext.H:74
void Test(MPI_Request &, int &, MPI_Status &)
Definition AMReX_ParallelDescriptor.cpp:1224
Message Asend(const T *, size_t n, int pid, int tag)
Definition AMReX_ParallelDescriptor.H:1163
bool UseGpuAwareMpi()
Definition AMReX_ParallelDescriptor.H:113
void Waitall(Vector< MPI_Request > &, Vector< MPI_Status > &)
Definition AMReX_ParallelDescriptor.cpp:1312
int SeqNum() noexcept
Returns sequential message sequence numbers, usually used as tags for send/recv.
Definition AMReX_ParallelDescriptor.H:678
Message Arecv(T *, size_t n, int pid, int tag)
Definition AMReX_ParallelDescriptor.H:1205
int MPI_Comm
Definition AMReX_ccse-mpi.H:51
static constexpr int MPI_REQUEST_NULL
Definition AMReX_ccse-mpi.H:57
Definition AMReX_Amr.cpp:50
@ make_alias
Definition AMReX_MakeType.H:7
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
void FillBoundary_finish(Vector< MF * > const &mf)
Wait for outstanding FillBoundary_nowait operations launched with the vector helper to complete.
Definition AMReX_FabArrayCommI.H:1171
void Copy(FabArray< DFAB > &dst, FabArray< SFAB > const &src, int srccomp, int dstcomp, int numcomp, int nghost)
Definition AMReX_FabArray.H:219
void Add(FabArray< FAB > &dst, FabArray< FAB > const &src, int srccomp, int dstcomp, int numcomp, int nghost)
Definition AMReX_FabArray.H:278
__host__ __device__ Array4< T > makeArray4(T *p, Box const &bx, int ncomp) noexcept
Definition AMReX_BaseFab.H:137
std::unique_ptr< char, TheFaArenaDeleter > TheFaArenaPointer
Definition AMReX_FabArray.H:118
BoxArray const & boxArray(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.boxArray().
Definition AMReX_FabArrayBase.cpp:3038
DistributionMapping const & DistributionMap(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.DistributionMap().
Definition AMReX_FabArrayBase.cpp:3043
void FillBoundary_nowait(Vector< MF * > const &mf, Vector< int > const &scomp, Vector< int > const &ncomp, Vector< IntVect > const &nghost, Vector< Periodicity > const &period, Vector< int > const &cross={})
Launch FillBoundary_nowait across a vector of FabArrays.
Definition AMReX_FabArrayCommI.H:1125
void FillBoundaryAndSync_nowait(Vector< MF * > const &mf, Vector< int > const &scomp, Vector< int > const &ncomp, Vector< IntVect > const &nghost, Vector< Periodicity > const &period)
Launch FillBoundaryAndSync_nowait across a vector of FabArrays.
Definition AMReX_FabArrayCommI.H:1194
void ParallelCopy_finish(FabArray< FAB > &dest, CommHandler handler, const FabArrayBase::CommMetaData &cmd, const DataPacking &data_packing)
Definition AMReX_NonLocalBC.H:914
void FillBoundaryAndSync(Vector< MF * > const &mf, Vector< int > const &scomp, Vector< int > const &ncomp, Vector< IntVect > const &nghost, Vector< Periodicity > const &period)
Perform FillBoundaryAndSync on a batch of FabArrays (e.g., MultiFabs).
Definition AMReX_FabArrayCommI.H:1545
void ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:202
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
void ParallelCopy(MF &dst, MF const &src, int scomp, int dcomp, int ncomp, IntVect const &ng_src=IntVect(0), IntVect const &ng_dst=IntVect(0), Periodicity const &period=Periodicity::NonPeriodic())
dst = src w/ MPI communication
Definition AMReX_FabArrayUtility.H:2264
@ x
Definition AMReX_Orientation.H:14
IntVect nGrowVect(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.nGrowVect().
Definition AMReX_FabArrayBase.cpp:3033
CommHandler ParallelCopy_nowait(NoLocalCopy, FabArray< FAB > &dest, const FabArray< FAB > &src, const FabArrayBase::CommMetaData &cmd, const DataPacking &data_packing)
Definition AMReX_NonLocalBC.H:822
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
void RemoveDuplicates(Vector< T > &vec)
Definition AMReX_Vector.H:210
void FillBoundaryAndSync_finish(Vector< MF * > const &mf)
Wait for outstanding FillBoundaryAndSync_nowait operations launched with the vector helper to complet...
Definition AMReX_FabArrayCommI.H:1236
void FillBoundary(Vector< MF * > const &mf, Vector< int > const &scomp, Vector< int > const &ncomp, Vector< IntVect > const &nghost, Vector< Periodicity > const &period, Vector< int > const &cross={})
Perform FillBoundary on a batch of FabArrays (e.g., MultiFabs).
Definition AMReX_FabArrayCommI.H:1262
Definition AMReX_TagParallelFor.H:26
A simple struct holding 3 int values for a 3D index.
Definition AMReX_Dim3.H:24
parallel copy or add
Definition AMReX_FabArrayBase.H:651
std::uint64_t m_id
Definition AMReX_FabArrayBase.H:666
Definition AMReX_FabArrayBase.H:563
bool m_threadsafe_rcv
Definition AMReX_FabArrayBase.H:566
std::unique_ptr< MapOfCopyComTagContainers > m_RcvTags
Definition AMReX_FabArrayBase.H:569
std::unique_ptr< MapOfCopyComTagContainers > m_SndTags
Definition AMReX_FabArrayBase.H:568
std::unique_ptr< CopyComTagsContainer > m_LocTags
Definition AMReX_FabArrayBase.H:567
FillBoundary.
Definition AMReX_FabArrayBase.H:588
std::uint64_t m_id
Definition AMReX_FabArrayBase.H:594
IntVect m_sb_snghost
Definition AMReX_FabArrayBase.H:598
Definition AMReX_TypeTraits.H:51
Definition AMReX_TypeTraits.H:61
Definition AMReX_TypeTraits.H:277
FabArray memory allocation information.
Definition AMReX_FabArray.H:73