Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_FBI.H
Go to the documentation of this file.
1#ifndef AMREX_FBI_H_
2#define AMREX_FBI_H_
3
8
9namespace amrex {
10
17template <class FAB>
18struct FabCopyTag {
19 FAB const* sfab;
21 IntVect offset; // sbox.smallEnd() - dbox.smallEnd()
22};
23
26 char const* p;
28};
29
31namespace detail {
32
33#ifdef AMREX_USE_GPU
34
35template <class T0, class T1>
36struct CellStore
37{
39 operator() (T0* d, T1 s) const noexcept
40 {
41 *d = static_cast<T0>(s);
42 }
43};
44
45template <class T0, class T1>
46struct CellAdd
47{
49 operator() (T0* d, T1 s) const noexcept
50 {
51 *d += static_cast<T0>(s);
52 }
53};
54
55template <class T0, class T1>
56struct CellAtomicAdd
57{
58 template<class U0=T0>
59 requires (amrex::HasAtomicAdd<U0>::value)
61 operator() (U0* d, T1 s) const noexcept
62 {
63 Gpu::Atomic::AddNoRet(d, static_cast<U0>(s));
64 }
65};
66
67template <class T0, class T1, class F>
68void
69fab_to_fab (Vector<Array4CopyTag<T0, T1> > const& copy_tags, int scomp, int dcomp, int ncomp,
70 F && f)
71{
72 TagVector<Array4CopyTag<T0, T1>> tv{copy_tags};
73
74 detail::ParallelFor_doit(tv,
76 int icell, int ncells, int i, int j, int k, Array4CopyTag<T0, T1> const& tag) noexcept
77 {
78 if (icell < ncells) {
79 for (int n = 0; n < ncomp; ++n) {
80 f(&(tag.dfab(i,j,k,n+dcomp)),
81 tag.sfab(i+tag.offset.x,j+tag.offset.y,k+tag.offset.z,n+scomp));
82 }
83 }
84 });
85}
86
87template <class TagType, class F>
88void
89fab_to_fab_store (Vector<TagType> const& tags, int scomp, int dcomp, int ncomp, F&&f)
90{
92 [=] AMREX_GPU_DEVICE (int i, int j, int k, TagType const& tag) noexcept
93 {
94 int m = Gpu::Atomic::Add(&(tag.mask(i,j,k)), 1);
95 if (m == 0) {
96 for (int n = 0; n < ncomp; ++n) {
97 f(&(tag.dfab(i,j,k,n+dcomp)),
98 tag.sfab(i+tag.offset.x,j+tag.offset.y,k+tag.offset.z,n+scomp));
99 }
100 }
101 });
102}
103
104template <class TagType, class F>
105void
106fab_to_fab_other (Vector<TagType> const& tags, int scomp, int dcomp, int ncomp, F&&f)
107{
109 [=] AMREX_GPU_DEVICE (int i, int j, int k, TagType const& tag) noexcept
110 {
111 int* m = &(tag.mask(i,j,k));
112 bool my_turn = false;
113 do {
114#if defined(AMREX_USE_SYCL)
115 my_turn = (Gpu::Atomic::Exch(m, 1) == 0);
116#else
117 my_turn = (Gpu::Atomic::CAS(m, 0, 1) == 0);
118#endif
119 if (my_turn) {
120#if defined(AMREX_USE_SYCL)
121 sycl::atomic_fence(sycl::memory_order::acq_rel, sycl::memory_scope::device);
122#else
123 __threadfence();
124#endif
125 for (int n = 0; n < ncomp; ++n) {
126 f(&(tag.dfab(i,j,k,n+dcomp)),
127 tag.sfab(i+tag.offset.x,j+tag.offset.y,k+tag.offset.z,n+scomp));
128 }
129#if defined(AMREX_USE_SYCL)
130 sycl::atomic_fence(sycl::memory_order::acq_rel, sycl::memory_scope::device);
131#else
132 __threadfence(); // It appears that we need this fence too if a GPU is shared.
133#endif
134 Gpu::Atomic::Exch(m, 0);
135 }
136 else {
137#if defined(AMREX_USE_CUDA)
138
139#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 700)
140#if defined(_WIN32)
141 volatile int tmp; // prevent optimization
142 for (int c = 0; c < 2; ++c) {
143 ++tmp;
144 }
145#else
146 for (int c = 0; c < 2; ++c) {
147 __asm__ volatile(""); // prevent optimization
148 }
149#endif
150#else
151 __nanosleep(1);
152#endif
153
154#elif defined(AMREX_USE_HIP)
155
156 __builtin_amdgcn_s_sleep(1);
157
158#elif defined(AMREX_USE_SYCL)
159
160 for (int c = 0; c < 2; ++c) {
161 __asm__ volatile(""); // prevent optimization
162 }
163
164#endif
165 }
166 } while (!my_turn);
167 });
168}
169
170template <class T0, class T1, class F>
171void
172fab_to_fab (Vector<Array4CopyTag<T0, T1> > const& copy_tags, int scomp, int dcomp,
173 int ncomp, F && f, Vector<Array4Tag<int> > const& masks)
174{
175 using TagType = Array4MaskCopyTag<T0, T1>;
176 Vector<TagType> tags;
177 const int N = copy_tags.size();
178 tags.reserve(N);
179 for (int i = 0; i < N; ++i) {
180 tags.push_back(TagType{.dfab = copy_tags[i].dfab, .sfab = copy_tags[i].sfab,
181 .mask = masks[i].dfab, .dbox = copy_tags[i].dbox,
182 .offset = copy_tags[i].offset});
183 }
184
185 if constexpr (std::is_same_v<F, CellStore<T0,T1>>)
186 {
187 fab_to_fab_store(tags, scomp, dcomp, ncomp, std::forward<F>(f));
188 }
189 else
190 {
191 fab_to_fab_other(tags, scomp, dcomp, ncomp, std::forward<F>(f));
192 }
193 Gpu::streamSynchronize(); // masks are temporaries
194}
195
196template <typename T0, typename T1>
197requires (amrex::IsStoreAtomic<T0>::value)
198void
199fab_to_fab_atomic_cpy (Vector<Array4CopyTag<T0, T1> > const& copy_tags, int scomp,
200 int dcomp, int ncomp, Vector<Array4Tag<int> > const&)
201{
202 fab_to_fab<T0, T1>(copy_tags, scomp, dcomp, ncomp, CellStore<T0, T1>());
203}
204
205template <typename T0, typename T1>
206requires (!amrex::IsStoreAtomic<T0>::value)
207void
208fab_to_fab_atomic_cpy (Vector<Array4CopyTag<T0, T1> > const& copy_tags, int scomp,
209 int dcomp, int ncomp, Vector<Array4Tag<int> > const& masks)
210{
211 fab_to_fab(copy_tags, scomp, dcomp, ncomp, CellStore<T0, T1>(), masks);
212}
213
214template <typename T0, typename T1>
215requires (amrex::HasAtomicAdd<T0>::value)
216void
217fab_to_fab_atomic_add (Vector<Array4CopyTag<T0, T1> > const& copy_tags, int scomp,
218 int dcomp, int ncomp, Vector<Array4Tag<int> > const&)
219{
220 fab_to_fab(copy_tags, scomp, dcomp, ncomp, CellAtomicAdd<T0, T1>());
221}
222
223template <typename T0, typename T1>
224requires (!amrex::HasAtomicAdd<T0>::value)
225void
226fab_to_fab_atomic_add (Vector<Array4CopyTag<T0, T1> > const& copy_tags, int scomp,
227 int dcomp, int ncomp, Vector<Array4Tag<int> > const& masks)
228{
229 fab_to_fab(copy_tags, scomp, dcomp, ncomp, CellAdd<T0, T1>(), masks);
230}
231
232template <typename T0, typename T1, class F>
233void deterministic_fab_to_fab (Vector<Array4CopyTag<T0,T1>> const& a_tags, int scomp,
234 int dcomp, int ncomp, F const& f)
235{
236 if (a_tags.empty()) { return; }
237
238 using TagType = Array4CopyTag<T0,T1>;
239
240 struct TiledTag {
241 int tag_index;
242 std::pair<int,Box> dindex_tilebox;
243 bool operator< (TiledTag const& rhs) const noexcept {
244 return this->dindex_tilebox < rhs.dindex_tilebox;
245 }
246 bool operator!= (TiledTag const& rhs) const noexcept {
247 return this->dindex_tilebox != rhs.dindex_tilebox;
248 }
249 };
250 Vector<TiledTag> tiled_tags;
251
252 auto const ixtype = a_tags[0].dbox.ixType();
253
254 constexpr int tile_size = 64;
255 for (int itag = 0; itag < a_tags.size(); ++itag) {
256 auto const& tag = a_tags[itag];
257 auto const& dlo = tag.dbox.smallEnd();
258 auto const& dhi = tag.dbox.bigEnd();
265#if (AMREX_SPACEDIM == 3)
266 for (int kt = tlo[2]; kt <= thi[2]; ++kt)
267#endif
268 {
269#if (AMREX_SPACEDIM >= 2)
270 for (int jt = tlo[1]; jt <= thi[1]; ++jt)
271#endif
272 {
273 for (int it = tlo[0]; it <= thi[0]; ++it)
274 {
275 IntVect lo(AMREX_D_DECL(it*tile_size,
276 jt*tile_size,
277 kt*tile_size));
278 tiled_tags.push_back(TiledTag{
279 .tag_index = itag,
280 .dindex_tilebox = std::make_pair(tag.dindex, Box(lo, lo+(tile_size-1), ixtype))
281 });
282 }
283 }
284 }
285 }
286
287 std::sort(tiled_tags.begin(), tiled_tags.end());
288
291 h_tags.reserve(tiled_tags.size());
292
293 for (unsigned int itag = 0; itag < tiled_tags.size(); ++itag) {
294 if (itag == 0) {
295 h_ntags.push_back(0);
296 } else if (tiled_tags[itag-1] != tiled_tags[itag]) {
297 h_ntags.push_back(itag);
298 }
299 auto const& ttag = tiled_tags[itag];
300 auto const& btag = a_tags[ttag.tag_index];
301 h_tags.push_back(TagType{.dfab = btag.dfab, .dindex = btag.dindex, .sfab = btag.sfab,
302 .dbox = btag.dbox & ttag.dindex_tilebox.second,
303 .offset = btag.offset});
304 }
305 h_ntags.push_back((unsigned int)tiled_tags.size());
306
307 Gpu::DeviceVector<TagType> d_tags(h_tags.size());
308 Gpu::DeviceVector<unsigned int> d_ntags(h_ntags.size());
309 Gpu::copyAsync(Gpu::hostToDevice,h_tags.begin(),h_tags.end(),d_tags.begin());
310 Gpu::copyAsync(Gpu::hostToDevice,h_ntags.begin(),h_ntags.end(),d_ntags.begin());
311 auto const* ptag = d_tags.data();
312 auto const* pntags = d_ntags.data();
313 auto const nblocks = int(h_ntags.size()-1);
314 constexpr auto nthreads = 256;
316#ifdef AMREX_USE_SYCL
317 [=] AMREX_GPU_DEVICE (sycl::nd_item<1> const& item) noexcept
318 [[sycl::reqd_work_group_size(nthreads)]]
319#else
320 [=] AMREX_GPU_DEVICE () noexcept
321#endif
322 {
323#ifdef AMREX_USE_SYCL
324 Dim1 blockIdx{item.get_group_linear_id()};
325 Dim1 threadIdx{item.get_local_linear_id()};
326#endif
327
328 for (unsigned int itag = pntags[blockIdx.x]; itag < pntags[blockIdx.x+1]; ++itag) {
329 auto const tag = ptag[itag];
330 auto ncells = int(tag.dbox.numPts());
331 const auto len = amrex::length(tag.dbox);
332 const auto lo = amrex::lbound(tag.dbox);
333 for (int icell = int(threadIdx.x); icell < ncells; icell += nthreads) {
334 int k = icell / (len.x*len.y);
335 int j = (icell - k*(len.x*len.y)) / len.x;
336 int i = (icell - k*(len.x*len.y)) - j*len.x;
337 i += lo.x;
338 j += lo.y;
339 k += lo.z;
340 for (int n = 0; n < ncomp; ++n) {
341 f(tag.dfab.ptr(i,j,k,n+dcomp),
342 tag.sfab(i + tag.offset.x,
343 j + tag.offset.y,
344 k + tag.offset.z, n+scomp));
345 }
346 }
347
348 if (itag+1 < pntags[blockIdx.x+1]) {
349#ifdef AMREX_USE_SYCL
350 sycl::group_barrier(item.get_group());
351#else
352 __syncthreads();
353#endif
354 }
355 }
356 });
358}
359
360template <typename B, typename V, typename TT>
361requires (amrex::HasAtomicAdd<V>::value)
362void unpack_recv_buffer_gpu_atomic_add (char* pbuffer, TagVector<TT> const& tv,
363 int dcomp, int ncomp)
364{
365 detail::ParallelFor_doit(tv,
366 [=] AMREX_GPU_DEVICE (
367 int icell, int ncells, int i, int j, int k, TT const& tag) noexcept
368 {
369 if (icell < ncells) {
370 Array4<B const> sfab{(B const*)(pbuffer+tag.poff),
371 amrex::begin(tag.bx), amrex::end(tag.bx), ncomp};
372 for (int n = 0; n < ncomp; ++n) {
373 Gpu::Atomic::AddNoRet(tag.dfab.ptr(i,j,k,n+dcomp),
374 (V)sfab(i,j,k,n));
375 }
376 }
377 });
378}
379
380template <typename B, typename V, typename TT>
381requires (!amrex::HasAtomicAdd<V>::value)
382void unpack_recv_buffer_gpu_atomic_add (char* pbuffer, TagVector<TT> const& tv,
383 int dcomp, int ncomp)
384{
385 amrex::ignore_unused(pbuffer, tv, dcomp, ncomp);
386 amrex::Abort("unpack_recv_buffer_gpu: should NOT get here");
387}
388
389#endif /* AMREX_USE_GPU */
390
391}
393
394template <class FAB>
395void
396FabArray<FAB>::FB_local_copy_cpu (const FB& TheFB, int scomp, int ncomp)
397{
398 auto const& LocTags = *(TheFB.m_LocTags);
399 auto N_locs = static_cast<int>(LocTags.size());
400 if (N_locs == 0) { return; }
401 bool is_thread_safe = TheFB.m_threadsafe_loc;
402 if (is_thread_safe)
403 {
404#ifdef AMREX_USE_OMP
405#pragma omp parallel for
406#endif
407 for (int i = 0; i < N_locs; ++i)
408 {
409 const CopyComTag& tag = LocTags[i];
410#ifdef BL_USE_TEAM
411 // With MPI teams, the tags include teammates' FABs. Only write our own.
412 if (distributionMap[tag.dstIndex] != ParallelDescriptor::MyProc()) { continue; }
413#endif
414
417
418 const FAB* sfab = &(get(tag.srcIndex));
419 FAB* dfab = &(get(tag.dstIndex));
420 dfab->template copy<RunOn::Host>(*sfab, tag.sbox, scomp, tag.dbox, scomp, ncomp);
421 }
422#ifdef BL_USE_TEAM
423 // Teammates may read our FABs next.
425#endif
426 }
427 else
428 {
430 for (int i = 0; i < N_locs; ++i)
431 {
432 const CopyComTag& tag = LocTags[i];
433#ifdef BL_USE_TEAM
434 // With MPI teams, the tags include teammates' FABs. Only write our own.
435 if (distributionMap[tag.dstIndex] != ParallelDescriptor::MyProc()) { continue; }
436#endif
437
440
441 loc_copy_tags[tag.dstIndex].push_back
442 ({this->fabPtr(tag.srcIndex), tag.dbox, tag.sbox.smallEnd()-tag.dbox.smallEnd()});
443 }
444#ifdef AMREX_USE_OMP
445#pragma omp parallel
446#endif
447 for (MFIter mfi(*this); mfi.isValid(); ++mfi)
448 {
449 const auto& tags = loc_copy_tags[mfi];
450 auto dfab = this->array(mfi);
451 for (auto const & tag : tags)
452 {
453 auto const sfab = tag.sfab->array();
454 const auto offset = tag.offset.dim3();
455 amrex::LoopConcurrentOnCpu(tag.dbox, ncomp,
456 [=] (int i, int j, int k, int n) noexcept
457 {
458 dfab(i,j,k,n+scomp) = sfab(i+offset.x,j+offset.y,k+offset.z,n+scomp);
459 });
460 }
461 }
462 }
463}
464
465template <class FAB>
466void
467FabArray<FAB>::FB_local_add_cpu (const FB& TheFB, int scomp, int ncomp)
468{
469 auto const& LocTags = *(TheFB.m_LocTags);
470 auto N_locs = static_cast<int>(LocTags.size());
471 if (N_locs == 0) { return; }
472
474 // We must make a temporary copy of the data first to avoid race condition.
475 std::vector<FAB> src_fabs(N_locs);
476 for (int itag = 0; itag < N_locs; ++itag) {
477 const CopyComTag& tag = LocTags[itag];
478 src_fabs[itag].resize(tag.sbox,ncomp);
479 loc_copy_tags[tag.dstIndex].push_back
480 (FabCopyTag<FAB>{&(src_fabs[itag]),
481 tag.dbox, tag.sbox.smallEnd()-tag.dbox.smallEnd()});
482 }
483
484#ifdef AMREX_USE_OMP
485#pragma omp parallel for
486#endif
487 for (int itag = 0; itag < N_locs; ++itag) {
488 const CopyComTag& tag = LocTags[itag];
489 src_fabs[itag].template copy<RunOn::Host>(this->operator[](tag.srcIndex), scomp, 0, ncomp);
490 }
491
492#ifdef AMREX_USE_OMP
493#pragma omp parallel
494#endif
495 for (MFIter mfi(*this); mfi.isValid(); ++mfi)
496 {
497 const auto& tags = loc_copy_tags[mfi];
498 const auto& dfab = this->array(mfi);
499 for (auto const & tag : tags)
500 {
501 auto const sfab = tag.sfab->array();
502 const auto offset = tag.offset.dim3();
503 amrex::LoopConcurrentOnCpu(tag.dbox, ncomp,
504 [&] (int i, int j, int k, int n) noexcept
505 {
506 dfab(i,j,k,n+scomp) += sfab(i+offset.x,j+offset.y,k+offset.z,n);
507 });
508 }
509 }
510}
511
512#ifdef AMREX_USE_GPU
513
514template <class FAB>
517{
518 auto const& LocTags = *(TheFB.m_LocTags);
519 int N_locs = LocTags.size();
520
521 using TagType = Array4CopyTag<value_type>;
522
524 if (auto it = m_fb_local_copy_handler.find(TheFB.m_id);
525 it != m_fb_local_copy_handler.end())
526 {
527 tv = it->second.get();
528 } else {
529 Vector<TagType> loc_copy_tags;
530 loc_copy_tags.reserve(N_locs);
531
532 for (int i = 0; i < N_locs; ++i)
533 {
534 const CopyComTag& tag = LocTags[i];
535
538
539 int li = this->localindex(tag.dstIndex);
540 loc_copy_tags.push_back
541 (TagType{.dfab = this->atLocalIdx(li).array(),
542 .dindex = tag.dstIndex,
543 .sfab = this->fabPtr(tag.srcIndex)->const_array(),
544 .dbox = tag.dbox,
545 .offset = (tag.sbox.smallEnd()-tag.dbox.smallEnd()).dim3()});
546 }
547
548 auto utv = std::make_unique<TagVector<TagType>>(loc_copy_tags);
549 tv = utv.get();
550 m_fb_local_copy_handler[TheFB.m_id] = std::move(utv);
551 }
552
553 return tv;
554}
555
556template <class FAB>
557void
558FabArray<FAB>::FB_local_copy_gpu (const FB& TheFB, int scomp, int ncomp)
559{
560 auto const& LocTags = *(TheFB.m_LocTags);
561 int N_locs = LocTags.size();
562 if (N_locs == 0) { return; }
563 bool is_thread_safe = TheFB.m_threadsafe_loc;
564
565 using TagType = Array4CopyTag<value_type>;
566
567 if (is_thread_safe || amrex::IsStoreAtomic<value_type>::value)
568 {
569 auto* tv = FB_get_local_copy_tag_vector(TheFB);
570
571 detail::ParallelFor_doit(*tv,
572 [=] AMREX_GPU_DEVICE (
573 int icell, int ncells, int i, int j, int k, TagType const& tag) noexcept
574 {
575 if (icell < ncells) {
576 for (int n = 0; n < ncomp; ++n) {
577 tag.dfab(i,j,k,n+scomp) = tag.sfab(i+tag.offset.x,
578 j+tag.offset.y,
579 k+tag.offset.z,n+scomp);
580 }
581 }
582 });
583 }
584 else
585 {
586 Vector<TagType> loc_copy_tags;
587 loc_copy_tags.reserve(N_locs);
588
589 Vector<BaseFab<int>> maskfabs(this->local_size());
590 Vector<Array4Tag<int> > masks_unique;
591 masks_unique.reserve(this->local_size());
592 Vector<Array4Tag<int> > masks;
593 masks.reserve(N_locs);
594
595 for (int i = 0; i < N_locs; ++i)
596 {
597 const CopyComTag& tag = LocTags[i];
598
601
602 int li = this->localindex(tag.dstIndex);
603 loc_copy_tags.push_back
604 (TagType{.dfab = this->atLocalIdx(li).array(),
605 .dindex = tag.dstIndex,
606 .sfab = this->fabPtr(tag.srcIndex)->const_array(),
607 .dbox = tag.dbox,
608 .offset = (tag.sbox.smallEnd()-tag.dbox.smallEnd()).dim3()});
609
610 if (!maskfabs[li].isAllocated()) {
611 maskfabs[li].resize(this->atLocalIdx(li).box());
612 masks_unique.emplace_back(Array4Tag<int>{.dfab = maskfabs[li].array()});
613 }
614 masks.emplace_back(Array4Tag<int>{.dfab = maskfabs[li].array()});
615 }
616
617 amrex::ParallelFor(masks_unique,
618 [=] AMREX_GPU_DEVICE (int i, int j, int k, Array4Tag<int> const& msk) noexcept
619 {
620 msk.dfab(i,j,k) = 0;
621 });
622
623 detail::fab_to_fab_atomic_cpy<value_type, value_type>(
624 loc_copy_tags, scomp, scomp, ncomp, masks);
625 }
626}
627
628template <class FAB>
629void
630FabArray<FAB>::FB_local_add_gpu (const FB& TheFB, int scomp, int ncomp, bool deterministic)
631{
632 auto const& LocTags = *(TheFB.m_LocTags);
633 int N_locs = LocTags.size();
634 if (N_locs == 0) { return; }
635
636 using TagType = Array4CopyTag<value_type>;
637
638 Vector<TagType> loc_copy_tags_1, loc_copy_tags_2;
639 loc_copy_tags_1.reserve(N_locs);
640 loc_copy_tags_2.reserve(N_locs);
641
642 // The source data are copied to one temporary buffer first.
643 Long npts = 0;
644 for (auto const& tag : LocTags) {
645 npts += tag.sbox.numPts();
646 }
647 auto* src_buf = static_cast<value_type*>
648 (The_Arena()->alloc(std::size_t(npts)*std::size_t(ncomp)*sizeof(value_type)));
649 auto* p = src_buf;
650 for (int itag = 0; itag < N_locs; ++itag) {
651 const CopyComTag& tag = LocTags[itag];
652 auto const& src = amrex::makeArray4(p, tag.sbox, ncomp);
653 p += tag.sbox.numPts()*ncomp;
654 loc_copy_tags_1.push_back(
655 TagType{.dfab = src, .dindex = -1,
656 .sfab = this->const_array(tag.srcIndex,scomp), .dbox = tag.sbox,
657 .offset = Dim3{.x = 0, .y = 0, .z = 0}});
658 loc_copy_tags_2.push_back(
659 TagType{.dfab = this->array(tag.dstIndex,scomp), .dindex = tag.dstIndex,
660 .sfab = src, .dbox = tag.dbox,
661 .offset = (tag.sbox.smallEnd()-tag.dbox.smallEnd()).dim3()});
662 }
663
664 // Note that we have shifted the starting component to zero in the code above.
665
666 // TODO: We could try to cache the tags like in FillBoundary.
667
668 detail::fab_to_fab(loc_copy_tags_1, 0, 0, ncomp,
669 detail::CellStore<value_type, value_type>{});
670 const bool use_deterministic = deterministic || ! amrex::HasAtomicAdd<value_type>::value;
671 if (use_deterministic) {
672 detail::deterministic_fab_to_fab(loc_copy_tags_2, 0, 0, ncomp,
673 detail::CellAdd<value_type,value_type>{});
674 } else {
676 detail::fab_to_fab(loc_copy_tags_2, 0, 0, ncomp,
677 detail::CellAtomicAdd<value_type, value_type>{});
678 }
679 ((void)0);
680 }
681
682 // The deterministic sum always synchronizes the stream.
683 if (Gpu::inNoSyncRegion() && !use_deterministic) {
684 Gpu::freeAsync(The_Arena(), src_buf);
685 } else {
687 The_Arena()->free(src_buf);
688 }
689}
690
691template <class FAB>
692void
694 const CommMetaData& thecmd, int scomp, int ncomp)
695{
696 auto const& LocTags = *(thecmd.m_LocTags);
697 int N_locs = LocTags.size();
698 if (N_locs == 0) { return; }
699 bool is_thread_safe = thecmd.m_threadsafe_loc;
700
701 using TagType = Array4BoxTag<value_type>;
702 Vector<TagType> loc_setval_tags;
703 loc_setval_tags.reserve(N_locs);
704
706
707 for (int i = 0; i < N_locs; ++i)
708 {
709 const CopyComTag& tag = LocTags[i];
711 loc_setval_tags.push_back(TagType{.dfab = this->array(tag.dstIndex), .dbox = tag.dbox});
712 }
713
714 amrex::ParallelFor(loc_setval_tags, ncomp,
715 [x,scomp] AMREX_GPU_DEVICE (int i, int j, int k, int n, TagType const& tag) noexcept
716 {
717 tag.dfab(i,j,k,n+scomp) = x;
718 });
719}
720
721template <class FAB>
722void
724 const CommMetaData& thecmd, int scomp, int ncomp)
725{
726 auto const& RcvTags = *(thecmd.m_RcvTags);
727 bool is_thread_safe = thecmd.m_threadsafe_rcv;
728
729 using TagType = Array4BoxTag<value_type>;
730 Vector<TagType> rcv_setval_tags;
731
732 for (auto it = RcvTags.begin(); it != RcvTags.end(); ++it) {
733 for (auto const& tag: it->second) {
734 rcv_setval_tags.push_back(TagType{.dfab = this->array(tag.dstIndex), .dbox = tag.dbox});
735 }
736 }
737
738 if (rcv_setval_tags.empty()) { return; }
739
741
742 amrex::ParallelFor(rcv_setval_tags, ncomp,
743 [x,scomp] AMREX_GPU_DEVICE (int i, int j, int k, int n, TagType const& tag) noexcept
744 {
745 tag.dfab(i,j,k,n+scomp) = x;
746 });
747}
748
749#if defined(__CUDACC__) && defined (AMREX_USE_CUDA)
750template <class FAB>
751void
752FabArray<FAB>::FB_local_copy_cuda_graph_1 (const FB& TheFB, int scomp, int ncomp)
753{
754 const int N_locs = (*TheFB.m_LocTags).size();
756 for (int i = 0; i < N_locs; ++i)
757 {
758 const CopyComTag& tag = (*TheFB.m_LocTags)[i];
759
760 BL_ASSERT(distributionMap[tag.dstIndex] == ParallelDescriptor::MyProc());
761 BL_ASSERT(distributionMap[tag.srcIndex] == ParallelDescriptor::MyProc());
762
763 loc_copy_tags[tag.dstIndex].push_back
764 ({this->fabPtr(tag.srcIndex), tag.dbox, tag.sbox.smallEnd()-tag.dbox.smallEnd()});
765 }
766
767 // Create Graph if one is needed.
768 if ( !(TheFB.m_localCopy.ready()) )
769 {
770 const_cast<FB&>(TheFB).m_localCopy.resize(N_locs);
771
772 int idx = 0;
773 // Record the graph.
774 for (MFIter mfi(*this, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi)
775 {
776 amrex::Gpu::Device::startGraphRecording( (mfi.LocalIndex() == 0),
777 const_cast<FB&>(TheFB).m_localCopy.getHostPtr(0),
778 (TheFB).m_localCopy.getDevicePtr(0),
779 std::size_t(sizeof(CopyMemory)*N_locs) );
780
781 const auto& tags = loc_copy_tags[mfi];
782 for (auto const & tag : tags)
783 {
784 const auto offset = tag.offset.dim3();
785 CopyMemory* cmem = TheFB.m_localCopy.getDevicePtr(idx++);
786 AMREX_HOST_DEVICE_FOR_3D (tag.dbox, i, j, k,
787 {
788 // Build the Array4's.
789 auto const dst = cmem->getDst<value_type>();
790 auto const src = cmem->getSrc<value_type>();
791 for (int n = 0; n < cmem->ncomp; ++n) {
792 dst(i,j,k,(cmem->scomp)+n) = src(i+offset.x,j+offset.y,k+offset.z,(cmem->scomp)+n);
793 }
794 });
795 }
796
797 bool last_iter = mfi.LocalIndex() == (this->local_size()-1);
798 cudaGraphExec_t graphExec = amrex::Gpu::Device::stopGraphRecording(last_iter);
799 if (last_iter) { const_cast<FB&>(TheFB).m_localCopy.setGraph( graphExec ); }
800 }
801 }
802
803 // Setup Launch Parameters
804 // This is perfectly threadable, right?
805 // Additional optimization -> Check to see whether values need to be reset?
806 // Can then remove this setup and memcpy from CudaGraph::executeGraph.
807 int idx = 0;
808 for (MFIter mfi(*this); mfi.isValid(); ++mfi)
809 {
810 auto const dst_array = this->array(mfi);
811 const auto& tags = loc_copy_tags[mfi];
812 for (auto const & tag : tags)
813 {
814 const_cast<FB&>(TheFB).m_localCopy.setParams(idx++, makeCopyMemory(tag.sfab->array(),
815 dst_array,
816 scomp, ncomp));
817 }
818 }
819
820 // Launch Graph
821 TheFB.m_localCopy.executeGraph();
822}
823
824#ifdef AMREX_USE_MPI
825template <class FAB>
826void
827FabArray<FAB>::FB_local_copy_cuda_graph_n (const FB& TheFB, int scomp, int ncomp)
828{
829 const int N_locs = TheFB.m_LocTags->size();
830
831 int launches = 0; // Used for graphs only.
832 LayoutData<Vector<FabCopyTag<FAB> > > loc_copy_tags(boxArray(),DistributionMap());
833 for (int i = 0; i < N_locs; ++i)
834 {
835 const CopyComTag& tag = (*TheFB.m_LocTags)[i];
836
837 BL_ASSERT(ParallelDescriptor::sameTeam(distributionMap[tag.dstIndex]));
838 BL_ASSERT(ParallelDescriptor::sameTeam(distributionMap[tag.srcIndex]));
839
840 if (distributionMap[tag.dstIndex] == ParallelDescriptor::MyProc())
841 {
842 loc_copy_tags[tag.dstIndex].push_back
843 ({this->fabPtr(tag.srcIndex), tag.dbox, tag.sbox.smallEnd()-tag.dbox.smallEnd()});
844 launches++;
845 }
846 }
847
848 FillBoundary_test();
849
850 if ( !(TheFB.m_localCopy.ready()) )
851 {
852 const_cast<FB&>(TheFB).m_localCopy.resize(launches);
853
854 int idx = 0;
855 int cuda_stream = 0;
856 for (MFIter mfi(*this, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi)
857 {
858 const auto& tags = loc_copy_tags[mfi];
859 for (int t = 0; t<tags.size(); ++t)
860 {
861 Gpu::Device::setStreamIndex(cuda_stream++);
862 amrex::Gpu::Device::startGraphRecording( (idx == 0),
863 const_cast<FB&>(TheFB).m_localCopy.getHostPtr(0),
864 (TheFB).m_localCopy.getDevicePtr(0),
865 std::size_t(sizeof(CopyMemory)*launches) );
866
867 const auto& tag = tags[t];
868 const Dim3 offset = tag.offset.dim3();
869
870 CopyMemory* cmem = TheFB.m_localCopy.getDevicePtr(idx++);
871 AMREX_HOST_DEVICE_FOR_3D(tag.dbox, i, j, k,
872 {
873 auto const dst = cmem->getDst<value_type>();
874 auto const src = cmem->getSrc<value_type>();
875 for (int n = 0; n < cmem->ncomp; ++n) {
876 dst(i,j,k,(cmem->scomp)+n) = src(i+offset.x,j+offset.y,k+offset.z,(cmem->scomp)+n);
877 }
878 });
879
880 bool last_iter = idx == launches;
881 cudaGraphExec_t graphExec = Gpu::Device::stopGraphRecording(last_iter);
882 if (last_iter) { const_cast<FB&>(TheFB).m_localCopy.setGraph( graphExec ); }
883 }
884 }
885 }
886
887 // Setup Launch Parameters
888 // This is perfectly threadable, right?
889 int idx = 0;
890 for (MFIter mfi(*this); mfi.isValid(); ++mfi)
891 {
892 const auto& dst_array = this->array(mfi);
893 const auto& tags = loc_copy_tags[mfi];
894 for (auto const & tag : tags)
895 {
896 const_cast<FB&>(TheFB).m_localCopy.setParams(idx++, makeCopyMemory(tag.sfab->array(),
897 dst_array,
898 scomp, ncomp));
899 }
900 }
901
902 // Launch Graph without synch. Local work is entirely independent.
903 TheFB.m_localCopy.executeGraph(false);
904}
905#endif /* AMREX_USE_MPI */
906
907#endif /* __CUDACC__ */
908
909#endif /* AMREX_USE_GPU */
910
911#ifdef AMREX_USE_MPI
912
913#ifdef AMREX_USE_GPU
914
915#if defined(__CUDACC__) && defined(AMREX_USE_CUDA)
916
917template <class FAB>
918void
919FabArray<FAB>::FB_pack_send_buffer_cuda_graph (const FB& TheFB, int scomp, int ncomp,
920 Vector<char*>& send_data,
921 Vector<std::size_t> const& send_size,
922 Vector<typename FabArray<FAB>::CopyComTagsContainer const*> const& send_cctc)
923{
924 const int N_snds = send_data.size();
925 if (N_snds == 0) { return; }
926
927 if ( !(TheFB.m_copyToBuffer.ready()) )
928 {
929 // Set size of CudaGraph buffer.
930 // Is the conditional ever expected false?
931 int launches = 0;
932 for (int send = 0; send < N_snds; ++send) {
933 if (send_size[send] > 0) {
934 launches += send_cctc[send]->size();
935 }
936 }
937 const_cast<FB&>(TheFB).m_copyToBuffer.resize(launches);
938
939 // Record the graph.
940 int idx = 0;
941 for (Gpu::StreamIter sit(N_snds,Gpu::StreamItInfo().DisableDeviceSync());
942 sit.isValid(); ++sit)
943 {
944 amrex::Gpu::Device::startGraphRecording( (sit() == 0),
945 const_cast<FB&>(TheFB).m_copyToBuffer.getHostPtr(0),
946 (TheFB).m_copyToBuffer.getDevicePtr(0),
947 std::size_t(sizeof(CopyMemory)*launches) );
948
949 const int j = sit();
950 if (send_size[j] > 0)
951 {
952 auto const& cctc = *send_cctc[j];
953 for (auto const& tag : cctc)
954 {
955 const Box& bx = tag.sbox;
956 CopyMemory* cmem = TheFB.m_copyToBuffer.getDevicePtr(idx++);
957 AMREX_HOST_DEVICE_FOR_3D (bx, ii, jj, kk,
958 {
959 auto const pfab = cmem->getDst<value_type>();
960 auto const sfab = cmem->getSrc<value_type>();
961 for (int n = 0; n < cmem->ncomp; ++n)
962 {
963 pfab(ii,jj,kk,n) = sfab(ii,jj,kk,n+(cmem->scomp));
964 }
965 });
966 }
967 }
968
969 bool last_iter = sit() == (N_snds-1);
970 cudaGraphExec_t graphExec = amrex::Gpu::Device::stopGraphRecording(last_iter);
971 if (last_iter) { const_cast<FB&>(TheFB).m_copyToBuffer.setGraph( graphExec ); }
972 }
973 }
974
975 // Setup Launch Parameters
976 int idx = 0;
977 for (int send = 0; send < N_snds; ++send)
978 {
979 const int j = send;
980 if (send_size[j] > 0)
981 {
982 char* dptr = send_data[j];
983 auto const& cctc = *send_cctc[j];
984 for (auto const& tag : cctc)
985 {
986 const_cast<FB&>(TheFB).m_copyToBuffer.setParams(idx++, makeCopyMemory(this->array(tag.srcIndex),
987 amrex::makeArray4((value_type*)(dptr),
988 tag.sbox,
989 ncomp),
990 scomp, ncomp));
991
992 dptr += (tag.sbox.numPts() * ncomp * sizeof(value_type));
993 }
994 amrex::ignore_unused(send_size);
995 BL_ASSERT(dptr <= send_data[j] + send_size[j]);
996 }
997 }
998
999 // Launch Graph synched, so copyToBuffer is complete prior to posting sends.
1000 TheFB.m_copyToBuffer.executeGraph();
1001}
1002
1003template <class FAB>
1004void
1005FabArray<FAB>::FB_unpack_recv_buffer_cuda_graph (const FB& TheFB, int dcomp, int ncomp,
1006 Vector<char*> const& recv_data,
1007 Vector<std::size_t> const& recv_size,
1008 Vector<CopyComTagsContainer const*> const& recv_cctc,
1009 bool /*is_thread_safe*/)
1010{
1011 const int N_rcvs = recv_cctc.size();
1012 if (N_rcvs == 0) { return; }
1013
1014 int launches = 0;
1015 LayoutData<Vector<VoidCopyTag> > recv_copy_tags(boxArray(),DistributionMap());
1016 for (int k = 0; k < N_rcvs; ++k)
1017 {
1018 if (recv_size[k] > 0)
1019 {
1020 const char* dptr = recv_data[k];
1021 auto const& cctc = *recv_cctc[k];
1022 for (auto const& tag : cctc)
1023 {
1024 recv_copy_tags[tag.dstIndex].push_back(VoidCopyTag{.p = dptr, .dbox = tag.dbox});
1025 dptr += tag.dbox.numPts() * ncomp * sizeof(value_type);
1026 launches++;
1027 }
1028 amrex::ignore_unused(recv_size);
1029 BL_ASSERT(dptr <= recv_data[k] + recv_size[k]);
1030 }
1031 }
1032
1033 if ( !(TheFB.m_copyFromBuffer.ready()) )
1034 {
1035 const_cast<FB&>(TheFB).m_copyFromBuffer.resize(launches);
1036
1037 int idx = 0;
1038 for (MFIter mfi(*this, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi)
1039 {
1040 amrex::Gpu::Device::startGraphRecording( (mfi.LocalIndex() == 0),
1041 const_cast<FB&>(TheFB).m_copyFromBuffer.getHostPtr(0),
1042 (TheFB).m_copyFromBuffer.getDevicePtr(0),
1043 std::size_t(sizeof(CopyMemory)*launches) );
1044
1045 const auto& tags = recv_copy_tags[mfi];
1046 for (auto const & tag : tags)
1047 {
1048 CopyMemory* cmem = TheFB.m_copyFromBuffer.getDevicePtr(idx++);
1049 AMREX_HOST_DEVICE_FOR_3D (tag.dbox, i, j, k,
1050 {
1051 auto const pfab = cmem->getSrc<value_type>();
1052 auto const dfab = cmem->getDst<value_type>();
1053 for (int n = 0; n < cmem->ncomp; ++n)
1054 {
1055 dfab(i,j,k,n+(cmem->scomp)) = pfab(i,j,k,n);
1056 }
1057 });
1058 }
1059
1060 bool last_iter = mfi.LocalIndex() == (this->local_size()-1);
1061 cudaGraphExec_t graphExec = amrex::Gpu::Device::stopGraphRecording(last_iter);
1062 if (last_iter) { const_cast<FB&>(TheFB).m_copyFromBuffer.setGraph( graphExec ); }
1063 }
1064 }
1065
1066 // Setup graph.
1067 int idx = 0;
1068 for (MFIter mfi(*this); mfi.isValid(); ++mfi)
1069 {
1070 auto dst_array = this->array(mfi);
1071 const auto & tags = recv_copy_tags[mfi];
1072 for (auto const & tag : tags)
1073 {
1074 const_cast<FB&>(TheFB).m_copyFromBuffer.setParams(idx++, makeCopyMemory(amrex::makeArray4((value_type*)(tag.p),
1075 tag.dbox,
1076 ncomp),
1077 dst_array,
1078 dcomp, ncomp));
1079 }
1080 }
1081
1082 // Launch Graph - synced because next action is freeing recv buffer.
1083 TheFB.m_copyFromBuffer.executeGraph();
1084}
1085
1086#endif /* __CUDACC__ */
1087
1088template <class FAB>
1089template <typename BUF>
1090auto
1092 Vector<std::size_t> const& send_size,
1093 Vector<CopyComTagsContainer const*> const& send_cctc,
1094 int ncomp, std::uint64_t id) const
1096{
1097 using TagType = CommSendBufTag<value_type>;
1098
1099 auto kit = std::find_if(send_cctc.begin(), send_cctc.end(),
1100 [] (CopyComTagsContainer const* p) { return p != nullptr; });
1101 if (kit == send_cctc.end()) {
1102 return nullptr;
1103 }
1104
1105 auto get_tags = [&] () -> Vector<TagType>
1106 {
1107 Vector<TagType> snd_copy_tags;
1108 char* pbuf = send_data[0];
1109 const int N_snds = send_data.size();
1110 for (int j = 0; j < N_snds; ++j)
1111 {
1112 if (send_size[j] > 0)
1113 {
1114 char* dptr = send_data[j];
1115 auto const& cctc = *send_cctc[j];
1116 for (auto const& tag : cctc)
1117 {
1118 snd_copy_tags.emplace_back
1119 (TagType{.sfab = this->const_array(tag.srcIndex), .poff = dptr-pbuf, .bx = tag.sbox});
1120 dptr += (tag.sbox.numPts() * ncomp * sizeof(BUF));
1121 }
1122 }
1123 }
1124 return snd_copy_tags;
1125 };
1126
1128 std::tuple<std::uint64_t,std::size_t,int> key{id, sizeof(BUF), ncomp};
1129
1130 if (auto it = m_send_copy_handler.find(key); it != m_send_copy_handler.end()) {
1131 tv = it->second.get();
1132 } else {
1133 if (m_send_copy_handler.size() > 32) {
1134 // Just in case. If this is used in ParallelCopy, it's possible
1135 // that the sending FabArray is the same, but the receiving
1136 // FabArray is different every time. Then the size of this map
1137 // could increase indefinitely.
1138 m_send_copy_handler.clear();
1139 }
1140 auto snd_copy_tags = get_tags();
1141 auto utv = std::make_unique<TagVector<TagType>>(snd_copy_tags);
1142 tv = utv.get();
1143 m_send_copy_handler[key] = std::move(utv);
1144 }
1145
1146 return tv;
1147}
1148
1149template <class FAB>
1150template <typename BUF>
1151void
1152FabArray<FAB>::pack_send_buffer_gpu (FabArray<FAB> const& src, int scomp, int ncomp,
1153 Vector<char*> const& send_data,
1154 Vector<std::size_t> const& send_size,
1155 Vector<CopyComTagsContainer const*> const& send_cctc,
1156 std::uint64_t id)
1157{
1158 const int N_snds = send_data.size();
1159 if (N_snds == 0) { return; }
1160
1161 using TagType = CommSendBufTag<value_type>;
1162
1163 auto* tv = src.template get_send_copy_tag_vector<BUF>
1164 (send_data, send_size, send_cctc, ncomp, id);
1165 if (tv == nullptr) { return; }
1166
1167 char* pbuffer = send_data[0];
1168
1169 detail::ParallelFor_doit(*tv,
1170 [=] AMREX_GPU_DEVICE (
1171 int icell, int ncells, int i, int j, int k, TagType const& tag) noexcept
1172 {
1173 if (icell < ncells) {
1174 Array4<BUF> dfab{(BUF*)(pbuffer+tag.poff),
1175 amrex::begin(tag.bx), amrex::end(tag.bx), ncomp};
1176 for (int n = 0; n < ncomp; ++n) {
1177 dfab(i,j,k,n) = (BUF)tag.sfab(i,j,k,n+scomp);
1178 }
1179 }
1180 });
1181
1183}
1184
1185template <class FAB>
1186template <typename BUF>
1187auto
1189 Vector<std::size_t> const& recv_size,
1190 Vector<CopyComTagsContainer const*> const& recv_cctc,
1191 int ncomp, std::uint64_t id)
1193{
1194 using TagType = CommRecvBufTag<value_type>;
1195
1196 auto kit = std::find_if(recv_cctc.begin(), recv_cctc.end(),
1197 [] (CopyComTagsContainer const* p) { return p != nullptr; });
1198 if (kit == recv_cctc.end()) {
1199 return nullptr;
1200 }
1201
1202 auto get_tags = [&] () -> Vector<TagType>
1203 {
1204 Vector<TagType> recv_copy_tags;
1205 char* pbuf = recv_data[0];
1206 const int N_rcvs = recv_cctc.size();
1207 for (int k = 0; k < N_rcvs; ++k)
1208 {
1209 if (recv_size[k] > 0)
1210 {
1211 char* dptr = recv_data[k];
1212 auto const& cctc = *recv_cctc[k];
1213 for (auto const& tag : cctc)
1214 {
1215 const int li = this->localindex(tag.dstIndex);
1216 recv_copy_tags.emplace_back
1217 (TagType{.dfab = this->atLocalIdx(li).array(), .poff = dptr-pbuf, .bx = tag.dbox});
1218 dptr += tag.dbox.numPts() * ncomp * sizeof(BUF);
1219 }
1220 }
1221 }
1222 return recv_copy_tags;
1223 };
1224
1226 std::tuple<std::uint64_t,std::size_t,int> key{id, sizeof(BUF), ncomp};
1227
1228 if (auto it = m_recv_copy_handler.find(key); it != m_recv_copy_handler.end()) {
1229 tv = it->second.get();
1230 } else {
1231 if (m_recv_copy_handler.size() > 32) {
1232 // Just in case. If this is used in ParallelCopy, it's possible
1233 // that the receiving FabArray is the same, but the sending
1234 // FabArray is different every time. Then the size of this map
1235 // could increase indefinitely.
1236 m_recv_copy_handler.clear();
1237 }
1238 auto recv_copy_tags = get_tags();
1239 auto utv = std::make_unique<TagVector<TagType>>(recv_copy_tags);
1240 tv = utv.get();
1241 m_recv_copy_handler[key] = std::move(utv);
1242 }
1243
1244 return tv;
1245}
1246
1247template <class FAB>
1248template <typename BUF>
1249void
1251 Vector<char*> const& recv_data,
1252 Vector<std::size_t> const& recv_size,
1253 Vector<CopyComTagsContainer const*> const& recv_cctc,
1254 CpOp op, bool is_thread_safe, std::uint64_t id,
1255 bool deterministic)
1256{
1257 const int N_rcvs = recv_cctc.size();
1258 if (N_rcvs == 0) { return; }
1259
1260 bool use_mask = false;
1261 if (!is_thread_safe)
1262 {
1264 (op == FabArrayBase::ADD && !amrex::HasAtomicAdd <value_type>::value))
1265 {
1266 use_mask = true;
1267 }
1268 }
1269
1270 if (deterministic)
1271 {
1272 AMREX_ALWAYS_ASSERT(op == FabArrayBase::ADD); // Only ADD for now
1273 using TagType = Array4CopyTag<value_type,BUF>;
1275 tags.reserve(N_rcvs);
1276 for (int k = 0; k < N_rcvs; ++k) {
1277 if (recv_size[k] > 0) {
1278 char const* dptr = recv_data[k];
1279 auto const& cctc = *recv_cctc[k];
1280 for (auto const& tag : cctc) {
1281 tags.emplace_back(
1282 TagType{.dfab = dst.array(tag.dstIndex), .dindex = tag.dstIndex,
1283 .sfab = Array4<BUF const>((BUF const*)dptr,
1284 amrex::begin(tag.dbox),
1285 amrex::end(tag.dbox), ncomp),
1286 .dbox = tag.dbox,
1287 .offset = Dim3{.x = 0, .y = 0, .z = 0}});
1288 dptr += tag.dbox.numPts() * ncomp * sizeof(BUF);
1289 }
1290 }
1291 }
1293 detail::deterministic_fab_to_fab<value_type,BUF>
1294 (tags, 0, dcomp, ncomp, detail::CellAdd<value_type,BUF>{});
1295 } else {
1296 amrex::Abort("SumBoundary requires operator+=");
1297 }
1298 }
1299 else if (!use_mask)
1300 {
1301 using TagType = CommRecvBufTag<value_type>;
1302 auto* tv = dst.template get_recv_copy_tag_vector<BUF>
1303 (recv_data, recv_size, recv_cctc, ncomp, id);
1304 if (tv == nullptr) { return; }
1305
1306 char* pbuffer = recv_data[0];
1307
1308 if (op == FabArrayBase::COPY)
1309 {
1310 detail::ParallelFor_doit(*tv,
1311 [=] AMREX_GPU_DEVICE (
1312 int icell, int ncells, int i, int j, int k, TagType const& tag) noexcept
1313 {
1314 if (icell < ncells) {
1315 Array4<BUF const> sfab{(BUF const*)(pbuffer+tag.poff),
1316 amrex::begin(tag.bx), amrex::end(tag.bx), ncomp};
1317 for (int n = 0; n < ncomp; ++n) {
1318 tag.dfab(i,j,k,n+dcomp) = (value_type)sfab(i,j,k,n);
1319 }
1320 }
1321 });
1322 }
1323 else
1324 {
1325 if (is_thread_safe) {
1326 detail::ParallelFor_doit(*tv,
1327 [=] AMREX_GPU_DEVICE (
1328 int icell, int ncells, int i, int j, int k, TagType const& tag) noexcept
1329 {
1330 if (icell < ncells) {
1331 Array4<BUF const> sfab{(BUF const*)(pbuffer+tag.poff),
1332 amrex::begin(tag.bx), amrex::end(tag.bx), ncomp};
1333 for (int n = 0; n < ncomp; ++n) {
1334 tag.dfab(i,j,k,n+dcomp) += (value_type)sfab(i,j,k,n);
1335 }
1336 }
1337 });
1338 } else {
1339 detail::unpack_recv_buffer_gpu_atomic_add<BUF, value_type>
1340 (pbuffer, *tv, dcomp, ncomp);
1341 }
1342 }
1343 // In a no-sync region the caller frees the buffer in stream order.
1344 if (!Gpu::inNoSyncRegion()) {
1346 }
1347 }
1348 else
1349 {
1350 char* pbuffer = recv_data[0];
1351
1352 using TagType = Array4CopyTag<value_type, BUF>;
1353 Vector<TagType> recv_copy_tags;
1354 recv_copy_tags.reserve(N_rcvs);
1355
1356 Vector<BaseFab<int> > maskfabs(dst.local_size());
1357 Vector<Array4Tag<int> > masks_unique;
1358 masks_unique.reserve(dst.local_size());
1359 Vector<Array4Tag<int> > masks;
1360
1361 for (int k = 0; k < N_rcvs; ++k)
1362 {
1363 if (recv_size[k] > 0)
1364 {
1365 std::size_t offset = recv_data[k]-recv_data[0];
1366 const char* dptr = pbuffer + offset;
1367 auto const& cctc = *recv_cctc[k];
1368 for (auto const& tag : cctc)
1369 {
1370 const int li = dst.localindex(tag.dstIndex);
1371 recv_copy_tags.emplace_back(TagType{
1372 .dfab = dst.atLocalIdx(li).array(), .dindex = tag.dstIndex,
1373 .sfab = amrex::makeArray4((BUF const*)(dptr), tag.dbox, ncomp),
1374 .dbox = tag.dbox,
1375 .offset = Dim3{.x = 0, .y = 0, .z = 0}
1376 });
1377 dptr += tag.dbox.numPts() * ncomp * sizeof(BUF);
1378
1379 if (!maskfabs[li].isAllocated()) {
1380 maskfabs[li].resize(dst.atLocalIdx(li).box());
1381 masks_unique.emplace_back(Array4Tag<int>{.dfab = maskfabs[li].array()});
1382 }
1383 masks.emplace_back(Array4Tag<int>{.dfab = maskfabs[li].array()});
1384 }
1385 BL_ASSERT(dptr <= pbuffer + offset + recv_size[k]);
1386 }
1387 }
1388
1389 amrex::ParallelFor(masks_unique,
1390 [=] AMREX_GPU_DEVICE (int i, int j, int k, Array4Tag<int> const& msk) noexcept
1391 {
1392 msk.dfab(i,j,k) = 0;
1393 });
1394
1395 if (op == FabArrayBase::COPY)
1396 {
1397 detail::fab_to_fab_atomic_cpy<value_type, BUF>(
1398 recv_copy_tags, 0, dcomp, ncomp, masks);
1399 }
1400 else
1401 {
1402 detail::fab_to_fab_atomic_add<value_type, BUF>(
1403 recv_copy_tags, 0, dcomp, ncomp, masks);
1404 }
1405
1406 // There is Gpu::streamSynchronize in fab_to_fab.
1407 }
1408}
1409
1410#endif /* AMREX_USE_GPU */
1411
1412template <class FAB>
1413template <typename BUF>
1414void
1415FabArray<FAB>::pack_send_buffer_cpu (FabArray<FAB> const& src, int scomp, int ncomp,
1416 Vector<char*> const& send_data,
1417 Vector<std::size_t> const& send_size,
1418 Vector<CopyComTagsContainer const*> const& send_cctc)
1419{
1420 amrex::ignore_unused(send_size);
1421
1422 auto const N_snds = static_cast<int>(send_data.size());
1423 if (N_snds == 0) { return; }
1424
1425#ifdef AMREX_USE_OMP
1426 if (omp_get_max_threads() > 1)
1427 {
1428 // Threads share tags, not neighbor ranks, which are often fewer than threads.
1430 for (int j = 0; j < N_snds; ++j)
1431 {
1432 if (send_size[j] > 0)
1433 {
1434 char* dptr = send_data[j];
1435 for (auto const& tag : *send_cctc[j])
1436 {
1437 tags.emplace_back(dptr, &tag);
1438 dptr += (tag.sbox.numPts() * ncomp * sizeof(BUF));
1439 }
1440 BL_ASSERT(dptr <= send_data[j] + send_size[j]);
1441 }
1442 }
1443
1444 auto const N_tags = static_cast<int>(tags.size());
1445 // Small chunks keep threads on nearby boxes, which share cache lines.
1446 int const chunk = std::clamp(N_tags/omp_get_max_threads(), 1, 64);
1447#pragma omp parallel for schedule(static,chunk)
1448 for (int i = 0; i < N_tags; ++i)
1449 {
1450 auto const& tag = *tags[i].second;
1451 const Box& bx = tag.sbox;
1452 auto const sfab = src.array(tag.srcIndex);
1453 auto pfab = amrex::makeArray4((BUF*)(tags[i].first),bx,ncomp);
1454 amrex::LoopConcurrentOnCpu( bx, ncomp,
1455 [=] (int ii, int jj, int kk, int n) noexcept
1456 {
1457 pfab(ii,jj,kk,n) = static_cast<BUF>(sfab(ii,jj,kk,n+scomp));
1458 });
1459 }
1460 return;
1461 }
1462#endif
1463
1464 for (int j = 0; j < N_snds; ++j)
1465 {
1466 if (send_size[j] > 0)
1467 {
1468 char* dptr = send_data[j];
1469 auto const& cctc = *send_cctc[j];
1470 for (auto const& tag : cctc)
1471 {
1472 const Box& bx = tag.sbox;
1473 auto const sfab = src.array(tag.srcIndex);
1474 auto pfab = amrex::makeArray4((BUF*)(dptr),bx,ncomp);
1475 amrex::LoopConcurrentOnCpu( bx, ncomp,
1476 [=] (int ii, int jj, int kk, int n) noexcept
1477 {
1478 pfab(ii,jj,kk,n) = static_cast<BUF>(sfab(ii,jj,kk,n+scomp));
1479 });
1480 dptr += (bx.numPts() * ncomp * sizeof(BUF));
1481 }
1482 BL_ASSERT(dptr <= send_data[j] + send_size[j]);
1483 }
1484 }
1485}
1486
1487template <class FAB>
1488template <typename BUF>
1489void
1491 Vector<char*> const& recv_data,
1492 Vector<std::size_t> const& recv_size,
1493 Vector<CopyComTagsContainer const*> const& recv_cctc,
1494 CpOp op, bool is_thread_safe)
1495{
1496 amrex::ignore_unused(recv_size);
1497
1498 auto const N_rcvs = static_cast<int>(recv_cctc.size());
1499 if (N_rcvs == 0) { return; }
1500
1501 if (is_thread_safe)
1502 {
1503#ifdef AMREX_USE_OMP
1504 if (omp_get_max_threads() > 1)
1505 {
1506 // Threads share tags, not neighbor ranks, which are often fewer than threads.
1508 for (int k = 0; k < N_rcvs; ++k)
1509 {
1510 if (recv_size[k] > 0)
1511 {
1512 const char* dptr = recv_data[k];
1513 for (auto const& tag : *recv_cctc[k])
1514 {
1515 tags.emplace_back(dptr, &tag);
1516 dptr += tag.dbox.numPts() * ncomp * sizeof(BUF);
1517 }
1518 BL_ASSERT(dptr <= recv_data[k] + recv_size[k]);
1519 }
1520 }
1521
1522 auto const N_tags = static_cast<int>(tags.size());
1523 // Small chunks keep threads on nearby boxes, which share cache lines.
1524 int const chunk = std::clamp(N_tags/omp_get_max_threads(), 1, 64);
1525#pragma omp parallel for schedule(static,chunk)
1526 for (int i = 0; i < N_tags; ++i)
1527 {
1528 auto const& tag = *tags[i].second;
1529 FAB& dfab = dst[tag.dstIndex];
1530 if (op == FabArrayBase::COPY)
1531 {
1532 dfab.template copyFromMem<RunOn::Host, BUF>(tag.dbox, dcomp, ncomp, tags[i].first);
1533 }
1534 else
1535 {
1536 dfab.template addFromMem<RunOn::Host, BUF>(tag.dbox, dcomp, ncomp, tags[i].first);
1537 }
1538 }
1539 return;
1540 }
1541#endif
1542
1543 for (int k = 0; k < N_rcvs; ++k)
1544 {
1545 if (recv_size[k] > 0)
1546 {
1547 const char* dptr = recv_data[k];
1548 auto const& cctc = *recv_cctc[k];
1549 for (auto const& tag : cctc)
1550 {
1551 const Box& bx = tag.dbox;
1552 FAB& dfab = dst[tag.dstIndex];
1553 if (op == FabArrayBase::COPY)
1554 {
1555 dfab.template copyFromMem<RunOn::Host, BUF>(bx, dcomp, ncomp, dptr);
1556 }
1557 else
1558 {
1559 dfab.template addFromMem<RunOn::Host, BUF>(tag.dbox, dcomp, ncomp, dptr);
1560 }
1561 dptr += bx.numPts() * ncomp * sizeof(BUF);
1562 }
1563 BL_ASSERT(dptr <= recv_data[k] + recv_size[k]);
1564 }
1565 }
1566 }
1567 else
1568 {
1569 LayoutData<Vector<VoidCopyTag> > recv_copy_tags;
1570 recv_copy_tags.define(dst.boxArray(),dst.DistributionMap());
1571 for (int k = 0; k < N_rcvs; ++k)
1572 {
1573 if (recv_size[k] > 0)
1574 {
1575 const char* dptr = recv_data[k];
1576 auto const& cctc = *recv_cctc[k];
1577 for (auto const& tag : cctc)
1578 {
1579 recv_copy_tags[tag.dstIndex].push_back(VoidCopyTag{.p = dptr, .dbox = tag.dbox});
1580 dptr += tag.dbox.numPts() * ncomp * sizeof(BUF);
1581 }
1582 BL_ASSERT(dptr <= recv_data[k] + recv_size[k]);
1583 }
1584 }
1585
1586#ifdef AMREX_USE_OMP
1587#pragma omp parallel
1588#endif
1589 for (MFIter mfi(dst); mfi.isValid(); ++mfi)
1590 {
1591 const auto& tags = recv_copy_tags[mfi];
1592 auto dfab = dst.array(mfi);
1593 for (auto const & tag : tags)
1594 {
1595 auto pfab = amrex::makeArray4((BUF*)(tag.p), tag.dbox, ncomp);
1596 if (op == FabArrayBase::COPY)
1597 {
1598 amrex::LoopConcurrentOnCpu(tag.dbox, ncomp,
1599 [=] (int i, int j, int k, int n) noexcept
1600 {
1601 dfab(i,j,k,n+dcomp) = pfab(i,j,k,n);
1602 });
1603 }
1604 else
1605 {
1606 amrex::LoopConcurrentOnCpu(tag.dbox, ncomp,
1607 [=] (int i, int j, int k, int n) noexcept
1608 {
1609 dfab(i,j,k,n+dcomp) += pfab(i,j,k,n);
1610 });
1611 }
1612 }
1613 }
1614 }
1615}
1616
1617#endif /* AMREX_USE_MPI */
1618
1619}
1620
1621#endif
#define BL_ASSERT(EX)
Definition AMReX_BLassert.H:39
#define AMREX_ALWAYS_ASSERT(EX)
Definition AMReX_BLassert.H:50
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
#define AMREX_HOST_DEVICE_FOR_3D(...)
Definition AMReX_GpuLaunchMacrosC.nolint.H:106
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
while parser_exe_t p
Definition AMReX_Parser_Exe_Body.H:33
#define AMREX_D_DECL(a, b, c)
Definition AMReX_SPACE.H:171
virtual void free(void *pt)=0
Free a previously allocated block pointed to by pt.
virtual void * alloc(std::size_t sz)=0
Allocate sz bytes from this arena.
__host__ __device__ Long numPts() const noexcept
Return the number of points contained in the BoxND.
Definition AMReX_Box.H:385
__host__ __device__ const IntVectND< dim > & smallEnd() const &noexcept
Return the inclusive lower bound of the box.
Definition AMReX_Box.H:124
friend class MFIter
Definition AMReX_FabArrayBase.H:48
CopyComTag::CopyComTagsContainer CopyComTagsContainer
Definition AMReX_FabArrayBase.H:236
int localindex(int K) const noexcept
Return local index in the vector of FABs.
Definition AMReX_FabArrayBase.H:124
const DistributionMapping & DistributionMap() const noexcept
Return constant reference to associated DistributionMapping.
Definition AMReX_FabArrayBase.H:135
int local_size() const noexcept
Return the number of local FABs in the FabArray.
Definition AMReX_FabArrayBase.H:118
CpOp
parallel copy or add
Definition AMReX_FabArrayBase.H:432
@ ADD
Definition AMReX_FabArrayBase.H:432
@ COPY
Definition AMReX_FabArrayBase.H:432
Box box(int K) const noexcept
Return the Kth Box in the BoxArray. That is, the valid region of the Kth grid.
Definition AMReX_FabArrayBase.H:106
DistributionMapping distributionMap
Definition AMReX_FabArrayBase.H:536
const BoxArray & boxArray() const noexcept
Return a constant reference to the BoxArray that defines the valid region associated with this FabArr...
Definition AMReX_FabArrayBase.H:100
An Array of FortranArrayBox(FAB)-like Objects.
Definition AMReX_FabArray.H:383
TagVector< CommSendBufTag< value_type > > const * get_send_copy_tag_vector(Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< CopyComTagsContainer const * > const &send_cctc, int ncomp, std::uint64_t id) const
typename std::conditional_t< IsBaseFab< FAB >::value, FAB, FABType >::value_type value_type
Definition AMReX_FabArray.H:394
void CMD_remote_setVal_gpu(value_type x, const CommMetaData &thecmd, int scomp, int ncomp)
Definition AMReX_FBI.H:723
static void unpack_recv_buffer_gpu(FabArray< FAB > &dst, int dcomp, int ncomp, Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< const CopyComTagsContainer * > const &recv_cctc, CpOp op, bool is_thread_safe, std::uint64_t id, bool deterministic)
Unpack received messages into dst on the GPU.
void FB_local_add_cpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:467
static void unpack_recv_buffer_cpu(FabArray< FAB > &dst, int dcomp, int ncomp, Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< const CopyComTagsContainer * > const &recv_cctc, CpOp op, bool is_thread_safe)
void FB_local_add_gpu(const FB &TheFB, int scomp, int ncomp, bool deterministic)
Definition AMReX_FBI.H:630
void FB_local_copy_gpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:558
void copy(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition AMReX_FabArray.H:1005
const Vector< std::string > & tags() const noexcept
Memory-usage tags recorded for this FabArray's allocations.
Definition AMReX_FabArray.H:516
void CMD_local_setVal_gpu(value_type x, const CommMetaData &thecmd, int scomp, int ncomp)
Definition AMReX_FBI.H:693
static void pack_send_buffer_gpu(FabArray< FAB > const &src, int scomp, int ncomp, Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< const CopyComTagsContainer * > const &send_cctc, std::uint64_t id)
Array4< typename FabArray< FAB >::value_type const > array(const MFIter &mfi) const noexcept
Read-only Array4 view for the FAB referenced by iterator mfi.
Definition AMReX_FabArray.H:649
void FB_local_copy_cpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:396
TagVector< Array4CopyTag< value_type > > const * FB_get_local_copy_tag_vector(const FB &TheFB)
Definition AMReX_FBI.H:515
static void pack_send_buffer_cpu(FabArray< FAB > const &src, int scomp, int ncomp, Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< const CopyComTagsContainer * > const &send_cctc)
FAB * fabPtr(const MFIter &mfi) noexcept
Return pointer to FAB.
Definition AMReX_FabArray.H:2118
TagVector< CommRecvBufTag< value_type > > const * get_recv_copy_tag_vector(Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< CopyComTagsContainer const * > const &recv_cctc, int ncomp, std::uint64_t id)
FAB & atLocalIdx(int L) noexcept
Return a reference to the FAB associated with local index L.
Definition AMReX_FabArray.H:607
Array4< typename FabArray< FAB >::value_type const > const_array(const MFIter &mfi) const noexcept
Synonym for array(const MFIter&) that highlights read-only semantics.
Definition AMReX_FabArray.H:677
a one-thingy-per-box distributed object
Definition AMReX_LayoutData.H:13
void define(const BoxArray &a_grids, const DistributionMapping &a_dm)
Definition AMReX_LayoutData.H:24
Iterator for looping ever tiles and boxes of amrex::FabArray based containers.
Definition AMReX_MFIter.H:88
bool isValid() const noexcept
Is the iterator valid i.e. is it associated with a FAB?
Definition AMReX_MFIter.H:176
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
Long size() const noexcept
Definition AMReX_Vector.H:54
amrex_long Long
Definition AMReX_INT.H:30
__host__ __device__ Dim3 length(Array4< T > const &a) noexcept
Return the spatial extents of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1391
ArrayND< T, 4, true > Array4
Convenient alias for a 4D ArrayND with three spatial dimensions plus a component dimension.
Definition AMReX_Array4.H:1352
__host__ __device__ Dim3 lbound(Array4< T > const &a) noexcept
Return the inclusive lower bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1363
__host__ __device__ BoxND< dim > coarsen(const BoxND< dim > &b, int ref_ratio) noexcept
Return a copy of b coarsened by the isotropic ratio ref_ratio.
Definition AMReX_Box.H:1471
PinnedVector< T > HostVector
A PODVector that uses pinned host memory. Same as PinnedVector. For a vector class that uses std::all...
Definition AMReX_GpuContainers.H:67
PODVector< T, ArenaAllocator< T > > DeviceVector
A PODVector that uses the standard memory Arena. Note that the memory might or might not be managed d...
Definition AMReX_GpuContainers.H:24
Arena * The_Arena()
Definition AMReX_Arena.cpp:839
int MyProc() noexcept
Definition AMReX_ParallelDescriptor.H:128
__host__ __device__ AMREX_FORCE_INLINE T Exch(T *address, T val) noexcept
Definition AMReX_GpuAtomic.H:542
__host__ __device__ AMREX_FORCE_INLINE T CAS(T *const address, T compare, T const val) noexcept
Definition AMReX_GpuAtomic.H:568
__host__ __device__ AMREX_FORCE_INLINE void AddNoRet(T *sum, T value) noexcept
Definition AMReX_GpuAtomic.H:322
__host__ __device__ AMREX_FORCE_INLINE T Add(T *sum, T value) noexcept
Definition AMReX_GpuAtomic.H:239
void freeAsync(Arena *arena, void *mem) noexcept
Definition AMReX_GpuDevice.H:364
void copyAsync(HostToDevice, InIter begin, InIter end, OutIter result) noexcept
A host-to-device copy routine. Note this is just a wrapper around memcpy, so it assumes contiguous st...
Definition AMReX_GpuContainers.H:228
static constexpr HostToDevice hostToDevice
Definition AMReX_GpuContainers.H:105
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
bool inNoSyncRegion() noexcept
Definition AMReX_GpuControl.H:148
gpuStream_t gpuStream() noexcept
Definition AMReX_GpuDevice.H:310
bool sameTeam(int rank) noexcept
Definition AMReX_ParallelDescriptor.H:345
int TeamSize() noexcept
Definition AMReX_ParallelDescriptor.H:310
const ProcessTeam & MyTeam() noexcept
Definition AMReX_ParallelDescriptor.H:365
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
@ F
Definition AMReX_Order.H:7
__host__ __device__ Array4< T > makeArray4(T *p, Box const &bx, int ncomp) noexcept
Definition AMReX_BaseFab.H:137
BoxArray const & boxArray(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.boxArray().
Definition AMReX_FabArrayBase.cpp:3038
DistributionMapping const & DistributionMap(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.DistributionMap().
Definition AMReX_FabArrayBase.cpp:3043
__host__ __device__ Dim3 begin(BoxND< dim > const &box) noexcept
Return the iterator begin coordinate of box as Dim3.
Definition AMReX_Box.H:2241
bool NeighborParticleContainer_impl< T_ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor >::use_mask
Definition AMReX_NeighborParticlesI.H:13
void launch(T const &n, L const &f) noexcept
Definition AMReX_GpuLaunchFunctsC.H:122
void ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:202
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
@ x
Definition AMReX_Orientation.H:14
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
void LoopConcurrentOnCpu(Dim3 lo, Dim3 hi, F const &f) noexcept
Definition AMReX_Loop.H:388
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
const int[]
Definition AMReX_BLProfiler.cpp:1665
__host__ __device__ Dim3 end(BoxND< dim > const &box) noexcept
Return the iterator end coordinate of box as Dim3.
Definition AMReX_Box.H:2259
__host__ __device__ constexpr int get(IntVectND< dim > const &iv) noexcept
Get the Idx-th element of IntVectND<dim>.
Definition AMReX_IntVect.H:1367
Definition AMReX_TagParallelFor.H:58
Definition AMReX_TagParallelFor.H:26
Definition AMReX_TagParallelFor.H:50
Array4< T > dfab
Definition AMReX_TagParallelFor.H:51
Definition AMReX_TagParallelFor.H:106
Definition AMReX_TagParallelFor.H:116
A simple struct holding 3 int values for a 3D index.
Definition AMReX_Dim3.H:24
Definition AMReX_FabArrayBase.H:563
bool m_threadsafe_loc
Definition AMReX_FabArrayBase.H:565
bool m_threadsafe_rcv
Definition AMReX_FabArrayBase.H:566
std::unique_ptr< MapOfCopyComTagContainers > m_RcvTags
Definition AMReX_FabArrayBase.H:569
std::unique_ptr< CopyComTagsContainer > m_LocTags
Definition AMReX_FabArrayBase.H:567
Used by a bunch of routines when communicating via MPI.
Definition AMReX_FabArrayBase.H:211
Box sbox
Definition AMReX_FabArrayBase.H:213
int srcIndex
Definition AMReX_FabArrayBase.H:215
Box dbox
Definition AMReX_FabArrayBase.H:212
int dstIndex
Definition AMReX_FabArrayBase.H:214
FillBoundary.
Definition AMReX_FabArrayBase.H:588
Describes a source FAB region participating in a copy.
Definition AMReX_FBI.H:18
IntVect offset
Definition AMReX_FBI.H:21
Box dbox
Definition AMReX_FBI.H:20
FAB const * sfab
Definition AMReX_FBI.H:19
Definition AMReX_TypeTraits.H:51
Definition AMReX_TypeTraits.H:61
Definition AMReX_TypeTraits.H:277
Definition AMReX_MFIter.H:20
void MemoryBarrier() const
memory fence
Definition AMReX_ParallelDescriptor.H:161
Definition AMReX_TagParallelFor.H:156
Lightweight tag describing raw pointer copies (used for pack/unpack).
Definition AMReX_FBI.H:25
Box dbox
Definition AMReX_FBI.H:27
char const * p
Definition AMReX_FBI.H:26