35template <
class T0,
class T1>
39 operator() (T0* d, T1 s)
const noexcept
41 *d =
static_cast<T0
>(s);
45template <
class T0,
class T1>
49 operator() (T0* d, T1 s)
const noexcept
51 *d +=
static_cast<T0
>(s);
55template <
class T0,
class T1>
59 requires (amrex::HasAtomicAdd<U0>::value)
61 operator() (U0* d, T1 s)
const noexcept
63 Gpu::Atomic::AddNoRet(d,
static_cast<U0
>(s));
67template <
class T0,
class T1,
class F>
69fab_to_fab (Vector<Array4CopyTag<T0, T1> >
const& copy_tags,
int scomp,
int dcomp,
int ncomp,
72 TagVector<Array4CopyTag<T0, T1>> tv{copy_tags};
74 detail::ParallelFor_doit(tv,
76 int icell,
int ncells,
int i,
int j,
int k, Array4CopyTag<T0, T1>
const& tag)
noexcept
79 for (
int n = 0; n < ncomp; ++n) {
80 f(&(tag.dfab(i,j,k,n+dcomp)),
81 tag.sfab(i+tag.offset.x,j+tag.offset.y,k+tag.offset.z,n+scomp));
87template <
class TagType,
class F>
89fab_to_fab_store (Vector<TagType>
const& tags,
int scomp,
int dcomp,
int ncomp,
F&&f)
96 for (
int n = 0; n < ncomp; ++n) {
97 f(&(tag.dfab(i,j,k,n+dcomp)),
98 tag.sfab(i+tag.offset.x,j+tag.offset.y,k+tag.offset.z,n+scomp));
104template <
class TagType,
class F>
106fab_to_fab_other (Vector<TagType>
const& tags,
int scomp,
int dcomp,
int ncomp,
F&&f)
111 int* m = &(tag.mask(i,j,k));
112 bool my_turn =
false;
114#if defined(AMREX_USE_SYCL)
120#if defined(AMREX_USE_SYCL)
121 sycl::atomic_fence(sycl::memory_order::acq_rel, sycl::memory_scope::device);
125 for (
int n = 0; n < ncomp; ++n) {
126 f(&(tag.dfab(i,j,k,n+dcomp)),
127 tag.sfab(i+tag.offset.x,j+tag.offset.y,k+tag.offset.z,n+scomp));
129#if defined(AMREX_USE_SYCL)
130 sycl::atomic_fence(sycl::memory_order::acq_rel, sycl::memory_scope::device);
137#if defined(AMREX_USE_CUDA)
139#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 700)
142 for (
int c = 0; c < 2; ++c) {
146 for (
int c = 0; c < 2; ++c) {
147 __asm__
volatile(
"");
154#elif defined(AMREX_USE_HIP)
156 __builtin_amdgcn_s_sleep(1);
158#elif defined(AMREX_USE_SYCL)
160 for (
int c = 0; c < 2; ++c) {
161 __asm__
volatile(
"");
170template <
class T0,
class T1,
class F>
172fab_to_fab (Vector<Array4CopyTag<T0, T1> >
const& copy_tags,
int scomp,
int dcomp,
173 int ncomp,
F && f, Vector<Array4Tag<int> >
const& masks)
175 using TagType = Array4MaskCopyTag<T0, T1>;
176 Vector<TagType> tags;
177 const int N = copy_tags.size();
179 for (
int i = 0; i < N; ++i) {
180 tags.push_back(TagType{.dfab = copy_tags[i].dfab, .sfab = copy_tags[i].sfab,
181 .mask = masks[i].dfab, .dbox = copy_tags[i].dbox,
182 .offset = copy_tags[i].offset});
185 if constexpr (std::is_same_v<F, CellStore<T0,T1>>)
187 fab_to_fab_store(tags, scomp, dcomp, ncomp, std::forward<F>(f));
191 fab_to_fab_other(tags, scomp, dcomp, ncomp, std::forward<F>(f));
196template <
typename T0,
typename T1>
197requires (amrex::IsStoreAtomic<T0>::value)
199fab_to_fab_atomic_cpy (Vector<Array4CopyTag<T0, T1> >
const& copy_tags,
int scomp,
200 int dcomp,
int ncomp, Vector<Array4Tag<int> >
const&)
202 fab_to_fab<T0, T1>(copy_tags, scomp, dcomp, ncomp, CellStore<T0, T1>());
205template <
typename T0,
typename T1>
206requires (!amrex::IsStoreAtomic<T0>::value)
208fab_to_fab_atomic_cpy (Vector<Array4CopyTag<T0, T1> >
const& copy_tags,
int scomp,
209 int dcomp,
int ncomp, Vector<Array4Tag<int> >
const& masks)
211 fab_to_fab(copy_tags, scomp, dcomp, ncomp, CellStore<T0, T1>(), masks);
214template <
typename T0,
typename T1>
215requires (amrex::HasAtomicAdd<T0>::value)
217fab_to_fab_atomic_add (Vector<Array4CopyTag<T0, T1> >
const& copy_tags,
int scomp,
218 int dcomp,
int ncomp, Vector<Array4Tag<int> >
const&)
220 fab_to_fab(copy_tags, scomp, dcomp, ncomp, CellAtomicAdd<T0, T1>());
223template <
typename T0,
typename T1>
224requires (!amrex::HasAtomicAdd<T0>::value)
226fab_to_fab_atomic_add (Vector<Array4CopyTag<T0, T1> >
const& copy_tags,
int scomp,
227 int dcomp,
int ncomp, Vector<Array4Tag<int> >
const& masks)
229 fab_to_fab(copy_tags, scomp, dcomp, ncomp, CellAdd<T0, T1>(), masks);
232template <
typename T0,
typename T1,
class F>
233void deterministic_fab_to_fab (Vector<Array4CopyTag<T0,T1>>
const& a_tags,
int scomp,
234 int dcomp,
int ncomp,
F const& f)
236 if (a_tags.empty()) {
return; }
238 using TagType = Array4CopyTag<T0,T1>;
242 std::pair<int,Box> dindex_tilebox;
243 bool operator< (TiledTag
const& rhs)
const noexcept {
244 return this->dindex_tilebox < rhs.dindex_tilebox;
246 bool operator!= (TiledTag
const& rhs)
const noexcept {
247 return this->dindex_tilebox != rhs.dindex_tilebox;
250 Vector<TiledTag> tiled_tags;
252 auto const ixtype = a_tags[0].dbox.ixType();
254 constexpr int tile_size = 64;
255 for (
int itag = 0; itag < a_tags.size(); ++itag) {
256 auto const& tag = a_tags[itag];
257 auto const& dlo = tag.dbox.smallEnd();
258 auto const& dhi = tag.dbox.bigEnd();
265#if (AMREX_SPACEDIM == 3)
266 for (
int kt = tlo[2]; kt <= thi[2]; ++kt)
269#if (AMREX_SPACEDIM >= 2)
270 for (
int jt = tlo[1]; jt <= thi[1]; ++jt)
273 for (
int it = tlo[0]; it <= thi[0]; ++it)
278 tiled_tags.push_back(TiledTag{
280 .dindex_tilebox = std::make_pair(tag.dindex,
Box(lo, lo+(tile_size-1), ixtype))
287 std::sort(tiled_tags.begin(), tiled_tags.end());
291 h_tags.reserve(tiled_tags.size());
293 for (
unsigned int itag = 0; itag < tiled_tags.size(); ++itag) {
295 h_ntags.push_back(0);
296 }
else if (tiled_tags[itag-1] != tiled_tags[itag]) {
297 h_ntags.push_back(itag);
299 auto const& ttag = tiled_tags[itag];
300 auto const& btag = a_tags[ttag.tag_index];
301 h_tags.push_back(TagType{.dfab = btag.dfab, .dindex = btag.dindex, .sfab = btag.sfab,
302 .dbox = btag.dbox & ttag.dindex_tilebox.second,
303 .offset = btag.offset});
305 h_ntags.push_back((
unsigned int)tiled_tags.size());
311 auto const* ptag = d_tags.data();
312 auto const* pntags = d_ntags.data();
313 auto const nblocks =
int(h_ntags.size()-1);
314 constexpr auto nthreads = 256;
318 [[sycl::reqd_work_group_size(nthreads)]]
324 Dim1 blockIdx{item.get_group_linear_id()};
325 Dim1 threadIdx{item.get_local_linear_id()};
328 for (
unsigned int itag = pntags[blockIdx.x]; itag < pntags[blockIdx.x+1]; ++itag) {
329 auto const tag = ptag[itag];
330 auto ncells =
int(tag.dbox.numPts());
333 for (
int icell =
int(threadIdx.x); icell < ncells; icell += nthreads) {
334 int k = icell / (len.x*len.y);
335 int j = (icell - k*(len.x*len.y)) / len.x;
336 int i = (icell - k*(len.x*len.y)) - j*len.x;
340 for (
int n = 0; n < ncomp; ++n) {
341 f(tag.dfab.ptr(i,j,k,n+dcomp),
342 tag.sfab(i + tag.offset.x,
344 k + tag.offset.z, n+scomp));
348 if (itag+1 < pntags[blockIdx.x+1]) {
350 sycl::group_barrier(item.get_group());
360template <
typename B,
typename V,
typename TT>
361requires (amrex::HasAtomicAdd<V>::value)
362void unpack_recv_buffer_gpu_atomic_add (
char* pbuffer, TagVector<TT>
const& tv,
363 int dcomp,
int ncomp)
365 detail::ParallelFor_doit(tv,
367 int icell,
int ncells,
int i,
int j,
int k, TT
const& tag)
noexcept
369 if (icell < ncells) {
372 for (
int n = 0; n < ncomp; ++n) {
380template <
typename B,
typename V,
typename TT>
381requires (!amrex::HasAtomicAdd<V>::value)
382void unpack_recv_buffer_gpu_atomic_add (
char* pbuffer, TagVector<TT>
const& tv,
383 int dcomp,
int ncomp)
386 amrex::Abort(
"unpack_recv_buffer_gpu: should NOT get here");
398 auto const& LocTags = *(TheFB.
m_LocTags);
399 auto N_locs =
static_cast<int>(LocTags.size());
400 if (N_locs == 0) {
return; }
405#pragma omp parallel for
407 for (
int i = 0; i < N_locs; ++i)
430 for (
int i = 0; i < N_locs; ++i)
441 loc_copy_tags[tag.
dstIndex].push_back
447 for (
MFIter mfi(*
this); mfi.isValid(); ++mfi)
449 const auto&
tags = loc_copy_tags[mfi];
450 auto dfab = this->
array(mfi);
451 for (
auto const & tag :
tags)
453 auto const sfab = tag.sfab->array();
454 const auto offset = tag.offset.dim3();
456 [=] (
int i,
int j,
int k,
int n)
noexcept
458 dfab(i,j,k,n+scomp) = sfab(i+offset.x,j+offset.y,k+offset.z,n+scomp);
469 auto const& LocTags = *(TheFB.
m_LocTags);
470 auto N_locs =
static_cast<int>(LocTags.size());
471 if (N_locs == 0) {
return; }
475 std::vector<FAB> src_fabs(N_locs);
476 for (
int itag = 0; itag < N_locs; ++itag) {
478 src_fabs[itag].resize(tag.
sbox,ncomp);
479 loc_copy_tags[tag.
dstIndex].push_back
485#pragma omp parallel for
487 for (
int itag = 0; itag < N_locs; ++itag) {
495 for (
MFIter mfi(*
this); mfi.isValid(); ++mfi)
497 const auto&
tags = loc_copy_tags[mfi];
498 const auto& dfab = this->
array(mfi);
499 for (
auto const & tag :
tags)
501 auto const sfab = tag.sfab->array();
502 const auto offset = tag.offset.dim3();
504 [&] (
int i,
int j,
int k,
int n)
noexcept
506 dfab(i,j,k,n+scomp) += sfab(i+offset.x,j+offset.y,k+offset.z,n);
518 auto const& LocTags = *(TheFB.m_LocTags);
519 int N_locs = LocTags.size();
524 if (
auto it = m_fb_local_copy_handler.find(TheFB.m_id);
525 it != m_fb_local_copy_handler.end())
527 tv = it->second.get();
530 loc_copy_tags.reserve(N_locs);
532 for (
int i = 0; i < N_locs; ++i)
540 loc_copy_tags.push_back
541 (TagType{.dfab = this->
atLocalIdx(li).array(),
548 auto utv = std::make_unique<TagVector<TagType>>(loc_copy_tags);
550 m_fb_local_copy_handler[TheFB.m_id] = std::move(utv);
560 auto const& LocTags = *(TheFB.
m_LocTags);
561 int N_locs = LocTags.size();
562 if (N_locs == 0) {
return; }
571 detail::ParallelFor_doit(*tv,
573 int icell,
int ncells,
int i,
int j,
int k, TagType
const& tag)
noexcept
575 if (icell < ncells) {
576 for (
int n = 0; n < ncomp; ++n) {
577 tag.dfab(i,j,k,n+scomp) = tag.sfab(i+tag.offset.x,
579 k+tag.offset.z,n+scomp);
587 loc_copy_tags.reserve(N_locs);
593 masks.reserve(N_locs);
595 for (
int i = 0; i < N_locs; ++i)
603 loc_copy_tags.push_back
604 (TagType{.dfab = this->
atLocalIdx(li).array(),
610 if (!maskfabs[li].isAllocated()) {
612 masks_unique.emplace_back(
Array4Tag<int>{.dfab = maskfabs[li].array()});
623 detail::fab_to_fab_atomic_cpy<value_type, value_type>(
624 loc_copy_tags, scomp, scomp, ncomp, masks);
632 auto const& LocTags = *(TheFB.
m_LocTags);
633 int N_locs = LocTags.size();
634 if (N_locs == 0) {
return; }
639 loc_copy_tags_1.reserve(N_locs);
640 loc_copy_tags_2.reserve(N_locs);
644 for (
auto const& tag : LocTags) {
645 npts += tag.sbox.numPts();
650 for (
int itag = 0; itag < N_locs; ++itag) {
654 loc_copy_tags_1.push_back(
655 TagType{.dfab = src, .dindex = -1,
657 .offset =
Dim3{.x = 0, .y = 0, .z = 0}});
658 loc_copy_tags_2.push_back(
660 .sfab = src, .dbox = tag.
dbox,
668 detail::fab_to_fab(loc_copy_tags_1, 0, 0, ncomp,
669 detail::CellStore<value_type, value_type>{});
671 if (use_deterministic) {
672 detail::deterministic_fab_to_fab(loc_copy_tags_2, 0, 0, ncomp,
673 detail::CellAdd<value_type,value_type>{});
676 detail::fab_to_fab(loc_copy_tags_2, 0, 0, ncomp,
677 detail::CellAtomicAdd<value_type, value_type>{});
696 auto const& LocTags = *(thecmd.
m_LocTags);
697 int N_locs = LocTags.size();
698 if (N_locs == 0) {
return; }
703 loc_setval_tags.reserve(N_locs);
707 for (
int i = 0; i < N_locs; ++i)
711 loc_setval_tags.push_back(TagType{.dfab = this->
array(tag.
dstIndex), .dbox = tag.
dbox});
715 [
x,scomp]
AMREX_GPU_DEVICE (
int i,
int j,
int k,
int n, TagType
const& tag)
noexcept
717 tag.dfab(i,j,k,n+scomp) =
x;
726 auto const& RcvTags = *(thecmd.
m_RcvTags);
732 for (
auto it = RcvTags.begin(); it != RcvTags.end(); ++it) {
733 for (
auto const& tag: it->second) {
734 rcv_setval_tags.push_back(TagType{.dfab = this->
array(tag.dstIndex), .dbox = tag.dbox});
738 if (rcv_setval_tags.empty()) {
return; }
743 [
x,scomp]
AMREX_GPU_DEVICE (
int i,
int j,
int k,
int n, TagType
const& tag)
noexcept
745 tag.dfab(i,j,k,n+scomp) =
x;
749#if defined(__CUDACC__) && defined (AMREX_USE_CUDA)
754 const int N_locs = (*TheFB.m_LocTags).size();
756 for (
int i = 0; i < N_locs; ++i)
758 const CopyComTag& tag = (*TheFB.m_LocTags)[i];
763 loc_copy_tags[tag.dstIndex].push_back
764 ({this->fabPtr(tag.srcIndex), tag.dbox, tag.sbox.smallEnd()-tag.dbox.smallEnd()});
768 if ( !(TheFB.m_localCopy.ready()) )
770 const_cast<FB&
>(TheFB).m_localCopy.resize(N_locs);
776 amrex::Gpu::Device::startGraphRecording( (mfi.LocalIndex() == 0),
777 const_cast<FB&
>(TheFB).m_localCopy.getHostPtr(0),
778 (TheFB).m_localCopy.getDevicePtr(0),
779 std::size_t(
sizeof(CopyMemory)*N_locs) );
781 const auto& tags = loc_copy_tags[mfi];
782 for (
auto const & tag : tags)
784 const auto offset = tag.offset.dim3();
785 CopyMemory* cmem = TheFB.m_localCopy.getDevicePtr(idx++);
789 auto const dst = cmem->getDst<value_type>();
790 auto const src = cmem->getSrc<value_type>();
791 for (int n = 0; n < cmem->ncomp; ++n) {
792 dst(i,j,k,(cmem->scomp)+n) = src(i+offset.x,j+offset.y,k+offset.z,(cmem->scomp)+n);
797 bool last_iter = mfi.LocalIndex() == (this->local_size()-1);
798 cudaGraphExec_t graphExec = amrex::Gpu::Device::stopGraphRecording(last_iter);
799 if (last_iter) {
const_cast<FB&
>(TheFB).m_localCopy.setGraph( graphExec ); }
810 auto const dst_array = this->array(mfi);
811 const auto& tags = loc_copy_tags[mfi];
812 for (
auto const & tag : tags)
814 const_cast<FB&
>(TheFB).m_localCopy.setParams(idx++, makeCopyMemory(tag.sfab->array(),
821 TheFB.m_localCopy.executeGraph();
827FabArray<FAB>::FB_local_copy_cuda_graph_n (
const FB& TheFB,
int scomp,
int ncomp)
829 const int N_locs = TheFB.m_LocTags->size();
833 for (
int i = 0; i < N_locs; ++i)
835 const CopyComTag& tag = (*TheFB.m_LocTags)[i];
837 BL_ASSERT(ParallelDescriptor::sameTeam(distributionMap[tag.dstIndex]));
838 BL_ASSERT(ParallelDescriptor::sameTeam(distributionMap[tag.srcIndex]));
840 if (distributionMap[tag.dstIndex] == ParallelDescriptor::MyProc())
842 loc_copy_tags[tag.dstIndex].push_back
843 ({this->fabPtr(tag.srcIndex), tag.dbox, tag.sbox.smallEnd()-tag.dbox.smallEnd()});
850 if ( !(TheFB.m_localCopy.ready()) )
852 const_cast<FB&
>(TheFB).m_localCopy.resize(launches);
856 for (MFIter mfi(*
this, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi)
858 const auto& tags = loc_copy_tags[mfi];
859 for (
int t = 0; t<tags.size(); ++t)
861 Gpu::Device::setStreamIndex(cuda_stream++);
862 amrex::Gpu::Device::startGraphRecording( (idx == 0),
863 const_cast<FB&
>(TheFB).m_localCopy.getHostPtr(0),
864 (TheFB).m_localCopy.getDevicePtr(0),
865 std::size_t(
sizeof(CopyMemory)*launches) );
867 const auto& tag = tags[t];
868 const Dim3 offset = tag.offset.dim3();
870 CopyMemory* cmem = TheFB.m_localCopy.getDevicePtr(idx++);
873 auto const dst = cmem->getDst<value_type>();
874 auto const src = cmem->getSrc<value_type>();
875 for (int n = 0; n < cmem->ncomp; ++n) {
876 dst(i,j,k,(cmem->scomp)+n) = src(i+offset.x,j+offset.y,k+offset.z,(cmem->scomp)+n);
880 bool last_iter = idx == launches;
881 cudaGraphExec_t graphExec = Gpu::Device::stopGraphRecording(last_iter);
882 if (last_iter) {
const_cast<FB&
>(TheFB).m_localCopy.setGraph( graphExec ); }
890 for (MFIter mfi(*
this); mfi.isValid(); ++mfi)
892 const auto& dst_array = this->array(mfi);
893 const auto& tags = loc_copy_tags[mfi];
894 for (
auto const & tag : tags)
896 const_cast<FB&
>(TheFB).m_localCopy.setParams(idx++, makeCopyMemory(tag.sfab->array(),
903 TheFB.m_localCopy.executeGraph(
false);
915#if defined(__CUDACC__) && defined(AMREX_USE_CUDA)
919FabArray<FAB>::FB_pack_send_buffer_cuda_graph (
const FB& TheFB,
int scomp,
int ncomp,
920 Vector<char*>& send_data,
921 Vector<std::size_t>
const& send_size,
922 Vector<
typename FabArray<FAB>::CopyComTagsContainer
const*>
const& send_cctc)
924 const int N_snds = send_data.size();
925 if (N_snds == 0) {
return; }
927 if ( !(TheFB.m_copyToBuffer.ready()) )
932 for (
int send = 0; send < N_snds; ++send) {
933 if (send_size[send] > 0) {
934 launches += send_cctc[send]->size();
937 const_cast<FB&
>(TheFB).m_copyToBuffer.resize(launches);
941 for (Gpu::StreamIter sit(N_snds,Gpu::StreamItInfo().DisableDeviceSync());
942 sit.isValid(); ++sit)
944 amrex::Gpu::Device::startGraphRecording( (sit() == 0),
945 const_cast<FB&
>(TheFB).m_copyToBuffer.getHostPtr(0),
946 (TheFB).m_copyToBuffer.getDevicePtr(0),
947 std::size_t(
sizeof(CopyMemory)*launches) );
950 if (send_size[j] > 0)
952 auto const& cctc = *send_cctc[j];
953 for (
auto const& tag : cctc)
955 const Box& bx = tag.sbox;
956 CopyMemory* cmem = TheFB.m_copyToBuffer.getDevicePtr(idx++);
959 auto const pfab = cmem->getDst<value_type>();
960 auto const sfab = cmem->getSrc<value_type>();
961 for (
int n = 0; n < cmem->ncomp; ++n)
963 pfab(ii,jj,kk,n) = sfab(ii,jj,kk,n+(cmem->scomp));
969 bool last_iter = sit() == (N_snds-1);
970 cudaGraphExec_t graphExec = amrex::Gpu::Device::stopGraphRecording(last_iter);
971 if (last_iter) {
const_cast<FB&
>(TheFB).m_copyToBuffer.setGraph( graphExec ); }
977 for (
int send = 0; send < N_snds; ++send)
980 if (send_size[j] > 0)
982 char* dptr = send_data[j];
983 auto const& cctc = *send_cctc[j];
984 for (
auto const& tag : cctc)
986 const_cast<FB&
>(TheFB).m_copyToBuffer.setParams(idx++, makeCopyMemory(this->array(tag.srcIndex),
992 dptr += (tag.sbox.numPts() * ncomp *
sizeof(value_type));
995 BL_ASSERT(dptr <= send_data[j] + send_size[j]);
1000 TheFB.m_copyToBuffer.executeGraph();
1005FabArray<FAB>::FB_unpack_recv_buffer_cuda_graph (
const FB& TheFB,
int dcomp,
int ncomp,
1006 Vector<char*>
const& recv_data,
1007 Vector<std::size_t>
const& recv_size,
1008 Vector<CopyComTagsContainer const*>
const& recv_cctc,
1011 const int N_rcvs = recv_cctc.size();
1012 if (N_rcvs == 0) {
return; }
1016 for (
int k = 0; k < N_rcvs; ++k)
1018 if (recv_size[k] > 0)
1020 const char* dptr = recv_data[k];
1021 auto const& cctc = *recv_cctc[k];
1022 for (
auto const& tag : cctc)
1024 recv_copy_tags[tag.dstIndex].push_back(VoidCopyTag{.p = dptr, .dbox = tag.dbox});
1025 dptr += tag.dbox.numPts() * ncomp *
sizeof(value_type);
1029 BL_ASSERT(dptr <= recv_data[k] + recv_size[k]);
1033 if ( !(TheFB.m_copyFromBuffer.ready()) )
1035 const_cast<FB&
>(TheFB).m_copyFromBuffer.resize(launches);
1038 for (MFIter mfi(*
this, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi)
1040 amrex::Gpu::Device::startGraphRecording( (mfi.LocalIndex() == 0),
1041 const_cast<FB&
>(TheFB).m_copyFromBuffer.getHostPtr(0),
1042 (TheFB).m_copyFromBuffer.getDevicePtr(0),
1043 std::size_t(
sizeof(CopyMemory)*launches) );
1045 const auto& tags = recv_copy_tags[mfi];
1046 for (
auto const & tag : tags)
1048 CopyMemory* cmem = TheFB.m_copyFromBuffer.getDevicePtr(idx++);
1051 auto const pfab = cmem->getSrc<value_type>();
1052 auto const dfab = cmem->getDst<value_type>();
1053 for (int n = 0; n < cmem->ncomp; ++n)
1055 dfab(i,j,k,n+(cmem->scomp)) = pfab(i,j,k,n);
1060 bool last_iter = mfi.LocalIndex() == (this->local_size()-1);
1061 cudaGraphExec_t graphExec = amrex::Gpu::Device::stopGraphRecording(last_iter);
1062 if (last_iter) {
const_cast<FB&
>(TheFB).m_copyFromBuffer.setGraph( graphExec ); }
1068 for (MFIter mfi(*
this); mfi.isValid(); ++mfi)
1070 auto dst_array = this->array(mfi);
1071 const auto & tags = recv_copy_tags[mfi];
1072 for (
auto const & tag : tags)
1074 const_cast<FB&
>(TheFB).m_copyFromBuffer.setParams(idx++, makeCopyMemory(
amrex::makeArray4((value_type*)(tag.p),
1083 TheFB.m_copyFromBuffer.executeGraph();
1089template <
typename BUF>
1094 int ncomp, std::uint64_t
id)
const
1099 auto kit = std::find_if(send_cctc.begin(), send_cctc.end(),
1101 if (kit == send_cctc.end()) {
1108 char* pbuf = send_data[0];
1109 const int N_snds = send_data.
size();
1110 for (
int j = 0; j < N_snds; ++j)
1112 if (send_size[j] > 0)
1114 char* dptr = send_data[j];
1115 auto const& cctc = *send_cctc[j];
1116 for (
auto const& tag : cctc)
1118 snd_copy_tags.emplace_back
1119 (TagType{.sfab = this->
const_array(tag.srcIndex), .poff = dptr-pbuf, .bx = tag.sbox});
1120 dptr += (tag.sbox.numPts() * ncomp *
sizeof(BUF));
1124 return snd_copy_tags;
1128 std::tuple<std::uint64_t,std::size_t,int> key{id,
sizeof(BUF), ncomp};
1130 if (
auto it = m_send_copy_handler.find(key); it != m_send_copy_handler.end()) {
1131 tv = it->second.get();
1133 if (m_send_copy_handler.size() > 32) {
1138 m_send_copy_handler.clear();
1140 auto snd_copy_tags = get_tags();
1141 auto utv = std::make_unique<TagVector<TagType>>(snd_copy_tags);
1143 m_send_copy_handler[key] = std::move(utv);
1150template <
typename BUF>
1158 const int N_snds = send_data.
size();
1159 if (N_snds == 0) {
return; }
1164 (send_data, send_size, send_cctc, ncomp,
id);
1165 if (tv ==
nullptr) {
return; }
1167 char* pbuffer = send_data[0];
1169 detail::ParallelFor_doit(*tv,
1171 int icell,
int ncells,
int i,
int j,
int k, TagType
const& tag)
noexcept
1173 if (icell < ncells) {
1176 for (
int n = 0; n < ncomp; ++n) {
1177 dfab(i,j,k,n) = (BUF)tag.sfab(i,j,k,n+scomp);
1186template <
typename BUF>
1191 int ncomp, std::uint64_t
id)
1196 auto kit = std::find_if(recv_cctc.begin(), recv_cctc.end(),
1198 if (kit == recv_cctc.end()) {
1205 char* pbuf = recv_data[0];
1206 const int N_rcvs = recv_cctc.
size();
1207 for (
int k = 0; k < N_rcvs; ++k)
1209 if (recv_size[k] > 0)
1211 char* dptr = recv_data[k];
1212 auto const& cctc = *recv_cctc[k];
1213 for (
auto const& tag : cctc)
1215 const int li = this->
localindex(tag.dstIndex);
1216 recv_copy_tags.emplace_back
1217 (TagType{.dfab = this->
atLocalIdx(li).array(), .poff = dptr-pbuf, .bx = tag.dbox});
1218 dptr += tag.dbox.numPts() * ncomp *
sizeof(BUF);
1222 return recv_copy_tags;
1226 std::tuple<std::uint64_t,std::size_t,int> key{id,
sizeof(BUF), ncomp};
1228 if (
auto it = m_recv_copy_handler.find(key); it != m_recv_copy_handler.end()) {
1229 tv = it->second.get();
1231 if (m_recv_copy_handler.size() > 32) {
1236 m_recv_copy_handler.clear();
1238 auto recv_copy_tags = get_tags();
1239 auto utv = std::make_unique<TagVector<TagType>>(recv_copy_tags);
1241 m_recv_copy_handler[key] = std::move(utv);
1248template <
typename BUF>
1254 CpOp op,
bool is_thread_safe, std::uint64_t
id,
1257 const int N_rcvs = recv_cctc.
size();
1258 if (N_rcvs == 0) {
return; }
1261 if (!is_thread_safe)
1275 tags.reserve(N_rcvs);
1276 for (
int k = 0; k < N_rcvs; ++k) {
1277 if (recv_size[k] > 0) {
1278 char const* dptr = recv_data[k];
1279 auto const& cctc = *recv_cctc[k];
1280 for (
auto const& tag : cctc) {
1282 TagType{.dfab = dst.
array(tag.dstIndex), .dindex = tag.dstIndex,
1287 .offset =
Dim3{.x = 0, .y = 0, .z = 0}});
1288 dptr += tag.dbox.numPts() * ncomp *
sizeof(BUF);
1293 detail::deterministic_fab_to_fab<value_type,BUF>
1294 (
tags, 0, dcomp, ncomp, detail::CellAdd<value_type,BUF>{});
1303 (recv_data, recv_size, recv_cctc, ncomp,
id);
1304 if (tv ==
nullptr) {
return; }
1306 char* pbuffer = recv_data[0];
1310 detail::ParallelFor_doit(*tv,
1312 int icell,
int ncells,
int i,
int j,
int k, TagType
const& tag)
noexcept
1314 if (icell < ncells) {
1317 for (
int n = 0; n < ncomp; ++n) {
1318 tag.dfab(i,j,k,n+dcomp) = (
value_type)sfab(i,j,k,n);
1325 if (is_thread_safe) {
1326 detail::ParallelFor_doit(*tv,
1328 int icell,
int ncells,
int i,
int j,
int k, TagType
const& tag)
noexcept
1330 if (icell < ncells) {
1333 for (
int n = 0; n < ncomp; ++n) {
1334 tag.dfab(i,j,k,n+dcomp) += (
value_type)sfab(i,j,k,n);
1339 detail::unpack_recv_buffer_gpu_atomic_add<BUF, value_type>
1340 (pbuffer, *tv, dcomp, ncomp);
1350 char* pbuffer = recv_data[0];
1354 recv_copy_tags.reserve(N_rcvs);
1361 for (
int k = 0; k < N_rcvs; ++k)
1363 if (recv_size[k] > 0)
1365 std::size_t offset = recv_data[k]-recv_data[0];
1366 const char* dptr = pbuffer + offset;
1367 auto const& cctc = *recv_cctc[k];
1368 for (
auto const& tag : cctc)
1371 recv_copy_tags.emplace_back(TagType{
1372 .dfab = dst.
atLocalIdx(li).array(), .dindex = tag.dstIndex,
1375 .offset =
Dim3{.x = 0, .y = 0, .z = 0}
1377 dptr += tag.dbox.numPts() * ncomp *
sizeof(BUF);
1379 if (!maskfabs[li].isAllocated()) {
1380 maskfabs[li].resize(dst.
atLocalIdx(li).box());
1381 masks_unique.emplace_back(
Array4Tag<int>{.dfab = maskfabs[li].array()});
1383 masks.emplace_back(
Array4Tag<int>{.dfab = maskfabs[li].array()});
1385 BL_ASSERT(dptr <= pbuffer + offset + recv_size[k]);
1392 msk.
dfab(i,j,k) = 0;
1397 detail::fab_to_fab_atomic_cpy<value_type, BUF>(
1398 recv_copy_tags, 0, dcomp, ncomp, masks);
1402 detail::fab_to_fab_atomic_add<value_type, BUF>(
1403 recv_copy_tags, 0, dcomp, ncomp, masks);
1413template <
typename BUF>
1422 auto const N_snds =
static_cast<int>(send_data.
size());
1423 if (N_snds == 0) {
return; }
1426 if (omp_get_max_threads() > 1)
1430 for (
int j = 0; j < N_snds; ++j)
1432 if (send_size[j] > 0)
1434 char* dptr = send_data[j];
1435 for (
auto const& tag : *send_cctc[j])
1437 tags.emplace_back(dptr, &tag);
1438 dptr += (tag.sbox.numPts() * ncomp *
sizeof(BUF));
1440 BL_ASSERT(dptr <= send_data[j] + send_size[j]);
1444 auto const N_tags =
static_cast<int>(
tags.size());
1446 int const chunk = std::clamp(N_tags/omp_get_max_threads(), 1, 64);
1447#pragma omp parallel for schedule(static,chunk)
1448 for (
int i = 0; i < N_tags; ++i)
1450 auto const& tag = *
tags[i].second;
1451 const Box& bx = tag.sbox;
1452 auto const sfab = src.
array(tag.srcIndex);
1455 [=] (
int ii,
int jj,
int kk,
int n)
noexcept
1457 pfab(ii,jj,kk,n) =
static_cast<BUF
>(sfab(ii,jj,kk,n+scomp));
1464 for (
int j = 0; j < N_snds; ++j)
1466 if (send_size[j] > 0)
1468 char* dptr = send_data[j];
1469 auto const& cctc = *send_cctc[j];
1470 for (
auto const& tag : cctc)
1472 const Box& bx = tag.sbox;
1473 auto const sfab = src.
array(tag.srcIndex);
1476 [=] (
int ii,
int jj,
int kk,
int n)
noexcept
1478 pfab(ii,jj,kk,n) =
static_cast<BUF
>(sfab(ii,jj,kk,n+scomp));
1480 dptr += (bx.
numPts() * ncomp *
sizeof(BUF));
1482 BL_ASSERT(dptr <= send_data[j] + send_size[j]);
1488template <
typename BUF>
1494 CpOp op,
bool is_thread_safe)
1498 auto const N_rcvs =
static_cast<int>(recv_cctc.
size());
1499 if (N_rcvs == 0) {
return; }
1504 if (omp_get_max_threads() > 1)
1508 for (
int k = 0; k < N_rcvs; ++k)
1510 if (recv_size[k] > 0)
1512 const char* dptr = recv_data[k];
1513 for (
auto const& tag : *recv_cctc[k])
1515 tags.emplace_back(dptr, &tag);
1516 dptr += tag.dbox.numPts() * ncomp *
sizeof(BUF);
1518 BL_ASSERT(dptr <= recv_data[k] + recv_size[k]);
1522 auto const N_tags =
static_cast<int>(
tags.size());
1524 int const chunk = std::clamp(N_tags/omp_get_max_threads(), 1, 64);
1525#pragma omp parallel for schedule(static,chunk)
1526 for (
int i = 0; i < N_tags; ++i)
1528 auto const& tag = *
tags[i].second;
1529 FAB& dfab = dst[tag.dstIndex];
1532 dfab.template copyFromMem<RunOn::Host, BUF>(tag.dbox, dcomp, ncomp,
tags[i].first);
1536 dfab.template addFromMem<RunOn::Host, BUF>(tag.dbox, dcomp, ncomp,
tags[i].first);
1543 for (
int k = 0; k < N_rcvs; ++k)
1545 if (recv_size[k] > 0)
1547 const char* dptr = recv_data[k];
1548 auto const& cctc = *recv_cctc[k];
1549 for (
auto const& tag : cctc)
1551 const Box& bx = tag.dbox;
1552 FAB& dfab = dst[tag.dstIndex];
1555 dfab.template copyFromMem<RunOn::Host, BUF>(bx, dcomp, ncomp, dptr);
1559 dfab.template addFromMem<RunOn::Host, BUF>(tag.dbox, dcomp, ncomp, dptr);
1561 dptr += bx.
numPts() * ncomp *
sizeof(BUF);
1563 BL_ASSERT(dptr <= recv_data[k] + recv_size[k]);
1571 for (
int k = 0; k < N_rcvs; ++k)
1573 if (recv_size[k] > 0)
1575 const char* dptr = recv_data[k];
1576 auto const& cctc = *recv_cctc[k];
1577 for (
auto const& tag : cctc)
1579 recv_copy_tags[tag.dstIndex].push_back(
VoidCopyTag{.p = dptr, .dbox = tag.dbox});
1580 dptr += tag.dbox.numPts() * ncomp *
sizeof(BUF);
1582 BL_ASSERT(dptr <= recv_data[k] + recv_size[k]);
1589 for (
MFIter mfi(dst); mfi.isValid(); ++mfi)
1591 const auto&
tags = recv_copy_tags[mfi];
1592 auto dfab = dst.
array(mfi);
1593 for (
auto const & tag :
tags)
1599 [=] (
int i,
int j,
int k,
int n)
noexcept
1601 dfab(i,j,k,n+dcomp) = pfab(i,j,k,n);
1607 [=] (
int i,
int j,
int k,
int n)
noexcept
1609 dfab(i,j,k,n+dcomp) += pfab(i,j,k,n);
#define BL_ASSERT(EX)
Definition AMReX_BLassert.H:39
#define AMREX_ALWAYS_ASSERT(EX)
Definition AMReX_BLassert.H:50
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
#define AMREX_HOST_DEVICE_FOR_3D(...)
Definition AMReX_GpuLaunchMacrosC.nolint.H:106
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
while parser_exe_t p
Definition AMReX_Parser_Exe_Body.H:33
#define AMREX_D_DECL(a, b, c)
Definition AMReX_SPACE.H:171
virtual void free(void *pt)=0
Free a previously allocated block pointed to by pt.
virtual void * alloc(std::size_t sz)=0
Allocate sz bytes from this arena.
__host__ __device__ Long numPts() const noexcept
Return the number of points contained in the BoxND.
Definition AMReX_Box.H:385
__host__ __device__ const IntVectND< dim > & smallEnd() const &noexcept
Return the inclusive lower bound of the box.
Definition AMReX_Box.H:124
friend class MFIter
Definition AMReX_FabArrayBase.H:48
CopyComTag::CopyComTagsContainer CopyComTagsContainer
Definition AMReX_FabArrayBase.H:236
int localindex(int K) const noexcept
Return local index in the vector of FABs.
Definition AMReX_FabArrayBase.H:124
const DistributionMapping & DistributionMap() const noexcept
Return constant reference to associated DistributionMapping.
Definition AMReX_FabArrayBase.H:135
int local_size() const noexcept
Return the number of local FABs in the FabArray.
Definition AMReX_FabArrayBase.H:118
CpOp
parallel copy or add
Definition AMReX_FabArrayBase.H:432
@ ADD
Definition AMReX_FabArrayBase.H:432
@ COPY
Definition AMReX_FabArrayBase.H:432
Box box(int K) const noexcept
Return the Kth Box in the BoxArray. That is, the valid region of the Kth grid.
Definition AMReX_FabArrayBase.H:106
DistributionMapping distributionMap
Definition AMReX_FabArrayBase.H:536
const BoxArray & boxArray() const noexcept
Return a constant reference to the BoxArray that defines the valid region associated with this FabArr...
Definition AMReX_FabArrayBase.H:100
An Array of FortranArrayBox(FAB)-like Objects.
Definition AMReX_FabArray.H:383
TagVector< CommSendBufTag< value_type > > const * get_send_copy_tag_vector(Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< CopyComTagsContainer const * > const &send_cctc, int ncomp, std::uint64_t id) const
typename std::conditional_t< IsBaseFab< FAB >::value, FAB, FABType >::value_type value_type
Definition AMReX_FabArray.H:394
void CMD_remote_setVal_gpu(value_type x, const CommMetaData &thecmd, int scomp, int ncomp)
Definition AMReX_FBI.H:723
static void unpack_recv_buffer_gpu(FabArray< FAB > &dst, int dcomp, int ncomp, Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< const CopyComTagsContainer * > const &recv_cctc, CpOp op, bool is_thread_safe, std::uint64_t id, bool deterministic)
Unpack received messages into dst on the GPU.
void FB_local_add_cpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:467
static void unpack_recv_buffer_cpu(FabArray< FAB > &dst, int dcomp, int ncomp, Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< const CopyComTagsContainer * > const &recv_cctc, CpOp op, bool is_thread_safe)
void FB_local_add_gpu(const FB &TheFB, int scomp, int ncomp, bool deterministic)
Definition AMReX_FBI.H:630
void FB_local_copy_gpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:558
void copy(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition AMReX_FabArray.H:1005
const Vector< std::string > & tags() const noexcept
Memory-usage tags recorded for this FabArray's allocations.
Definition AMReX_FabArray.H:516
void CMD_local_setVal_gpu(value_type x, const CommMetaData &thecmd, int scomp, int ncomp)
Definition AMReX_FBI.H:693
static void pack_send_buffer_gpu(FabArray< FAB > const &src, int scomp, int ncomp, Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< const CopyComTagsContainer * > const &send_cctc, std::uint64_t id)
Array4< typename FabArray< FAB >::value_type const > array(const MFIter &mfi) const noexcept
Read-only Array4 view for the FAB referenced by iterator mfi.
Definition AMReX_FabArray.H:649
void FB_local_copy_cpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:396
TagVector< Array4CopyTag< value_type > > const * FB_get_local_copy_tag_vector(const FB &TheFB)
Definition AMReX_FBI.H:515
static void pack_send_buffer_cpu(FabArray< FAB > const &src, int scomp, int ncomp, Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< const CopyComTagsContainer * > const &send_cctc)
FAB * fabPtr(const MFIter &mfi) noexcept
Return pointer to FAB.
Definition AMReX_FabArray.H:2118
TagVector< CommRecvBufTag< value_type > > const * get_recv_copy_tag_vector(Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< CopyComTagsContainer const * > const &recv_cctc, int ncomp, std::uint64_t id)
FAB & atLocalIdx(int L) noexcept
Return a reference to the FAB associated with local index L.
Definition AMReX_FabArray.H:607
Array4< typename FabArray< FAB >::value_type const > const_array(const MFIter &mfi) const noexcept
Synonym for array(const MFIter&) that highlights read-only semantics.
Definition AMReX_FabArray.H:677
a one-thingy-per-box distributed object
Definition AMReX_LayoutData.H:13
void define(const BoxArray &a_grids, const DistributionMapping &a_dm)
Definition AMReX_LayoutData.H:24
Iterator for looping ever tiles and boxes of amrex::FabArray based containers.
Definition AMReX_MFIter.H:88
bool isValid() const noexcept
Is the iterator valid i.e. is it associated with a FAB?
Definition AMReX_MFIter.H:176
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
Long size() const noexcept
Definition AMReX_Vector.H:54
amrex_long Long
Definition AMReX_INT.H:30
__host__ __device__ Dim3 length(Array4< T > const &a) noexcept
Return the spatial extents of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1391
ArrayND< T, 4, true > Array4
Convenient alias for a 4D ArrayND with three spatial dimensions plus a component dimension.
Definition AMReX_Array4.H:1352
__host__ __device__ Dim3 lbound(Array4< T > const &a) noexcept
Return the inclusive lower bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1363
__host__ __device__ BoxND< dim > coarsen(const BoxND< dim > &b, int ref_ratio) noexcept
Return a copy of b coarsened by the isotropic ratio ref_ratio.
Definition AMReX_Box.H:1471
PinnedVector< T > HostVector
A PODVector that uses pinned host memory. Same as PinnedVector. For a vector class that uses std::all...
Definition AMReX_GpuContainers.H:67
PODVector< T, ArenaAllocator< T > > DeviceVector
A PODVector that uses the standard memory Arena. Note that the memory might or might not be managed d...
Definition AMReX_GpuContainers.H:24
Arena * The_Arena()
Definition AMReX_Arena.cpp:839
int MyProc() noexcept
Definition AMReX_ParallelDescriptor.H:128
__host__ __device__ AMREX_FORCE_INLINE T Exch(T *address, T val) noexcept
Definition AMReX_GpuAtomic.H:542
__host__ __device__ AMREX_FORCE_INLINE T CAS(T *const address, T compare, T const val) noexcept
Definition AMReX_GpuAtomic.H:568
__host__ __device__ AMREX_FORCE_INLINE void AddNoRet(T *sum, T value) noexcept
Definition AMReX_GpuAtomic.H:322
__host__ __device__ AMREX_FORCE_INLINE T Add(T *sum, T value) noexcept
Definition AMReX_GpuAtomic.H:239
void freeAsync(Arena *arena, void *mem) noexcept
Definition AMReX_GpuDevice.H:364
void copyAsync(HostToDevice, InIter begin, InIter end, OutIter result) noexcept
A host-to-device copy routine. Note this is just a wrapper around memcpy, so it assumes contiguous st...
Definition AMReX_GpuContainers.H:228
static constexpr HostToDevice hostToDevice
Definition AMReX_GpuContainers.H:105
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
bool inNoSyncRegion() noexcept
Definition AMReX_GpuControl.H:148
gpuStream_t gpuStream() noexcept
Definition AMReX_GpuDevice.H:310
bool sameTeam(int rank) noexcept
Definition AMReX_ParallelDescriptor.H:345
int TeamSize() noexcept
Definition AMReX_ParallelDescriptor.H:310
const ProcessTeam & MyTeam() noexcept
Definition AMReX_ParallelDescriptor.H:365
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
@ F
Definition AMReX_Order.H:7
__host__ __device__ Array4< T > makeArray4(T *p, Box const &bx, int ncomp) noexcept
Definition AMReX_BaseFab.H:137
BoxArray const & boxArray(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.boxArray().
Definition AMReX_FabArrayBase.cpp:3038
DistributionMapping const & DistributionMap(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.DistributionMap().
Definition AMReX_FabArrayBase.cpp:3043
__host__ __device__ Dim3 begin(BoxND< dim > const &box) noexcept
Return the iterator begin coordinate of box as Dim3.
Definition AMReX_Box.H:2241
bool NeighborParticleContainer_impl< T_ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor >::use_mask
Definition AMReX_NeighborParticlesI.H:13
void launch(T const &n, L const &f) noexcept
Definition AMReX_GpuLaunchFunctsC.H:122
void ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:202
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
@ x
Definition AMReX_Orientation.H:14
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
void LoopConcurrentOnCpu(Dim3 lo, Dim3 hi, F const &f) noexcept
Definition AMReX_Loop.H:388
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
const int[]
Definition AMReX_BLProfiler.cpp:1665
__host__ __device__ Dim3 end(BoxND< dim > const &box) noexcept
Return the iterator end coordinate of box as Dim3.
Definition AMReX_Box.H:2259
__host__ __device__ constexpr int get(IntVectND< dim > const &iv) noexcept
Get the Idx-th element of IntVectND<dim>.
Definition AMReX_IntVect.H:1367
Definition AMReX_TagParallelFor.H:58
Definition AMReX_TagParallelFor.H:26
Definition AMReX_TagParallelFor.H:50
Array4< T > dfab
Definition AMReX_TagParallelFor.H:51
Definition AMReX_TagParallelFor.H:106
Definition AMReX_TagParallelFor.H:116
A simple struct holding 3 int values for a 3D index.
Definition AMReX_Dim3.H:24
Used by a bunch of routines when communicating via MPI.
Definition AMReX_FabArrayBase.H:211
Box sbox
Definition AMReX_FabArrayBase.H:213
int srcIndex
Definition AMReX_FabArrayBase.H:215
Box dbox
Definition AMReX_FabArrayBase.H:212
int dstIndex
Definition AMReX_FabArrayBase.H:214
FillBoundary.
Definition AMReX_FabArrayBase.H:588
Describes a source FAB region participating in a copy.
Definition AMReX_FBI.H:18
IntVect offset
Definition AMReX_FBI.H:21
Box dbox
Definition AMReX_FBI.H:20
FAB const * sfab
Definition AMReX_FBI.H:19
Definition AMReX_TypeTraits.H:51
Definition AMReX_TypeTraits.H:61
Definition AMReX_TypeTraits.H:277
Definition AMReX_MFIter.H:20
void MemoryBarrier() const
memory fence
Definition AMReX_ParallelDescriptor.H:161
Definition AMReX_TagParallelFor.H:156
Lightweight tag describing raw pointer copies (used for pack/unpack).
Definition AMReX_FBI.H:25
Box dbox
Definition AMReX_FBI.H:27
char const * p
Definition AMReX_FBI.H:26