Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_NonLocalBC.H
Go to the documentation of this file.
1#ifndef AMREX_NONLOCAL_BC_H_
2#define AMREX_NONLOCAL_BC_H_
3#include <AMReX_Config.H>
4#include <AMReX_Concepts.H>
5#include <AMReX_TypeTraits.H>
6#include <AMReX_FabArray.H>
7#include <AMReX_FArrayBox.H>
8
9namespace amrex::NonLocalBC {
10
12// [concept.IndexMapping]
13//
14
16template <typename T, typename... Args>
17using Inverse_t = decltype(std::declval<T>().Inverse(std::declval<Args>()...));
18
20template <typename T>
21struct HasInverseMemFn : IsDetectedExact<Dim3, Inverse_t, T, Dim3> {};
22
25template <class IndexMap>
27 : Conjunction<IsCallableR<Dim3, IndexMap&, const Dim3&>,
28 HasInverseMemFn<const IndexMap&>> {};
29
37template <class IndexMap>
38struct IsMonotoneIndexMapping : std::false_type {};
39
41// [IndexMapping.MultiBlockIndexMapping]
42//
43
55
62 [[nodiscard]] AMREX_GPU_HOST_DEVICE Dim3 operator()(Dim3 i) const noexcept {
63 int iv[3]{i.x, i.y, i.z};
64 int iv_new[3]{};
65 for (int d = 0; d < AMREX_SPACEDIM; ++d) {
66 iv_new[d] = sign[d] * (iv[permutation[d]] - offset[d]);
67 }
68 return Dim3{.x = iv_new[0], .y = iv_new[1], .z = iv_new[2]};
69 }
70
77 [[nodiscard]] AMREX_GPU_HOST_DEVICE Dim3 Inverse(Dim3 i) const noexcept {
78 int iv_new[3]{i.x, i.y, i.z};
79 int iv[3]{};
80 for (int d = 0; d < AMREX_SPACEDIM; ++d) {
81 AMREX_ASSERT(sign[d] == 1 || sign[d] == -1);
82 iv[permutation[d]] = iv_new[d] * sign[d] + offset[d];
83 }
84 return Dim3{.x = iv[0], .y = iv[1], .z = iv[2]};
85 }
86
87 [[nodiscard]] IndexType operator()(IndexType it) const noexcept {
88 return IndexType{IntVect{AMREX_D_DECL(it[permutation[0]], it[permutation[1]], it[permutation[2]])}};
89 }
90
91 [[nodiscard]] IndexType Inverse(IndexType it) const noexcept {
92 IntVect inverse_permutation;
93 for (int i = 0; i < AMREX_SPACEDIM; ++i) {
94 inverse_permutation[permutation[i]] = i;
95 }
96 return IndexType{IntVect{AMREX_D_DECL(it[inverse_permutation[0]], it[inverse_permutation[1]], it[inverse_permutation[2]])}};
97 }
98};
99
112template <typename DTOS>
115Apply(DTOS const& dtos, const IntVect& iv)
116{
117 Dim3 i = dtos(iv.dim3());
118 return IntVect{AMREX_D_DECL(i.x, i.y, i.z)};
119}
120
130template <typename DTOS>
132Box
133Image (DTOS const& dtos, const Box& box)
134{
135 IntVect mapped_smallEnd = Apply(dtos, box.smallEnd());
136 IntVect mapped_bigEnd = Apply(dtos, box.bigEnd());
137 IntVect smallEnd;
138 IntVect bigEnd;
139 for (int d = 0; d < AMREX_SPACEDIM; ++d) {
140 smallEnd[d] = std::min(mapped_smallEnd[d], mapped_bigEnd[d]);
141 bigEnd[d] = std::max(mapped_smallEnd[d], mapped_bigEnd[d]);
142 }
143 return Box{smallEnd, bigEnd, box.ixType()};
144}
145
155template <typename DTOS>
157Box
158Image (DTOS const& dtos, const Box& box)
159{
160 // "Forget" the index type mapping and invoke Image without changing the index type.
161 Box srcbox = Image([&dtos](Dim3 d) { return dtos(d); }, box);
162 // Fix the index type of the resulting box
163 srcbox.setType(dtos(box.ixType()));
164 return srcbox;
165}
166
179template <typename DTOS>
180requires (HasInverseMemFn<DTOS>::value)
182ApplyInverse(DTOS const& dtos, const IntVect& iv)
183{
184 return Apply([&dtos](Dim3 i) { return dtos.Inverse(i); }, iv);
185}
186
196template <typename DTOS>
197requires (HasInverseMemFn<DTOS>::value && !IsCallableR<IndexType, DTOS, IndexType>::value)
198Box
199InverseImage (DTOS const& dtos, const Box& box)
200{
201 return Image([&dtos](Dim3 i) { return dtos.Inverse(i); }, box);
202}
203
213template <typename DTOS>
214requires (HasInverseMemFn<DTOS>::value && IsCallableR<IndexType, DTOS, IndexType>::value)
215Box
216InverseImage (DTOS const& dtos, const Box& box)
217{
218 return Image([&dtos](auto&& i) { return dtos.Inverse(i); }, box);
219}
220
222static_assert(IsIndexMapping<MultiBlockIndexMapping>(), // NOLINT(bugprone-throw-keyword-missing)
223 "MultiBlockIndexMapping is expected to satisfy IndexMapping");
224
226template <> struct IsMonotoneIndexMapping<MultiBlockIndexMapping> : std::true_type {};
227
229// [class.MultiBlockCommMetaData]
230//
231
238
242 template <typename DTOS>
243 requires (IsIndexMapping<DTOS>::value)
244 MultiBlockCommMetaData(const FabArrayBase& dst, const Box& dstbox, const FabArrayBase& src,
245 const IntVect& ngrow, DTOS const& dtos);
246
250 template <typename DTOS>
253 const Box& dstbox, const BoxArray& srcba,
254 const DistributionMapping& srcdm, const IntVect& ngrow, DTOS const& dtos);
255
257
281 template <typename DTOS>
283 void define(const BoxArray& dstba, const DistributionMapping& dstdm, const Box& dstbox,
284 const BoxArray& srcba, const DistributionMapping& srcdm, const IntVect& ngrow,
285 DTOS const& dtos);
286};
287
289// [concept.FabProjection]
290//
291
296template <typename P, BaseFabType FAB>
298 : Disjunction<
299 IsCallableR<typename FAB::value_type, P, Array4<const typename FAB::value_type>, Dim3, int>,
300 IsCallableR<typename FAB::value_type, P, Array4<const typename FAB::value_type>, Dim3, int, Dim3>>
301{};
302
303namespace detail {
305 template <typename Proj, typename T>
307 T call_projection (Proj const& proj, Array4<T const> const& src, Dim3 const& src_index, int comp,
308 Dim3 const& dst_index) noexcept
309 {
310 if constexpr (IsCallableR<T, Proj, Array4<T const>, Dim3, int, Dim3>::value) {
311 return proj(src, src_index, comp, dst_index);
312 } else {
313 amrex::ignore_unused(dst_index);
314 return proj(src, src_index, comp);
315 }
316 }
317}
318
320// [FabProjection.Identity]
321// [IndexMapping.Identity]
322
326struct Identity {
328 constexpr Dim3 operator()(Dim3 i) const noexcept { return i; }
330 [[nodiscard]] static constexpr Dim3 Inverse(Dim3 i) noexcept { return i; }
331
333 template <typename T>
334 constexpr T operator()(Array4<const T> array, Dim3 i, int comp = 0) const
335 noexcept(noexcept(array(i.x, i.y, i.z, comp))) {
336 return array(i.x, i.y, i.z, comp);
337 }
338
340 constexpr int operator()(int i) const noexcept { return i; }
341};
342static constexpr Identity identity{};
343
344static_assert(sizeof(Identity) == 1 );
345static_assert(std::is_trivially_default_constructible_v<Identity> );
346static_assert(std::is_trivially_copy_assignable_v<Identity> );
347static_assert(std::is_trivially_copy_constructible_v<Identity> );
348static_assert(IsIndexMapping<Identity>() ); // NOLINT(bugprone-throw-keyword-missing)
349static_assert(IsFabProjection<Identity, FArrayBox>() ); // NOLINT(bugprone-throw-keyword-missing)
350
352template <> struct IsMonotoneIndexMapping<Identity> : std::true_type {};
353
355// [FabProjection.MapComponents]
356
363template <typename Base, typename Map = Identity> struct MapComponents {
364 static_assert(IsCallable<Base, Array4<const Real>, Dim3, int>::value,
365 "Base needs to be a callable function: (Array4<const T>, Dim3, i) -> auto.");
366
368 "Map needs to be a callable function: int -> int.");
369
370 Base base;
371 Map map;
372
373 template <typename T>
375 constexpr decltype(auto) operator()(Array4<const T> array, Dim3 i, int comp) const
376 noexcept(noexcept(base(array, i, map(comp))))
377 {
378 return base(array, i, map(comp));
379 }
380};
381
382static_assert(std::is_trivially_copy_assignable<MapComponents<Identity>>() ); // NOLINT(bugprone-throw-keyword-missing)
383static_assert(std::is_trivially_copy_constructible<MapComponents<Identity>>() ); // NOLINT(bugprone-throw-keyword-missing)
384static_assert(IsFabProjection<MapComponents<Identity>, FArrayBox>() ); // NOLINT(bugprone-throw-keyword-missing)
385
387// [FabProjection.MapComponents.SwapComponents]
388
394template <int CompI, int CompJ> struct SwapComponents {
395 static_assert(CompI >= 0 && CompJ >= 0, "CompI >= 0 && CompJ >= 0");
396
400 constexpr int operator()(int i) const noexcept {
401 const int map[2] = {CompI, CompJ};
402 return i == CompI || i == CompJ ? map[std::size_t(i == CompI)] : i;
403 }
404};
405
406template <int CompI> struct SwapComponents<CompI, -1> {
407 static_assert(CompI >= 0, "CompI >= 0");
408
409 int J;
410 constexpr int operator()(int i) const noexcept {
411 const int map[2] = {CompI, J};
412 return i == CompI || i == J ? map[std::size_t(i == CompI)] : i;
413 }
414};
415
416template <int CompJ> struct SwapComponents<-1, CompJ> {
417 static_assert(CompJ >= 0, "CompJ >= 0");
418
419 int I;
420 constexpr int operator()(int i) const noexcept {
421 const int map[2] = {I, CompJ};
422 return i == I || i == CompJ ? map[std::size_t(i == I)] : i;
423 }
424};
425
426template <> struct SwapComponents<-1, -1> {
427 int I;
428 int J;
429 constexpr int operator()(int i) const noexcept {
430 const int map[2] = {I, J};
431 return i == I || i == J ? map[std::size_t(i == I)] : i;
432 }
433};
434
436
437template <int CompI, int CompJ> static constexpr SwapComponents<CompI, CompJ> swap_indices{};
438
439static_assert(sizeof(SwapComponents<0, 1>) == 1 );
440static_assert(sizeof(DynamicSwapComponents) == 2 * sizeof(int) );
441static_assert(sizeof(SwapComponents<0, -1>) == sizeof(int) );
442static_assert(sizeof(SwapComponents<-1, 1>) == sizeof(int) );
443static_assert(std::is_trivially_default_constructible<MapComponents<Identity, SwapComponents<0, 1>>>() ); // NOLINT(bugprone-throw-keyword-missing)
444static_assert(std::is_trivially_copy_assignable<MapComponents<Identity, SwapComponents<0, 1>>>() ); // NOLINT(bugprone-throw-keyword-missing)
445static_assert(std::is_trivially_copy_constructible<MapComponents<Identity, SwapComponents<0, 1>>>() ); // NOLINT(bugprone-throw-keyword-missing)
446static_assert(IsFabProjection<MapComponents<Identity, SwapComponents<0, 1>>, FArrayBox>() ); // NOLINT(bugprone-throw-keyword-missing)
447
448static_assert(swap_indices<0, 1>(0) == 1 );
449static_assert(swap_indices<0, 1>(1) == 0 );
450static_assert(swap_indices<0, 1>(2) == 2 );
451static_assert(DynamicSwapComponents{0, 1}(0) == 1 );
452static_assert(DynamicSwapComponents{0, 1}(1) == 0 );
453static_assert(DynamicSwapComponents{0, 1}(2) == 2 );
454
456// [class.CommData]
457// [class.CommHandler]
458
492
493#ifdef AMREX_USE_MPI
496 int n_components, std::size_t object_size, std::size_t align);
497
499void PostRecvs(CommData& recv, int mpi_tag);
500
502void PostSends(CommData& send, int mpi_tag);
503
508void TranslateRanks(CommData& comm, MPI_Comm c);
509
515
521void Alltoallv(CommData const& send, CommData& recv, MPI_Comm c);
522
527 int n_components, std::size_t object_size, std::size_t block_size,
528 MPI_Comm c);
529
534void Alltoall(CommData const& send, CommData& recv, std::size_t block_size, MPI_Comm c);
535
537void ReleaseCommBuffer(CommData& comm);
538
540void AcquireCommBuffer(CommData& comm);
541#endif
542
547#ifdef AMREX_USE_MPI
548 int mpi_tag{};
551#endif
552};
553
555// [concept.DataPacking]
556//
557template <typename... Args>
558using PrepareSendBuffers_t = decltype(PrepareSendBuffers(std::declval<Args>()...));
559
560template <typename... Args>
561using PrepareRecvBuffers_t = decltype(PrepareRecvBuffers(std::declval<Args>()...));
562
563template <typename... Args>
564using PackSendBuffers_t = decltype(PackSendBuffers(std::declval<Args>()...));
565
566template <typename... Args>
567using UnpackRecvBuffers_t = decltype(UnpackRecvBuffers(std::declval<Args>()...));
568
569template <typename... Args>
570using LocalCopy_t = decltype(LocalCopy(std::declval<Args>()...));
571
573#if defined(AMREX_USE_CUDA) && defined(_WIN32)
574template <typename DP, BaseFabType FAB> struct IsDataPacking : std::true_type {};
575#else
576template <typename DP, BaseFabType FAB>
579 IsDetected<LocalCopy_t, DP&, FabArray<FAB>&, const FabArray<FAB>&, const FabArrayBase::CopyComTagsContainer&>
580#ifdef AMREX_USE_MPI
581 ,IsDetected<PrepareSendBuffers_t, DP&, FabArray<FAB>&, const FabArray<FAB>&, CommData&, const FabArrayBase::MapOfCopyComTagContainers&>,
582 IsDetected<PrepareRecvBuffers_t, DP&, FabArray<FAB>&, const FabArray<FAB>&, CommData&, const FabArrayBase::MapOfCopyComTagContainers&>,
583 IsDetected<PackSendBuffers_t, DP&, const FabArray<FAB>&, CommData&>,
584 IsDetected<UnpackRecvBuffers_t, DP&, FabArray<FAB>&, CommData&>
585#endif
586 > {};
587#endif
588
589template <BaseFabType FAB, class DTOS = Identity, class Proj = Identity>
591void
592local_copy_cpu (FabArray<FAB>& dest, const FabArray<FAB>& src, int dcomp, int scomp, int ncomp,
593 FabArrayBase::CopyComTagsContainer const& local_tags, DTOS const& dtos = DTOS{},
594 Proj const& proj = Proj{}) noexcept;
595
596
597template <BaseFabType FAB, class DTOS = Identity, class Proj = Identity>
598requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
599void
600unpack_recv_buffer_cpu (FabArray<FAB>& mf, int dcomp, int ncomp, Vector<char*> const& recv_data,
601 Vector<std::size_t> const& recv_size,
603 DTOS const& dtos = DTOS{}, Proj const& proj = Proj{}) noexcept;
604
605
606#ifdef AMREX_USE_GPU
607template <BaseFabType FAB, class DTOS = Identity, class Proj = Identity>
608requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
609void
610local_copy_gpu (FabArray<FAB>& dest, const FabArray<FAB>& src, int dcomp, int scomp, int ncomp,
611 FabArrayBase::CopyComTagsContainer const& local_tags, DTOS const& dtos = DTOS{},
612 Proj const& proj = Proj{}) noexcept;
613
614
615template <BaseFabType FAB, class DTOS = Identity, class Proj = Identity>
616requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
617void
618unpack_recv_buffer_gpu (FabArray<FAB>& mf, int scomp, int ncomp,
619 Vector<char*> const& recv_data,
620 Vector<std::size_t> const& recv_size,
622 DTOS const& dtos = DTOS{}, Proj const& proj = Proj{});
623#endif
624
626// [DataPacking.PackComponents]
627//
628// PackComponents is the simplest data packing policy.
629// This provides us with sane default behaviour that we can use when defining new
630// data packing policies.
631
638
640template <BaseFabType FAB>
641void
642LocalCopy (const PackComponents& components, FabArray<FAB>& dest, const FabArray<FAB>& src,
643 const FabArrayBase::CopyComTagsContainer& local_tags)
644{
645#ifdef AMREX_USE_GPU
646 if (Gpu::inLaunchRegion()) {
647 local_copy_gpu(dest, src, components.dest_component, components.src_component,
648 components.n_components, local_tags);
649 } else
650#endif
651 {
652 local_copy_cpu(dest, src, components.dest_component, components.src_component,
653 components.n_components, local_tags);
654 }
655}
656
657#ifdef AMREX_USE_MPI
659template <BaseFabType FAB>
660void
661PrepareSendBuffers (const PackComponents& components, FabArray<FAB>& dest, const FabArray<FAB>& src,
663{
664 using T = typename FAB::value_type;
665 ignore_unused(dest, src);
666 PrepareCommBuffers(comm, cctc, components.n_components, sizeof(T), alignof(T));
667}
668
670template <BaseFabType FAB>
671void
672PrepareRecvBuffers (const PackComponents& components, FabArray<FAB>& dest, const FabArray<FAB>& src,
674{
675 using T = typename FAB::value_type;
676 ignore_unused(dest, src);
677 PrepareCommBuffers(comm, cctc, components.n_components, sizeof(T), alignof(T));
678}
679
681template <BaseFabType FAB>
682void
683PackSendBuffers (const PackComponents& components, const FabArray<FAB>& src, CommData& send)
684{
685#ifdef AMREX_USE_GPU
686 if (Gpu::inLaunchRegion()) {
688 send.data, send.size, send.cctc, send.id);
689 } else
690#endif // AMREX_USE_GPU
691 {
693 send.data, send.size, send.cctc);
694 }
695}
696
698template <BaseFabType FAB>
699void
700UnpackRecvBuffers (const PackComponents& components, FabArray<FAB>& dest, const CommData& recv)
701{
702#ifdef AMREX_USE_GPU
703 if (Gpu::inLaunchRegion()) {
704 unpack_recv_buffer_gpu(dest, components.dest_component, components.n_components, recv.data,
705 recv.size, recv.cctc);
706 } else
707#endif // AMREX_USE_GPU
708 {
709 unpack_recv_buffer_cpu(dest, components.dest_component, components.n_components, recv.data,
710 recv.size, recv.cctc);
711 }
712}
713#endif // AMREX_USE_MPI
714
715static_assert(IsDataPacking<PackComponents, FArrayBox>(), // NOLINT(bugprone-throw-keyword-missing)
716 "PackComponents is expected to satisfy the concept DataPacking.");
717
719// [DataPacking.ApplyDtosAndProjectionOnReciever]
720//
725template <typename DTOS = Identity, typename FabProj = Identity>
727 constexpr ApplyDtosAndProjectionOnReciever() = default;
728 constexpr ApplyDtosAndProjectionOnReciever(const PackComponents& components, DTOS dtos_ = DTOS{}, FabProj proj_ = FabProj{})
729 : PackComponents(components), dtos(std::move(dtos_)), proj(std::move(proj_)) {}
730
731 DTOS dtos;
732 FabProj proj;
733
734 static_assert(IsCallableR<Dim3, DTOS, Dim3>::value, "DTOS needs to be a callable: Dim3 -> Dim3");
735 static_assert(IsFabProjection<FabProj, FArrayBox>(), "FabProj needs to be at least a projection on FArrayBox.");
736};
737
739template <BaseFabType FAB, typename DTOS, typename FabProj>
740void
742 const FabArray<FAB>& src, const FabArrayBase::CopyComTagsContainer& local_tags)
743{
744 static_assert(IsFabProjection<FabProj, FAB>(), "FabProj needs to be a projection for given FAB type.");
745 // Received data is indexed from 0, but local data from src_component.
746 AMREX_ALWAYS_ASSERT_WITH_MESSAGE((std::is_same_v<FabProj,Identity>) ||
747 (std::is_same_v<FabProj,MapComponents<Identity>>) ||
748 packing.src_component == 0,
749 "NonLocalBC: src_component must be 0 unless the projection is Identity");
750#ifdef AMREX_USE_GPU
751 if (Gpu::inLaunchRegion()) {
752 local_copy_gpu(dest, src, packing.dest_component, packing.src_component,
753 packing.n_components, local_tags, packing.dtos, packing.proj);
754 } else
755#endif
756 {
757 local_copy_cpu(dest, src, packing.dest_component, packing.src_component,
758 packing.n_components, local_tags, packing.dtos, packing.proj);
759 }
760}
761
762#ifdef AMREX_USE_MPI
764template <BaseFabType FAB, typename DTOS, typename FabProj>
765void
767 FabArray<FAB>& dest, const CommData& recv)
768{
769 // If FAB is not FArrayBox we have not checked for the correct types yet.
770 static_assert(IsFabProjection<FabProj, FAB>(), "FabProj needs to be a projection for given FAB type.");
771 AMREX_ALWAYS_ASSERT_WITH_MESSAGE((std::is_same_v<FabProj,Identity>) ||
772 (std::is_same_v<FabProj,MapComponents<Identity>>) ||
773 packing.src_component == 0,
774 "NonLocalBC: src_component must be 0 unless the projection is Identity");
775#ifdef AMREX_USE_GPU
776 if (Gpu::inLaunchRegion()) {
777 unpack_recv_buffer_gpu(dest, packing.dest_component, packing.n_components, recv.data,
778 recv.size, recv.cctc, packing.dtos, packing.proj);
779 } else
780#endif // AMREX_USE_GPU
781 {
782 unpack_recv_buffer_cpu(dest, packing.dest_component, packing.n_components, recv.data,
783 recv.size, recv.cctc, packing.dtos, packing.proj);
784 }
785}
786#endif // AMREX_USE_MPI
787
788static_assert(IsDataPacking<ApplyDtosAndProjectionOnReciever<>, FArrayBox>(), // NOLINT(bugprone-throw-keyword-missing)
789 "ApplyDtosAndProjectionOnReciever<> is expected to satisfy the DataPacking concept.");
790
792// [ParallelCopy_nowait]
793
794static constexpr struct NoLocalCopy {} no_local_copy{};
795static constexpr struct DoLocalCopy {} do_local_copy{};
796
818template <BaseFabType FAB, typename DataPacking>
819requires (IsDataPacking<DataPacking, FAB>::value)
820#ifdef AMREX_USE_MPI
821[[nodiscard]] CommHandler
823 const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing)
824{
825 CommHandler handler{};
826 if (ParallelContext::NProcsSub() == 1) {
827 return handler;
828 }
829 //
830 // Do this before prematurely exiting if running in parallel.
831 // Otherwise sequence numbers will not match across MPI processes.
832 //
834
835 if (cmd.m_RcvTags && !(cmd.m_RcvTags->empty())) {
836 PrepareRecvBuffers(data_packing, dest, src, handler.recv, *cmd.m_RcvTags);
837 PostRecvs(handler.recv, handler.mpi_tag);
838 }
839
840 if (cmd.m_SndTags && !(cmd.m_SndTags->empty())) {
841 PrepareSendBuffers(data_packing, dest, src, handler.send, *cmd.m_SndTags);
842 PackSendBuffers(data_packing, src, handler.send);
843 PostSends(handler.send, handler.mpi_tag);
844 }
845 return handler;
846}
847#else
849 const FabArrayBase::CommMetaData&, const DataPacking&) {
850 return CommHandler{};
851}
852#endif
853
875template <BaseFabType FAB, typename DataPacking>
877#ifdef AMREX_USE_MPI
878[[nodiscard]]
879#endif
882 const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing)
883{
884 CommHandler comm = ParallelCopy_nowait(no_local_copy, dest, src, cmd, data_packing);
885 // Eagerly do the local work and hope for some overlap with communication
886 if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
887 LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
888 }
889 return comm;
890}
891
892// [ParallelCopy_finish]
893
910template <BaseFabType FAB, typename DataPacking>
911requires (IsDataPacking<DataPacking, FAB>::value)
912void
913#ifdef AMREX_USE_MPI
915 const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing)
916{
917 // If any FabArray is empty we have nothing to do.
918 if (dest.empty()) {
919 return;
920 }
921 // Return if nothing do
922 if (ParallelContext::NProcsSub() == 1) {
923 return;
924 }
925 // Unpack receives
926 if (cmd.m_RcvTags && !(cmd.m_RcvTags->empty())) {
928#ifdef AMREX_DEBUG
929 if (!CheckRcvStats(handler.recv.stats, handler.recv.size, handler.mpi_tag)) {
930 amrex::Abort("NonLocalPC::ParallelCopy_finish failed with wrong message size");
931 }
932#endif
933 UnpackRecvBuffers(data_packing, dest, handler.recv);
934 }
935
936 // Wait for all sends to be done
937 if (cmd.m_SndTags && !(cmd.m_SndTags->empty())) {
939 }
940}
941#else
943#endif
944
962template <BaseFabType FAB, typename DataPacking>
963requires (IsDataPacking<DataPacking, FAB>::value)
964void
966 const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing)
967{
968 // Eagerly do the local work and hope for some overlap with communication
969 if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
970 LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
971 }
972 ParallelCopy_finish(dest, std::move(handler), cmd, data_packing); // NOLINT
973}
974
976// [ParallelCopy_alltoallv]
977
1002template <BaseFabType FAB, typename DataPacking>
1003requires (IsDataPacking<DataPacking, FAB>::value)
1004void
1006 const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing,
1007 CommHandler* cached = nullptr, MPI_Comm comm = MPI_COMM_NULL)
1008{
1009 if (dest.empty()) { return; }
1010
1011#ifdef AMREX_USE_MPI
1012 if (comm == MPI_COMM_NULL) { comm = ParallelContext::CommunicatorSub(); }
1013 int nprocs_comm;
1014 MPI_Comm_size(comm, &nprocs_comm);
1015 if (nprocs_comm > 1)
1016 {
1017 CommHandler local_handler{};
1018 CommHandler& handler = cached ? *cached : local_handler;
1019
1020 static const FabArrayBase::MapOfCopyComTagContainers no_tags{};
1021 auto const& rcv_tags = cmd.m_RcvTags ? *cmd.m_RcvTags : no_tags;
1022 auto const& snd_tags = cmd.m_SndTags ? *cmd.m_SndTags : no_tags;
1023
1024 // PrepareCommBuffers sets the id even when there is nothing to communicate.
1025 if (handler.send.id == std::numeric_limits<std::uint64_t>::max()) {
1026 PrepareRecvBuffers(data_packing, dest, src, handler.recv, rcv_tags);
1027 PrepareSendBuffers(data_packing, dest, src, handler.send, snd_tags);
1028 TranslateRanks(handler.recv, comm);
1029 TranslateRanks(handler.send, comm);
1030 PrepareAlltoallvCounts(handler.recv, comm);
1031 PrepareAlltoallvCounts(handler.send, comm);
1032 } else {
1033 AcquireCommBuffer(handler.recv);
1034 AcquireCommBuffer(handler.send);
1035 }
1036
1037 if (!snd_tags.empty()) {
1038 PackSendBuffers(data_packing, src, handler.send);
1039 }
1040
1041 // The local copy comes first. MPI_Ialltoallv would allow overlapping
1042 // it, but it was slower than the blocking call on the systems tested.
1043 if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
1044 LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
1045 }
1046
1047 Alltoallv(handler.send, handler.recv, comm);
1048
1049 if (!rcv_tags.empty()) {
1050 UnpackRecvBuffers(data_packing, dest, handler.recv);
1051 }
1052 if (cached) {
1053 ReleaseCommBuffer(handler.send);
1054 ReleaseCommBuffer(handler.recv);
1055 }
1056 return;
1057 }
1058#else
1059 amrex::ignore_unused(cached, comm);
1060#endif
1061
1062 if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
1063 LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
1064 }
1065}
1066
1071template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
1072requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
1073void
1075 const FabArrayBase::CommMetaData& cmd,
1076 int srccomp, int destcomp, int numcomp,
1077 DTOS const& dtos = DTOS{}, Proj const& proj = Proj{},
1078 CommHandler* cached = nullptr, MPI_Comm comm = MPI_COMM_NULL)
1079{
1080 PackComponents components{};
1081 components.dest_component = destcomp;
1082 components.src_component = srccomp;
1083 components.n_components = numcomp;
1084 ApplyDtosAndProjectionOnReciever<DTOS, Proj> packing{components, dtos, proj};
1085 ParallelCopy_alltoallv(dest, src, cmd, packing, cached, comm);
1086}
1087
1089// [ParallelCopy_alltoall]
1090
1098template <BaseFabType FAB, typename DataPacking>
1100void
1102 const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing,
1103 std::size_t block_size, CommHandler* cached = nullptr,
1104 MPI_Comm comm = MPI_COMM_NULL)
1105{
1106 if (dest.empty()) { return; }
1107
1108#ifdef AMREX_USE_MPI
1109 if (comm == MPI_COMM_NULL) { comm = ParallelContext::CommunicatorSub(); }
1110 int nprocs_comm;
1111 MPI_Comm_size(comm, &nprocs_comm);
1112 if (nprocs_comm > 1)
1113 {
1114 using T = typename FAB::value_type;
1115
1116 CommHandler local_handler{};
1117 CommHandler& handler = cached ? *cached : local_handler;
1118
1119 static const FabArrayBase::MapOfCopyComTagContainers no_tags{};
1120 auto const& rcv_tags = cmd.m_RcvTags ? *cmd.m_RcvTags : no_tags;
1121 auto const& snd_tags = cmd.m_SndTags ? *cmd.m_SndTags : no_tags;
1122
1123 if (handler.send.id == std::numeric_limits<std::uint64_t>::max()) {
1124 PrepareCommBuffersUniform(handler.recv, rcv_tags, data_packing.n_components,
1125 sizeof(T), block_size, comm);
1126 PrepareCommBuffersUniform(handler.send, snd_tags, data_packing.n_components,
1127 sizeof(T), block_size, comm);
1128 } else {
1129 AcquireCommBuffer(handler.recv);
1130 AcquireCommBuffer(handler.send);
1131 }
1132
1133 if (!snd_tags.empty()) {
1134 PackSendBuffers(data_packing, src, handler.send);
1135 }
1136
1137 if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
1138 LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
1139 }
1140
1141 Alltoall(handler.send, handler.recv, block_size, comm);
1142
1143 if (!rcv_tags.empty()) {
1144 UnpackRecvBuffers(data_packing, dest, handler.recv);
1145 }
1146 if (cached) {
1147 ReleaseCommBuffer(handler.send);
1148 ReleaseCommBuffer(handler.recv);
1149 }
1150 return;
1151 }
1152#else
1153 amrex::ignore_unused(block_size, cached, comm);
1154#endif
1155
1156 if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
1157 LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
1158 }
1159}
1160
1164template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
1165requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
1166void
1168 const FabArrayBase::CommMetaData& cmd,
1169 int srccomp, int destcomp, int numcomp,
1170 DTOS const& dtos, Proj const& proj,
1171 std::size_t block_size, CommHandler* cached = nullptr,
1172 MPI_Comm comm = MPI_COMM_NULL)
1173{
1174 PackComponents components{};
1175 components.dest_component = destcomp;
1176 components.src_component = srccomp;
1177 components.n_components = numcomp;
1178 ApplyDtosAndProjectionOnReciever<DTOS, Proj> packing{components, dtos, proj};
1179 ParallelCopy_alltoall(dest, src, cmd, packing, block_size, cached, comm);
1180}
1181
1205template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
1206requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
1207void
1209 SrcComp srccomp, DestComp destcomp, NumComps numcomp, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{})
1210{
1211 PackComponents components{};
1212 components.dest_component = destcomp.i;
1213 components.src_component = srccomp.i;
1214 components.n_components = numcomp.n;
1215#if defined(__GNUC__) && !defined(__clang__)
1216#pragma GCC diagnostic push
1217#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
1218#endif
1219 ApplyDtosAndProjectionOnReciever<DTOS, Proj> packing{components, dtos, proj};
1220 CommHandler handler = ParallelCopy_nowait(dest, src, cmd, packing);
1221 ParallelCopy_finish(dest, std::move(handler), cmd, packing); // NOLINT
1222#if defined(__GNUC__) && !defined(__clang__)
1223#pragma GCC diagnostic pop
1224#endif
1225}
1226
1250template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
1251requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
1252void
1254 int srccomp, int destcomp, int numcomp, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{})
1255{
1256 ParallelCopy(dest, src, cmd, SrcComp(srccomp), DestComp(destcomp), NumComps(numcomp), dtos, proj);
1257}
1258
1289template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
1292ParallelCopy (FabArray<FAB>& dest, const Box& destbox, const FabArray<FAB>& src, SrcComp srccomp,
1293 DestComp destcomp, NumComps numcomp, const IntVect& ngrow, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{})
1294{
1295 MultiBlockCommMetaData cmd(dest, destbox, src, ngrow, dtos);
1296 ParallelCopy(dest, src, cmd, srccomp, destcomp, numcomp, dtos, proj);
1297 return cmd;
1298}
1299
1330template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
1333ParallelCopy (FabArray<FAB>& dest, const Box& destbox, const FabArray<FAB>& src, int srccomp,
1334 int destcomp, int numcomp, const IntVect& ngrow, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{})
1335{
1336 return ParallelCopy(dest, destbox, src, SrcComp(srccomp), DestComp(destcomp), NumComps(numcomp), ngrow, dtos, proj);
1337}
1338
1339// Rotate90 fills the lo-x and lo-y boundary regions by rotating the data
1340// around (x=0,y=0) by 90 degrees in either direction. It also fills the
1341// corner of lo-x and lo-y boundary region by rotating the data by 180
1342// degrees.
1343
1344template <BaseFabType FAB>
1345void
1346Rotate90 (FabArray<FAB>& mf, int scomp, int ncomp, IntVect const& nghost, Box const& domain);
1347
1348template <BaseFabType FAB>
1349void
1350Rotate90 (FabArray<FAB>& mf, Box const& domain);
1351
1352// Rotate180 fills the lo-x boundary by rotating the data around
1353// (x=0,y=L_y/2) by 180 degrees.
1354
1355template <BaseFabType FAB>
1356void
1357Rotate180 (FabArray<FAB>& mf, int scomp, int ncomp, IntVect const& nghost, Box const& domain);
1358
1359template <BaseFabType FAB>
1360void
1361Rotate180 (FabArray<FAB>& mf, Box const& domain);
1362
1363// Fill the polar boundaries of the spherical coordinates (theta, phi, r).
1364// The lo-x boundary is filled with f(-x,y) = f(x,mod(y+pi,2*pi)), and
1365// the hi-x boundary is filled with f(pi+x,y) = f(pi-x,mod(y+pi,2*pi)).
1366
1367template <BaseFabType FAB>
1368void
1369FillPolar (FabArray<FAB>& mf, int scomp, int ncomp, IntVect const& nghost, Box const& domain);
1370
1371template <BaseFabType FAB>
1372void
1373FillPolar (FabArray<FAB>& mf, Box const& domain);
1374
1412template <BaseFabType FAB, typename DTOS, typename Proj = Identity>
1414[[nodiscard]]
1417 int scomp, int ncomp, DTOS const& dtos,
1418 Proj const& proj = Proj{});
1419
1456template <BaseFabType FAB, typename DTOS, typename Proj = Identity>
1457requires (IsCallableR<Dim3,DTOS,Dim3>::value && IsFabProjection<Proj,FAB>::value)
1458void
1461 int scomp, int ncomp, DTOS const& dtos,
1462 Proj const& proj = Proj{});
1463
1496template <BaseFabType FAB, typename DTOS, typename Proj = Identity>
1497requires (IsCallableR<Dim3,DTOS,Dim3>::value && IsFabProjection<Proj,FAB>::value)
1498void
1500 int scomp, int ncomp, DTOS const& dtos, Proj const& proj = Proj{})
1501{
1502 BL_PROFILE("FillBoundary(cmd)");
1503 auto handler = FillBoundary_nowait(mf, cmd, scomp, ncomp, dtos, proj);
1504 FillBoundary_finish(std::move(handler), mf, cmd, scomp, ncomp, dtos, proj);
1505}
1506
1521template <BaseFabType FAB, typename DTOS>
1522requires (IsCallableR<Dim3,DTOS,Dim3>::value)
1523[[nodiscard]]
1524FabArrayBase::CommMetaData
1526 Geometry const& geom, DTOS const& dtos);
1527
1528}
1529
1530#include <AMReX_NonLocalBCImpl.H>
1531
1532namespace amrex {
1537 using NonLocalBC::SphThetaPhiRIndexMapping;
1538 using NonLocalBC::SphThetaPhiRComponentMapping;
1547}
1548
1549#endif
#define BL_PROFILE(a)
Definition AMReX_BLProfiler.H:593
#define AMREX_ALWAYS_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:49
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
Floating-point specialization of BaseFab plus FAB I/O helpers.
Distributed container of FAB objects plus copy and fill utilities.
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
#define AMREX_D_DECL(a, b, c)
Definition AMReX_SPACE.H:171
Reference-counted collection of Boxes.
Definition AMReX_BoxArray.H:681
__host__ __device__ const IntVectND< dim > & bigEnd() const &noexcept
Return the inclusive upper bound of the box.
Definition AMReX_Box.H:136
__host__ __device__ IndexTypeND< dim > ixType() const noexcept
Return the indexing type.
Definition AMReX_Box.H:148
__host__ __device__ BoxND & setType(const IndexTypeND< dim > &t) noexcept
Set indexing type.
Definition AMReX_Box.H:534
__host__ __device__ const IntVectND< dim > & smallEnd() const &noexcept
Return the inclusive lower bound of the box.
Definition AMReX_Box.H:124
Calculates the distribution of FABs to MPI processes.
Definition AMReX_DistributionMapping.H:55
A Fortran Array of REALs.
Definition AMReX_FArrayBox.H:237
Base class for FabArray.
Definition AMReX_FabArrayBase.H:47
CopyComTag::CopyComTagsContainer CopyComTagsContainer
Definition AMReX_FabArrayBase.H:236
CopyComTag::MapOfCopyComTagContainers MapOfCopyComTagContainers
Definition AMReX_FabArrayBase.H:237
bool empty() const noexcept
Definition AMReX_FabArrayBase.H:94
An Array of FortranArrayBox(FAB)-like Objects.
Definition AMReX_FabArray.H:383
static void pack_send_buffer_gpu(FabArray< FAB > const &src, int scomp, int ncomp, Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< const CopyComTagsContainer * > const &send_cctc, std::uint64_t id)
static void pack_send_buffer_cpu(FabArray< FAB > const &src, int scomp, int ncomp, Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< const CopyComTagsContainer * > const &send_cctc)
Rectangular problem domain geometry.
Definition AMReX_Geometry.H:87
__host__ __device__ constexpr Dim3 dim3() const noexcept
Definition AMReX_IntVect.H:269
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
ArrayND< T, 4, true > Array4
Convenient alias for a 4D ArrayND with three spatial dimensions plus a component dimension.
Definition AMReX_Array4.H:1352
bool inLaunchRegion() noexcept
Definition AMReX_GpuControl.H:88
Definition AMReX_NonLocalBC.cpp:53
void AcquireCommBuffer(CommData &comm)
Allocate the_data of a prepared CommData again after ReleaseCommBuffer.
Definition AMReX_NonLocalBC.cpp:244
IntVect Apply(DTOS const &dtos, const IntVect &iv)
Applies the Dim3 to Dim3 mapping onto IntVects.
Definition AMReX_NonLocalBC.H:115
void Rotate90(FabArray< FAB > &mf, int scomp, int ncomp, IntVect const &nghost, Box const &domain)
decltype(PrepareRecvBuffers(std::declval< Args >()...)) PrepareRecvBuffers_t
Definition AMReX_NonLocalBC.H:561
void ReleaseCommBuffer(CommData &comm)
Free the_data of a prepared CommData, keeping the rest of the meta-data.
Definition AMReX_NonLocalBC.cpp:238
FabArrayBase::CommMetaData makeFillBoundaryMetaData(FabArray< FAB > &mf, IntVect const &nghost, Geometry const &geom, DTOS const &dtos)
Make metadata for FillBoundary.
void unpack_recv_buffer_cpu(FabArray< FAB > &mf, int dcomp, int ncomp, Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< FabArrayBase::CopyComTagsContainer const * > const &recv_cctc, DTOS const &dtos=DTOS{}, Proj const &proj=Proj{}) noexcept
void Alltoallv(CommData const &send, CommData &recv, MPI_Comm comm)
Exchange all send buffers for all recv buffers with one MPI_Alltoallv.
Definition AMReX_NonLocalBC.cpp:284
IntVect ApplyInverse(DTOS const &dtos, const IntVect &iv)
Applies the Dim3 to Dim3 invserse mapping onto IntVects.
Definition AMReX_NonLocalBC.H:182
void PrepareCommBuffers(CommData &comm, const FabArrayBase::MapOfCopyComTagContainers &cctc, int n_components, std::size_t object_size, std::size_t align)
Fill all class member variables of comm but the request and the stats vector.
Definition AMReX_NonLocalBC.cpp:57
Box InverseImage(DTOS const &dtos, const Box &box)
Applies the inverse Dim3 to Dim3 mapping onto Boxes without changing the index type.
Definition AMReX_NonLocalBC.H:199
void PackSendBuffers(const PackComponents &components, const FabArray< FAB > &src, CommData &send)
Serializes FAB data without any knowledge of a DTOS nor a projection.
Definition AMReX_NonLocalBC.H:683
decltype(std::declval< T >().Inverse(std::declval< Args >()...)) Inverse_t
Return type of an InverseImage class member function.
Definition AMReX_NonLocalBC.H:17
CommHandler FillBoundary_nowait(FabArray< FAB > &mf, const FabArrayBase::CommMetaData &cmd, int scomp, int ncomp, DTOS const &dtos, Proj const &proj=Proj{})
Start communication to fill boundary.
void Alltoall(CommData const &send, CommData &recv, std::size_t block_size, MPI_Comm comm)
Exchange uniform blocks of block_size bytes with one MPI_Alltoall.
Definition AMReX_NonLocalBC.cpp:260
decltype(PrepareSendBuffers(std::declval< Args >()...)) PrepareSendBuffers_t
Definition AMReX_NonLocalBC.H:558
void ParallelCopy_finish(FabArray< FAB > &dest, CommHandler handler, const FabArrayBase::CommMetaData &cmd, const DataPacking &data_packing)
Definition AMReX_NonLocalBC.H:914
void Rotate180(FabArray< FAB > &mf, int scomp, int ncomp, IntVect const &nghost, Box const &domain)
void PrepareAlltoallvCounts(CommData &comm, MPI_Comm c)
Fill comm.counts and comm.displs for MPI_Alltoallv on communicator c.
Definition AMReX_NonLocalBC.cpp:171
void LocalCopy(const PackComponents &components, FabArray< FAB > &dest, const FabArray< FAB > &src, const FabArrayBase::CopyComTagsContainer &local_tags)
Dispatch local copies to the default behaviour that knows no DTOS nor projection.
Definition AMReX_NonLocalBC.H:642
static constexpr SwapComponents< CompI, CompJ > swap_indices
Definition AMReX_NonLocalBC.H:437
void PostSends(CommData &send, int mpi_tag)
Initiate all sends with MPI_Isend calls associated with tag mpi_tag.
Definition AMReX_NonLocalBC.cpp:135
void local_copy_cpu(FabArray< FAB > &dest, const FabArray< FAB > &src, int dcomp, int scomp, int ncomp, FabArrayBase::CopyComTagsContainer const &local_tags, DTOS const &dtos=DTOS{}, Proj const &proj=Proj{}) noexcept
void PostRecvs(CommData &recv, int mpi_tag)
Initiate all recvieves with MPI_Irecv calls associated with tag mpi_tag.
Definition AMReX_NonLocalBC.cpp:119
SwapComponents<-1, -1 > DynamicSwapComponents
Definition AMReX_NonLocalBC.H:435
void FillBoundary_finish(CommHandler handler, FabArray< FAB > &mf, const FabArrayBase::CommMetaData &cmd, int scomp, int ncomp, DTOS const &dtos, Proj const &proj=Proj{})
Finish communication started by FillBoundary_nowait.
void FillPolar(FabArray< FAB > &mf, int scomp, int ncomp, IntVect const &nghost, Box const &domain)
void FillBoundary(FabArray< FAB > &mf, const FabArrayBase::CommMetaData &cmd, int scomp, int ncomp, DTOS const &dtos, Proj const &proj=Proj{})
Fill ghost cells for FabArray/MultiFab.
Definition AMReX_NonLocalBC.H:1499
void UnpackRecvBuffers(const PackComponents &components, FabArray< FAB > &dest, const CommData &recv)
De-serializes FAB data without any knowledge of a DTOS nor a projection.
Definition AMReX_NonLocalBC.H:700
static constexpr struct amrex::NonLocalBC::DoLocalCopy do_local_copy
decltype(LocalCopy(std::declval< Args >()...)) LocalCopy_t
Definition AMReX_NonLocalBC.H:570
void local_copy_gpu(FabArray< FAB > &dest, const FabArray< FAB > &src, int dcomp, int scomp, int ncomp, FabArrayBase::CopyComTagsContainer const &local_tags, DTOS const &dtos=DTOS{}, Proj const &proj=Proj{}) noexcept
template MultiBlockCommMetaData ParallelCopy(FabArray< FArrayBox > &dest, const Box &destbox, const FabArray< FArrayBox > &src, int destcomp, int srccomp, int numcomp, const IntVect &ngrow, MultiBlockIndexMapping const &, Identity const &)
CommHandler ParallelCopy_nowait(NoLocalCopy, FabArray< FAB > &dest, const FabArray< FAB > &src, const FabArrayBase::CommMetaData &cmd, const DataPacking &data_packing)
Definition AMReX_NonLocalBC.H:822
static constexpr Identity identity
Definition AMReX_NonLocalBC.H:342
decltype(PackSendBuffers(std::declval< Args >()...)) PackSendBuffers_t
Definition AMReX_NonLocalBC.H:564
Box Image(DTOS const &dtos, const Box &box)
Applies the Dim3 to Dim3 mapping onto Boxes but does not change the index type.
Definition AMReX_NonLocalBC.H:133
decltype(UnpackRecvBuffers(std::declval< Args >()...)) UnpackRecvBuffers_t
Definition AMReX_NonLocalBC.H:567
void PrepareRecvBuffers(const PackComponents &components, FabArray< FAB > &dest, const FabArray< FAB > &src, CommData &comm, const FabArrayBase::MapOfCopyComTagContainers &cctc)
Calls PrepareComBuffers.
Definition AMReX_NonLocalBC.H:672
void ParallelCopy_alltoallv(FabArray< FAB > &dest, const FabArray< FAB > &src, const FabArrayBase::CommMetaData &cmd, const DataPacking &data_packing, CommHandler *cached=nullptr, MPI_Comm comm=MPI_COMM_NULL)
ParallelCopy that exchanges all messages with a single MPI_Alltoallv.
Definition AMReX_NonLocalBC.H:1005
static constexpr struct amrex::NonLocalBC::NoLocalCopy no_local_copy
void PrepareSendBuffers(const PackComponents &components, FabArray< FAB > &dest, const FabArray< FAB > &src, CommData &comm, const FabArrayBase::MapOfCopyComTagContainers &cctc)
Calls PrepareComBuffers.
Definition AMReX_NonLocalBC.H:661
void unpack_recv_buffer_gpu(FabArray< FAB > &mf, int scomp, int ncomp, Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< FabArrayBase::CopyComTagsContainer const * > const &recv_cctc, DTOS const &dtos=DTOS{}, Proj const &proj=Proj{})
void ParallelCopy_alltoall(FabArray< FAB > &dest, const FabArray< FAB > &src, const FabArrayBase::CommMetaData &cmd, const DataPacking &data_packing, std::size_t block_size, CommHandler *cached=nullptr, MPI_Comm comm=MPI_COMM_NULL)
ParallelCopy that exchanges uniform blocks with a single MPI_Alltoall.
Definition AMReX_NonLocalBC.H:1101
void PrepareCommBuffersUniform(CommData &comm, const FabArrayBase::MapOfCopyComTagContainers &cctc, int n_components, std::size_t object_size, std::size_t block_size, MPI_Comm c)
Like PrepareCommBuffers, but every process in communicator c gets a block of block_size bytes in the ...
Definition AMReX_NonLocalBC.cpp:185
void TranslateRanks(CommData &comm, MPI_Comm c)
Fill comm.comm_rank with the ranks of comm.rank (global ranks) in communicator c.
Definition AMReX_NonLocalBC.cpp:151
MPI_Comm CommunicatorSub() noexcept
sub-communicator for current frame
Definition AMReX_ParallelContext.H:70
int NProcsSub() noexcept
number of ranks in current frame
Definition AMReX_ParallelContext.H:74
void Waitall(Vector< MPI_Request > &, Vector< MPI_Status > &)
Definition AMReX_ParallelDescriptor.cpp:1312
int SeqNum() noexcept
Returns sequential message sequence numbers, usually used as tags for send/recv.
Definition AMReX_ParallelDescriptor.H:678
int MPI_Comm
Definition AMReX_ccse-mpi.H:51
static constexpr int MPI_COMM_NULL
Definition AMReX_ccse-mpi.H:59
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
std::unique_ptr< char, TheFaArenaDeleter > TheFaArenaPointer
Definition AMReX_FabArray.H:118
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
@ y
Definition AMReX_Orientation.H:14
@ x
Definition AMReX_Orientation.H:14
@ z
Definition AMReX_Orientation.H:14
IndexTypeND< 3 > IndexType
IndexType is an alias for amrex::IndexTypeND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:41
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
const int[]
Definition AMReX_BLProfiler.cpp:1665
std::is_same< Expected, Detected_t< Op, Args... > > IsDetectedExact
Definition AMReX_TypeTraits.H:186
This class stores both recv and send buffers with an associated MPI tag.
Definition AMReX_NonLocalBC.H:546
Logical traits let us combine multiple type requirements.
Definition AMReX_TypeTraits.H:232
Destination-component descriptor.
Definition AMReX_BaseFab.H:105
int i
Definition AMReX_BaseFab.H:108
A simple struct holding 3 int values for a 3D index.
Definition AMReX_Dim3.H:24
int x
Definition AMReX_Dim3.H:24
int z
Definition AMReX_Dim3.H:24
int y
Definition AMReX_Dim3.H:24
Definition AMReX_TypeTraits.H:238
Definition AMReX_FabArrayBase.H:563
std::unique_ptr< MapOfCopyComTagContainers > m_RcvTags
Definition AMReX_FabArrayBase.H:569
std::unique_ptr< MapOfCopyComTagContainers > m_SndTags
Definition AMReX_FabArrayBase.H:568
std::unique_ptr< CopyComTagsContainer > m_LocTags
Definition AMReX_FabArrayBase.H:567
Test if a given type T is callable with arguments of type Args...
Definition AMReX_TypeTraits.H:214
Test if a given type T is callable with arguments of type Args...
Definition AMReX_TypeTraits.H:208
This class stores data dependencies for an inter-block communication.
Definition AMReX_NonLocalBC.H:236
MultiBlockCommMetaData(const FabArrayBase &dst, const Box &dstbox, const FabArrayBase &src, const IntVect &ngrow, DTOS const &dtos)
Build global meta data by calling the define() member function.
This class specializes behaviour on local copies and unpacking receive buffers.
Definition AMReX_NonLocalBC.H:726
constexpr ApplyDtosAndProjectionOnReciever(const PackComponents &components, DTOS dtos_=DTOS{}, FabProj proj_=FabProj{})
Definition AMReX_NonLocalBC.H:728
FabProj proj
Definition AMReX_NonLocalBC.H:732
DTOS dtos
Definition AMReX_NonLocalBC.H:731
This class holds data buffers for either immediate MPI send or recv calls.
Definition AMReX_NonLocalBC.H:464
Vector< std::size_t > size
The size in bytes for each data transaction.
Definition AMReX_NonLocalBC.H:475
Vector< char * > data
Pointers to the_data that can be used for each single data transaction.
Definition AMReX_NonLocalBC.H:471
Vector< const FabArrayBase::CopyComTagsContainer * > cctc
For each request the copy comm tags for the corresponding data FABs.
Definition AMReX_NonLocalBC.H:481
std::size_t total_bytes
Size of the_data in bytes, so that it can be released and reacquired.
Definition AMReX_NonLocalBC.H:488
Vector< MPI_Status > stats
For each request the corresponding MPI_status, used for debugging.
Definition AMReX_NonLocalBC.H:479
Vector< MPI_Request > request
The associated MPI_Request for each data transaction.
Definition AMReX_NonLocalBC.H:477
TheFaArenaPointer the_data
Holds 'em all in one data pointer.
Definition AMReX_NonLocalBC.H:467
std::uint64_t id
Definition AMReX_NonLocalBC.H:489
Vector< int > rank
Stores MPI ranks. For recvs it is the 'from' rank and for sends it is the 'to' rank.
Definition AMReX_NonLocalBC.H:469
Vector< int > displs
Definition AMReX_NonLocalBC.H:486
Vector< std::size_t > offset
All offsets of data in the_data.
Definition AMReX_NonLocalBC.H:473
Vector< int > counts
size and offset per rank of that communicator, as MPI_Alltoallv wants them.
Definition AMReX_NonLocalBC.H:485
Vector< int > comm_rank
rank translated into the communicator used by Alltoall and Alltoallv.
Definition AMReX_NonLocalBC.H:483
This class stores both recv and send buffers with an associated MPI tag.
Definition AMReX_NonLocalBC.H:546
CommData send
Definition AMReX_NonLocalBC.H:550
int mpi_tag
Definition AMReX_NonLocalBC.H:548
CommData recv
Definition AMReX_NonLocalBC.H:549
Definition AMReX_NonLocalBC.H:795
Type trait that tests if T has an InverseImage class member function.
Definition AMReX_NonLocalBC.H:21
This class acts as a default no-op operator.
Definition AMReX_NonLocalBC.H:326
constexpr Dim3 operator()(Dim3 i) const noexcept
The identity function for Dim3.
Definition AMReX_NonLocalBC.H:328
constexpr T operator()(Array4< const T > array, Dim3 i, int comp=0) const noexcept(noexcept(array(i.x, i.y, i.z, comp)))
Definition AMReX_NonLocalBC.H:334
constexpr int operator()(int i) const noexcept
The identity for int.
Definition AMReX_NonLocalBC.H:340
static constexpr Dim3 Inverse(Dim3 i) noexcept
The identity function for Dim3.
Definition AMReX_NonLocalBC.H:330
This type trait tests if a given type DP satisfies the DataPacking concept for type FAB.
Definition AMReX_NonLocalBC.H:586
This type trait tests if a type P is a projection for FAB.
Definition AMReX_NonLocalBC.H:301
Tests if a given type IndexMap is usable as an index mapping between two index based coordinate syste...
Definition AMReX_NonLocalBC.H:28
Type trait for index mappings that are monotone in each component over the whole index space,...
Definition AMReX_NonLocalBC.H:38
This class takes a projection and a component map and combines them to form a new projection.
Definition AMReX_NonLocalBC.H:363
Base base
Definition AMReX_NonLocalBC.H:370
Map map
Definition AMReX_NonLocalBC.H:371
constexpr decltype(auto) operator()(Array4< const T > array, Dim3 i, int comp) const noexcept(noexcept(base(array, i, map(comp))))
Definition AMReX_NonLocalBC.H:375
This class stores data dependencies for an inter-block communication.
Definition AMReX_NonLocalBC.H:236
MultiBlockCommMetaData(const BoxArray &dstba, const DistributionMapping &dstdm, const Box &dstbox, const BoxArray &srcba, const DistributionMapping &srcdm, const IntVect &ngrow, DTOS const &dtos)
Build global meta data by calling the define() member function.
MultiBlockCommMetaData(const FabArrayBase &dst, const Box &dstbox, const FabArrayBase &src, const IntVect &ngrow, DTOS const &dtos)
Build global meta data by calling the define() member function.
void define(const BoxArray &dstba, const DistributionMapping &dstdm, const Box &dstbox, const BoxArray &srcba, const DistributionMapping &srcdm, const IntVect &ngrow, DTOS const &dtos)
Build global meta data that is being used to identify send and recv dependencies in communication rou...
This struct describes an affine index transformation for two coordinate systems.
Definition AMReX_NonLocalBC.H:48
IndexType operator()(IndexType it) const noexcept
Definition AMReX_NonLocalBC.H:87
IntVect sign
A vector of 1 and -1 describing the orientation in each component.
Definition AMReX_NonLocalBC.H:54
__host__ __device__ Dim3 operator()(Dim3 i) const noexcept
Applies this mapping on the index from destination space and returns an index in the source space.
Definition AMReX_NonLocalBC.H:62
__host__ __device__ Dim3 Inverse(Dim3 i) const noexcept
The inverse function is given by rearringing all above terms.
Definition AMReX_NonLocalBC.H:77
IntVect offset
The offset in the source index space.
Definition AMReX_NonLocalBC.H:52
IntVect permutation
This vector needs to be a valid permutation.
Definition AMReX_NonLocalBC.H:50
IndexType Inverse(IndexType it) const noexcept
Definition AMReX_NonLocalBC.H:91
Definition AMReX_NonLocalBC.H:794
Contains information about which components take part of the data transaction.
Definition AMReX_NonLocalBC.H:633
int dest_component
Definition AMReX_NonLocalBC.H:634
int src_component
Definition AMReX_NonLocalBC.H:635
int n_components
Definition AMReX_NonLocalBC.H:636
int J
Definition AMReX_NonLocalBC.H:428
int I
Definition AMReX_NonLocalBC.H:427
constexpr int operator()(int i) const noexcept
Definition AMReX_NonLocalBC.H:429
constexpr int operator()(int i) const noexcept
Definition AMReX_NonLocalBC.H:420
int I
Definition AMReX_NonLocalBC.H:419
constexpr int operator()(int i) const noexcept
Definition AMReX_NonLocalBC.H:410
int J
Definition AMReX_NonLocalBC.H:409
This is a permutation where only two components are swapped.
Definition AMReX_NonLocalBC.H:394
constexpr int operator()(int i) const noexcept
Swaps indices CompI and CompJ.
Definition AMReX_NonLocalBC.H:400
Number-of-components descriptor.
Definition AMReX_BaseFab.H:112
int n
Definition AMReX_BaseFab.H:115
Source-component descriptor.
Definition AMReX_BaseFab.H:98
int i
Definition AMReX_BaseFab.H:101